From c2a2bb94fb0c4c7174be42be5418661b3063820e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 18 Oct 2026 09:00:03 -0500 Subject: [PATCH 001/105] chore: open 0.16.0-alpha.1 on develop --- plugins/dw/.claude-plugin/plugin.json | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/dw/.claude-plugin/plugin.json b/plugins/dw/.claude-plugin/plugin.json index eb1bca4e..021c98a6 100644 --- a/plugins/dw/.claude-plugin/plugin.json +++ b/plugins/dw/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "dw", "description": "Compose MiniMax H3 video, MiniMax Music 3, LTX-2.5 and Kandinsky 6.0 workflows over a dw MCP server, and cut a multi-episode series from them: which template fits which shape, the hard rules, cost, and how to judge the output. Prompt format comes from the vendors' own guides.", - "version": "0.15.0", + "version": "0.16.0-alpha.1", "author": { "name": "Don Kackman" }, diff --git a/pyproject.toml b/pyproject.toml index 8c87583f..6ec884e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ name = "diffusers-workflow" # runtime (static in TOML because importing dw at build time would drag # torch into the build environment). Release tags must match it - see # docs/RELEASING.md -version = "0.15.0" +version = "0.16.0-alpha.1" description = "Declarative workflow engine and web UI for Hugging Face Diffusers" readme = "README.md" requires-python = ">=3.10" From b094a8ab13b4418f1832a0b0b72c78b1f3c6f45a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 16:32:54 -0500 Subject: [PATCH 002/105] docs(tasks): name generate_speech's TTS families where an agent looks get_task's summary and the generate-speech template description named no model beyond the Bark default, so a field agent concluded dw has no TTS but Bark and ran Kokoro locally. Name Bark, MMS/VITS and SpeechT5 with the voice control each takes in the docstring summary, the template description and the lead of TASKS.md's Speech Generation section. Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 10 +++++++++- dw/tasks/model_handlers.py | 3 ++- dw/tasks/speech_generation.py | 9 ++++++++- workflows/templates/generate-speech.json | 2 +- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 4127bb18..54cf2374 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -2912,7 +2912,15 @@ It is merged after everything else, so it can override `repetition_penalty` and ## Speech Generation -Speak a line of text with a local text-to-speech model. The result is a waveform carrying the rate its model generated at, so it composes with [`slice_audio`](#slice_audio), [`fade_audio`](#fade_audio) and [`pair_audio`](#pair_audio) directly ([`concat_videos`](#concat_videos) and `dissolve_videos` join videos — pair the track onto a video first). +Speak a line of text with a local text-to-speech model. Any transformers text-to-speech checkpoint can be named as `model_name`; three families are wired for their voice controls: + +| Family | Checkpoint | Voice | +|--------|-----------|-------| +| Bark | `suno/bark-small` (the command's default) | Distinct speakers by `voice_preset` | +| MMS / VITS | `facebook/mms-tts-eng`, and `mms-tts-` per language (the template's default) | One voice, a quarter Bark's size; a multi-speaker VITS takes `speaker_id` in `forward_params` | +| SpeechT5 | `microsoft/speecht5_tts` | The voice of a reference clip, given as `speaker_embedding` | + +The result is a waveform carrying the rate its model generated at, so it composes with [`slice_audio`](#slice_audio), [`fade_audio`](#fade_audio) and [`pair_audio`](#pair_audio) directly ([`concat_videos`](#concat_videos) and `dissolve_videos` join videos — pair the track onto a video first). ```json { diff --git a/dw/tasks/model_handlers.py b/dw/tasks/model_handlers.py index 50849db1..4a73a9c0 100644 --- a/dw/tasks/model_handlers.py +++ b/dw/tasks/model_handlers.py @@ -301,7 +301,8 @@ def _handle_text_generation(task, arguments, previous_pipelines): consumes_device=True, ) def _handle_speech_generation(task, arguments, previous_pipelines): - """Speak a line of text with a local text-to-speech model""" + """Speak a line of text with a local text-to-speech model (Bark, MMS/VITS + or SpeechT5)""" logger.debug("Generating speech") if ("text" in arguments) == ("messages" in arguments): raise ValueError( diff --git a/dw/tasks/speech_generation.py b/dw/tasks/speech_generation.py index 7692abbf..0034e98a 100644 --- a/dw/tasks/speech_generation.py +++ b/dw/tasks/speech_generation.py @@ -120,7 +120,14 @@ def _generation_settings(value, name): def generate_speech(text=None, device="cpu", seed=None, **kwargs): - """Speak a line of text with a local text-to-speech model. + """Speak a line of text with a local text-to-speech model - any + transformers text-to-speech checkpoint named by model_name. Three + families are wired for their voice controls: Bark (suno/bark-small, the + default; distinct speakers by voice_preset), MMS/VITS + (facebook/mms-tts-eng and the other per-language mms-tts-* checkpoints; + one voice, small and fast; a multi-speaker VITS takes speaker_id through + forward_params) and SpeechT5 (microsoft/speecht5_tts; speaks in the + voice of a reference clip given as speaker_embedding). Args: text: The line to speak. Mutually exclusive with messages - exactly diff --git a/workflows/templates/generate-speech.json b/workflows/templates/generate-speech.json index 4a4c8afe..b4f35a22 100644 --- a/workflows/templates/generate-speech.json +++ b/workflows/templates/generate-speech.json @@ -1,6 +1,6 @@ { "id": "GenerateSpeech", - "description": "Speaks a line of text with a local text-to-speech model and saves it as a wav. The result declares no 'sample_rate': a generated track carries the rate its model produced it at, so saying nothing is what saves it correctly.", + "description": "Speaks a line of text with a local text-to-speech model and saves it as a wav. model_name picks the voice: facebook/mms-tts-eng (the default, one voice; other languages are mms-tts-), suno/bark-small (distinct speakers via a voice_preset task argument) or microsoft/speecht5_tts (the voice of a reference clip via speaker_embedding). The result declares no 'sample_rate': a generated track carries the rate its model produced it at, so saying nothing is what saves it correctly.", "variables": { "text": "Somebody finished the pistachio and nobody is saying anything.", "model_name": "facebook/mms-tts-eng" From 70e6aba913722517a89e353d9f02cd8ad5a5f4a8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 16:50:58 -0500 Subject: [PATCH 003/105] fix(server): #944 - a deleted workspace's job rows stay resolvable by id forget_workspace retires every row to a ' (deleted)' workspace instead of deleting unkept ones, so get_job / get_job_workflow still answer; unscoped history listings skip retired rows. Co-Authored-By: Claude Sonnet 5.5 --- dw/server/job_history.py | 29 ++++++++++++++------------ tests/test_forget_workspace_history.py | 9 +++++--- tests/test_server.py | 25 ++++++++++++++++++++++ 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/dw/server/job_history.py b/dw/server/job_history.py index 6d9d9243..66b8cbc0 100644 --- a/dw/server/job_history.py +++ b/dw/server/job_history.py @@ -133,6 +133,11 @@ def _add_missing_columns(connection): return added +# Appended to the workspace of a retired workspace's rows (#944); no workspace +# name can contain a space and parenthesis. +DELETED_SUFFIX = " (deleted)" + + class JobHistory: """Finished jobs, persisted so the Jobs view survives server restarts. @@ -271,6 +276,9 @@ def recent_summaries(self, limit=200, workspace=None, statuses=None): if workspace: clauses.append("workspace = ?") params.append(workspace) + else: + clauses.append("workspace NOT LIKE ?") + params.append(f"%{DELETED_SUFFIX}") if statuses: statuses = list(statuses) placeholders = ", ".join("?" for _ in statuses) @@ -438,23 +446,18 @@ def mark_kept(self, job_id): connection.execute("UPDATE jobs SET kept = 1 WHERE id = ?", (job_id,)) def forget_workspace(self, workspace): - """Delete the rows recorded in this workspace (#913). A deleted - workspace takes its history with it, so a new workspace reusing the - name starts with none. A job a kept asset names (#917) survives, its - workspace renamed to one no workspace can be called, so - `get_job_workflow` still resolves the asset's provenance. Returns - the number of rows removed.""" + """Retire the rows recorded in this workspace (#913). A new workspace + reusing the name starts with none: every row moves to a workspace + name no workspace can be called, so no listing of the name finds it. + The rows stay, so a job id still resolves for `get_job` and + `get_job_workflow` (#944), which fold the recorded arguments into the + run's definition without the workspace. Returns the rows retired.""" with self._lock, self._connect() as connection: cursor = connection.execute( - "DELETE FROM jobs WHERE workspace = ? AND COALESCE(kept, 0) = 0", - (workspace,), - ) - removed = cursor.rowcount - connection.execute( "UPDATE jobs SET workspace = ? WHERE workspace = ?", - (f"{workspace} (deleted)", workspace), + (f"{workspace}{DELETED_SUFFIX}", workspace), ) - return removed + return cursor.rowcount def events_for(self, job_id): """A finished job's persisted event tail. [] for a job recorded diff --git a/tests/test_forget_workspace_history.py b/tests/test_forget_workspace_history.py index 2708e1cc..2cff49f3 100644 --- a/tests/test_forget_workspace_history.py +++ b/tests/test_forget_workspace_history.py @@ -30,7 +30,10 @@ def test_forgetting_a_workspace_leaves_a_same_named_one_empty(tmp_path): assert history.recent_summaries(workspace="sweep") == [] assert [r["id"] for r in history.recent_summaries(workspace="other")] == ["c"] - assert history.get("a") is None + assert history.get("a")["status"] == "succeeded" + assert history.recent_summaries() and [ + r["id"] for r in history.recent_summaries() + ] == ["c"] def test_a_job_a_kept_asset_names_survives_its_workspace(tmp_path): @@ -39,8 +42,8 @@ def test_a_job_a_kept_asset_names_survives_its_workspace(tmp_path): history.record(_job("b", "sweep")) history.mark_kept("a") - assert history.forget_workspace("sweep") == 1 + assert history.forget_workspace("sweep") == 2 assert history.recent_summaries(workspace="sweep") == [] - assert history.get("b") is None + assert history.get("b")["status"] == "succeeded" assert history.get("a")["status"] == "succeeded" diff --git a/tests/test_server.py b/tests/test_server.py index ba71e331..f3b24460 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -557,6 +557,31 @@ def test_job_workflow_returns_an_inline_definition(server): assert historical["definition"] == valid_workflow() +def test_job_workflow_of_a_deleted_workspace_folds_the_arguments(server): + """#944: a job id still resolves once its workspace is deleted - the + definition comes back unrealized with the recorded arguments folded in.""" + workflow = valid_workflow("fold") + workflow["variables"] = {**workflow.get("variables", {}), "n": 1, "label": "x"} + with server(success_script) as client: + response = client.post( + "/api/jobs", + json={"workflow": workflow, "arguments": {"n": "30", "label": "coerced"}}, + ) + assert response.status_code == 201, response.json() + job = response.json() + wait_for_status(client, job["id"], TERMINAL_STATES) + manager = client.app.state.job_manager + manager.forget_workspace(manager.get(job["id"]).spec["workspace"]) + manager.jobs.pop(job["id"], None) + + assert client.get(f"/api/jobs/{job['id']}").status_code == 200 + body = client.get(f"/api/jobs/{job['id']}/workflow").json() + assert body["realized"] is False + variables = body["definition"]["variables"] + assert (variables["n"], variables["label"]) == (30, "coerced") + assert "n, label" in body["note"] or "label, n" in body["note"] + + def test_job_workflow_reads_back_a_job_launched_from_a_file(server): """A job submitted by path names no definition - the file it ran is re-read from the root the job was confined to.""" From 47001f3ec5101c967790c095ce62e9653584f270 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 17:00:20 -0500 Subject: [PATCH 004/105] feat(catalog): #945 - models/z-image declares a vram_estimate over batch x width x height Calibrated on an RTX 3090 with offload: model (15.8 GiB at 4x1344x768). Co-Authored-By: Claude Sonnet 5.5 --- tests/test_zimage_vram.py | 61 +++++++++++++++++++++++++++++++++++ workflows/models/z-image.json | 6 ++++ 2 files changed, 67 insertions(+) create mode 100644 tests/test_zimage_vram.py diff --git a/tests/test_zimage_vram.py b/tests/test_zimage_vram.py new file mode 100644 index 00000000..867ca14f --- /dev/null +++ b/tests/test_zimage_vram.py @@ -0,0 +1,61 @@ +"""The Z-Image catalog entry's VRAM estimate (#945): a batch-size voxel +variable projects like frames do, and the numbers match what lem measured.""" + +import json +import os + +from dw.vram_estimate import required_gb, vram_estimate_errors + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _entry(): + with open(os.path.join(REPO_ROOT, "workflows", "models", "z-image.json")) as f: + return json.load(f) + + +def _at(entry, **overrides): + """The entry as the run executes it: variables substituted into the step.""" + entry["variables"].update(overrides) + arguments = entry["steps"][0]["pipeline"]["arguments"] + for key, value in arguments.items(): + if isinstance(value, str) and value.startswith("variable:"): + arguments[key] = entry["variables"][value.split(":", 1)[1]] + return entry + + +def test_the_projection_matches_the_lem_measurements(): + estimate = _entry()["vram_estimate"] + for n, w, h, measured in [ + (1, 512, 512, 12.16), + (1, 1024, 1024, 12.88), + (4, 960, 544, 13.83), + (4, 1344, 768, 15.80), + ]: + projected = required_gb( + estimate, {"num_images_per_prompt": n, "width": w, "height": h} + ) + assert abs(projected - measured) < 0.4, (n, w, h, projected) + + +def test_batch_size_moves_the_projection_and_is_not_a_frame_count(): + estimate = _entry()["vram_estimate"] + one = required_gb( + estimate, {"num_images_per_prompt": 1, "width": 1344, "height": 768} + ) + eight = required_gb( + estimate, {"num_images_per_prompt": 8, "width": 1344, "height": 768} + ) + assert eight > one + 5 + + +def test_the_default_and_the_field_report_shape_fit_a_24gb_card(): + for overrides in ({}, {"width": 1344, "height": 768}): + entry = _at(_entry(), **overrides) + assert vram_estimate_errors(entry, device_type="cuda", capacity_gb=23.6) == [] + + +def test_a_batch_past_the_card_warns(): + entry = _at(_entry(), width=1344, height=768, num_images_per_prompt=16) + errors = vram_estimate_errors(entry, device_type="cuda", capacity_gb=23.6) + assert errors and "RTX 3090" in errors[0]["message"] diff --git a/workflows/models/z-image.json b/workflows/models/z-image.json index 9c968927..a1cc0210 100644 --- a/workflows/models/z-image.json +++ b/workflows/models/z-image.json @@ -16,6 +16,12 @@ "cost": [ {"device": "cuda", "name": "RTX 3090", "vram_gb": 24, "minutes": 1.4} ], + "vram_estimate": { + "base_gb": 11.8, + "bytes_per_voxel": 1031, + "voxel_variables": ["num_images_per_prompt", "width", "height"], + "reason": "peak reserved VRAM on an RTX 3090 with offload: model, bf16, 9 steps (#945): 12.2 GiB at 1x512x512, 12.9 at 1x1024x1024, 13.8 at 4x960x544, 15.8 at 4x1344x768 - batch size multiplies the latent and VAE decode tensors as pixels do. Calibrated WITH offload: model; a workflow that omits it keeps the whole pipeline resident (~22.9 GiB at 4x1344x768 on lem) and runs out of memory well below this projection" + }, "configures": "templates/text-to-image", "steps": [ { From db1cd2ab7e1045fc34f2c4e325f04fbf544be7f2 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 17:02:00 -0500 Subject: [PATCH 005/105] fix(plan): #948 - a task-only workflow's unknown estimate names why it is unpriced Co-Authored-By: Claude Sonnet 5.5 --- docs/SERVER.md | 3 ++- dw/plan.py | 23 +++++++++++++++++++++++ tests/test_inherited_cost.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/docs/SERVER.md b/docs/SERVER.md index 5dd42d32..d814d997 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -467,7 +467,8 @@ The editor's forms come from these; they are just as usable from scripts: naming the drivers that moved, because it was measured at another setting of them; `other_device` (no entry for the serving backend; the first entry's figure, which is a warning rather - than a quote) or `unknown` (no cost block, or more than one list changed + than a quote) or `unknown` (no cost block - a workflow of task steps alone says so in + `unpriced_reason`, since it has no pipeline to inherit from - or more than one list changed so there is nothing honest to extrapolate along) or `inherited` (an inline workflow has no catalog name, so its price is this server's `observed` runs of the catalog template that loads the same pipeline - diff --git a/dw/plan.py b/dw/plan.py index c01eac7b..6f31e713 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -708,9 +708,32 @@ def estimate( # say which template and why, so it reads differently from no history result["inherited_from"] = observed.get("inherited_from") result["withheld"] = withheld + elif result["basis"] == UNKNOWN and result["minutes"] is None: + reason = _unpriced_reason(definition) + if reason: + result["unpriced_reason"] = reason return result +def _unpriced_reason(definition): + """Why a workflow with no price at all has none, or None when the cause + is not a shape this can name (#948). #797 prices an inline workflow from + the catalog template with the same pipeline identity, so a workflow of + task steps alone has nothing to match and its cost is simply unmeasured - + which says nothing about whether the run is seconds or an hour.""" + steps = [s for s in definition.get("steps") or [] if isinstance(s, dict)] + tasks = [s for s in steps if isinstance(s.get("task"), dict)] + if not tasks or len(tasks) != len(steps): + return None + commands = sorted({str(s["task"].get("command", "unknown")) for s in tasks}) + return ( + "task-only workflow (" + ", ".join(commands) + "): no pipeline step " + "to inherit a price from a catalog template, no cost block and no " + "run history of its own. The run's length depends on its input " + "media, which nothing here measures" + ) + + def _observed(observed, device): """This box's history as an estimate, or None when it has nothing to quote from. diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 882cd0d2..544fd418 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -464,6 +464,39 @@ def test_the_plan_carries_the_withheld_reason(self, tmp_path): assert "withheld" in _inherited_cost_warnings(estimate)[0] +class TestTaskOnlyWorkflowSaysWhyUnpriced: + """#948: a workflow of task steps has no pipeline identity to inherit a + price from, and its unknown estimate says so.""" + + def test_the_plan_names_the_reason(self, tmp_path): + inline = { + "id": "stems", + "steps": [ + { + "id": "split", + "task": {"command": "separate_stems", "arguments": {}}, + } + ], + } + candidate = workflow_from_definition( + inline, str(tmp_path), str(tmp_path), str(tmp_path) + ) + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + plan = build_plan(candidate, {}, device="cpu", lookup_sizes=False) + estimate = plan["estimate"] + assert estimate["basis"] == "unknown" and estimate["minutes"] is None + assert "separate_stems" in estimate["unpriced_reason"] + assert "task-only" in estimate["unpriced_reason"] + + def test_a_pipeline_workflow_gets_no_task_only_reason(self, tmp_path): + candidate = workflow_from_definition( + definition("inline"), str(tmp_path), str(tmp_path), str(tmp_path) + ) + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + plan = build_plan(candidate, {}, device="cpu", lookup_sizes=False) + assert "unpriced_reason" not in plan["estimate"] + + def sized(name, steps=9, width=960, height=544, **extra): """`definition` whose step also sets inference steps and a canvas.""" workflow = definition(name, **extra) From 2ff8a662334adbb75b59612510ca0b78ddee08d6 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 17:08:36 -0500 Subject: [PATCH 006/105] fix(server): #948 - declare unpriced_reason in the plan estimate response model Co-Authored-By: Claude Sonnet 5.5 --- dw/server/api_models.py | 5 +++++ ui/src/lib/generated/api-schema.ts | 5 +++++ ui/src/lib/generated/openapi.json | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/dw/server/api_models.py b/dw/server/api_models.py index f11d1881..eafffa42 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -666,6 +666,11 @@ class PlanEstimate(ApiModel): "could not be re-priced for this workflow (its size cannot be compared " "to the template's)." ) + unpriced_reason: str = sometimes( + "With basis `unknown` and no `withheld`: why a workflow of task steps " + "alone has no price - it has no pipeline to inherit one from, and its " + "length follows its input media (#948)." + ) priced_for: str = sometimes( "The card the run would be dispatched to, whose history the figure is - " "as `cuda:1 NVIDIA GeForce RTX 3090`." diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 1a97f4cf..88ed7b04 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -3347,6 +3347,11 @@ export interface components { * @description What contributed nothing to `minutes` when `partial` is true - the workflow's own id when its own steps went unpriced, else the path of each composed child with no cost block or observed history. When `minutes` is null it still names the composed children that left it unpriced (`partial` is then false); empty when nothing composed went unpriced. */ unpriced: string[]; + /** + * Unpriced Reason + * @description With basis `unknown` and no `withheld`: why a workflow of task steps alone has no price - it has no pipeline to inherit one from, and its length follows its input media (#948). + */ + unpriced_reason?: string; /** * Withheld * @description With basis `unknown` and `inherited_from` set: why the template's runs could not be re-priced for this workflow (its size cannot be compared to the template's). diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index d3148376..8ea26122 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -3685,6 +3685,11 @@ "title": "Unpriced", "type": "array" }, + "unpriced_reason": { + "description": "With basis `unknown` and no `withheld`: why a workflow of task steps alone has no price - it has no pipeline to inherit one from, and its length follows its input media (#948).", + "title": "Unpriced Reason", + "type": "string" + }, "withheld": { "description": "With basis `unknown` and `inherited_from` set: why the template's runs could not be re-priced for this workflow (its size cannot be compared to the template's).", "title": "Withheld", From 7c7f46aa815938be3afc09e9afd238f0b3a23d58 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 17:26:05 -0500 Subject: [PATCH 007/105] fix(gallery): #928 - a file of a run with no recorded role lists as intermediate when in intermediate/ or read by a later run Co-Authored-By: Claude Sonnet 5.5 --- dw/server/routes/gallery.py | 9 ++++++++- tests/test_gallery_roles.py | 12 +++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/dw/server/routes/gallery.py b/dw/server/routes/gallery.py index 947e4366..d7872a29 100644 --- a/dw/server/routes/gallery.py +++ b/dw/server/routes/gallery.py @@ -135,6 +135,12 @@ def _version(folder, run_id): if split else None ) + # A run from before roles were recorded has none: what its layout + # or a later run's reads still say is enough to call a file + # intermediate (#928) + if run_id and entry["role"] is None: + if subfolder == "intermediate" or relative_name in consumed: + entry["role"] = "intermediate" if split: if split[0] not in notes_by_run: notes_by_run[split[0]] = read_notes(split[0]) @@ -246,7 +252,8 @@ def gallery( Each entry carries `role`, `final` or `intermediate` (null for a flat-layout file or one no manifest lists); a file a later run read lists as intermediate unless its step declared it final - (dw/roles.py rule 4). `role` is a repeatable + (dw/roles.py rule 4). A run that recorded no role lists a file in its + `intermediate/` subfolder, or one a later run read, as intermediate. `role` is a repeatable filter; a null role counts as final. Omitted, every file is listed.""" if role: unknown = sorted(set(role) - set(ROLES)) diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index 70da0182..d5291cf6 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -24,8 +24,9 @@ def test_recorded_roles_reach_the_listing(gallery_client, output_root, write_run assert files["b.png"]["role"] == "final" -def test_older_run_has_no_role(gallery_client, output_root, write_run): - # A manifest from before roles: b reads a, but nothing is guessed +def test_older_run_reads_its_intermediate_subfolder_as_intermediate(gallery_client, output_root, write_run): + # A manifest from before roles: a sits in intermediate/, so it lists as + # intermediate (#928); b has nothing to say and stays unknown realized = { "id": "wf", "steps": [ @@ -51,7 +52,7 @@ def test_older_run_has_no_role(gallery_client, output_root, write_run): files=["intermediate/a.png", "b.png"], ) files = {f["label"]: f for f in gallery_client.get("/api/gallery").json()["files"]} - assert files["a.png"]["role"] is None + assert files["a.png"]["role"] == "intermediate" assert files["b.png"]["role"] is None @@ -350,7 +351,7 @@ def test_a_declared_final_stays_final(gallery_client, output_root, write_run): } -def test_a_pre_roles_file_a_later_run_read_stays_unknown( +def test_a_pre_roles_file_a_later_run_read_lists_as_intermediate( gallery_client, output_root, write_run ): write_run( @@ -368,7 +369,8 @@ def test_a_pre_roles_file_a_later_run_read_stays_unknown( files=["cut.png"], inputs=[{"reference": f"output:shots/{SHOT_RUN}/shot.png", "kind": "output"}], ) - assert _roles(gallery_client)["shot.png"] is None + assert _roles(gallery_client)["shot.png"] == "intermediate" + assert _roles(gallery_client, role="final") == {"cut.png": "final"} def test_an_asset_kept_from_a_file_demotes_it(tmp_path, write_run): From cea2574ac3952a577045b4962ec751d4c940ce21 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 17:26:30 -0500 Subject: [PATCH 008/105] style: #928 - ruff format Co-Authored-By: Claude Sonnet 5.5 --- tests/test_gallery_roles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index d5291cf6..dfaca828 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -24,7 +24,9 @@ def test_recorded_roles_reach_the_listing(gallery_client, output_root, write_run assert files["b.png"]["role"] == "final" -def test_older_run_reads_its_intermediate_subfolder_as_intermediate(gallery_client, output_root, write_run): +def test_older_run_reads_its_intermediate_subfolder_as_intermediate( + gallery_client, output_root, write_run +): # A manifest from before roles: a sits in intermediate/, so it lists as # intermediate (#928); b has nothing to say and stays unknown realized = { From a9aa90e0ad9581814a963a04e9166b8e463bbf29 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 17:29:53 -0500 Subject: [PATCH 009/105] chore(api): #928 - regenerate the OpenAPI document Co-Authored-By: Claude Sonnet 5.5 --- ui/src/lib/generated/api-schema.ts | 3 ++- ui/src/lib/generated/openapi.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 88ed7b04..ad1bb8bd 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -343,7 +343,8 @@ export interface paths { * Each entry carries `role`, `final` or `intermediate` (null for a * flat-layout file or one no manifest lists); a file a later run read * lists as intermediate unless its step declared it final - * (dw/roles.py rule 4). `role` is a repeatable + * (dw/roles.py rule 4). A run that recorded no role lists a file in its + * `intermediate/` subfolder, or one a later run read, as intermediate. `role` is a repeatable * filter; a null role counts as final. Omitted, every file is listed. */ get: operations["gallery_api_gallery_get"]; diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 8ea26122..ff75000a 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -5992,7 +5992,7 @@ }, "/api/gallery": { "get": { - "description": "A page of media files in the output directory, newest first.\nStateless by design - the gallery survives server restarts because\nit reads the directory tree, not job history. 'folders' lists every\ndistinct workflow folder present (over the whole directory, not just\nthis page), for the UI's folder filter - a run id is not a folder of\nits own, so a workflow's runs group together; '' stands for files\nsaved directly at the output root, and is itself always a member so\nthat folder-less outputs stay selectable once anything is nested.\n'subfolders' is the other axis, over the whole directory the same\nway: the in-run subfolders steps wrote into ('final',\n'intermediate'), '' for files at a run's root. `folder` and\n`subfolder` filter independently and intersect when both are given.\n`version` narrows to the runs holding that ordinal - with `folder`,\nthe one run \"v4\" names; without it, that run of every workflow.\n\n`only_orphans=true` inverts the whole call: instead of media files,\nit returns run directories holding nothing but their own\nbookkeeping (manifest.json, workflow.json, job.json) as `runs`,\neach `{name, mtime}` - a run that wrote any file at all, a\ntext-shape prompt or a utility's side output included, is not\nlisted. `folder`/`subfolder` and the `folders`/`subfolders` facets\ndo not apply in this mode, since an orphan run has no file to\ncarry either. `name` is exactly what `DELETE /api/gallery/{name}`\naccepts, so listing and deleting an orphan is a two-call round\ntrip (#170).\n\n`media=true` adds `duration_seconds` to each audio/video entry,\nprobed the same way `get_gallery_metadata` reports it - which two\ntakes of the same workflow otherwise have no way to be told apart\nby, since size and mtime are misleading proxies for length (#356).\nOff by default and bounded by `limit`: only the page actually\nreturned is probed, not the whole listing, so the cost of asking\nstays proportional to the page size rather than the library size.\n\nEach entry carries `role`, `final` or `intermediate` (null for a\nflat-layout file or one no manifest lists); a file a later run read\nlists as intermediate unless its step declared it final\n(dw/roles.py rule 4). `role` is a repeatable\nfilter; a null role counts as final. Omitted, every file is listed.", + "description": "A page of media files in the output directory, newest first.\nStateless by design - the gallery survives server restarts because\nit reads the directory tree, not job history. 'folders' lists every\ndistinct workflow folder present (over the whole directory, not just\nthis page), for the UI's folder filter - a run id is not a folder of\nits own, so a workflow's runs group together; '' stands for files\nsaved directly at the output root, and is itself always a member so\nthat folder-less outputs stay selectable once anything is nested.\n'subfolders' is the other axis, over the whole directory the same\nway: the in-run subfolders steps wrote into ('final',\n'intermediate'), '' for files at a run's root. `folder` and\n`subfolder` filter independently and intersect when both are given.\n`version` narrows to the runs holding that ordinal - with `folder`,\nthe one run \"v4\" names; without it, that run of every workflow.\n\n`only_orphans=true` inverts the whole call: instead of media files,\nit returns run directories holding nothing but their own\nbookkeeping (manifest.json, workflow.json, job.json) as `runs`,\neach `{name, mtime}` - a run that wrote any file at all, a\ntext-shape prompt or a utility's side output included, is not\nlisted. `folder`/`subfolder` and the `folders`/`subfolders` facets\ndo not apply in this mode, since an orphan run has no file to\ncarry either. `name` is exactly what `DELETE /api/gallery/{name}`\naccepts, so listing and deleting an orphan is a two-call round\ntrip (#170).\n\n`media=true` adds `duration_seconds` to each audio/video entry,\nprobed the same way `get_gallery_metadata` reports it - which two\ntakes of the same workflow otherwise have no way to be told apart\nby, since size and mtime are misleading proxies for length (#356).\nOff by default and bounded by `limit`: only the page actually\nreturned is probed, not the whole listing, so the cost of asking\nstays proportional to the page size rather than the library size.\n\nEach entry carries `role`, `final` or `intermediate` (null for a\nflat-layout file or one no manifest lists); a file a later run read\nlists as intermediate unless its step declared it final\n(dw/roles.py rule 4). A run that recorded no role lists a file in its\n`intermediate/` subfolder, or one a later run read, as intermediate. `role` is a repeatable\nfilter; a null role counts as final. Omitted, every file is listed.", "operationId": "gallery_api_gallery_get", "parameters": [ { From b3a16fa10fd1dd034b86465e1cee4b752a071b49 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 17:40:23 -0500 Subject: [PATCH 010/105] fix(roles): #928 - a run that recorded no role takes its file's subfolder role, then rule 4, in dw/roles.py Moves the fallback out of the gallery route into listed_role, the owner the map names, so the listing and the metadata route agree. Updates the roles docstring, the map row, and the role wording in the tool and API docs; regenerates the OpenAPI document. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/MCP.md | 2 +- docs/SERVER.md | 5 +++-- dw/roles.py | 24 ++++++++++++++------ dw/server/outputs.py | 8 +++++-- dw/server/routes/gallery.py | 12 +++------- dw_mcp/catalog.py | 4 ++-- dw_mcp/tools_catalog.py | 2 +- tests/test_gallery_roles.py | 35 ++++++++++++++++++++++++++++++ tests/test_roles.py | 21 +++++++++++++++++- ui/src/lib/generated/api-schema.ts | 6 ++--- ui/src/lib/generated/openapi.json | 2 +- 12 files changed, 93 insertions(+), 30 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 920f76cb..edc8ab0c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -67,7 +67,7 @@ one to open. | Concept | Owner | Rule | Enforced by | | --- | --- | --- | --- | | Run directories | `dw/runs.py`: `open_run`, `workflow_identity`, `new_run_id` | Each execution writes into `///` with a `manifest.json`, unless the flat layout is chosen (the test pins the flat-layout exception). | `tests/test_runs.py::TestRunDirectories::test_the_flat_layout_writes_where_it_always_did` | -| A file's role (final / intermediate / input) | `dw/roles.py` | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end, with `role_declared` where result.role stated it; the server reports only recorded roles, so an older run has none. At read time the gallery (`GET /api/gallery`, its `role=` filter, the metadata route) also demotes a file a later run of the same workspace read - an `output:` input (an unpinned `latest` is the newest run started before the reader) or an `asset:` keep_output kept from it - unless its role was declared (`consumed_outputs`, `listed_role`, once per listing; no manifest is rewritten). | `tests/test_roles.py`, `tests/test_roles.py::test_a_file_a_later_run_read_is_consumed`, `tests/test_roles.py::test_listed_role`, `tests/test_gallery_roles.py::test_a_file_a_later_run_read_lists_as_intermediate`, `tests/test_gallery_roles.py::test_a_declared_final_stays_final` | +| A file's role (final / intermediate / input) | `dw/roles.py` | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end, with `role_declared` where result.role stated it; a run that predates roles recorded none, so the listing falls back on the file's own `final`/`intermediate` subfolder, then the demotion below, and a file neither places has none. At read time the gallery (`GET /api/gallery`, its `role=` filter, the metadata route) also demotes a file a later run of the same workspace read - an `output:` input (an unpinned `latest` is the newest run started before the reader) or an `asset:` keep_output kept from it - unless its role was declared (`consumed_outputs`, `listed_role`, once per listing; no manifest is rewritten). | `tests/test_roles.py`, `tests/test_roles.py::test_a_file_a_later_run_read_is_consumed`, `tests/test_roles.py::test_listed_role`, `tests/test_roles.py::test_listed_role_of_a_run_that_recorded_none`, `tests/test_gallery_roles.py::test_older_run_reads_its_intermediate_subfolder_as_intermediate`, `tests/test_gallery_roles.py::test_a_pre_roles_file_a_later_run_read_lists_as_intermediate`, `tests/test_gallery_roles.py::test_metadata_agrees_with_the_listing_for_an_older_run`, `tests/test_gallery_roles.py::test_a_file_a_later_run_read_lists_as_intermediate`, `tests/test_gallery_roles.py::test_a_declared_final_stays_final` | | Notes on runs, files, assets, workspaces | dw/notes.py | notes.json beside what it describes (run dir or asset dir; a file's own note else the run's), workspace.json at a workspace root; only the server writes them, never the engine; ≤4000 chars, plain text. | tests/test_notes.py | | Run versions | `dw/runs.py`: `open_run`, `run_versions`, `record_run_versions`, `copy_runs_in` | A run takes its number once, when it opens (or, copied in from another workspace, when `copy_runs_in` places it), so deleting a middle run leaves a gap rather than renumbering. | `tests/test_runs.py::TestRunVersions::test_a_deleted_middle_run_leaves_a_gap_rather_than_renumbering`, `tests/test_runs.py::TestRunVersions::test_runs_started_in_the_same_second_still_number_upward` | | Run-version surfaces | `dw/runs.py`: `run_versions`, `dw/server/routes/gallery.py`, `dw/server/outputs.py`, `dw_mcp/tools_catalog.py`: `list_gallery`, `ui/src/lib/pages/GalleryPage.svelte` | Only `dw/runs.py` computes the number, and every other module reads it back. | `tests/test_runs.py::TestRunVersions::test_an_output_reference_can_name_a_run_by_its_version` | diff --git a/docs/MCP.md b/docs/MCP.md index 94f72e32..f614b32a 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -233,7 +233,7 @@ when no single workflow covers it. | `get_health()` | — | Check that the server is alive, and which machine answered: `version`, `device`, whether any model process is resident (`worker_alive`), and the queue depth. The engine runs one job per GPU: `workers` has one entry per card - `device`, `name`, `vram_gb`, `alive`, its own `current_job`, and the worker's `host_memory_rss_mb` while it runs - and the top-level `current_job` is the longest-running job. `worker_alive: false` is the normal idle state on a server that has not run a job since startup or the last memory clear - not a fault - the on-demand worker starts with the next job (#206) | | `get_server_info()` | — | What this installation can do and where it keeps things: `device` (the accelerator a run will use), `version`, the `workspace` this session is working in and the workflow/asset/output/prompt `directories` of *that* workspace, the bind address and port, whether a token is required, and whether MCP is mounted. Check the device before authoring - a CUDA-only choice (bitsandbytes, `torch.compile`, flash attention) is not available on an `mps` or `cpu` server. `runtime` (#222) reports the Python version, torch version and the CUDA version torch was built against, the NVIDIA driver version (when `nvidia-smi` is reachable), and the installed versions of diffusers, transformers, accelerate, bitsandbytes, peft, safetensors and sentencepiece (`null` for one not installed) - for diagnosing an environment mismatch between boxes without shelling in. `caches` lists dw's disk caches (`inductor`, `triton`, `quantized`), each with `name`, `bytes`, `files`, `enabled`, `clearable` and `disk_free`, plus `entry_count` for a cache with entries, and `cache_entries=true` lists them (`id`, `model`, `subfolder`, `component`, `bytes`, `files`, `unreadable`, `created`, `last_hit`; the quantizer and dtype are not in `/api/caches`); it is omitted when the server has no `/api/caches`. There is no MCP tool to clear one - that is `DELETE /api/caches/{name}` or `.../entries/{entry}` ([SERVER.md](SERVER.md)) | | `list_jobs(limit=20, status=None, workspace=None)` | optional `limit` (newest N), `status` (one state or a comma-separated set of `queued`, `running`, `succeeded`, `failed`, `cancelled`), `workspace` | List queued, running and recent jobs, **newest first** (a queued job held for host RAM carries `waiting_for`). Bounded by default: the unbounded listing was over a client's tool-result limit on a server with a few months of history, which made it a tool that could not be called at all. `total` says how many matched and `truncated`/`next` say so when the answer was cut - raise `limit` or narrow with `status`. Without `workspace`, a named workspace lists its own jobs and the default one lists every job the server holds | -| `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None, media=False, role=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version`, `media`, `role` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with no run; `intermediate` once a later step or a later run read the file, unless `result.role` declared it); `role=["final"]` lists only those, and the `role` filter defaults to every file. See `dw/roles.py` for how a role is decided. A file starred in the web UI's gallery (its keep mark) carries `starred: true`; an unstarred one carries no `starred`. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | +| `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None, media=False, role=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version`, `media`, `role` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with no run, or an older run's file that neither its subfolder nor a later run's reads place; `intermediate` once a later step or a later run read the file, unless `result.role` declared it); `role=["final"]` lists only those, and the `role` filter defaults to every file. See `dw/roles.py` for how a role is decided. A file starred in the web UI's gallery (its keep mark) carries `starred: true`; an unstarred one carries no `starred`. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | | `get_gallery_metadata(name, envelope=False, workspace=None, interval_seconds=None)` | `name`, `workspace` | Get the metadata embedded in a generated file — or, when `name` is an `asset:` reference, what an *input* asset holds (`source` says which; `job` is null for an asset). Reading an input's duration, frame count, fps and sample rate before a run is how a caller learns the `total_frames`, `fps` and `sample_rate` a workflow expects it to supply: the exact workflow and arguments that produced it, and, for audio/video, a `media` block (duration, rate, channels, fps, size, peak/mean dBFS) and `findings` - each level problem the server measured against `dw/audio_qc.py`'s thresholds (`full_scale`, `near_silent`), with the fix. Only an image (PNG/JPEG/WebP) embeds `metadata` this way — it is always null for audio and video, and `next` then points at `get_job_workflow(job_id)` when `job` is known, or says a kept asset carries no provenance at all when it isn't. `envelope=true` adds `media.envelope` — `rms_dbfs` and `peak_dbfs` one entry per second, or per `interval_seconds` (default 1.0, 0.05 to 3600; 0.1 places a vocal onset; a finer-than-default width that would need over 2400 bins is refused with the interval that fits) — which is what locates something in a track rather than measuring the whole of it. `media.shots` is set on an output joined from shots: one `{name, start_frame, num_frames, start_sample, num_samples}` per shot, as the join measured them (see the run manifest in docs/WORKFLOW_GUIDE.md), else null. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | ### Media diff --git a/docs/SERVER.md b/docs/SERVER.md index d814d997..08b4672d 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -596,8 +596,9 @@ The editor's forms come from these; they are just as usable from scripts: `subfolders` list every distinct value over the whole tree, `''` always a member of each so root-level files stay selectable. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with - no run; `intermediate` once a later run read the file, unless the step's - `result.role` declared it), and `GET /api/gallery?role=final` lists only + no run, or a file of a run that recorded no role whose subfolder says + nothing and that no later run read; `intermediate` once a later run read + the file, unless the step's `result.role` declared it), and `GET /api/gallery?role=final` lists only those (repeatable; a null role counts as final). See `dw/roles.py` for how a role is decided; the metadata route reports the file's `role` and the run's `inputs` too diff --git a/dw/roles.py b/dw/roles.py index abf8d346..9812d718 100644 --- a/dw/roles.py +++ b/dw/roles.py @@ -17,8 +17,11 @@ A composed child's rolled-up manifest entries carry `parent_step` and take that step's role. The engine records roles when a run ends, from the -`role_basis` it took as the run opened; the server reads only what was -recorded, so a run that predates roles has none - unknown, not guessed. Inputs +`role_basis` it took as the run opened. A run that predates roles recorded +none, and its realized workflow no longer shows its data flow (a run +resolves its references in place), so a listing falls back on what is +still readable: rule 2 from the file's own subfolder, then rule 4 below. A +file neither says anything about stays unknown. Inputs are the asset: and output: references the expanded steps carry - a run's 'made from' list. @@ -34,14 +37,15 @@ 'latest' is the newest run that started before it), or through an asset: reference to an asset keep_output kept from that file. A role rule 1 declared is recorded as such (`role_declared`) and never demoted; an entry - from before that flag is demotable, and a file with no recorded role - stays unknown. + from before that flag is demotable, and so is a file with no recorded + role, which lists as intermediate once read. This module names no model. """ import json import os +import posixpath from .references import ( ASSET, @@ -227,11 +231,17 @@ def run_inputs(steps): return seen -def listed_role(role, declared, consumed): +def listed_role(role, declared, consumed, name=None): """Rule 4: the role a listing shows for a file whose run recorded `role` - intermediate once a later run `consumed` it, unless rule 1 - `declared` the role. An unknown role stays unknown.""" - if role is None or declared or not consumed: + `declared` the role. With no recorded role (a run that predates roles), + rule 2 reads the subfolder of `name`, the file's '/' separated path + inside its run, before rule 4 applies; with neither, it stays None.""" + if role is None: + role = _subfolder_role(posixpath.dirname(name or "") or None) + if role is None: + return INTERMEDIATE if consumed else None + if declared or not consumed: return role return INTERMEDIATE diff --git a/dw/server/outputs.py b/dw/server/outputs.py index 908879f7..d39da564 100644 --- a/dw/server/outputs.py +++ b/dw/server/outputs.py @@ -547,9 +547,13 @@ def declared_files(run_dir): def listed_file_role(run_dir, name, consumed): """The role a listing shows for file `name` of a run: its recorded role, - demoted when `consumed` (dw/roles.py rule 4).""" + demoted when `consumed` (dw/roles.py rule 4), or for a run that recorded + none, what the file's subfolder and `consumed` still say.""" return listed_role( - run_roles(run_dir).get(name), name in declared_files(run_dir), consumed + run_roles(run_dir).get(name), + name in declared_files(run_dir), + consumed, + name, ) diff --git a/dw/server/routes/gallery.py b/dw/server/routes/gallery.py index d7872a29..4893390e 100644 --- a/dw/server/routes/gallery.py +++ b/dw/server/routes/gallery.py @@ -135,12 +135,6 @@ def _version(folder, run_id): if split else None ) - # A run from before roles were recorded has none: what its layout - # or a later run's reads still say is enough to call a file - # intermediate (#928) - if run_id and entry["role"] is None: - if subfolder == "intermediate" or relative_name in consumed: - entry["role"] = "intermediate" if split: if split[0] not in notes_by_run: notes_by_run[split[0]] = read_notes(split[0]) @@ -252,9 +246,9 @@ def gallery( Each entry carries `role`, `final` or `intermediate` (null for a flat-layout file or one no manifest lists); a file a later run read lists as intermediate unless its step declared it final - (dw/roles.py rule 4). A run that recorded no role lists a file in its - `intermediate/` subfolder, or one a later run read, as intermediate. `role` is a repeatable - filter; a null role counts as final. Omitted, every file is listed.""" + (dw/roles.py rule 4). A run that recorded no role takes the role its + file's subfolder names, then rule 4; with neither it is null. `role` + is a repeatable filter; a null role counts as final. Omitted, every file is listed.""" if role: unknown = sorted(set(role) - set(ROLES)) if unknown: diff --git a/dw_mcp/catalog.py b/dw_mcp/catalog.py index d2551fc5..9d320f3e 100644 --- a/dw_mcp/catalog.py +++ b/dw_mcp/catalog.py @@ -270,8 +270,8 @@ def list_gallery( enough to pick between two takes without one metadata call per candidate. Off by default; a plain call carries no `duration_seconds`. - Each file carries `role` (`final`, `intermediate`, or null with no run); - `role=[...]` keeps only those, null counting as final.""" + Each file carries `role` (`final`, `intermediate`, or null when nothing + says); `role=[...]` keeps only those, null counting as final.""" params = {"limit": limit} if role: params["role"] = list(role) diff --git a/dw_mcp/tools_catalog.py b/dw_mcp/tools_catalog.py index 09a01e7d..affbe598 100644 --- a/dw_mcp/tools_catalog.py +++ b/dw_mcp/tools_catalog.py @@ -272,7 +272,7 @@ def list_gallery( carries `folder` (the workflow), `subfolder` (the step's chosen subfolder, '' for none; `subfolder=` filters on it) and `role`: `final` is a deliverable, `intermediate` a file a later step - or run consumed, null with no run. `role=` filters. Each entry's `url` is workspace-scoped; use it as given. + or run consumed, null if unknown. `role=` filters. Each entry's `url` is workspace-scoped; use it as given. Entries carry `run_id` and `version`, the run's stable ordinal (the web UI's `v5`) - quote it to a person. `folder=` plus diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index dfaca828..32e494b7 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -56,6 +56,8 @@ def test_older_run_reads_its_intermediate_subfolder_as_intermediate( files = {f["label"]: f for f in gallery_client.get("/api/gallery").json()["files"]} assert files["a.png"]["role"] == "intermediate" assert files["b.png"]["role"] is None + assert _roles(gallery_client, role="intermediate") == {"a.png": "intermediate"} + assert _roles(gallery_client, role="final") == {"b.png": None} def test_role_filter(gallery_client, output_root, write_run): @@ -375,6 +377,39 @@ def test_a_pre_roles_file_a_later_run_read_lists_as_intermediate( assert _roles(gallery_client, role="final") == {"cut.png": "final"} +def test_metadata_agrees_with_the_listing_for_an_older_run( + gallery_client, output_root, write_run +): + # A run that recorded no role: the metadata route goes through the same + # owner as the listing, so the two say the same thing (#928) + write_run( + output_root, + "shots", + SHOT_RUN, + [ + {"step": "clip", "files": ["intermediate/clip.mp4"]}, + {"step": "still", "files": ["final/still.png"]}, + {"step": "shot", "files": ["shot.png"]}, + ], + files=["intermediate/clip.mp4", "final/still.png", "shot.png"], + ) + listed = _roles(gallery_client) + assert listed == { + "clip.mp4": "intermediate", + "still.png": "final", + "shot.png": None, + } + for name, label in ( + ("intermediate/clip.mp4", "clip.mp4"), + ("final/still.png", "still.png"), + ("shot.png", "shot.png"), + ): + meta = gallery_client.get( + f"/api/gallery/shots/{SHOT_RUN}/{name}/metadata" + ).json() + assert meta["role"] == listed[label] + + def test_an_asset_kept_from_a_file_demotes_it(tmp_path, write_run): from fastapi.testclient import TestClient diff --git a/tests/test_roles.py b/tests/test_roles.py index b85f7a9b..8ef160b5 100644 --- a/tests/test_roles.py +++ b/tests/test_roles.py @@ -359,8 +359,27 @@ def test_an_unresolvable_reference_consumes_nothing(tmp_path, write_run): (FINAL, True, True, FINAL), (FINAL, False, False, FINAL), (INTERMEDIATE, False, True, INTERMEDIATE), - (None, False, True, None), + (None, False, True, INTERMEDIATE), ], ) def test_listed_role(role, declared, consumed, listed): assert listed_role(role, declared, consumed) == listed + + +@pytest.mark.parametrize( + "name, consumed, listed", + [ + ("intermediate/clip.mp4", False, INTERMEDIATE), + ("intermediate/frames/0.png", False, INTERMEDIATE), + ("final/still.png", False, FINAL), + ("final/still.png", True, INTERMEDIATE), + ("shot.png", True, INTERMEDIATE), + ("shot.png", False, None), + ("final", False, None), + ("intermediate_frames/a.png", False, None), + (None, False, None), + ], +) +def test_listed_role_of_a_run_that_recorded_none(name, consumed, listed): + # A run from before roles: rule 2 from the file's subfolder, then rule 4 + assert listed_role(None, False, consumed, name) == listed diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index ad1bb8bd..eaf4a92f 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -343,9 +343,9 @@ export interface paths { * Each entry carries `role`, `final` or `intermediate` (null for a * flat-layout file or one no manifest lists); a file a later run read * lists as intermediate unless its step declared it final - * (dw/roles.py rule 4). A run that recorded no role lists a file in its - * `intermediate/` subfolder, or one a later run read, as intermediate. `role` is a repeatable - * filter; a null role counts as final. Omitted, every file is listed. + * (dw/roles.py rule 4). A run that recorded no role takes the role its + * file's subfolder names, then rule 4; with neither it is null. `role` + * is a repeatable filter; a null role counts as final. Omitted, every file is listed. */ get: operations["gallery_api_gallery_get"]; put?: never; diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index ff75000a..8122d26b 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -5992,7 +5992,7 @@ }, "/api/gallery": { "get": { - "description": "A page of media files in the output directory, newest first.\nStateless by design - the gallery survives server restarts because\nit reads the directory tree, not job history. 'folders' lists every\ndistinct workflow folder present (over the whole directory, not just\nthis page), for the UI's folder filter - a run id is not a folder of\nits own, so a workflow's runs group together; '' stands for files\nsaved directly at the output root, and is itself always a member so\nthat folder-less outputs stay selectable once anything is nested.\n'subfolders' is the other axis, over the whole directory the same\nway: the in-run subfolders steps wrote into ('final',\n'intermediate'), '' for files at a run's root. `folder` and\n`subfolder` filter independently and intersect when both are given.\n`version` narrows to the runs holding that ordinal - with `folder`,\nthe one run \"v4\" names; without it, that run of every workflow.\n\n`only_orphans=true` inverts the whole call: instead of media files,\nit returns run directories holding nothing but their own\nbookkeeping (manifest.json, workflow.json, job.json) as `runs`,\neach `{name, mtime}` - a run that wrote any file at all, a\ntext-shape prompt or a utility's side output included, is not\nlisted. `folder`/`subfolder` and the `folders`/`subfolders` facets\ndo not apply in this mode, since an orphan run has no file to\ncarry either. `name` is exactly what `DELETE /api/gallery/{name}`\naccepts, so listing and deleting an orphan is a two-call round\ntrip (#170).\n\n`media=true` adds `duration_seconds` to each audio/video entry,\nprobed the same way `get_gallery_metadata` reports it - which two\ntakes of the same workflow otherwise have no way to be told apart\nby, since size and mtime are misleading proxies for length (#356).\nOff by default and bounded by `limit`: only the page actually\nreturned is probed, not the whole listing, so the cost of asking\nstays proportional to the page size rather than the library size.\n\nEach entry carries `role`, `final` or `intermediate` (null for a\nflat-layout file or one no manifest lists); a file a later run read\nlists as intermediate unless its step declared it final\n(dw/roles.py rule 4). A run that recorded no role lists a file in its\n`intermediate/` subfolder, or one a later run read, as intermediate. `role` is a repeatable\nfilter; a null role counts as final. Omitted, every file is listed.", + "description": "A page of media files in the output directory, newest first.\nStateless by design - the gallery survives server restarts because\nit reads the directory tree, not job history. 'folders' lists every\ndistinct workflow folder present (over the whole directory, not just\nthis page), for the UI's folder filter - a run id is not a folder of\nits own, so a workflow's runs group together; '' stands for files\nsaved directly at the output root, and is itself always a member so\nthat folder-less outputs stay selectable once anything is nested.\n'subfolders' is the other axis, over the whole directory the same\nway: the in-run subfolders steps wrote into ('final',\n'intermediate'), '' for files at a run's root. `folder` and\n`subfolder` filter independently and intersect when both are given.\n`version` narrows to the runs holding that ordinal - with `folder`,\nthe one run \"v4\" names; without it, that run of every workflow.\n\n`only_orphans=true` inverts the whole call: instead of media files,\nit returns run directories holding nothing but their own\nbookkeeping (manifest.json, workflow.json, job.json) as `runs`,\neach `{name, mtime}` - a run that wrote any file at all, a\ntext-shape prompt or a utility's side output included, is not\nlisted. `folder`/`subfolder` and the `folders`/`subfolders` facets\ndo not apply in this mode, since an orphan run has no file to\ncarry either. `name` is exactly what `DELETE /api/gallery/{name}`\naccepts, so listing and deleting an orphan is a two-call round\ntrip (#170).\n\n`media=true` adds `duration_seconds` to each audio/video entry,\nprobed the same way `get_gallery_metadata` reports it - which two\ntakes of the same workflow otherwise have no way to be told apart\nby, since size and mtime are misleading proxies for length (#356).\nOff by default and bounded by `limit`: only the page actually\nreturned is probed, not the whole listing, so the cost of asking\nstays proportional to the page size rather than the library size.\n\nEach entry carries `role`, `final` or `intermediate` (null for a\nflat-layout file or one no manifest lists); a file a later run read\nlists as intermediate unless its step declared it final\n(dw/roles.py rule 4). A run that recorded no role takes the role its\nfile's subfolder names, then rule 4; with neither it is null. `role`\nis a repeatable filter; a null role counts as final. Omitted, every file is listed.", "operationId": "gallery_api_gallery_get", "parameters": [ { From e428f63136534a7a1de2ea9178c9db9e0c46e850 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 18:37:07 -0500 Subject: [PATCH 011/105] fix(locations): #951 - validate checks a bare relative media name; refusal names output: for an own output Co-Authored-By: Claude Sonnet 5.5 --- dw/locations.py | 28 +++++++++++++++++++++++++++- tests/test_locations.py | 23 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/dw/locations.py b/dw/locations.py index 6d5047c9..196fcdc4 100644 --- a/dw/locations.py +++ b/dw/locations.py @@ -202,6 +202,7 @@ def validate_media_path( if not workflows_are_trusted(): roots = media_roots(base_dir) if not any(_within(resolved, root) for root in roots): + own = _own_output_hint(location) raise PathTraversalError( f"Refusing to read {what} at '{location}': it resolves " f"outside every directory this workflow may read " @@ -215,7 +216,7 @@ def validate_media_path( f"'asset:' reference, use keep_output(shared=True) to copy " f"a generated file into the library every workspace shares " f"if it needs to cross that boundary on purpose, or pass " - f"--trust-workflows if you trust this workflow's source." + f"--trust-workflows if you trust this workflow's source." + own ) if require_exists and not os.path.exists(resolved): @@ -223,6 +224,26 @@ def validate_media_path( return resolved +def _own_output_hint(location): + """A sentence naming `output:` when the bare relative name is a + file in this workspace's own outputs, else an empty string.""" + text = str(location) + if os.path.isabs(text) or ".." in text.replace("\\", "/").split("/"): + return "" + try: + from .runs import output_root + + if os.path.isfile(os.path.join(output_root(), text)): + return ( + f" This file is one of this workspace's own outputs, so " + f"you most likely want 'output:{text}' - no copy into the " + f"asset library is needed." + ) + except Exception: + logger.debug("Could not check the output root for a hint", exc_info=True) + return "" + + def _joined(location, base_dir): return os.path.join(base_dir, str(location)) if base_dir else str(location) @@ -620,6 +641,11 @@ def _check(value, base_dir, what): validate_media_url(value, what) elif os.path.isabs(value): validate_media_path(value, base_dir, what, require_exists=False) + elif ".." not in value.replace("\\", "/").split("/"): + # A bare relative name resolves against the workflow directory and + # is refused at run time when that lands outside every root (a + # manifest file name from get_job is the usual case) + validate_media_path(value, base_dir, what, require_exists=False) elif ".." in value.replace("\\", "/").split("/"): # A relative path is under base_dir by construction unless it # climbs out, and validate_path refuses '..' - but only when the diff --git a/tests/test_locations.py b/tests/test_locations.py index 2d8414f0..0a22b65f 100644 --- a/tests/test_locations.py +++ b/tests/test_locations.py @@ -865,3 +865,26 @@ def test_a_record_handed_on_as_a_dict_is_left_alone( ): definition = self._step(command, key, {"mode": "crop"}) assert location_errors(definition, base_dir=workflow_dir) == [] + + +class TestOwnOutputHint: + """#951: a bare manifest file name is refused with `output:` as the fix.""" + + def test_bare_name_of_an_own_output_names_the_output_reference( + self, untrusted, workflow_dir + ): + from dw import runs + + with tempfile.TemporaryDirectory() as outputs: + sub = os.path.join(outputs, "wf", "run") + os.makedirs(sub) + _image(os.path.join(sub, "a.png")) + token = runs.activate_output_root(outputs) + try: + # workflow_dir stands in for a base that is not the output root + with patch("dw.locations.media_roots", return_value=[workflow_dir]): + with pytest.raises(PathTraversalError) as refusal: + validate_media_path("wf/run/a.png", None, "an image argument") + finally: + runs.deactivate_output_root(token) + assert "output:wf/run/a.png" in str(refusal.value) From 5c154813ad240ae217075e3c8ff74df652f220a0 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 18:45:07 -0500 Subject: [PATCH 012/105] fix(locations): #951 - own-output hint resolves through runs.resolve_output_reference and references.make_ref Co-Authored-By: Claude Sonnet 5.5 --- dw/locations.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/dw/locations.py b/dw/locations.py index 196fcdc4..b0bfcc73 100644 --- a/dw/locations.py +++ b/dw/locations.py @@ -231,14 +231,17 @@ def _own_output_hint(location): if os.path.isabs(text) or ".." in text.replace("\\", "/").split("/"): return "" try: - from .runs import output_root - - if os.path.isfile(os.path.join(output_root(), text)): - return ( - f" This file is one of this workspace's own outputs, so " - f"you most likely want 'output:{text}' - no copy into the " - f"asset library is needed." - ) + from .runs import resolve_output_reference + + ref = references.make_ref(references.OUTPUT, text) + resolve_output_reference(ref) + return ( + f" This file is one of this workspace's own outputs, so " + f"you most likely want '{ref}' - no copy into the " + f"asset library is needed." + ) + except (InvalidInputError, PathTraversalError, ValueError): + pass except Exception: logger.debug("Could not check the output root for a hint", exc_info=True) return "" @@ -646,7 +649,7 @@ def _check(value, base_dir, what): # is refused at run time when that lands outside every root (a # manifest file name from get_job is the usual case) validate_media_path(value, base_dir, what, require_exists=False) - elif ".." in value.replace("\\", "/").split("/"): + else: # A relative path is under base_dir by construction unless it # climbs out, and validate_path refuses '..' - but only when the # loader reaches it, which is a queued job and three seconds in From 015ddd22039d651685fa87e3109f8bb96315d899 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 18:48:54 -0500 Subject: [PATCH 013/105] test: #951 - drop the media_roots patch from the own-output hint test (patch-target ratchet) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_locations.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_locations.py b/tests/test_locations.py index 0a22b65f..308f9c6a 100644 --- a/tests/test_locations.py +++ b/tests/test_locations.py @@ -881,10 +881,8 @@ def test_bare_name_of_an_own_output_names_the_output_reference( _image(os.path.join(sub, "a.png")) token = runs.activate_output_root(outputs) try: - # workflow_dir stands in for a base that is not the output root - with patch("dw.locations.media_roots", return_value=[workflow_dir]): - with pytest.raises(PathTraversalError) as refusal: - validate_media_path("wf/run/a.png", None, "an image argument") + with pytest.raises(PathTraversalError) as refusal: + validate_media_path("wf/run/a.png", None, "an image argument") finally: runs.deactivate_output_root(token) assert "output:wf/run/a.png" in str(refusal.value) From 19773c4dfb167aad5efcef3c17bbdd63429e8bcc Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 19:00:37 -0500 Subject: [PATCH 014/105] fix(locations): #951 - validate refuses a bare own-output name not found beside the workflow, naming output: validate joined the bare name onto the workflow directory, which is contained by construction, so it passed; the run's check_script reads the raw string against the server's working directory and refuses it. Co-Authored-By: Claude Opus 5.5 --- dw/locations.py | 16 ++++++++++++- tests/test_locations.py | 52 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/dw/locations.py b/dw/locations.py index b0bfcc73..3fd7b158 100644 --- a/dw/locations.py +++ b/dw/locations.py @@ -648,7 +648,21 @@ def _check(value, base_dir, what): # A bare relative name resolves against the workflow directory and # is refused at run time when that lands outside every root (a # manifest file name from get_job is the usual case) - validate_media_path(value, base_dir, what, require_exists=False) + resolved = validate_media_path(value, base_dir, what, require_exists=False) + if not os.path.exists(resolved): + # Contained under the workflow directory is not found there: + # a task reading the raw string resolves it against the + # server's working directory and refuses it, a pipeline + # argument finds nothing. Either way the run fails, so an own + # output named without its prefix is refused now (#951) + own = _own_output_hint(value) + if own: + raise InvalidInputError( + f"Refusing to read {what} at '{value}': a bare name " + f"is not looked up in the output root, and no file " + f"by that name is beside the workflow, so the run " + f"would refuse it." + own + ) else: # A relative path is under base_dir by construction unless it # climbs out, and validate_path refuses '..' - but only when the diff --git a/tests/test_locations.py b/tests/test_locations.py index 308f9c6a..177f0144 100644 --- a/tests/test_locations.py +++ b/tests/test_locations.py @@ -886,3 +886,55 @@ def test_bare_name_of_an_own_output_names_the_output_reference( finally: runs.deactivate_output_root(token) assert "output:wf/run/a.png" in str(refusal.value) + + def test_validate_refuses_a_bare_own_output_name_with_the_hint( + self, untrusted, workflow_dir + ): + """The tester's repro: check-script's input_audio given a manifest + file name without its prefix fails validation, not the run.""" + import shutil + + from dw import runs + from dw.workflow import workflow_from_file + + template = os.path.join(workflow_dir, "check-script.json") + shutil.copy( + os.path.join( + os.path.dirname(__file__), + "..", + "workflows", + "templates", + "check-script.json", + ), + template, + ) + with tempfile.TemporaryDirectory() as outputs: + sub = os.path.join(outputs, "cut", "run", "final") + os.makedirs(sub) + with open(os.path.join(sub, "episode.mp4"), "wb") as f: + f.write(b"\0") + token = runs.activate_output_root(outputs) + try: + workflow = workflow_from_file(template, outputs) + bare = workflow.validation_errors( + {"input_audio": "cut/run/final/episode.mp4", "lines": ["x"]} + ) + prefixed = workflow.validation_errors( + { + "input_audio": "output:cut/run/final/episode.mp4", + "lines": ["x"], + } + ) + finally: + runs.deactivate_output_root(token) + messages = [e["message"] for e in bare if e["path"].endswith(".audio")] + assert messages, bare + assert "'output:cut/run/final/episode.mp4'" in messages[0] + assert not [e for e in prefixed if e["path"].endswith(".audio")], prefixed + + def test_validate_keeps_a_bare_name_beside_the_workflow( + self, untrusted, workflow_dir + ): + _image(os.path.join(workflow_dir, "a.png")) + definition = {"steps": [{"name": "s", "arguments": {"image": "a.png"}}]} + assert location_errors(definition, None, workflow_dir) == [] From 9d483ff094e02795cffb9a48af70ee0d25ebedcb Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 19:09:56 -0500 Subject: [PATCH 015/105] fix(locations): #952 - validate refuses any bare relative media name that resolves nowhere, hint only for own outputs Co-Authored-By: Claude Sonnet 5.5 --- dw/locations.py | 19 +++++++++---------- tests/test_locations.py | 9 +++++++++ tests/test_reference_limits.py | 7 ++++++- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/dw/locations.py b/dw/locations.py index 3fd7b158..8062276b 100644 --- a/dw/locations.py +++ b/dw/locations.py @@ -653,16 +653,15 @@ def _check(value, base_dir, what): # Contained under the workflow directory is not found there: # a task reading the raw string resolves it against the # server's working directory and refuses it, a pipeline - # argument finds nothing. Either way the run fails, so an own - # output named without its prefix is refused now (#951) - own = _own_output_hint(value) - if own: - raise InvalidInputError( - f"Refusing to read {what} at '{value}': a bare name " - f"is not looked up in the output root, and no file " - f"by that name is beside the workflow, so the run " - f"would refuse it." + own - ) + # argument finds nothing. Either way the run fails, so it is + # refused now, with the output: hint when it names an own + # output (#951, #952) + raise InvalidInputError( + f"Refusing to read {what} at '{value}': a bare name " + f"is not looked up in the output root, and no file " + f"by that name is beside the workflow, so the run " + f"would refuse it." + _own_output_hint(value) + ) else: # A relative path is under base_dir by construction unless it # climbs out, and validate_path refuses '..' - but only when the diff --git a/tests/test_locations.py b/tests/test_locations.py index 177f0144..36f80093 100644 --- a/tests/test_locations.py +++ b/tests/test_locations.py @@ -938,3 +938,12 @@ def test_validate_keeps_a_bare_name_beside_the_workflow( _image(os.path.join(workflow_dir, "a.png")) definition = {"steps": [{"name": "s", "arguments": {"image": "a.png"}}]} assert location_errors(definition, None, workflow_dir) == [] + + def test_validate_refuses_a_bare_name_that_resolves_nowhere( + self, untrusted, workflow_dir + ): + definition = {"steps": [{"name": "s", "arguments": {"image": "no-such.png"}}]} + errors = location_errors(definition, None, workflow_dir) + assert len(errors) == 1, errors + assert "no file by that name is beside the workflow" in errors[0]["message"] + assert "output:" not in errors[0]["message"] diff --git a/tests/test_reference_limits.py b/tests/test_reference_limits.py index 30d8bbf3..21722305 100644 --- a/tests/test_reference_limits.py +++ b/tests/test_reference_limits.py @@ -128,7 +128,12 @@ def test_an_audio_only_shot_is_refused_before_the_run(self): "references": [reference(AUDIO)], } ] - errors = self._template().validation_errors({"shots": shots}) + errors = [ + e + for e in self._template().validation_errors({"shots": shots}) + # the placeholder file name resolves nowhere, which is its own error + if not e["path"].endswith(".from_file") + ] assert len(errors) == 1 assert errors[0]["path"].endswith("pipeline.arguments.references") assert "shot@audio_only" in errors[0]["message"] From 19474c09b3fe2072f125a67866c467f36de6cef4 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 19:17:27 -0500 Subject: [PATCH 016/105] fix(check-script): #953 - line_clipped_at_end needs the tail within 15 dB of the last word, so a score bed is not a clip Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 3 ++- dw/tasks/script_check.py | 26 ++++++++++++++++++++------ tests/test_script_check.py | 18 ++++++++++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 54cf2374..65e95eb7 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3043,7 +3043,7 @@ Markup stripped from an expected line: `[Language] ...`, ``, | ---- | ---------- | --------------------- | ---- | | `line_mismatch` | A line's similarity is below `similarity`; a dropped line comes back with `heard: ""` | The line's similarity / `similarity` | `{line, seconds, word}` - the line's first heard word, or where it should have been | | `tag_spoken` | A word from the line's stripped markup was heard, and is not also a word of the line's dialogue | The word heard / none | `{line, seconds, word}` | -| `line_clipped_at_end` | A line's last heard word ends inside the final `CLIP_TAIL_SECONDS` of its shot (when the line has one) or of the file, and that tail is still above `GUARD_FLOOR_DBFS` | The tail's level in dBFS / `GUARD_FLOOR_DBFS` | `{line, seconds, word, shot}` - `shot` when the line has one | +| `line_clipped_at_end` | A line's last heard word ends inside the final `CLIP_TAIL_SECONDS` of its shot (when the line has one) or of the file, and that tail is still above `GUARD_FLOOR_DBFS` and within `CLIP_TAIL_BELOW_WORD_DB` of the last word's own loudest window (a quieter tail is a score bed or room tone, not the word - #953) | The tail's level in dBFS / `GUARD_FLOOR_DBFS` | `{line, seconds, word, shot}` - `shot` when the line has one | | `speech_where_silent` | `lines` is `[]` and words were heard above the floor and outside a repetition loop | Number of words heard / 0 | `{line: null, seconds, word}` - the first; `shot` when shots are known | | `speech_in_silent_shot` | Shots are known, at least one line names a shot, and words were heard above the floor and outside a repetition loop in a shot no line names. A word is in the shot its midpoint falls in; one a line in another shot matched (a word straddling the cut) is that line's, not counted | Number of words heard in the shot / 0 | `{line: null, seconds, word, shot}` - the shot's first counted word, `seconds` no earlier than the shot's start | | `unscripted_speech` | Shots are known and a line names the shot; words were heard inside it that align to no line (a line spoken twice, an ad-lib) | Number of unmatched words in the shot / 0 | `{line: null, seconds, end, word, shot, repeats_line}` - the first such word; `repeats_line` is true when those words repeat or contain one of the shot's lines | @@ -3060,6 +3060,7 @@ The thresholds are module constants in `dw/tasks/script_check.py`, deliberately | `UNHEARD_MIN_SECONDS` / `UNHEARD_MARGIN_SECONDS` / `UNHEARD_FLOOR_DBFS` | 1.0 s / 0.25 s / -40 dBFS | A gap in a line's words (or a stretched word) this long, with a window above the floor in its margin-trimmed interior, is a `speech_not_transcribed` | | `GUARD_WINDOW_SECONDS` | 0.05 s (= `DEAD_AIR_WINDOW`, `shot_dead_air`'s window) | The window a heard word is measured in - its loudest one, so a span overhanging a pause does not average a real word away | | `CLIP_TAIL_SECONDS` | 0.25 s | The final stretch of the file a line's last word must end in to be `line_clipped_at_end` | +| `CLIP_TAIL_BELOW_WORD_DB` | 15 dB | A tail more than this far under the last word's loudest window is not the word still sounding (a bed under a finished line), so not `line_clipped_at_end` | | `REPEAT_RUN_MIN` | 6 | Heard words repeating the same unit this many times over, back to back, are a repetition loop: every chunk holding one is discarded (`reason: "repetition"`), however loud | | `REPEAT_MAX_PERIOD` | 4 words | The longest repeating unit a loop is looked for in - `pre pre pre` has period 1, `thank you thank you` period 2 | diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index c3c12dfd..2a57e062 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -85,6 +85,10 @@ # GUARD_FLOOR_DBFS - still voiced when the file stops, not a word that # finished into a quiet tail CLIP_TAIL_SECONDS = 0.25 +# ... and within CLIP_TAIL_BELOW_WORD_DB of the last word's own loudest window: +# a word cut mid-sound leaves a tail near its level, a score bed or room tone +# under a word that finished sits far below it (#953) +CLIP_TAIL_BELOW_WORD_DB = 15.0 # Audible speech the transcript has no words for (#923): Whisper can drop a # passage from its text on a longer input (the ad-lib heard in a short file # is gone), and the word alignment hands that time to a neighbouring word, @@ -104,6 +108,7 @@ "guard_floor_dbfs": GUARD_FLOOR_DBFS, "guard_window_seconds": GUARD_WINDOW_SECONDS, "clip_tail_seconds": CLIP_TAIL_SECONDS, + "clip_tail_below_word_db": CLIP_TAIL_BELOW_WORD_DB, "unheard_min_seconds": UNHEARD_MIN_SECONDS, "unheard_margin_seconds": UNHEARD_MARGIN_SECONDS, "unheard_floor_dbfs": UNHEARD_FLOOR_DBFS, @@ -159,7 +164,9 @@ def mismatch_rule(similarity): GUARD_FLOOR_DBFS, "the line's last word ends inside the final" f" {CLIP_TAIL_SECONDS} s of its shot (or of the file) and that tail is" - " still voiced at this level - the shot stops mid-word", + f" still voiced at this level, within {CLIP_TAIL_BELOW_WORD_DB} dB of the" + " word's own level (a quieter tail is a bed or room tone) - the shot" + " stops mid-word", ) SPEECH_IN_SILENT_SHOT = _rule( "speech_in_silent_shot", @@ -374,17 +381,22 @@ def _at(line, heard_word=None, seconds=None, shot=None): return at -def _clipped_tail(mono, sample_rate, end, tails): +def _clipped_tail(mono, sample_rate, end, tails, word=None): """The first of `tails` - (shot name or None, the second it ends at) - that a line ending at `end` is clipped by: the word ends inside its final CLIP_TAIL_SECONDS (within that much past it too, a word timestamp - being approximate) and the tail is voiced above GUARD_FLOOR_DBFS. - Returns (shot, level) or None.""" + being approximate) and the tail is voiced above GUARD_FLOOR_DBFS and + within CLIP_TAIL_BELOW_WORD_DB of `word`'s (start, end) loudest window, + when given: a tail far under the word is a bed, not the word. Returns (shot, level) or None.""" + word_db = word_level(mono, sample_rate, *word) if word else None + floor = GUARD_FLOOR_DBFS + if word_db is not None: + floor = max(floor, word_db - CLIP_TAIL_BELOW_WORD_DB) for shot, tail_end in tails: if not tail_end - CLIP_TAIL_SECONDS <= end <= tail_end + CLIP_TAIL_SECONDS: continue level = tail_level(mono, sample_rate, tail_end) - if level is not None and level > GUARD_FLOOR_DBFS: + if level is not None and level > floor: return shot, level return None @@ -625,7 +637,9 @@ def _clipped_findings(mono, sample_rate, tails, index, own, shot): """line_clipped_at_end for one heard line, as a list of zero or one finding: its last word ends in a voiced tail of its shot or the file.""" end = own[-1]["end"] - clipped = _clipped_tail(mono, sample_rate, end, tails) + clipped = _clipped_tail( + mono, sample_rate, end, tails, (own[-1]["start"], own[-1]["end"]) + ) if clipped is None: return [] at = {"line": index, "seconds": _round(end), "word": own[-1]["text"]} diff --git a/tests/test_script_check.py b/tests/test_script_check.py index f653f0a6..5d4d631f 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -772,6 +772,24 @@ def test_skipped_when_no_line_names_a_shot(self): self.assertIn("no line names", skip["reason"]) +class TestClippedUnderBed(unittest.TestCase): + CHUNKS = [ + {"start": 0.5, "end": 0.9, "text": " hello"}, + {"start": 1.6, "end": 1.95, "text": " world"}, + ] + + def test_bed_under_a_finished_word_is_not_clipped(self): + # voice (-20 dBFS) up to 1.7 s, then a bed ~30 dB lower to the end + mono = numpy.concatenate([voiced(1.7), voiced(0.3, amplitude=0.003)]) + answer = run(["hello world"], self.CHUNKS, mono) + self.assertEqual(rules(answer, "line_clipped_at_end"), []) + + def test_bed_under_the_whole_word_still_clips_a_voiced_tail(self): + mono = voiced(2.0) + voiced(2.0, amplitude=0.003) + answer = run(["hello world"], self.CHUNKS, mono) + self.assertEqual(len(rules(answer, "line_clipped_at_end")), 1) + + class TestClippedAtShotEnd(unittest.TestCase): LINES = [{"text": "hello world", "shot": "A"}] From 48cfa718678b200d932c13b35373cd74f331633b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 20:58:08 -0500 Subject: [PATCH 017/105] test(mcp): fixed surface ceiling, guide-pointer resolution test, skills under their cap SURFACE_BUDGET was a ratchet raised to each measurement plus <1 token, so every docstring change paid for itself by trimming another tool's text, and at least once the trim took a pointer agents needed (#574 re-broke #282). It is now a fixed 16_000 with a stated policy: move narrative to a guide and leave a pointer before raising it. tests/test_guide_pointers.py checks that every guide pointer in the MCP instructions, tool descriptions, skills and references opens a real section, and that the tools pointing at a guide today keep their pointer. The three skills within ~100 bytes of the 12 KiB cap move situational detail into references, linked from SKILL.md with when to read them: minimax-h3 staging rules, minimax-music3 memory check, series-episodes cross-episode loudness. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 1 + dw_mcp/CLAUDE.md | 2 +- plugins/dw/skills/minimax-h3/SKILL.md | 15 +- .../skills/minimax-h3/references/staging.md | 17 ++ plugins/dw/skills/minimax-music3/SKILL.md | 19 +- .../minimax-music3/references/memory.md | 23 +++ plugins/dw/skills/series-episodes/SKILL.md | 38 +--- .../series-episodes/references/loudness.md | 47 +++++ tests/test_guide_pointers.py | 132 ++++++++++++++ tests/test_mcp_server.py | 170 +++--------------- 10 files changed, 256 insertions(+), 208 deletions(-) create mode 100644 plugins/dw/skills/minimax-h3/references/staging.md create mode 100644 plugins/dw/skills/minimax-music3/references/memory.md create mode 100644 plugins/dw/skills/series-episodes/references/loudness.md create mode 100644 tests/test_guide_pointers.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index edc8ab0c..0ca9e84e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -178,6 +178,7 @@ one to open. | `dw_mcp` stays torch-free | `dw_mcp/` | `dw_mcp` reaches `dw.serve` over HTTP and imports no `dw` module, because `dw/__init__.py` pulls in torch. | `tests/test_mcp_server.py::TestStartupWeight::test_the_server_starts_without_importing_the_engine` | | Tool surface | `dw_mcp/server.py`, `dw_mcp/tools_*.py` | Only these modules import the MCP SDK, and each tool body is a one-line call into a handler. | `tests/test_mcp_server.py::test_the_wiring_table_covers_every_registered_tool`, `tests/test_mcp_server.py::test_the_stated_tool_count_is_the_registered_one` | | Surface text budget | the tool docstrings in `dw_mcp/tools_*.py`, the instructions in `dw_mcp/server.py` | The instructions and each tool description stay at or under 2,048 characters, and the whole surface stays within its token budget. | `tests/test_mcp_server.py`: `SURFACE_BUDGET`, `CLIENT_TEXT_LIMIT`; `tests/test_mcp_server.py::test_no_text_the_agent_reads_is_cut_off_by_the_client`, `tests/test_mcp_server.py::test_the_tool_surface_fits_the_budget` | +| Guide pointers in resident text | the tool docstrings in `dw_mcp/tools_*.py`, the instructions in `dw_mcp/server.py`, each plugin `SKILL.md` and its `references/` | Every `get_guide(...)` / guide-section pointer opens a real section through `dw/server/guides.py`'s `get_guide`, and the tools that point at a guide keep their pointer when text is trimmed to fit the budget. | `tests/test_guide_pointers.py` (`TOOLS_THAT_POINT`) | | Spending needs consent | each handler behind a tool that takes `acknowledged_cost` (`dw_mcp/diagnose.py`, `models.py`, `prompts.py`, `workspaces.py`) | Every tool that takes `acknowledged_cost` refuses until it is set; `delete_workspace`'s refusal is the server's 409. | `tests/test_mcp_twins.py::test_every_tool_that_takes_acknowledged_cost_refuses_without_it` | | Copies of engine rules | the constants and word lists in `dw_mcp/` and `dw/run.py` | `dw_mcp` cannot import its owners, so each copy equals its owner; a copy that needs no twin is deleted, not pinned. | `tests/test_mcp_twins.py` | | The CLI is a client of the server | `dw/run.py` | `dw.run` queues on `dw.serve` through `dw_mcp.client`, so `dw` imports `dw_mcp` and never the reverse. | `tests/test_mcp_server.py::TestStartupWeight::test_the_server_starts_without_importing_the_engine`, `tests/test_mcp_twins.py` | diff --git a/dw_mcp/CLAUDE.md b/dw_mcp/CLAUDE.md index 39e63313..16c6ab1d 100644 --- a/dw_mcp/CLAUDE.md +++ b/dw_mcp/CLAUDE.md @@ -4,5 +4,5 @@ Guidance for the `dw_mcp/` stdio MCP server package. What it covers and how its ## Writing text an agent reads -- A tool description (the docstring in `dw_mcp/tools_*.py`) or the instructions in `dw_mcp/server.py` must put what an agent has to act on first, and point at a guide section rather than restate it. `tests/test_mcp_server.py` pins the length (`CLIENT_TEXT_LIMIT`, `SURFACE_BUDGET`) but not the ordering, and the instructions and `list_workflows` sit within a few dozen characters of the limit. +- A tool description (the docstring in `dw_mcp/tools_*.py`) or the instructions in `dw_mcp/server.py` must put what an agent has to act on first, and point at a guide section rather than restate it. `tests/test_mcp_server.py` pins the length (`CLIENT_TEXT_LIMIT`, `SURFACE_BUDGET`) but not the ordering, and the instructions and `list_workflows` sit within a few dozen characters of the limit. `SURFACE_BUDGET` is a fixed ceiling with headroom, not a ratchet: when it fails, move narrative to a guide and leave a pointer before raising it. `tests/test_guide_pointers.py` checks every pointer resolves and that a tool which points at a guide keeps its pointer. - `MAX_WAIT_SECONDS` (`dw_mcp/diagnose.py`) is interpolated into the tool descriptions, so no doc or skill quotes the number. diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index d1fa8fab..3c4f81b8 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -100,18 +100,9 @@ section). `validate_workflow` refuses it and warns on a `weight_name` naming neither. - Smeary faces (int8 base) or draft -> final tiers: `references/quality-and-speed.md`. - Backend: default `sparse` (dense off CUDA); same one at draft and final. -- Fit a crowd's action to what it holds: candle-holders asked to clap - rendered three-handed people. -- A directed crowd move needs its landmark in frame: "marches up to the - house" put it behind camera. Track from the side, landmark ahead. -- Two people in frame can lip-sync the wrong one, seemingly by whichever - voice enters first: credit a shout to the crowd, spell the sung line in - `` tags, frame the singer face-on at medium close-up. -- State a crowd's age range; unstated skews young. -- A three-reference shot can open on the portraits' backdrop - for ~1.5s (about 1 in 25): check a shot's first frames and say in - `subject_definitions` that the portrait's backdrop, pose and framing are - not reused. +- A crowd, two people in frame, or a three-reference shot: read + `references/staging.md` before the prompt - who lip-syncs, where the + landmark is, and the portrait backdrop that can open a shot. - Ref2VA reaches 1344x768 too (trade-off: the `recipes` guide's MiniMax-H3 section). There the 24 GB ceiling on the `17n+5` grid drops with each reference: 243 frames at one, 209 at two, 175 at three, 141 at four. `validate_workflow` refuses diff --git a/plugins/dw/skills/minimax-h3/references/staging.md b/plugins/dw/skills/minimax-h3/references/staging.md new file mode 100644 index 00000000..93a79d9c --- /dev/null +++ b/plugins/dw/skills/minimax-h3/references/staging.md @@ -0,0 +1,17 @@ +# MiniMax H3: staging crowds, two people, and reference shots + +Part of the `minimax-h3` skill; its *Hard rules* say when to read this. Each +rule came from a render that went wrong on our server (#484). + +- Fit a crowd's action to what it holds: candle-holders asked to clap + rendered three-handed people. +- A directed crowd move needs its landmark in frame: "marches up to the + house" put it behind camera. Track from the side, landmark ahead. +- Two people in frame can lip-sync the wrong one, seemingly by whichever + voice enters first: credit a shout to the crowd, spell the sung line in + `` tags, frame the singer face-on at medium close-up. +- State a crowd's age range; unstated skews young. +- A three-reference shot can open on the portraits' backdrop + for ~1.5s (about 1 in 25): check a shot's first frames and say in + `subject_definitions` that the portrait's backdrop, pose and framing are + not reused. diff --git a/plugins/dw/skills/minimax-music3/SKILL.md b/plugins/dw/skills/minimax-music3/SKILL.md index c7fa081f..09f566af 100644 --- a/plugins/dw/skills/minimax-music3/SKILL.md +++ b/plugins/dw/skills/minimax-music3/SKILL.md @@ -18,21 +18,10 @@ shapes; do not author a new workflow until the shape decision below fails. 2. `list_workflows(shape="audio")`, and `shape="sequence"` for the music video. Trust the names below only after the listing confirms them. 3. `get_workflow` on the one chosen, for its variables and defaults. -4. Before a long track or the music video - anything near the - card's ceiling - `get_memory` with the server idle and read a `live: true` - reading's `gpu_memory_allocated_mb`. Only a live reading is the worker's own - and only live readings compare: `info: null` means nothing - is resident, go ahead, while a `live: false` reading with a populated `info` - is cached from another moment - it reads low while a job is loading a model, - so ask again once idle rather than trusting it. A non-trivial - idle figure is what an earlier run left behind, coming off the ~22 GB - these templates need. With the server idle, `clear_memory` clears it - (refused while a job is queued or running) - it also drops the step cache, - so the next run, including a seeded rerun, is cold and regenerates. Re-read - `get_memory` afterwards to confirm. Don't retry into a failed attempt - without clearing first - a failed attempt is what leaves weight - resident, so an immediate retry starts from less than the attempt that - failed had. +4. Before a long track or the music video - anything near the card's + ceiling, the ~22 GB these templates need - read `references/memory.md`: + which `get_memory` reading to trust, when `clear_memory` first, and why + never to retry into a failed attempt without it. ## Which shape is the request diff --git a/plugins/dw/skills/minimax-music3/references/memory.md b/plugins/dw/skills/minimax-music3/references/memory.md new file mode 100644 index 00000000..5bd1d2ee --- /dev/null +++ b/plugins/dw/skills/minimax-music3/references/memory.md @@ -0,0 +1,23 @@ +# MiniMax Music 3: checking memory before a long run + +Part of the `minimax-music3` skill; step 4 of *Before anything* says when to +read this - before a long track or the music video, anything near the card's +ceiling. + +`get_memory` with the server idle, and read a `live: true` reading's +`gpu_memory_allocated_mb`. Only a live reading is the worker's own and only +live readings compare: + +- `info: null` means nothing is resident: go ahead. +- A `live: false` reading with a populated `info` is cached from another + moment. It reads low while a job is loading a model, so ask again once idle + rather than trusting it. +- A non-trivial idle figure is what an earlier run left behind, coming off + the ~22 GB these templates need. With the server idle, `clear_memory` + clears it (refused while a job is queued or running). It also drops the + step cache, so the next run, including a seeded rerun, is cold and + regenerates. Re-read `get_memory` afterwards to confirm. + +Don't retry into a failed attempt without clearing first: a failed attempt is +what leaves weight resident, so an immediate retry starts from less than the +attempt that failed had. diff --git a/plugins/dw/skills/series-episodes/SKILL.md b/plugins/dw/skills/series-episodes/SKILL.md index ac97f197..72989120 100644 --- a/plugins/dw/skills/series-episodes/SKILL.md +++ b/plugins/dw/skills/series-episodes/SKILL.md @@ -117,26 +117,11 @@ composes into, not something to re-derive: applied before the score is passed in. - **normalize**: the mixed world sound and score are normalized together (`assemble-and-score`'s `balanced` step, -3 dBFS peak ceiling, always - enforced) so one episode is not louder than the next. A shared peak - ceiling does not mean a shared loudness - a sparse, dialogue-only episode - and a dense, score-heavy one can both sit at -3 dBFS peak and still read - as very different volumes. `assemble-and-score`'s `target_lufs` variable - (passed through to `normalize_audio`) gains the mix toward a measured - loudness before the ceiling is applied - but gain down succeeds only - while the mix's own peak is below the ceiling; a single loud peak (a - laugh track, a sting) caps gain up the same way, and can pull a gain-down - episode lower still. Match episodes **downward** anyway: pick one series `target_lufs` - that every episode can reach - in practice, about the quietest episode - the ceiling holds back - and pass that same value on every episode's - `assemble-and-score` run. A `target_lufs_capped` warning on an - episode means that episode's own peak sets the series' ceiling; lower the - series `target_lufs` rather than accepting the mismatch. Only when the - series should sit louder than its most dynamic episode allows (e.g. -16 - for streaming) pass `limit: true` beside the same `target_lufs` on every - episode: -3 dBFS becomes a true-peak ceiling a look-ahead limiter holds, - so the laugh is limited rather than setting the gain. A `limiter_heavy` - warning on an episode means the series target is squashing it audibly - - lower the series `target_lufs` rather than living with it. + enforced), but a shared peak is not a shared loudness. Pass one series + `target_lufs` on every episode, matched **downward** to what the quietest + episode can reach; before episode 1, read `references/loudness.md` for how + to pick it and what `target_lufs_capped`, `limit: true` and + `limiter_heavy` mean. - **pair**: the normalized track is muxed onto the cut - the episode's deliverable. @@ -171,16 +156,9 @@ does, `keep_output` it and reference the resulting `asset:` instead. Beyond `minimax-h3`'s own judging checklist (a character that changes between shots, a portrait imposing its framing, a voice without affect): watch two episodes back to back and check the cast reads as the same -people - the failure mode step 0 exists to prevent. `get_gallery_metadata` -on each episode's final file for duration and loudness, so a level -mismatch between episodes shows up before a viewer notices it - its -`peak_dbfs` is a single sample and does not say how loud the episode reads -as a whole; `integrated_lufs` (BS.1770, whole-track) is the field that -answers that, and is what to compare across episodes. A peak-normalized -episode with one loud outlier (a studio-audience laugh, a sting) can sit -1-2 LU quieter than its neighbors even at the same -3 dBFS peak ceiling - -that gap is `target_lufs`'s to close, matched downward (the **normalize** -bullet; `limit: true` for a series that must sit louder). To confirm every +people - the failure mode step 0 exists to prevent. Compare episodes' +`integrated_lufs` (`get_gallery_metadata`), not `peak_dbfs`; the second +half of `references/loudness.md` reads the gap. To confirm every line rendered rather than judging it by ear, run `templates/check-script` against the episode's script: `validate_workflow(name="templates/check-script", arguments={"input_audio": "output:", "lines": [...]})` first (free; it diff --git a/plugins/dw/skills/series-episodes/references/loudness.md b/plugins/dw/skills/series-episodes/references/loudness.md new file mode 100644 index 00000000..11f9f790 --- /dev/null +++ b/plugins/dw/skills/series-episodes/references/loudness.md @@ -0,0 +1,47 @@ +# Series episodes: one loudness across episodes + +Part of the `series-episodes` skill; its **normalize** beat and *Run and +judge* say when to read this - picking the series `target_lufs` before +episode 1, and comparing finished episodes. + +## Picking the series target + +`assemble-and-score`'s `balanced` step normalizes the mixed world sound and +score together to a -3 dBFS peak ceiling, always enforced. A shared peak +ceiling does not mean a shared loudness: a sparse, dialogue-only episode and +a dense, score-heavy one can both sit at -3 dBFS peak and still read as very +different volumes. + +`assemble-and-score`'s `target_lufs` variable (passed through to +`normalize_audio`) gains the mix toward a measured loudness before the +ceiling is applied. Gain down succeeds only while the mix's own peak is below +the ceiling; a single loud peak (a laugh track, a sting) caps gain up the +same way, and can pull a gain-down episode lower still. + +Match episodes **downward** anyway: pick one series `target_lufs` that every +episode can reach - in practice, about the quietest episode the ceiling holds +back - and pass that same value on every episode's `assemble-and-score` run. + +- A `target_lufs_capped` warning on an episode means that episode's own peak + sets the series' ceiling; lower the series `target_lufs` rather than + accepting the mismatch. +- Only when the series should sit louder than its most dynamic episode allows + (e.g. -16 for streaming), pass `limit: true` beside the same `target_lufs` + on every episode: -3 dBFS becomes a true-peak ceiling a look-ahead limiter + holds, so the laugh is limited rather than setting the gain. +- A `limiter_heavy` warning on an episode means the series target is + squashing it audibly - lower the series `target_lufs` rather than living + with it. + +## Comparing finished episodes + +`get_gallery_metadata` on each episode's final file, for duration and +loudness, so a level mismatch between episodes shows up before a viewer +notices it. Its `peak_dbfs` is a single sample and does not say how loud the +episode reads as a whole; `integrated_lufs` (BS.1770, whole-track) is the +field that answers that, and is what to compare across episodes. + +A peak-normalized episode with one loud outlier (a studio-audience laugh, a +sting) can sit 1-2 LU quieter than its neighbors even at the same -3 dBFS +peak ceiling. That gap is `target_lufs`'s to close, matched downward as +above; `limit: true` for a series that must sit louder. diff --git a/tests/test_guide_pointers.py b/tests/test_guide_pointers.py new file mode 100644 index 00000000..e224de43 --- /dev/null +++ b/tests/test_guide_pointers.py @@ -0,0 +1,132 @@ +"""Every pointer an agent is handed to a guide section opens that section. + +The resident text - the MCP server's instructions and tool descriptions, and +each plugin skill's SKILL.md - says what a tool or a family does and points +at a guide for the rest. A pointer is the whole of the discovery path: a +section renamed out from under one, or a pointer trimmed away to fit a +budget (#574 re-broke #282 that way), leaves an agent holding no way to the +detail. So every pointer must resolve through the same `get_guide` the +server answers with, and the tools that carry one now must keep it. +""" + +import asyncio +import glob +import os +import re + +import pytest + +from dw.server import guides +from tests.test_examples import REPO_ROOT + +pytest.importorskip("mcp", reason="the mcp extra is not installed") + +from tests.test_mcp_server import ok, server_over, tools_of # noqa: E402 + +SKILL_TEXTS = sorted( + glob.glob(os.path.join(REPO_ROOT, "plugins", "dw", "skills", "*", "SKILL.md")) + + glob.glob( + os.path.join(REPO_ROOT, "plugins", "dw", "skills", "*", "references", "*.md") + ) +) + +# The guide a file name stands for, for the "WORKFLOW_GUIDE's ..." form. +_BY_FILE = { + file_name.removesuffix(".md"): name + for name, (file_name, _summary) in guides.GUIDES.items() +} + +# The forms a pointer is written in, with whitespace already collapsed: +# get_guide("workflows", section="The loop") get_guide("loras") +# the `acceleration` guide, section "Reading Memory While Offloading" +# WORKFLOW_GUIDE's "The loop" +_CALL = re.compile( + r'get_guide\(\s*"(?P[a-z-]+)"(?:\s*,\s*section\s*=\s*"(?P
[^"]+)")?\s*\)' +) +_PROSE = re.compile(r'the `(?P[a-z-]+)` guide, section "(?P
[^"]+)"') +_FILE = re.compile(r'(?P[A-Z0-9_]+)\'s "(?P
[^"]+)"') + +# The tools whose description points at a guide today. A pointer trimmed +# away to fit the surface budget fails here rather than leaving the tool +# with no way to its detail; adding one to a tool means adding it here. +TOOLS_THAT_POINT = { + "get_job", + "get_memory", + "get_output_audio", + "save_lora", + "validate_workflow", + "wait_for_job", +} + + +def pointers(text): + """(guide name, section or None) for each pointer in `text`.""" + flat = " ".join(text.split()) + found = [(m["name"], m["section"]) for m in _CALL.finditer(flat)] + found += [(m["name"], m["section"]) for m in _PROSE.finditer(flat)] + found += [ + (_BY_FILE[m["file"]], m["section"]) + for m in _FILE.finditer(flat) + if m["file"] in _BY_FILE + ] + return found + + +def unresolved(text): + """The pointers in `text` that `get_guide` refuses.""" + bad = [] + for name, section in pointers(text): + try: + guides.get_guide(name, section) + except (guides.GuideError, KeyError) as error: + bad.append((name, section, str(error)[:120])) + return bad + + +def mcp_texts(): + server = server_over(ok({})) + tools = asyncio.run(tools_of(server)) + texts = {name: tool.description or "" for name, tool in tools.items()} + texts[""] = server.instructions + return texts + + +def test_every_pointer_in_the_mcp_surface_resolves(): + bad = {name: unresolved(text) for name, text in mcp_texts().items()} + assert not {name: b for name, b in bad.items() if b} + + +def test_the_tools_that_point_at_a_guide_still_do(): + texts = mcp_texts() + lost = sorted(name for name in TOOLS_THAT_POINT if not pointers(texts[name])) + assert not lost, f"these tools lost their guide pointer: {lost}" + + +def test_the_server_instructions_point_at_the_authoring_section(): + assert ("workflows", "Authoring a workflow from an agent") in pointers( + mcp_texts()[""] + ) + + +@pytest.mark.parametrize( + "path", SKILL_TEXTS, ids=lambda p: os.path.relpath(p, REPO_ROOT) +) +def test_every_pointer_in_a_skill_resolves(path): + with open(path, encoding="utf-8") as handle: + assert not unresolved(handle.read()) + + +def test_the_pointer_forms_are_recognised(): + """The extraction itself, so a form it stops matching cannot pass the + resolution tests by finding nothing.""" + text = ( + 'see get_guide("workflows",\n section="The loop") and the ' + '`acceleration` guide, section "Caching", and WORKFLOW_GUIDE\'s ' + '"References", and get_guide("loras")' + ) + assert pointers(text) == [ + ("workflows", "The loop"), + ("loras", None), + ("acceleration", "Caching"), + ("workflows", "References"), + ] diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index a609dd8e..8f496520 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1554,156 +1554,26 @@ def test_the_stated_tool_count_is_the_registered_one(): # Chars / 4, the way COMPACT_BUDGET in tests/test_catalog_structure.py is -# measured. This is not a listing an agent chooses to read - it is resident -# for the whole session before a single call, which is why it gets a ceiling -# at all and why the catalog listing, read once, has had one since #101. -# Measured 2026-09-19 at 13_225 before the list_prompts rewrite that landed -# in the same plan (descriptions 9_013, input schemas 3_212, instructions -# 999), and at 13_421 after it (9_138 / 3_283 / 1_000) - the three filter -# parameters and the docstring that says what the library is for. Set at -# 13_800, which is room for a tool or two and not room for a second -# validate_workflow. Those figures were taken on Python 3.14, which dedents -# docstrings at compile time; on 3.12 the same surface measured 14_574 -# (descriptions 10_281) until `tool()` began passing the cleandoc'd -# docstring as the description, since then 13_474 (9_181 / 3_293 / 1_000) -# on either interpreter. Measure on the interpreter CI runs, or on both. -# Worth knowing before raising it: four tools are a quarter of the -# descriptions (validate_workflow 774, wait_for_job 518, list_workflows 482, -# list_gallery 445), and validate_workflow's `plan.basis` taxonomy and -# wait_for_job's stall-diagnosis paragraph are both restated in -# WORKFLOW_GUIDE's "The loop" - which an agent fetches on demand. The -# question to ask first is whether the second copy has to be the resident one. -# Measured 2026-09-20 at 14_017 (9_386 / 3_618 / 1_014) after -# get_output_frames (#193, #210) - a `seams`/`boundaries`/`names` shape whose -# 971-char input schema alone is most of what a "tool or two" bought. Paid -# for, per the paragraph above, by actually cutting the two restated copies -# rather than raising the ceiling: wait_for_job's stall-diagnosis paragraph -# and validate_workflow's `plan.basis` taxonomy moved to WORKFLOW_GUIDE's -# "The loop" (steps 4 and 5), leaving a pointer plus the pinned words other -# tests still check for. Landed at 13_741 (9_110 / 3_618 / 1_014) - back -# under the original ceiling with room to spare. -# Measured 2026-09-20 at 13_784 (9_153 / 3_618 / 1_014) after -# get_output_frames's docstring grew a sentence saying where `boundaries` -# comes from before stage 2 (`shots` on the artifact) exists - the running -# sum of `get_gallery_metadata`'s `frame_count` for each shot. 16 tokens of -# headroom left; the next docstring change here should measure again rather -# than assume it still fits. -# Measured 2026-09-20 at 13_799.5 (9_144.5 / 3_641 / 1_014) after -# get_output_frames gained `hear` (soundtrack around each `at` moment, one -# AudioContent per tile that has one). Paid for by tightening two -# sentences: `get_output_frames`'s "(seconds, or "frame:N")" lost its -# comma, and `get_output_audio`'s "The text part reports the track's -# length and, for an excerpt, exactly what was cut, so a slice is never -# mistaken for the whole." shortened to "The text part says what was cut." -# 0.5 tokens of headroom left; the next docstring change here should -# measure again rather than assume it still fits. -# Merging feat/193-evidence into develop combined this branch's tightened -# get_output_audio/get_output_frames text with develop's own independent -# `crop` addition to get_output_image - two branches each under budget on -# their own pushed the merged surface to 13_937, since neither could see -# the other's growth. Paid for by cutting get_output_image's redundant -# "Images only: ..." cross-reference sentence (the other two tools already -# say where images live), shortening get_output_image's and -# get_output_audio's `workspace` sentence to the terse form -# get_output_frames already used ("pins this call to another workspace"), -# and trimming a few more words from get_output_image's crop/downscale -# sentences. Measured 2026-09-20 at 13_796.2 (9_110.8 / 3_671.5 / 1_014.0) -# post-merge. 3.8 tokens of headroom left; a merge that touches tool -# docstrings on both sides needs to measure again, not just each side's -# own pre-merge figure. -# #279/#280/#282 grew get_gallery_metadata's docstring (the peak_dbfs -# thresholds and a resolvable guide pointer) by ~92 tokens without -# measuring, to 13_888. Paid for inside the same docstring: the workspace -# sentence to the terse "pins this call" form, and the envelope and asset -# paragraphs said in fewer words with every fact kept. Measured 2026-09-21 -# at 13_760.5 (9_075.0 / 3_671.5 / 1_014.0). 39.5 tokens of headroom left. -# run_workflow's `wait_seconds` and delete_output's `job_id` (two schema -# parameters, ~50 tokens, plus the sentences that explain them) first -# measured at 14_097. Paid for inside the three docstrings they touch: -# run_workflow's wait paragraph said once and tersely, its spelling / -# arguments / workspace sentences shortened with every fact kept, -# wait_for_job's cap paragraph and stall sentence said in fewer words -# (the pinned phrases stay), delete_output's "leaves the workspace as you -# found it" clause dropped. Measured 2026-09-21 at 13_796.75 (9_061.5 / -# 3_721.25 / 1_014.0). 3.25 tokens of headroom left; measure again before -# the next docstring change. -# #303 gave get_output_frames a `crop` parameter (parity with -# get_output_image, applied per sampled tile before the aggregate -# shrink) - the schema growth alone first measured at 13_884. Paid for -# across all three media docstrings: get_output_frames' selector and -# boundaries sentences said in fewer words, get_output_image's crop/ -# downscale sentence tightened, and get_output_audio's intro and -# envelope sentences shortened (still passing "WAV"/"own encoding"). -# Measured 2026-09-21 at 13_798.25 (9_032.5 / 3_751.75 / 1_014.0). 1.75 -# tokens of headroom left; measure again before the next docstring change. -# The tester's #303 bounce (crop coordinates depended on max_dimension, and -# a count-mode crop was checked against the assembled sheet rather than the -# source frame) needed a docstring correction on top of that: get_output_ -# frames' and get_output_image's crop sentences now say "cut ... before the -# downscale" without the old space-filling clause, which pushed descriptions -# over budget first (13_820). Measured 2026-09-21 at 13_790.8 (9_025.0 / -# 3_751.8 / 1_014.0). 9.2 tokens of headroom left. -# Run versions added four sentences to list_gallery teaching `version` and -# `run_id` - the handle for naming one of several runs that wrote the same -# basename, which is the one thing the surface could not say before. Written -# as tightly as it can be said and still 44 tokens over, so the budget takes -# them deliberately rather than the sentence being cut to nothing. Measured -# 2026-09-22 at 13_844.0 (9_078.0 / 3_752.0 / 1_014.0). 6 tokens of headroom. -# Then list_gallery took `folder` and `version`, so "show me v5" is one call -# rather than a scan of the listing. The docstring paid for its own new -# sentence and then some (descriptions 9_078 -> 9_068, the `url` sentence -# said in fewer words); the two schema entries (+49) are what the budget -# takes, since no docstring can pay for a parameter's schema. Measured -# 2026-09-22 at 13_883.0 (9_068.0 / 3_801.0 / 1_014.0). 7 tokens of headroom. -# #376 moved model narrative out of the resident descriptions, each fact -# still stated where an agent reads it on demand: get_output_audio's -# transcription walkthrough to WORKFLOW_GUIDE's "The loop" step 6 (a pointer -# stays), and the H3 frame-grid example (validate_workflow, list_workflows), -# the family list (list_prompts) and get_job_events' silent-phase examples -# dropped, every rule kept. Measured 2026-09-24 at 13_883.0 before (9_529.0 / -# 3_845.75 / 508.25 - the instructions and descriptions split differently -# from the entry above, the total agrees) and 13_670.5 after (9_316.5 / -# 3_845.75 / 508.25). The ceiling stays: the 219.5 tokens of headroom are -# reserved for #388 (`assess_output`), which raises it only by any remainder. -# #388 spent it on the assess_output tool (description + schema). The design's -# optional pointer in the instructions' loop did not fit the client's 2_048 -# character cap on instructions, so get_gallery_metadata's hint carries it. -# Measured 2026-09-24 at 13_849.5 (9_395.0 / 3_946.25 / 508.25); the ceiling -# stays, 40.5 of headroom. -# 2026-10-03: the LoRA catalog added list_loras, save_lora and -# recommend_loras - measured at 14_255.0 (9_398.5 / 4_345.75 / 510.75); the -# ceiling moves to 14_265 with 10 of headroom. -# 2026-10-07 (#682): get_job and list_jobs name a job's `device`, paid for by -# trimming their own text - measured at 14_264.75; the ceiling stays. -# 2026-10-11 (#828 stage A): get_server_info's one clause on `caches` (16 -# tokens) - measured 14_246.25 before and 14_262.25 after (9_281.75 / -# 4_470.25 / 510.25). Text trimmed since 10-07 left room, so the ceiling -# stays, 2.75 of headroom. -# 2026-10-13 (#859, #853 stage A): copy_to_workspace (121.25 description + -# 84.75 schema) - measured at 14_470.25 (9_276.25 / 4_683.75 / 510.25); the -# ceiling moves to 14_471 with 0.75 of headroom. -# 2026-10-13 (#859 bounce): set_note, documented since the notes feature but -# never registered, is registered (66.75 description + 83.25 schema) - -# measured at 14_620.25 (9_343 / 4_767 / 510.25); the ceiling moves to -# 14_621 with 0.75 of headroom. -# 2026-10-13 (#860, #853 stage B): copy_to_workspace takes assets (its -# description names them; the schema gains the parameter, runs optional) - -# measured at 14_699.5 (9_381 / 4_808 / 510.5); the ceiling moves to 14_700 -# with 0.5 of headroom. -# 2026-10-13 (#885, #867 stage 1): get_job's description names a manifest -# entry's `timing` and the guide section for it (+28.5 description) - measured -# at 14_728 (9_409.5 / 4_808.25 / 510.25); the ceiling moves to 14_728 with -# none to spare. -# 2026-10-14 (#889, #871 stage A): compare_outputs (60.75 description + -# 200.75 schema - `crop` is one box or one per name) - measured at 14_989.25 -# (9_470 / 5_009 / 510.25); the ceiling moves to 14_990 with 0.75 of headroom. -# 2026-10-14 (#886, #867 stage 2): get_job_events' description names the -# `cache_stats` kind (+15 description) - measured at 15_004.25 (9_485 / -# 5_009 / 510.25); the ceiling moves to 15_005 with 0.75 of headroom. -# 2026-10-15 (#922): get_gallery_metadata gains `include_workflow` and says an -# image's `media` is its measured size (+46) - measured at 15_051.25; the -# ceiling moves to 15_052 with 0.75 of headroom. -SURFACE_BUDGET = 15_052 +# measured: the descriptions, input schemas and instructions a client can hold +# resident for the whole session before a single call. A client that defers +# tool schemas (Claude Code loads only the names until a tool is searched for) +# pays far less than this, but not every client does, so the ceiling stays. +# +# A fixed ceiling with headroom, not a ratchet. Until 2026-10-19 it was raised +# to each new measurement plus a token or less, so every docstring change paid +# for itself by trimming another tool's sentences - about twenty rounds of it +# (this file's history has each one) - and at least once the trim took the +# pointer an agent needed (#574 re-broke #282). Measured 2026-10-19 at +# 15_051.75 (65 tools); the ceiling is set at 16_000, about one large tool's +# worth of room. +# +# What a description holds: what the tool does, when to use it, the pinned +# words other tests check for, and a pointer to the guide section with the +# rest - never a second copy of that section. Pointers are checked to resolve +# in tests/test_guide_pointers.py. When this fails, move narrative to a guide +# and leave a pointer before raising the ceiling; raise it in a step of the +# same size when a new tool genuinely needs the room, and say why here. +SURFACE_BUDGET = 16_000 @pytest.mark.asyncio From c08ba4f0b9483cfc673349a2b2f714df64445f4a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 21:10:24 -0500 Subject: [PATCH 018/105] feat(catalog): #946 - models/z-image-inpaint, Z-Image Turbo masked inpaint configuring templates/inpaint Co-Authored-By: Claude Sonnet 5.5 --- workflows/models/z-image-inpaint.json | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 workflows/models/z-image-inpaint.json diff --git a/workflows/models/z-image-inpaint.json b/workflows/models/z-image-inpaint.json new file mode 100644 index 00000000..3ce28ee2 --- /dev/null +++ b/workflows/models/z-image-inpaint.json @@ -0,0 +1,54 @@ +{ + "cost_drivers": ["num_images_per_prompt", "num_inference_steps"], + "variables": { + "image": "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png", + "mask_image": "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup_mask.png", + "prompt": "a clean empty background that continues the surrounding scene", + "num_images_per_prompt": 1, + "num_inference_steps": 9, + "guidance_scale": 0.0, + "strength": 1.0, + "seed": null + }, + "seed": "variable:seed", + "id": "ZImageInpaint", + "description": "Masked inpainting and object removal with Z-Image Turbo (ZImageInpaintPipeline): white mask pixels are repainted from the prompt, the rest is kept. Uses the same Tongyi-MAI/Z-Image-Turbo checkpoint as models/z-image, so a server that has one has the other, and fits a 24 GB card with offload where FLUX.1 Fill (templates/inpaint) needs a 54 GB download. To remove an object, prompt for what should replace it (the background), not for the object being removed.", + "cost": [], + "vram_estimate": { + "base_gb": 11.8, + "bytes_per_voxel": 1031, + "voxel_variables": ["num_images_per_prompt"], + "reason": "reuses the models/z-image text-to-image calibration (#945): same transformer, text encoder and VAE with offload: model, so the resident weights match; the inpaint pass adds the encoded source image and mask latents, small beside the weights. Image size follows the input image, so the per-voxel term is only the batch count here. Calibrate on lem (#946)" + }, + "configures": "templates/inpaint", + "steps": [ + { + "name": "inpaint", + "pipeline": { + "configuration": { + "component_type": "ZImageInpaintPipeline", + "offload": "model" + }, + "from_pretrained_arguments": { + "model_name": "Tongyi-MAI/Z-Image-Turbo", + "torch_dtype": "torch.bfloat16", + "low_cpu_mem_usage": true + }, + "arguments": { + "image": {"location": "variable:image"}, + "mask_image": {"location": "variable:mask_image"}, + "prompt": "variable:prompt", + "strength": "variable:strength", + "num_inference_steps": "variable:num_inference_steps", + "num_images_per_prompt": "variable:num_images_per_prompt", + "guidance_scale": "variable:guidance_scale" + } + }, + "result": { + "content_type": "image/jpeg", + "embed_metadata": true, + "subfolder": "final" + } + } + ] +} From 4fdcc50fb185ecf11c23897e3259237aaa2a0889 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 21:11:04 -0500 Subject: [PATCH 019/105] fix(catalog): #946 - drop empty cost list from z-image-inpaint Co-Authored-By: Claude Sonnet 5.5 --- workflows/models/z-image-inpaint.json | 1 - 1 file changed, 1 deletion(-) diff --git a/workflows/models/z-image-inpaint.json b/workflows/models/z-image-inpaint.json index 3ce28ee2..39bd6daa 100644 --- a/workflows/models/z-image-inpaint.json +++ b/workflows/models/z-image-inpaint.json @@ -13,7 +13,6 @@ "seed": "variable:seed", "id": "ZImageInpaint", "description": "Masked inpainting and object removal with Z-Image Turbo (ZImageInpaintPipeline): white mask pixels are repainted from the prompt, the rest is kept. Uses the same Tongyi-MAI/Z-Image-Turbo checkpoint as models/z-image, so a server that has one has the other, and fits a 24 GB card with offload where FLUX.1 Fill (templates/inpaint) needs a 54 GB download. To remove an object, prompt for what should replace it (the background), not for the object being removed.", - "cost": [], "vram_estimate": { "base_gb": 11.8, "bytes_per_voxel": 1031, From 1c53c50fcf7b370e3ef7092fc39dd7e2f44d929c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 21:20:26 -0500 Subject: [PATCH 020/105] refactor(server): #949 - output_annotations builds an output file's role, note, starred and version for the listing and the metadata route gallery_metadata now reports starred. Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 1 + dw/server/api_models.py | 1 + dw/server/outputs.py | 34 +++++++++++++++++++++- dw/server/routes/gallery.py | 46 ++++++++---------------------- dw/server/routes/media.py | 23 ++++++++------- tests/test_gallery_roles.py | 27 ++++++++++++++++++ ui/src/lib/generated/api-schema.ts | 5 ++++ ui/src/lib/generated/openapi.json | 5 ++++ 8 files changed, 96 insertions(+), 46 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0ca9e84e..c505fca0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -71,6 +71,7 @@ one to open. | Notes on runs, files, assets, workspaces | dw/notes.py | notes.json beside what it describes (run dir or asset dir; a file's own note else the run's), workspace.json at a workspace root; only the server writes them, never the engine; ≤4000 chars, plain text. | tests/test_notes.py | | Run versions | `dw/runs.py`: `open_run`, `run_versions`, `record_run_versions`, `copy_runs_in` | A run takes its number once, when it opens (or, copied in from another workspace, when `copy_runs_in` places it), so deleting a middle run leaves a gap rather than renumbering. | `tests/test_runs.py::TestRunVersions::test_a_deleted_middle_run_leaves_a_gap_rather_than_renumbering`, `tests/test_runs.py::TestRunVersions::test_runs_started_in_the_same_second_still_number_upward` | | Run-version surfaces | `dw/runs.py`: `run_versions`, `dw/server/routes/gallery.py`, `dw/server/outputs.py`, `dw_mcp/tools_catalog.py`: `list_gallery`, `ui/src/lib/pages/GalleryPage.svelte` | Only `dw/runs.py` computes the number, and every other module reads it back. | `tests/test_runs.py::TestRunVersions::test_an_output_reference_can_name_a_run_by_its_version` | +| What the server says about an output file | `dw/server/outputs.py`: `output_annotations` | The gallery listing (`dw/server/routes/gallery.py`) and the metadata route (`dw/server/routes/media.py`) describe an output file's `role`, `note`, `note_source`, `starred` and `version` through `output_annotations`, and neither builds those fields on its own. | `tests/test_gallery_roles.py::test_metadata_agrees_with_the_listing_on_a_starred_noted_demoted_file` | | Result subfolders | `dw/subfolders.py`: `subfolder_errors`, `dw/security.py`: `SUBFOLDER_PATTERN` | A step's `result.subfolder` is checked for shape at validation. | `tests/test_subfolders.py::TestValidationErrorsIntegration::test_validation_errors_reports_a_bad_subfolder` | | Template subfolder roles | `workflows/templates/`, `dw/workflows/` | Every saving step of a template is marked `final` or `intermediate`, with at least one `final`, and packaged builtins stay unmarked. | `tests/test_template_subfolders.py::test_every_saving_step_of_a_template_names_its_role`, `tests/test_template_subfolders.py::test_the_packaged_builtins_stay_unmarked` | | JSON records beside media | `dw/media_types.py`: `JsonRecord`, `dw/result.py`: `Result.save_artifact` | A dict marked `JsonRecord` is saved whole as one `.json`, whatever content type the step declares, and is never split into one file per key. | `tests/test_face_track.py::test_json_record_is_saved_whole_under_a_video_content_type`, `tests/test_face_track.py::test_the_task_result_saves_crops_as_video_and_track_as_one_json` | diff --git a/dw/server/api_models.py b/dw/server/api_models.py index eafffa42..664c3139 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -829,6 +829,7 @@ class GalleryMetadata(ApiModel): note: str = sometimes("The effective note: the file's own, else its run's.") run_note: str = sometimes("The note of the run that wrote the file.") note_source: Literal["file", "run"] = sometimes("Whose note `note` is.") + starred: bool = sometimes("True when the file is starred; absent otherwise.") class AssetLibraryRoot(ApiModel): diff --git a/dw/server/outputs.py b/dw/server/outputs.py index d39da564..a581bf97 100644 --- a/dw/server/outputs.py +++ b/dw/server/outputs.py @@ -32,7 +32,7 @@ LibraryRoot, library_path, ) -from ..notes import read_notes +from ..notes import effective_note, read_notes, read_starred from ..roles import ROLE_DECLARED_KEY, consumed_outputs, entry_role, listed_role from ..runs import ( MANIFEST_FILE_NAME, @@ -557,6 +557,38 @@ def listed_file_role(run_dir, name, consumed): ) +def output_annotations(run_dir, file_name, consumed, cache): + """What the server says about one output file of a run: `role` (demoted + when `consumed`), `note` and `note_source` when it has a note, `starred` + when it is starred, and `version`, the run's ordinal among its workflow's + runs. The listing and the metadata route both describe a file through + this and build none of these themselves. + + `cache` is a dict the caller keeps for one listing, so each run's notes, + stars and each workflow's versions are read once, not once per file.""" + notes = cache.setdefault("notes", {}) + if run_dir not in notes: + notes[run_dir] = read_notes(run_dir) + stars = cache.setdefault("starred", {}) + if run_dir not in stars: + stars[run_dir] = read_starred(run_dir) + identity_dir = os.path.dirname(run_dir) + versions = cache.setdefault("versions", {}) + if identity_dir not in versions: + versions[identity_dir] = run_versions(identity_dir) + fields = { + "role": listed_file_role(run_dir, file_name, consumed), + "version": versions[identity_dir].get(os.path.basename(run_dir)), + } + note, source = effective_note(notes[run_dir], file_name) + if note: + fields["note"] = note + fields["note_source"] = source + if file_name in stars[run_dir]: + fields["starred"] = True + return fields + + def consumed_in(state, ws): """`consumed_outputs` over a workspace's outputs, its assets found down the search path a run resolves 'asset:' in. Once per listing.""" diff --git a/dw/server/routes/gallery.py b/dw/server/routes/gallery.py index 4893390e..ab967869 100644 --- a/dw/server/routes/gallery.py +++ b/dw/server/routes/gallery.py @@ -23,9 +23,6 @@ from ...notes import ( NOTES_FILE_NAME, drop_file_note, - effective_note, - read_notes, - read_starred, ) from ...roles import ROLES from ...runs import ( @@ -33,7 +30,6 @@ REALIZED_FILE_NAME, is_run_id, record_run_versions, - run_versions, split_run_path, ) from ...security import SecurityError, validate_path @@ -49,7 +45,7 @@ resolve_output_file, run_directory, consumed_in, - listed_file_role, + output_annotations, served_url, split_run_name, strip_output_prefix, @@ -74,20 +70,9 @@ def _gallery_entries(root, ws, consumed=frozenset()): files = list(iter_gallery_files(root)) except OSError: files = [] - # One read of each workflow's run ordinals per listing, not per file: - # a run of fifty files would otherwise re-read the same manifests - # fifty times - versions_by_folder = {} - # And one read of each run's notes.json, for the same reason - notes_by_run = {} - starred_by_run = {} - - def _version(folder, run_id): - if not run_id: - return None - if folder not in versions_by_folder: - versions_by_folder[folder] = run_versions(os.path.join(root, folder)) - return versions_by_folder[folder].get(run_id) + # The per-listing cache output_annotations reads each run's notes, + # stars and versions through, once rather than once per file + cache = {} for relative_name, folder, subfolder, run_id, kind, path in files: try: @@ -114,7 +99,7 @@ def _version(folder, run_id): # `name` is too long to quote. None under the flat # layout, which has no runs to number "run_id": run_id, - "version": _version(folder, run_id), + "version": None, # Quoted (slashes kept literal): a name carrying '#', '?' # or '%' would otherwise break the src the gallery # renders it into. The mtime still rides along for cache @@ -130,21 +115,14 @@ def _version(folder, run_id): } # None under the flat layout, and for a file no manifest lists split = split_run_name(root, relative_name) if run_id else None - entry["role"] = ( - listed_file_role(split[0], split[1], relative_name in consumed) - if split - else None - ) + entry["role"] = None if split: - if split[0] not in notes_by_run: - notes_by_run[split[0]] = read_notes(split[0]) - note = effective_note(notes_by_run[split[0]], split[1])[0] - if note: - entry["note"] = note - if split[0] not in starred_by_run: - starred_by_run[split[0]] = read_starred(split[0]) - if split[1] in starred_by_run[split[0]]: - entry["starred"] = True + annotated = output_annotations( + split[0], split[1], relative_name in consumed, cache + ) + # The listing does not report whose note it is + annotated.pop("note_source", None) + entry.update(annotated) absolute_url = absolute_served_url(output_path, ws, int(stat.st_mtime)) if absolute_url is not None: entry["absolute_url"] = absolute_url diff --git a/dw/server/routes/media.py b/dw/server/routes/media.py index e6bd3f4b..b02598b6 100644 --- a/dw/server/routes/media.py +++ b/dw/server/routes/media.py @@ -89,7 +89,7 @@ resolve_output_file, run_directory, consumed_in, - listed_file_role, + output_annotations, run_inputs, strip_output_prefix, validated_run_split, @@ -203,9 +203,18 @@ def gallery_metadata( consumed = relative.replace(os.sep, "/") in consumed_in( request.app.state, ws ) - role = listed_file_role(split[0], split[1], consumed) + annotated = output_annotations(split[0], split[1], consumed, {}) + role = annotated["role"] + version = annotated["version"] inputs = run_inputs(split[0]) - notes = _note_fields(read_notes(split[0]), split[1]) + notes = { + key: annotated[key] + for key in ("note", "note_source", "starred") + if key in annotated + } + run_note = read_notes(split[0])["run"] + if run_note: + notes["run_note"] = run_note metadata = read_embedded_metadata(path) if metadata is not None and not include_workflow and "workflow" in metadata: metadata = { @@ -269,14 +278,6 @@ def _image_media(path): return None -def _note_fields(notes, name): - """The metadata route's note keys for file `name` of a run, each only - when set.""" - note, source = effective_note(notes, name) - fields = {"note": note, "note_source": source, "run_note": notes["run"]} - return {key: value for key, value in fields.items() if value} - - @router.get("/api/gallery/{name:path}/assess") def gallery_assess( request: Request, diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index 32e494b7..82d8122b 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -410,6 +410,33 @@ def test_metadata_agrees_with_the_listing_for_an_older_run( assert meta["role"] == listed[label] +def test_metadata_agrees_with_the_listing_on_a_starred_noted_demoted_file( + gallery_client, output_root, write_run +): + # One owner (output_annotations) builds role, note, starred and version + # for both routes, so a file starred, noted and demoted by a later run + # reads the same from each + from dw.notes import set_file_note, set_file_starred + + _shot_and_cut(output_root, write_run, f"output:shots/{SHOT_RUN}/final/shot.png") + run_dir = os.path.join(output_root, "shots", SHOT_RUN) + set_file_note(run_dir, "final/shot.png", "the keeper") + set_file_starred(run_dir, "final/shot.png", True) + listed = { + f["label"]: f for f in gallery_client.get("/api/gallery").json()["files"] + }["shot.png"] + meta = gallery_client.get( + f"/api/gallery/shots/{SHOT_RUN}/final/shot.png/metadata" + ).json() + assert listed["starred"] is True + assert listed["role"] == "intermediate" + assert listed["note"] == "the keeper" + for key in ("role", "note", "starred", "version"): + assert meta[key] == listed[key], key + assert meta["note_source"] == "file" + assert meta["version"] is not None + + def test_an_asset_kept_from_a_file_demotes_it(tmp_path, write_run): from fastapi.testclient import TestClient diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index eaf4a92f..8a36dd5a 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -2478,6 +2478,11 @@ export interface components { * @enum {string} */ source: "output" | "asset"; + /** + * Starred + * @description True when the file is starred; absent otherwise. + */ + starred?: boolean; /** Version */ version: number | null; }; diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 8122d26b..b1e5cb53 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -1516,6 +1516,11 @@ "title": "Source", "type": "string" }, + "starred": { + "description": "True when the file is starred; absent otherwise.", + "title": "Starred", + "type": "boolean" + }, "version": { "anyOf": [ { From 46b103677af127d332ba0ab12f01ac106b4e1896 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 21:28:04 -0500 Subject: [PATCH 021/105] refactor(server): #950 - job history reads columns by name and records from one (column, getter) table; parity test against the live Job Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 1 + dw/server/job_history.py | 194 +++++++++++++++++-------------- tests/test_job_history_parity.py | 58 +++++++++ 3 files changed, 165 insertions(+), 88 deletions(-) create mode 100644 tests/test_job_history_parity.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c505fca0..d86a8eae 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -169,6 +169,7 @@ one to open. | dw disk caches | `dw/disk_caches.py`; `dw/server/jobs.py`: `JobManager.hold_dispatch`; `dw/server/routes/system.py`: `clear_cache`, `delete_cache_entry` | Members `inductor`, `triton`, `quantized`. Clear only a root carrying the `.dw-cache` marker, never while a job is running or queued (`quantized` also not during a model download); an entry delete takes only a 64-hex id, checked before any path is built; clear renames into `.clearing-*` then deletes. The busy check and the dispatch hold are one step in `JobManager.hold_dispatch`, kept for the whole clear or entry delete: a job submitted meanwhile is accepted but stays queued until it ends. | `tests/test_disk_caches.py`, `tests/test_disk_caches.py::test_job_submitted_during_a_clear_waits_for_it` | | Job-to-run link | `dw/server/job_record.py`, `dw/server/jobs.py`: `JobManager.realized` | A job records `run_id`, `run_dir` and `run_version`, and `JobManager.realized` reads that run's `workflow.json`, confined to the output root. | `tests/test_server_jobs.py::test_realized_reads_the_file_the_run_wrote`, `tests/test_server_jobs.py::test_realized_refuses_a_run_dir_that_escapes_the_output_root` | | Job history | `dw/server/job_history.py`: `JobHistory`, `_ADDED_COLUMNS` | Finished jobs persist in `jobs.sqlite`, in WAL mode, so the Jobs view survives a restart. A database from an older release gains each column it lacks from the `_ADDED_COLUMNS` table when it is opened. | `tests/test_server.py::test_job_history_survives_restart_and_reruns`, `tests/test_job_history_wal.py::test_the_jobs_database_uses_wal_mode`, `tests/test_server_jobs.py::test_a_v0_13_0_database_gains_the_columns_added_since` | +| Job fields, live and persisted | `dw/server/job_record.py`: `Job.summary`, `Job.detail`; `dw/server/job_history.py`: `_RECORDED`, `_ADDED_COLUMNS`, `recent_summaries`, `_to_detail` | A job field is declared once per side: on `Job` (and its `summary()`/`detail()`), and in history as an `_ADDED_COLUMNS` entry plus one `(column, getter)` in `_RECORDED`. History readers take columns by name, never by position, and a historical job serves the same summary and detail keys a live one does, apart from the live-only poll state and the history-only markers the parity test declares. | `tests/test_job_history_parity.py::test_history_serves_the_keys_a_live_job_does` | | HTTP security | `dw/server/http_security.py`: `install_middleware`, `query_token_ok` | Every API route needs the token, and only a route marked `query_token_ok` takes it as `?token=`. | `tests/test_security_auth.py::TestTheTokenGate::test_every_api_spelling_needs_the_token`, `tests/test_security_auth.py::TestTheTokenGate::test_the_query_token_is_refused_where_it_is_not_allowed` | | Prompt enhancement | `dw/server/enhancers.py`: `PRESETS`, `_T2I_SYSTEM_PROMPT` | `PRESETS` is the Enhance panel's menu (label, curated LLMs, preselect hints); a preset's family-specific knowledge, its spec, lives in a builtin under `dw/workflows/`, never in Python. `t2i`'s generic, family-agnostic system prompt stays in Python. A third preset moves the menu into builtin metadata (#696). | `tests/test_prompt_library.py::test_every_enhancer_preset_targets_a_known_family`, `tests/test_server.py::TestEnhance::test_presets_are_listed`, `tests/test_server.py::TestEnhance::test_enhance_workflows_are_schema_valid`, `tests/test_server.py::TestEnhance::test_an_unknown_preset_is_a_client_error` | diff --git a/dw/server/job_history.py b/dw/server/job_history.py index 66b8cbc0..37a4459f 100644 --- a/dw/server/job_history.py +++ b/dw/server/job_history.py @@ -106,6 +106,56 @@ def _memory_key_of(job): ("kept", "INTEGER DEFAULT 0"), ) + +def _json(value): + return json.dumps(value, default=str) + + +def _rerun_spec(job): + """The spec's workflow_name/warnings are derived; keep what rerun needs.""" + return {key: job.spec[key] for key in RERUN_SPEC_KEYS if key in job.spec} + + +# What `JobHistory.record` writes: one (column, getter) per recorded column, so +# a new job field is one entry here (plus its `_ADDED_COLUMNS` DDL), not a +# placeholder and a positional value. `getattr(..., None)` is for a test double +# or an older in-memory Job predating a column: absent means unknown, which the +# column itself allows (#243) +_RECORDED = ( + ("id", lambda job: job.id), + ("workflow", lambda job: job.workflow_name), + ("status", lambda job: job.status), + ("created_at", lambda job: job.created_at), + ("started_at", lambda job: job.started_at), + ("finished_at", lambda job: job.finished_at), + ("arguments", lambda job: _json(job.spec.get("arguments", {}))), + ("spec", lambda job: _json(_rerun_spec(job))), + ("manifest", lambda job: _json(job.manifest)), + ("warnings", lambda job: _json(job.warnings)), + ("error", lambda job: job.error), + ("events", lambda job: _json(job.events[-MAX_PERSISTED_EVENTS:])), + ("workspace", lambda job: job.spec.get("workspace") or DEFAULT_WORKSPACE_NAME), + ("workflow_name", lambda job: job.catalog_name), + ("run_id", lambda job: job.run_id), + ("run_dir", lambda job: job.run_dir), + ("acknowledged", lambda job: job.acknowledged), + ( + "host_memory_peak_rss_mb", + lambda job: getattr(job, "host_memory_peak_rss_mb", None), + ), + ( + "host_memory_job_peak_rss_mb", + lambda job: getattr(job, "host_memory_job_peak_rss_mb", None), + ), + ("run_version", lambda job: getattr(job, "run_version", None)), + ("device", lambda job: getattr(job, "device", None)), + ("device_ordinal", lambda job: getattr(job, "device_ordinal", None)), + ("device_card", lambda job: getattr(job, "device_card", None)), + ("note", lambda job: getattr(job, "note", None)), + ("memory_key", _memory_key_of), + ("rerun_of", lambda job: getattr(job, "rerun_of", None)), +) + # `device_ordinal`/`device_card` from `device`, in SQL: an ordinal never # holds a space, so the label splits at its first one, and a label with none # (`cpu`, `mps`) is all ordinal with no card @@ -191,54 +241,21 @@ def _connect(self): # one left to GC checkpoints the WAL whenever it is collected, # rewriting jobs.sqlite at an arbitrary later moment. with contextlib.closing(sqlite3.connect(self.db_path, timeout=5)) as connection: + # Readers take columns by name; a positional index is a silent + # miss the next time a column is added + connection.row_factory = sqlite3.Row connection.execute("PRAGMA journal_mode=WAL") with connection: yield connection def record(self, job): - # The spec's workflow_name/warnings are derived; keep what rerun needs - rerun_spec = {key: job.spec[key] for key in RERUN_SPEC_KEYS if key in job.spec} + columns = ", ".join(column for column, _ in _RECORDED) + placeholders = ", ".join("?" for _ in _RECORDED) + values = tuple(getter(job) for _, getter in _RECORDED) with self._lock, self._connect() as connection: connection.execute( - "INSERT OR REPLACE INTO jobs (id, workflow, status, created_at," - " started_at, finished_at, arguments, spec, manifest, warnings," - " error, events, workspace, workflow_name, run_id, run_dir," - " acknowledged, host_memory_peak_rss_mb," - " host_memory_job_peak_rss_mb, run_version, device," - " device_ordinal, device_card, note, memory_key, rerun_of) VALUES" - " (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - ( - job.id, - job.workflow_name, - job.status, - job.created_at, - job.started_at, - job.finished_at, - json.dumps(job.spec.get("arguments", {}), default=str), - json.dumps(rerun_spec, default=str), - json.dumps(job.manifest, default=str), - json.dumps(job.warnings, default=str), - job.error, - json.dumps(job.events[-MAX_PERSISTED_EVENTS:], default=str), - job.spec.get("workspace") or DEFAULT_WORKSPACE_NAME, - job.catalog_name, - job.run_id, - job.run_dir, - job.acknowledged, - # A test double or an older in-memory Job predating this - # column reports None here rather than failing record() - # (#243) - the same "absent means unknown" the column - # itself allows - getattr(job, "host_memory_peak_rss_mb", None), - getattr(job, "host_memory_job_peak_rss_mb", None), - getattr(job, "run_version", None), - getattr(job, "device", None), - getattr(job, "device_ordinal", None), - getattr(job, "device_card", None), - getattr(job, "note", None), - _memory_key_of(job), - getattr(job, "rerun_of", None), - ), + f"INSERT OR REPLACE INTO jobs ({columns}) VALUES ({placeholders})", + values, ) def peaks_for_key(self, key): @@ -247,11 +264,11 @@ def peaks_for_key(self, key): the workflow's name (#869). Empty for a key no run has reported.""" with self._lock, self._connect() as connection: rows = connection.execute( - "SELECT host_memory_job_peak_rss_mb FROM jobs WHERE status = ?" + "SELECT host_memory_job_peak_rss_mb AS peak FROM jobs WHERE status = ?" " AND memory_key = ? AND host_memory_job_peak_rss_mb IS NOT NULL", (SUCCEEDED, key), ).fetchall() - return [row[0] for row in rows] + return [row["peak"] for row in rows] def recent_summaries(self, limit=200, workspace=None, statuses=None): """Summary rows only - the jobs list is polled, and parsing four JSON @@ -267,7 +284,8 @@ def recent_summaries(self, limit=200, workspace=None, statuses=None): "SELECT id, workflow, status, created_at, started_at, finished_at," " workspace, workflow_name, run_id, acknowledged, run_version," " device, note, run_dir," - " CASE WHEN json_valid(spec) THEN json_extract(spec, '$.output_dir') END," + " CASE WHEN json_valid(spec) THEN json_extract(spec, '$.output_dir') END" + " AS output_dir," " rerun_of" " FROM jobs" ) @@ -292,26 +310,26 @@ def recent_summaries(self, limit=200, workspace=None, statuses=None): rows = connection.execute(query, params).fetchall() return [ { - "id": row[0], - "workflow": row[1], - "status": row[2], - "created_at": row[3], - "started_at": row[4], - "finished_at": row[5], - "workspace": row[6] or DEFAULT_WORKSPACE_NAME, - "workflow_name": row[7], - "run_id": row[8], - "acknowledged": row[9] or ACK_NONE, - "run_version": row[10], - "device": row[11], + "id": row["id"], + "workflow": row["workflow"], + "status": row["status"], + "created_at": row["created_at"], + "started_at": row["started_at"], + "finished_at": row["finished_at"], + "workspace": row["workspace"] or DEFAULT_WORKSPACE_NAME, + "workflow_name": row["workflow_name"], + "run_id": row["run_id"], + "acknowledged": row["acknowledged"] or ACK_NONE, + "run_version": row["run_version"], + "device": row["device"], "historical": True, # Only when set: JobSummary.note is absent, never null - **({"note": row[12]} if row[12] else {}), + **({"note": row["note"]} if row["note"] else {}), # Where the run is, for the list to read its current note # from; JobManager.list takes these off before serving - "_run_dir": row[13], - "_output_dir": row[14], - **({"rerun_of": row[15]} if row[15] else {}), + "_run_dir": row["run_dir"], + "_output_dir": row["output_dir"], + **({"rerun_of": row["rerun_of"]} if row["rerun_of"] else {}), } for row in rows ] @@ -334,7 +352,7 @@ def device_ordinal(self, job_id): row = connection.execute( "SELECT device_ordinal FROM jobs WHERE id = ?", (job_id,) ).fetchone() - return row[0] if row else None + return row["device_ordinal"] if row else None def watermark(self): """How far the table has got - what a derived figure caches against. @@ -351,10 +369,10 @@ def watermark(self): """ with self._lock, self._connect() as connection: row = connection.execute( - "SELECT COUNT(*), MAX(finished_at) FROM jobs" + "SELECT COUNT(*) AS runs, MAX(finished_at) AS last_finished FROM jobs" " WHERE workflow_name IS NOT NULL" ).fetchone() - return (row[0], row[1]) if row else (0, None) + return (row["runs"], row["last_finished"]) if row else (0, None) def finished_runs(self): """Every successful, named run grouped by (workspace, workflow name), @@ -469,10 +487,10 @@ def events_for(self, job_id): ).fetchone() if row is None: return None - if not row[0]: + if not row["events"]: return [] try: - return json.loads(row[0]) + return json.loads(row["events"]) except json.JSONDecodeError: return [] @@ -509,8 +527,8 @@ def job_for_file(self, file_name, workspace=None): with self._lock, self._connect() as connection: rows = connection.execute(query, params).fetchall() for row in rows: - if self._manifest_wrote(row[2], file_name): - return {"id": row[0], "status": row[1]} + if self._manifest_wrote(row["manifest"], file_name): + return {"id": row["id"], "status": row["status"]} return None @staticmethod @@ -554,35 +572,35 @@ def parse(text, fallback): except (TypeError, ValueError): return fallback - spec = parse(row[7], {}) + spec = parse(row["spec"], {}) # The persisted tail is capped at MAX_PERSISTED_EVENTS, and # get_job_events serves that same tail - so counting it, rather than # hardcoding 0, keeps event_count truthful about what a caller who # pages through get_job_events will actually see (#289) - events = parse(row[16], []) + events = parse(row["events"], []) return { - "id": row[0], - "workflow": row[1], - "status": row[2], - "created_at": row[3], - "started_at": row[4], - "finished_at": row[5], - "arguments": parse(row[6], {}), + "id": row["id"], + "workflow": row["workflow"], + "status": row["status"], + "created_at": row["created_at"], + "started_at": row["started_at"], + "finished_at": row["finished_at"], + "arguments": parse(row["arguments"], {}), "spec": spec, - "manifest": parse(row[8], []), - "warnings": parse(row[9], []), - "error": row[10], - "workspace": row[11] or DEFAULT_WORKSPACE_NAME, - "workflow_name": row[12], - "run_id": row[13], - "run_dir": row[14], - "run_version": row[17], - "device": row[18], - "acknowledged": row[15] or ACK_NONE, + "manifest": parse(row["manifest"], []), + "warnings": parse(row["warnings"], []), + "error": row["error"], + "workspace": row["workspace"] or DEFAULT_WORKSPACE_NAME, + "workflow_name": row["workflow_name"], + "run_id": row["run_id"], + "run_dir": row["run_dir"], + "run_version": row["run_version"], + "device": row["device"], + "acknowledged": row["acknowledged"] or ACK_NONE, "acknowledged_cost": (spec or {}).get("acknowledged_cost"), "traceback": None, "event_count": len(events) if isinstance(events, list) else 0, "historical": True, - **({"note": row[19]} if row[19] else {}), - **({"rerun_of": row[20]} if row[20] else {}), + **({"note": row["note"]} if row["note"] else {}), + **({"rerun_of": row["rerun_of"]} if row["rerun_of"] else {}), } diff --git a/tests/test_job_history_parity.py b/tests/test_job_history_parity.py new file mode 100644 index 00000000..a687d3a0 --- /dev/null +++ b/tests/test_job_history_parity.py @@ -0,0 +1,58 @@ +"""A job taken through `JobHistory.record` reads back the keys a live job +serves: a field added to `Job.summary()`/`detail()` and missed on the history +side fails here rather than going silently absent from `get_job` after a +restart.""" + +import pytest + +from dw.server.job_history import JobHistory +from dw.server.job_record import Job + +# The one place the two sides are allowed to differ. Live-only: the poll state +# a finished job no longer has. History-only: the marker, the rerun spec in +# detail, and the run location +# `JobManager.list` takes off before serving. +LIVE_ONLY = {"progress", "cancel_requested", "waiting_for"} +HISTORY_ONLY_SUMMARY = {"historical", "_run_dir", "_output_dir"} +HISTORY_ONLY_DETAIL = {"historical", "spec"} + + +def _job(**extra): + job = Job( + { + "workflow_name": "w", + "catalog_name": "w", + "arguments": {"prompt": "a cat"}, + "workspace": "default", + **extra, + } + ) + job.status = "succeeded" + job.started_at = 10.0 + job.finished_at = 70.0 + job.run_id = "run-1" + job.run_dir = "/tmp/run-1" + job.run_version = 3 + job.device_ordinal = "cuda:1" + job.device_card = "NVIDIA GeForce RTX 3090" + return job + + +@pytest.mark.parametrize( + "extra", + [{}, {"note": "a test", "rerun_of": "abc123def456"}], + ids=["optional-unset", "optional-set"], +) +def test_history_serves_the_keys_a_live_job_does(tmp_path, extra): + job = _job(**extra) + history = JobHistory(tmp_path / "jobs.sqlite") + history.record(job) + + (summary,) = history.recent_summaries() + detail = history.get(job.id) + + assert set(summary) - HISTORY_ONLY_SUMMARY == set(job.summary()) - LIVE_ONLY + assert set(detail) - HISTORY_ONLY_DETAIL == set(job.detail()) - LIVE_ONLY + for key, value in job.summary().items(): + assert summary[key] == value, key + assert detail[key] == value, key From ee1fc733db4bc45145316f18620140ec6a82d0e2 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 21:34:34 -0500 Subject: [PATCH 022/105] feat(validate): #954 - vram_config_differs note when offload/quantization differs from the calibrated template Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/WORKFLOW_GUIDE.md | 7 +- dw/validation.py | 3 +- dw/vram_estimate.py | 38 +++++++++++ dw/vram_inheritance.py | 121 ++++++++++++++++++++++++++++----- tests/test_vram_inheritance.py | 70 +++++++++++++++++++ 6 files changed, 222 insertions(+), 19 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d86a8eae..f5f7d19f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -103,7 +103,7 @@ one to open. | Raw workflow GET | `dw/server/routes/library.py`: `get_workflow` | The raw workflow GET is served verbatim, with no `observed`, because the editor saves what it reads. | — | | VRAM projection | `dw/vram_estimate.py`, `dw/guides.py`: `guide_lengths` | A template's `vram_estimate` is projected per step after `for_each` expansion, and only the largest step over the ceiling is reported. With `bytes_per_guide_voxel` declared, each guide the step lays in (its own non-null `guides` plus a guide chain's) adds its snapped frames times the canvas; an unprobeable clip is charged at `num_frames`, and validate, admission and run project the same number. | `tests/test_vram_estimate.py::test_only_the_largest_over_ceiling_step_is_reported_once_per_cost_entry`, `tests/test_vram_guides.py::test_validate_apply_and_required_agree_on_a_guided_definition` | | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | -| Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | +| Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. Under the ceiling it still gets a `vram_config_differs` warning when its offload or quantization differs from the template's, which the estimate was calibrated on (#954). | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers`, `tests/test_vram_inheritance.py::TestConfigDiffers::test_missing_offload_under_the_ceiling_notes_it` | | Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache, LoRA and reference count (the `references` list lengths, per-entry `item:` ones read off the `for_each` list; named, not scaled, #915) set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run (a `for_each` counted per member, against the list the bucket ran, so a fallback to the default bucket re-prices a longer list) re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestDifferences::test_a_reference_count_other_than_the_templates_is_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding`, `tests/test_inherited_cost.py::TestAListLongerThanTheFallbackBucket::test_an_inline_six_shot_copy_scales_by_members` | | Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit, unless the shape holds its pipeline across the list and the fit is the larger (the key leaves list length out, #910); with neither there is no figure. It never refuses; dispatch may delay a start on it (`dw/server/dispatch.py`: `Dispatcher._next_dispatch`, #869) (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestALongerListThanObserved::test_an_accumulating_list_takes_the_larger_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | | Acceleration warnings | `dw/acceleration_warnings.py`: `acceleration_warnings`, `CACHE_MIN_USEFUL_STEPS`, `CUDA_ONLY_ATTENTION_BACKENDS` (the last entry of `WARNING_CHECKS`) | An acceleration setting that can't pay off (a `first_block`/`mag` cache under `CACHE_MIN_USEFUL_STEPS`, any `taylorseer` cache, `compile.fullgraph` with a cache, a cache on a pipeline class with no transformer or whose transformer lacks `enable_cache()`, a CUDA-only `attention_backend` on a non-CUDA device) is a warning and never a refusal, and an unresolvable value (steps, backend, device) leaves it silent. | `tests/test_validation.py::TestAccelerationWarnings::test_warnings_never_refuse`, `tests/test_validation.py::TestAccelerationWarnings::test_steps_that_resolve_to_nothing_are_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unknown_device_is_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unresolvable_backend_is_silent_on_mps` | diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index e6695ce2..b1998b19 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -494,7 +494,12 @@ step (for a `for_each`, the heaviest member). It warns rather than refuses: your offload and quantization config may differ from the template's. Cut frames, size or references until it goes quiet, or declare your own `vram_estimate`, which always wins and is judged exactly as a -template's. A pipeline the catalog declares no ceiling for gets no warning - +template's. Under the ceiling it is not always silent: the template's +estimate was calibrated on its own offload and quantization, so a workflow +that sets either differently (no `offload` where the template sets +`model`, say) gets one warning of kind `vram_config_differs` naming both +settings and the projection - real usage may be higher. Over the ceiling the +one `vram_projection_inherited` warning names that difference instead. A pipeline the catalog declares no ceiling for gets no warning - silence there is not a clearance - and H3's `t2va` and `ref2va` are different identities with different ceilings. diff --git a/dw/validation.py b/dw/validation.py index 06d69260..ec31729a 100644 --- a/dw/validation.py +++ b/dw/validation.py @@ -656,7 +656,8 @@ def _null_variable_argument_warnings(context): # A step loading a pipeline the catalog declares a VRAM ceiling for, in a # workflow that declares none, projected past it - a warning, since this # workflow's offload/quantization may be leaner than the template's - # (#502) + # (#502); under it, a note when that offload/quantization differs from + # the one the estimate was calibrated on (#954) Check( "inherited_vram_warnings", # No index, nothing to inherit - and no expansion read for it diff --git a/dw/vram_estimate.py b/dw/vram_estimate.py index b2945db9..13498b7f 100644 --- a/dw/vram_estimate.py +++ b/dw/vram_estimate.py @@ -515,6 +515,44 @@ def vram_estimate_errors( return errors +def largest_projection( + definition, + arguments=None, + supplied=(), + device_type=None, + capacity_gb=None, + source_indices=None, + written=None, + capacity_label=None, + base_dir=None, + probe=None, +): + """The largest step a declared vram_estimate projects, as + {path, gb, card} (`card` names the smallest card checked against), over + or under the ceiling alike; None when nothing is projected. The same + arguments as `vram_estimate_errors`, for a note on a projection that + passes.""" + estimate = declared_estimate(definition) + if estimate is None: + return None + cost = definition.get("cost") + entries = _entries_for( + cost if isinstance(cost, list) else [], device_type, capacity_gb, capacity_label + ) + cards = [e for e in entries if _as_number(e.get("vram_gb")) is not None] + projections = list(_projections(definition, estimate, arguments, base_dir, probe)) + if not cards or not projections: + return None + index, step, _, _, _, projected = max(projections, key=lambda p: p[-1]) + card = min(cards, key=lambda e: e["vram_gb"]) + label = card.get("name") or card.get("device", "this device") + return { + "path": _where(estimate, index, step, supplied, source_indices, written), + "gb": projected, + "card": f"{card['vram_gb']} GB {label}", + } + + def required_vram_gb( definition, arguments=None, device_type=None, base_dir=None, probe=None ): diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index c6d37482..ff754911 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -23,7 +23,11 @@ An inherited ceiling warns and never refuses (#479, Q3): the hand-built workflow may offload or quantize differently from the template, and the escape hatch for a leaner config is declaring its own `vram_estimate`, which -always wins. There is no run-time backstop - the worker has no catalog, and +always wins. A projection *under* the ceiling is not silent when the +workflow's offload or quantization differs from the template's: the +estimate was calibrated on that configuration, so it gets a +`vram_config_differs` note (#954) naming both settings. Over the ceiling +the one warning names the difference instead. There is no run-time backstop - the worker has no catalog, and the pre-queue check covers every server submission. """ @@ -38,9 +42,16 @@ same_number, undeclared_variable_references, ) -from .vram_estimate import pipeline_identity, vram_estimate_errors +from .vram_estimate import ( + largest_projection, + pipeline_identity, + vram_estimate_errors, +) KIND = "vram_projection_inherited" +# A projection under the ceiling whose offload or quantization is not the +# one the template was calibrated on (#954) +CONFIG_KIND = "vram_config_differs" def template_identity(definition): @@ -285,10 +296,49 @@ def build_index(catalog): "vram_estimate": definition[ESTIMATE_KEY], "cost": cost, "template": name, + "definition": definition, } return found +# The differences that move a template's measured VRAM; frame count is scaled +# by the projection and cache, attention backend and LoRA are named elsewhere +_CALIBRATED = (("offload", "offload"), ("quantization", "quantization_config")) + + +def _setting_text(definition, label, key, arguments=None): + """How `definition` sets `key`, as 'offload: model' - or 'no offload' when + it sets none - with `variable:` references resolved.""" + values = sorted( + _found_expanded(definition, key, _compared_variables(definition, arguments)) + ) + if not values: + return f"no {label}" + shown = ", ".join( + value if len(value) <= 60 else value[:57] + "..." + for value in (v.strip('"') for v in values) + ) + return f"{key}: {shown}" + + +def calibration_differences(written, entry, arguments=None): + """(labels, calibrated text, written text) for the offload and + quantization `written` sets differently from the template `entry` was + built from; empty when they match or the template was not kept.""" + template = entry.get("definition") + if not isinstance(template, dict) or not isinstance(written, dict): + return [], "", "" + labels = [] + calibrated = [] + used = [] + for label, key in _CALIBRATED: + if sets_differently(written, template, key, arguments): + labels.append(label) + calibrated.append(_setting_text(template, label, key)) + used.append(_setting_text(written, label, key, arguments)) + return labels, "; ".join(calibrated), "; ".join(used) + + def inherited_vram_warnings( definition, index, @@ -316,6 +366,14 @@ def inherited_vram_warnings( variables = definition.get("variables") variables = variables if isinstance(variables, dict) else {} identities = [pipeline_identity(step, variables) for step in steps] + written_definition = written if isinstance(written, dict) else definition + written_variables = written_definition.get("variables") + written_variables = { + **(written_variables if isinstance(written_variables, dict) else {}), + **(arguments or {}), + } + written_steps = written_definition.get("steps") + written_steps = written_steps if isinstance(written_steps, list) else [] warnings = [] for identity in dict.fromkeys(i for i in identities if i in index): entry = index[identity] @@ -336,9 +394,20 @@ def inherited_vram_warnings( }, "cost": entry["cost"], } - errors = vram_estimate_errors( - matched, - arguments, + # Only this identity's written steps, so another pipeline's offload + # is not held against this one's calibration + own = { + **written_definition, + "steps": [ + step + if isinstance(step, dict) + and pipeline_identity(step, written_variables) == identity + else {} + for step in written_steps + ], + } + labels, calibrated, used = calibration_differences(own, entry, arguments) + common = dict( supplied=supplied, device_type=device_type, capacity_gb=capacity_gb, @@ -348,15 +417,35 @@ def inherited_vram_warnings( base_dir=base_dir, probe=probe, ) - if not errors: - continue - error = errors[0] - warnings.append( - f"{error['path']}: {KIND}: {error['message'].rstrip('.')}. The " - f"ceiling is inherited from '{entry['template']}', which loads " - f"the same pipeline; this workflow declares no vram_estimate and " - f"its offload and quantization config may differ from the " - f"template's, so this warns rather than refuses - declare a " - f"vram_estimate to be judged by your own numbers" - ) + errors = vram_estimate_errors(matched, arguments, **common) + if errors: + error = errors[0] + if labels: + config = ( + f"its {' and '.join(labels)} differs from the template's " + f"(calibrated with {calibrated}; this workflow sets {used})" + ) + else: + config = ( + "its offload and quantization config may differ from the template's" + ) + warnings.append( + f"{error['path']}: {KIND}: {error['message'].rstrip('.')}. The " + f"ceiling is inherited from '{entry['template']}', which loads " + f"the same pipeline; this workflow declares no vram_estimate and " + f"{config}, so this warns rather than refuses - declare a " + f"vram_estimate to be judged by your own numbers" + ) + elif labels: + projection = largest_projection(matched, arguments, **common) + if projection is None: + continue + warnings.append( + f"{projection['path']}: {CONFIG_KIND}: the VRAM projection is " + f"inherited from '{entry['template']}', calibrated with " + f"{calibrated}; this workflow sets {used}, so real usage may " + f"be higher than the projection ({projection['gb']:.1f} GB of " + f"{projection['card']}) - declare a vram_estimate to be " + f"judged by your own numbers" + ) return warnings diff --git a/tests/test_vram_inheritance.py b/tests/test_vram_inheritance.py index dde56d23..d7646ba4 100644 --- a/tests/test_vram_inheritance.py +++ b/tests/test_vram_inheritance.py @@ -442,3 +442,73 @@ def test_a_job_over_an_inherited_ceiling_queues_and_carries_the_warning(api): assert response.status_code == 201, response.text (warning,) = _inherited(response.json()["warnings"]) assert "templates/minimax/reference-to-video" in warning + + +# --- a note when the config differs from the calibrated one (#954) --------------- + + +def _with_config(step, **configuration): + step = copy.deepcopy(step) + step["pipeline"]["configuration"].update(configuration) + return step + + +def _calibrated_index(**configuration): + template = _template("ref2va", REF2VA_ESTIMATE) + template["steps"] = [_with_config(s, **configuration) for s in template["steps"]] + return build_index([("templates/calibrated", template)]) + + +class TestConfigDiffers: + def _kinds(self, warnings, kind): + return [w for w in warnings if f": {kind}:" in w] + + def test_missing_offload_under_the_ceiling_notes_it(self): + index = _calibrated_index(offload="model") + step = _h3_step("shot", num_frames=124) + warnings = _warnings(_inline(step), index=index) + (warning,) = warnings + assert "vram_config_differs" in warning + assert "templates/calibrated" in warning + assert "offload: model" in warning + assert "no offload" in warning + assert "24 GB" in warning + + def test_a_different_quantization_notes_it(self): + quantization = {"quantization_config": {"quant_type": "nf4"}} + index = _calibrated_index(**quantization) + warnings = _warnings(_inline(_h3_step("shot", num_frames=124)), index=index) + (warning,) = self._kinds(warnings, "vram_config_differs") + assert "quantization_config" in warning + assert "no quantization" in warning + + def test_the_same_config_says_nothing(self): + index = _calibrated_index(offload="model") + step = _with_config(_h3_step("shot", num_frames=124), offload="model") + assert _warnings(_inline(step), index=index) == [] + + def test_its_own_estimate_says_nothing(self): + index = _calibrated_index(offload="model") + definition = _inline( + _h3_step("shot", num_frames=124), + vram_estimate={**REF2VA_ESTIMATE, "base_gb": 4.0}, + cost=CARD, + ) + assert _warnings(definition, index=index) == [] + + def test_over_the_ceiling_one_warning_names_the_difference(self): + index = _calibrated_index(offload="model") + warnings = _warnings(_inline(_h3_step("shot")), index=index) + (warning,) = warnings + assert f": {KIND}:" in warning + assert "vram_config_differs" not in warning + assert "its offload differs from the template's" in warning + assert "offload: model" in warning and "no offload" in warning + + def test_offload_set_through_an_overridden_variable_is_compared_resolved(self): + index = _calibrated_index(offload="model") + step = _with_config(_h3_step("shot", num_frames=124), offload="variable:mode") + definition = _inline(step, variables={"mode": "model"}) + assert _warnings(definition, index=index) == [] + (warning,) = _warnings(definition, index=index, arguments={"mode": None}) + assert "vram_config_differs" in warning From bae4beed1ae6c43b0912e363a8f382e578718b0f Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 21:52:07 -0500 Subject: [PATCH 023/105] docs(catalog): #946 - z-image-inpaint is masked add/replace, not object removal; removal and outpaint stay on templates/inpaint (FLUX.1 Fill) Co-Authored-By: Claude Sonnet 5.5 --- workflows/models/z-image-inpaint.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/models/z-image-inpaint.json b/workflows/models/z-image-inpaint.json index 39bd6daa..2444666f 100644 --- a/workflows/models/z-image-inpaint.json +++ b/workflows/models/z-image-inpaint.json @@ -12,7 +12,7 @@ }, "seed": "variable:seed", "id": "ZImageInpaint", - "description": "Masked inpainting and object removal with Z-Image Turbo (ZImageInpaintPipeline): white mask pixels are repainted from the prompt, the rest is kept. Uses the same Tongyi-MAI/Z-Image-Turbo checkpoint as models/z-image, so a server that has one has the other, and fits a 24 GB card with offload where FLUX.1 Fill (templates/inpaint) needs a 54 GB download. To remove an object, prompt for what should replace it (the background), not for the object being removed.", + "description": "Masked add or replace with Z-Image Turbo (ZImageInpaintPipeline): white mask pixels are repainted with what the prompt names, the rest is kept. NOT for object removal: with a prompt it paints a new object or a mismatched patch rather than continuing the surrounding scene (#946); use templates/inpaint (FLUX.1 Fill) to remove an object or outpaint. Uses the same Tongyi-MAI/Z-Image-Turbo checkpoint as models/z-image, so a server that has one has the other, and fits a 24 GB card with offload where FLUX.1 Fill needs a 54 GB download. Prompt for the thing to put in the mask.", "vram_estimate": { "base_gb": 11.8, "bytes_per_voxel": 1031, From e26b893b03baaf0fdfbf73e909d5883adae15f73 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 21:59:51 -0500 Subject: [PATCH 024/105] fix(server): #957 - stamp manifest roles on the job at finish so history serves the same entries as a live job Co-Authored-By: Claude Sonnet 5.5 --- dw/server/jobs.py | 16 +++++++++++++- tests/test_server_jobs.py | 45 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/dw/server/jobs.py b/dw/server/jobs.py index 25015174..43fcff7f 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -62,7 +62,7 @@ from . import dispatch, job_results, pool, worker_memory from .dispatch import Dispatcher from .job_history import JobHistory -from .outputs import job_run_directory, with_run_note +from .outputs import job_run_directory, with_run_note, with_run_state from .job_record import ( ACK_NONE, CANCELLED, @@ -816,12 +816,26 @@ def _slot_running(self, job_id): def _finish(self, job, status, error=None, traceback_text=None): job.finish(status, error=error, traceback_text=traceback_text) + self._stamp_roles(job) try: self.history.record(job) except Exception as e: logger.warning(f"Could not persist job {job.id}: {e}") self._trim_terminal_jobs() + def _stamp_roles(self, job): + """Put each manifest entry's role on the job while its run directory + is read, so history serves the entries a live job does (#957) - the + run may be moved or deleted before the job is read back.""" + try: + location = self.run_location(job.id) + if location is not None and isinstance(job.manifest, list): + job.manifest = with_run_state({"manifest": job.manifest}, *location)[ + "manifest" + ] + except Exception as e: + logger.warning(f"Could not read roles for job {job.id}: {e}") + def forget_workspace(self, workspace): """Drop a deleted workspace's finished jobs, in memory and in history (#913), so a workspace later created under the same name lists none diff --git a/tests/test_server_jobs.py b/tests/test_server_jobs.py index 1f8449aa..f140d249 100644 --- a/tests/test_server_jobs.py +++ b/tests/test_server_jobs.py @@ -685,3 +685,48 @@ def test_run_location_is_the_jobs_own_root(manager, tmp_path): root, run_dir = manager.run_location("named") assert root == os.path.realpath(str(other)) assert run_dir == RUN_DIR + + +def test_history_keeps_the_roles_a_live_manifest_carries(tmp_path): + """#957: a finished job's manifest entries carry `role`, and history + reads them back even after the run directory is gone.""" + run_dir = tmp_path / "outputs" / RUN_DIR + name = f"{RUN_DIR}/final/a.png" + + def roled_script(command): + for message in tracked_script(command): + if message["type"] == "success": + (run_dir / "final").mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps( + { + "steps": [ + {"step": "s", "files": ["final/a.png"], "role": "final"} + ] + } + ) + ) + message = { + **message, + "manifest": [ + {"step": "s", "files": [str(tmp_path / "outputs" / name)]} + ], + } + yield message + + made = JobManager( + str(tmp_path / "outputs"), + worker_manager=ScriptedWorkerManager(roled_script), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=str(tmp_path), + ) + try: + job = finished_job(made) + live = made.describe(job)["manifest"] + assert live[0]["role"] == "final" + import shutil + + shutil.rmtree(run_dir) + assert made.history.get(job.id)["manifest"] == live + finally: + made.shutdown() From 50e56a9cf116f1caa1334524f868484b2ef22a1f Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 22:02:17 -0500 Subject: [PATCH 025/105] test(server): #957 - the job is stamped with roles at finish; assert history holds them Co-Authored-By: Claude Sonnet 5.5 --- tests/test_gallery_roles.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index 82d8122b..7b66ab91 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -220,7 +220,12 @@ def test_job_detail_entries_carry_the_run_roles(tmp_path): with TestClient(app, base_url="http://localhost") as client: job = client.post("/api/jobs", json={"workflow": valid_workflow()}).json() detail = wait_for_status(client, job["id"], ["succeeded"]) - assert all("role" not in e for e in manager.jobs[job["id"]].manifest) + # the job is stamped at finish, so history holds the roles too (#957) + stored = manager.history.get(job["id"])["manifest"] + assert {e["step"]: e["role"] for e in stored} == { + "a": "intermediate", + "b": "final", + } assert {e["step"]: e["role"] for e in detail["manifest"]} == { "a": "intermediate", "b": "final", From 5c05c1d10513b0ff4b8fe2d3ffecfa4b42e17170 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 22:07:00 -0500 Subject: [PATCH 026/105] feat(stems): #955 - separate_stems warns stacked_vocals over layered-stereo spans of the vocal stem Calibrated on two-minds Always On take A (AlwaysOnStems 494d1a7469b7): chorus 1:19-1:34 windows corr -0.276..0.251, side/mid 0.609..1.682; verse 1 corr 0.656..0.969, side/mid 0.016..0.208; verse 2 0.663..0.964, 0.018..0.208. Chosen: STACK_CORRELATION_MAX 0.45, STACK_SIDE_RATIO_MIN 0.4. Intro hum (0:08-0:12) decorrelates like the chorus (corr -0.06..0.21) but sits 16.5 dB or more under the stem's level; STACK_LEVEL_BELOW_DB 14 excludes it. Flags 1:19-1:34 and 1:59-2:09 (both choruses), nothing before 1:19. Co-Authored-By: Claude Opus 5.5 --- dw/tasks/voice_attribution.py | 116 ++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/dw/tasks/voice_attribution.py b/dw/tasks/voice_attribution.py index 2813f9d9..60a9ad7f 100644 --- a/dw/tasks/voice_attribution.py +++ b/dw/tasks/voice_attribution.py @@ -105,6 +105,31 @@ "voices_too_similar": VOICES_TOO_SIMILAR, } +# separate_stems' stacked-vocals check (#947), on the 44.1 kHz stereo vocal +# stem. Music 3 records a chorus as layered stereo doubles, and htdemucs +# returns them in the one vocal stem: one voice processed as one where two +# sing distorts (the 1:19 chorus of `two-minds` *Always On*). A window is +# stacked when it is sung - at least STACK_VOICED_SHARE of its frames +# voiced, and its mid level within STACK_LEVEL_BELOW_DB of the stem's level +# (the VOICED_LEVEL_PERCENTILE frame rms) - and its left and right channels +# disagree: their correlation below STACK_CORRELATION_MAX and the side/mid +# energy ratio above STACK_SIDE_RATIO_MIN. The level gate is there because +# faint content decorrelates as readily as a double: on *Always On* an +# intro hum 16.5 dB down measured as stacked as the chorus. Flagged windows +# merge across gaps of one window or less, and a span shorter than +# STACK_MIN_SPAN_SECONDS (a single doubled word) is dropped. Calibrated on +# *Always On* take A (job 494d1a7469b7): its chorus windows measure +# correlation -0.28 to 0.25 and side/mid 0.61 to 1.68, its verse windows +# 0.66 to 0.97 and 0.02 to 0.21 +STACK_WINDOW_SECONDS = 1.0 +STACK_VOICED_SHARE = 0.5 +STACK_LEVEL_BELOW_DB = 14.0 +STACK_CORRELATION_MAX = 0.45 +STACK_SIDE_RATIO_MIN = 0.4 +STACK_MIN_SPAN_SECONDS = 2.0 +# How many spans the warning names before "and N more" +STACK_SPANS_LISTED = 8 + def _missing(package, error): return RuntimeError( @@ -331,16 +356,22 @@ def voiced_mask(waveform, sample_rate): """Per-frame voiced flags for a mono waveform, the frame length in samples and the floor used: FRAME_SECONDS frames whose rms reaches voiced_floor_dbfs.""" - frame = max(1, int(round(FRAME_SECONDS * sample_rate))) - count = len(waveform) // frame - if count == 0: + rms, frame = _frame_rms(waveform, sample_rate) + if len(rms) == 0: return numpy.zeros(0, dtype=bool), frame, VOICED_FLOOR_MIN_DBFS - frames = waveform[: count * frame].reshape(count, frame) - rms = numpy.sqrt(numpy.mean(frames.astype(numpy.float64) ** 2, axis=1)) floor_dbfs = voiced_floor_dbfs(rms) return rms >= 10 ** (floor_dbfs / 20), frame, floor_dbfs +def _frame_rms(waveform, sample_rate): + """The rms of each FRAME_SECONDS frame of a mono waveform, and the frame + length in samples.""" + frame = max(1, int(round(FRAME_SECONDS * sample_rate))) + count = len(waveform) // frame + frames = waveform[: count * frame].reshape(count, frame) + return numpy.sqrt(numpy.mean(frames.astype(numpy.float64) ** 2, axis=1)), frame + + class _Voicing: """The voiced frames of the vocal stem, answering spans in seconds.""" @@ -868,6 +899,74 @@ def attribute_voices( STEMS_COMMAND = "separate_stems" +def stacked_spans(stereo, sample_rate): + """The spans of a stereo vocal stem that hold stacked (layered stereo) + voices, as (start, end) seconds: the STACK_* check above. A mono stem, or + one with nothing sung, has none.""" + audio = numpy.asarray(stereo, dtype=numpy.float64) + if audio.ndim != 2 or audio.shape[0] < 2: + return [] + left, right = audio[0], audio[1] + mid, side = (left + right) / 2, (left - right) / 2 + window = max(1, int(round(STACK_WINDOW_SECONDS * sample_rate))) + voiced, frame, _ = voiced_mask(mid, sample_rate) + rms, _ = _frame_rms(mid, sample_rate) + if not voiced.any(): + return [] + level = float(numpy.percentile(rms, VOICED_LEVEL_PERCENTILE)) + quietest = level * 10 ** (-STACK_LEVEL_BELOW_DB / 20) + flagged = [] + for index in range(len(mid) // window): + span = slice(index * window, (index + 1) * window) + frames = voiced[span.start // frame : span.stop // frame] + if len(frames) == 0 or frames.mean() < STACK_VOICED_SHARE: + continue + lw, rw, mw, sw = left[span], right[span], mid[span], side[span] + mid_energy = float(mw @ mw) + if math.sqrt(mid_energy / window) < quietest: + continue + denominator = math.sqrt(float(lw @ lw) * float(rw @ rw)) + correlation = float(lw @ rw) / denominator if denominator else 1.0 + if ( + correlation < STACK_CORRELATION_MAX + and float(sw @ sw) / mid_energy > STACK_SIDE_RATIO_MIN + ): + flagged.append(index) + runs = [] + for index in flagged: + # a gap of one window or less joins the run + if runs and index - runs[-1][1] <= 1: + runs[-1][1] = index + 1 + else: + runs.append([index, index + 1]) + seconds = window / sample_rate + return [ + (first * seconds, last * seconds) + for first, last in runs + if (last - first) * seconds >= STACK_MIN_SPAN_SECONDS + ] + + +def _minutes(seconds): + whole = int(seconds) + return f"{whole // 60}:{whole % 60:02d}" + + +def stacked_vocals_message(spans): + """The stacked_vocals warning for these spans.""" + listed = ", ".join( + f"{_minutes(start)}–{_minutes(end)}" + for start, end in spans[:STACK_SPANS_LISTED] + ) + if len(spans) > STACK_SPANS_LISTED: + listed += f" and {len(spans) - STACK_SPANS_LISTED} more" + return ( + f"{STEMS_COMMAND}: the vocals stem holds stacked (layered stereo) " + f"voices over {listed}; treat those spans as more than one voice " + "before processing them as one" + ) + + def separate_stems(audio, sample_rate=None, device="cpu"): """Task command: split a mix into its vocals, drums, bass and other stems. @@ -899,6 +998,13 @@ def separate_stems(audio, sample_rate=None, device="cpu"): stems, stem_rate = _separate_with_fallback( _run_separator_stems, STEMS_COMMAND, waveform, rate, device, torch.float32 ) + spans = stacked_spans(stems["vocals"], stem_rate) + if spans: + emit_warning( + stacked_vocals_message(spans), + kind="stacked_vocals", + command=STEMS_COMMAND, + ) return { name: as_track(array, stem_rate, STEMS_COMMAND) for name, array in stems.items() } From 5e8642beedb6eebeeecc64baa43ff1abd891d1aa Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 22:10:38 -0500 Subject: [PATCH 027/105] test(stems): #955 - stacked_vocals spans, merge, drop, truncation and the separate_stems path; TASKS.md names the warning Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 23 ++++++++ tests/test_separate_stems.py | 109 ++++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 65e95eb7..991ee1e7 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -2257,6 +2257,29 @@ The model is fixed (`htdemucs`, run in fp32), its weights download on first use, and it needs the `demucs` package. On MPS a failed separation falls back to the CPU and warns (`separation_cpu_fallback`). +**Stacked vocals.** A chorus is often recorded as layered stereo doubles - every +Music 3 chorus is - and htdemucs returns all of them in the one `vocals` stem. +When the stem's left and right channels sing different lines over a sung span, +the step warns `stacked_vocals` and names the spans in m:ss (up to 8, then "and +N more"): + +``` +separate_stems: the vocals stem holds stacked (layered stereo) voices over +1:19–1:34, 1:59–2:09; treat those spans as more than one voice before +processing them as one +``` + +Read it before working on a span as one voice: pitch or formant work on a +stacked span distorts it, and an `attribute_voices` window or a lip-sync shot +over one has two singers in it. Pick a span outside the flagged ones, or plan +the shot as a two-shot. The stems themselves are unchanged. The check is a 1 s +window test on the stereo stem: the window must be sung (half its frames voiced, +and within 14 dB of the stem's level), with an L/R correlation under 0.45 and a +side/mid energy ratio over 0.4. Windows merge across a one-window gap, and +spans under 2 s are dropped. A mono source, an instrumental and a single doubled +word never flag it. The thresholds are the `STACK_*` constants in +`dw/tasks/voice_attribution.py`. + ## Voice attribution Which reference voice sings each line of a song, by timbre - staging diff --git a/tests/test_separate_stems.py b/tests/test_separate_stems.py index 0dcd2aec..a36ab81c 100644 --- a/tests/test_separate_stems.py +++ b/tests/test_separate_stems.py @@ -14,7 +14,12 @@ from dw.introspection import describe_task, list_tasks from dw.media_types import AudioTrack from dw.tasks import model_cache -from dw.tasks.voice_attribution import separate_stems +from dw.tasks.voice_attribution import ( + STACK_SPANS_LISTED, + separate_stems, + stacked_spans, + stacked_vocals_message, +) SOURCES = ["drums", "bass", "other", "vocals"] RATE = 44100 @@ -101,5 +106,107 @@ def test_task_is_registered_and_described(self): self.assertIn("audio", names) +def vocals_only(model, mix, device=None, split=True): + """The whole (normalized) mix comes back as the vocal stem.""" + shares = torch.tensor([0.0, 0.0, 0.0, 1.0]).view(1, 4, 1, 1) + return mix[:, None] * shares + + +def sung(seconds, stacked=(), rate=8000, quiet=()): + """A stereo vocal stem, one second per window: the same voice on both + channels, except in `stacked` windows, where each channel sings its own + (uncorrelated) line. A `quiet` window sits 30 dB down.""" + t = numpy.arange(rate) / rate + lead = 0.5 * numpy.sin(2 * numpy.pi * 220 * t) + double = 0.5 * numpy.sin(2 * numpy.pi * 331 * t) + windows = [] + for index in range(seconds): + right = double if index in stacked else lead + gain = 10 ** (-30 / 20) if index in quiet else 1.0 + windows.append(numpy.stack([lead, right]) * gain) + return numpy.concatenate(windows, axis=1).astype(numpy.float32), rate + + +class TestStackedVocals(unittest.TestCase): + """#947: the vocal stem's stacked (layered stereo) spans.""" + + def test_decorrelated_channels_over_a_known_span_are_that_span(self): + self.assertEqual(stacked_spans(*sung(12, stacked=range(4, 8))), [(4.0, 8.0)]) + + def test_identical_channels_never_flag(self): + self.assertEqual(stacked_spans(*sung(12)), []) + + def test_silence_never_flags(self): + self.assertEqual(stacked_spans(numpy.zeros((2, 8000 * 10)), 8000), []) + + def test_a_mono_stem_never_flags(self): + stem, rate = sung(12, stacked=range(4, 8)) + self.assertEqual(stacked_spans(stem[:1], rate), []) + + def test_a_run_shorter_than_the_minimum_span_is_dropped(self): + self.assertEqual(stacked_spans(*sung(12, stacked={5})), []) + self.assertEqual(stacked_spans(*sung(12, stacked={5, 6})), [(5.0, 7.0)]) + + def test_windows_a_one_window_gap_apart_merge(self): + stem = sung(12, stacked={2, 3, 5, 6}) + self.assertEqual(stacked_spans(*stem), [(2.0, 7.0)]) + stem = sung(12, stacked={1, 2, 5, 6}) + self.assertEqual(stacked_spans(*stem), [(1.0, 3.0), (5.0, 7.0)]) + + def test_a_faint_decorrelated_stretch_is_not_a_second_voice(self): + stem = sung(12, stacked=range(2, 6), quiet=range(2, 6)) + self.assertEqual(stacked_spans(*stem), []) + + def test_more_than_the_listed_spans_say_how_many_more(self): + count = STACK_SPANS_LISTED + 2 + stacked = [ + start + offset for start in range(0, count * 4, 4) for offset in (0, 1) + ] + spans = stacked_spans(*sung(count * 4, stacked=set(stacked))) + self.assertEqual(len(spans), count) + message = stacked_vocals_message(spans) + self.assertIn("0:00–0:02, 0:04–0:06", message) + self.assertIn("0:28–0:30 and 2 more;", message) + self.assertNotIn("0:32", message) + + def test_spans_read_as_minutes_and_seconds(self): + message = stacked_vocals_message([(79.0, 94.0), (119.0, 129.0)]) + self.assertEqual( + message, + "separate_stems: the vocals stem holds stacked (layered stereo) " + "voices over 1:19–1:34, 1:59–2:09; treat those spans as more than " + "one voice before processing them as one", + ) + + +class TestSeparateStemsWarnsStackedVocals(unittest.TestCase): + """The check runs on the real separate_stems path; only demucs is faked.""" + + def setUp(self): + model_cache.clear_model_cache() + + def separate(self, source): + with ( + patch.dict(sys.modules, fake_demucs(vocals_only)), + patch("dw.tasks.voice_attribution.emit_warning") as warn, + ): + stems = separate_stems(AudioTrack(source, RATE), device="cpu") + return stems, warn + + def test_a_stacked_vocal_stem_warns_once_with_its_span(self): + source, _ = sung(10, stacked=range(2, 5), rate=RATE) + stems, warn = self.separate(source) + self.assertEqual(sorted(stems), sorted(SOURCES)) + warn.assert_called_once() + self.assertEqual(warn.call_args.kwargs["kind"], "stacked_vocals") + self.assertEqual(warn.call_args.kwargs["command"], "separate_stems") + self.assertIn("over 0:02–0:05;", warn.call_args.args[0]) + + def test_a_single_voice_does_not_warn(self): + stems, warn = self.separate(sung(10, rate=RATE)[0]) + self.assertEqual(len(stems), 4) + warn.assert_not_called() + + if __name__ == "__main__": unittest.main() From 3750bcff4ab296ac2c8ee1ca0530a62f95b203b6 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 22:29:10 -0500 Subject: [PATCH 028/105] docs(tasks): #958 - stem separation example and prose use file_base_name Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 991ee1e7..a5b4668a 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -2233,7 +2233,7 @@ score bed. "command": "separate_stems", "arguments": { "audio": "asset:song.mp3" } }, - "result": { "content_type": "audio/wav", "file_name": "stems" } + "result": { "content_type": "audio/wav", "file_base_name": "stems" } }, { "name": "lyrics", @@ -2250,7 +2250,7 @@ score bed. | `audio` | Yes | Path or URL of an audio file (or of a video file, whose soundtrack is taken), a waveform from a previous step, or an earlier step's video generated with a soundtrack | | `sample_rate` | With a waveform | Sample rate of a directly passed waveform (files carry their own) | -The result saves one file per stem, `-vocals`, `-drums`, `-bass` +The result saves one file per stem, `-vocals`, `-drums`, `-bass` and `-other`, and a later step reads one as `previous_result:.vocals`. Stems are 44.1 kHz stereo, whatever the source was, and sum back to the mix. The model is fixed (`htdemucs`, run in fp32), its weights download on first From cf9f136f30ac3aaec5c28fa314347fc4d91d072e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 22:32:30 -0500 Subject: [PATCH 029/105] docs(proposals): #947 - design record for the stacked_vocals warning Co-Authored-By: Claude Opus 5.5 --- .../stacked-vocals-warning-complete.md | 115 ++++++++++++++++++ docs/proposals/todo.md | 5 + 2 files changed, 120 insertions(+) create mode 100644 docs/proposals/complete/stacked-vocals-warning-complete.md diff --git a/docs/proposals/complete/stacked-vocals-warning-complete.md b/docs/proposals/complete/stacked-vocals-warning-complete.md new file mode 100644 index 00000000..0f590ced --- /dev/null +++ b/docs/proposals/complete/stacked-vocals-warning-complete.md @@ -0,0 +1,115 @@ +# A `stacked_vocals` warning from `separate_stems` (#947) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out, +2026-10-19). Plan v1 was approved by Don on 2026-10-20 (UTC). Stage A, +#955, shipped at `develop` @ `29158446` and was verified on lem. + +## The idea + +A field report from `two-minds` (*Always On*): MiniMax Music 3 can't sing +two distinct same-gender voices, and its choruses are stacked stereo +doubles. `separate_stems` returns them inside the one `vocals` stem. +Processing the 1:19 chorus as one voice is what distorted it. The issue +asked for two things: + +1. a voice/pitch conversion task (WORLD / `pyworld`) to re-voice a window + of a vocal stem; +2. a flag from `separate_stems` / `attribute_voices` saying a window holds + more than one voice. + +## Verdict and scope (Don, plan v1) + +**Build smaller:** ask (2) only, as a warning on `separate_stems`. + +- **Q1, warning only.** No `layering` result key; the four-stem dict is + unchanged (a plain `warnings` key would have broken the wav save and three + test pins). +- **Q2, voice conversion deferred.** The agent already re-voiced the stem + off-server with `pyworld` and brought it back, so the gap cost one local + script. It would add a C-extension core dependency, a new task surface and + a skill line the music3 SKILL.md has no room for. +- **Q3, `attribute_voices` untouched.** Its separation path collapses to + mono 16 kHz before a stereo check could run. +- **Cut:** the issue's "more than one pitch track" signal. Multi-pitch on a + sung stem is unreliable, and two singers in one register read as one + track anyway. + +## What was built (stage A, #955) + +- `stacked_spans()` and the `STACK_*` constants in + `dw/tasks/voice_attribution.py`, run in `separate_stems` on the 44.1 kHz + stereo vocals stem. Pure numpy. In 1 s windows that are voiced (at least + half their 20 ms frames pass `voiced_mask` on the mid signal) and within + 14 dB of the stem's level, a window is flagged when the uncentered L/R + correlation is below 0.45 **and** the side/mid energy ratio is above 0.4. + Flagged windows merge across a one-window gap; spans under 2 s drop. +- One `emit_warning(kind="stacked_vocals", command="separate_stems")`: + `the vocals stem holds stacked (layered stereo) voices over 1:19–1:34, + 1:59–2:09; treat those spans as more than one voice before processing them + as one`. Up to 8 spans in m:ss, then "and N more". +- `docs/TASKS.md` *Stem separation* names the warning and what to do with + it. +- Unit tests in `tests/test_separate_stems.py`: a known decorrelated span, + identical L/R, silence, mono, a short run dropped, a merge across a gap, + truncation, the m:ss text, and two runs of the real `separate_stems` path + with only demucs faked. + +### Calibration (the plan's gate: passed) + +On the *Always On* seed-7 vocals stem (`AlwaysOnStems` job `494d1a7469b7`): + +| section | windows | L/R correlation | side/mid | +|---|---|---|---| +| verse 1 (0:16–0:49) | 25 | 0.656 – 0.969 | 0.016 – 0.208 | +| verse 2 (0:49–1:19) | 24 | 0.663 – 0.964 | 0.018 – 0.208 | +| chorus 1 (1:19–1:34) | 15 | −0.276 – 0.251 | 0.609 – 1.682 | +| final chorus (1:59–2:09) | 10 | 0.060 – 0.206 | 0.667 – 0.891 | + +### Deviations from the plan + +1. **A level gate, `STACK_LEVEL_BELOW_DB = 14`.** htdemucs leaves the + intro's "electrical hum" pad in the vocals stem (0:08–0:12), and it + measures as decorrelated as the chorus. It sits 16.5 dB or more below the + stem's level, so the gate drops it. The margin is thin (2.5 dB on that + side, 4 dB to the quietest chorus window): retune it first if another + song disagrees. +2. **"Voiced window" means at least half its frames voiced** + (`STACK_VOICED_SHARE = 0.5`); the plan left the share open. +3. **The bridge (about 1:42–1:47) doesn't flag**: the level gate drops it. + +## Verification + +Cases C-F506 (the seed-7 known answer, plus the chained +`transcribe_audio`), C-F507 (mono, an instrumental re-separation and a +1.5 s slice don't warn; a 20 s slice warns in its own time) and C-F508 (the +guide names the warning; `separate_stems` keeps its three arguments) all +passed on lem against `29158446`. The fixture is the shared asset +`qa-cast/always-on-seed7.mp3`. + +The verify also found a stale example in the same TASKS.md section +(`file_name` for `file_base_name`), fixed separately in #958. + +## Cost and bounces + +| stage | bounces | cost | +|---|---|---| +| A (#955) | 0 | not recorded on the stage | + +## Deferred, and what brings it back + +- **The voice/pitch conversion task (ask 1).** Comes back on a second field + report needing a re-voice, from a piece other than *Always On*. The plan's + sketch: `shift_voice(audio, semitones, formant_shift, start_seconds, + duration_seconds, crossfade_seconds)`, WORLD analysis/resynthesis over a + window, crossfaded back, summed with the existing `mix_audio`; a + `pyworld` dependency after a wheel check on linux-cuda and macOS-arm64; it + would refuse a window this check flags as stacked. A learned + voice-conversion model stays out either way. +- **Structured spans (Q1).** A `layering` `JsonRecord` key saved as + `-layering.json` (the `crop_face_track` precedent), if a caller + needs the spans machine-readable. +- **The check in `attribute_voices` (Q3).** Needs the stereo stem kept + through `separate_vocals`. Worth it only if attribution on a stacked + chorus misleads a session. +- **A pitch-track signal.** Only if the stereo check proves blind to a real + stacked span. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index a992807a..9b1ebd68 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -329,6 +329,11 @@ remaining deferred fix is recorded in two-shot and lip-syncs the right singer; a last frame doesn't help. One `minimax-h3` skill line; no template, since `music-video` takes the recipe with arguments. Record: `complete/h3-keyframe-audio-complete.md`. +- **A `stacked_vocals` warning from `separate_stems`** (#947, stage #955), + shipped 2026-10-19. A stereo check on the vocals stem names the spans + that hold layered voices (a Music 3 chorus). Record, including what was + deferred (the voice/pitch conversion task, structured spans, the check in + `attribute_voices`): `complete/stacked-vocals-warning-complete.md`. ## Declined From b541b9ffe407193633f0b8931b554f5afe5bc3af Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 23:25:30 -0500 Subject: [PATCH 030/105] fix(check-script): #959 - judge the clipped tail against the line's loudest window, not a bed-stretched last word Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 9 ++++++--- tests/test_script_check.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 2a57e062..4a993ffc 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -386,7 +386,7 @@ def _clipped_tail(mono, sample_rate, end, tails, word=None): that a line ending at `end` is clipped by: the word ends inside its final CLIP_TAIL_SECONDS (within that much past it too, a word timestamp being approximate) and the tail is voiced above GUARD_FLOOR_DBFS and - within CLIP_TAIL_BELOW_WORD_DB of `word`'s (start, end) loudest window, + within CLIP_TAIL_BELOW_WORD_DB of the (start, end) span's loudest window, when given: a tail far under the word is a bed, not the word. Returns (shot, level) or None.""" word_db = word_level(mono, sample_rate, *word) if word else None floor = GUARD_FLOOR_DBFS @@ -635,10 +635,13 @@ def _untranscribed(mono, sample_rate, index, start, end, shot): def _clipped_findings(mono, sample_rate, tails, index, own, shot): """line_clipped_at_end for one heard line, as a list of zero or one - finding: its last word ends in a voiced tail of its shot or the file.""" + finding: its last word ends in a voiced tail of its shot or the file. The + tail is judged against the whole line's loudest window, not the last + word's: under a score the ASR stretches the last word over the bed, and a + bed measured against itself passes the guard (#959).""" end = own[-1]["end"] clipped = _clipped_tail( - mono, sample_rate, end, tails, (own[-1]["start"], own[-1]["end"]) + mono, sample_rate, end, tails, (own[0]["start"], own[-1]["end"]) ) if clipped is None: return [] diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 5d4d631f..67fcfa14 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -790,6 +790,18 @@ def test_bed_under_the_whole_word_still_clips_a_voiced_tail(self): self.assertEqual(len(rules(answer, "line_clipped_at_end")), 1) + def test_last_word_stretched_over_the_bed_is_not_clipped(self): + # the ASR places the last word wholly over the bed (#959): its own + # level is the bed's, the line's voice is 30 dB above the tail + mono = numpy.concatenate([voiced(1.2), voiced(0.8, amplitude=0.003)]) + chunks = [ + {"start": 0.5, "end": 0.9, "text": " hello"}, + {"start": 1.5, "end": 1.95, "text": " world"}, + ] + answer = run(["hello world"], chunks, mono) + self.assertEqual(rules(answer, "line_clipped_at_end"), []) + + class TestClippedAtShotEnd(unittest.TestCase): LINES = [{"text": "hello world", "shot": "A"}] From 04d81aa13885629b78f93cf0542ced4d689d3ba2 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 19 Oct 2026 23:25:54 -0500 Subject: [PATCH 031/105] style(tests): #959 - ruff format Co-Authored-By: Claude Sonnet 5.5 --- tests/test_script_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 67fcfa14..56a135a9 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -789,7 +789,6 @@ def test_bed_under_the_whole_word_still_clips_a_voiced_tail(self): answer = run(["hello world"], self.CHUNKS, mono) self.assertEqual(len(rules(answer, "line_clipped_at_end")), 1) - def test_last_word_stretched_over_the_bed_is_not_clipped(self): # the ASR places the last word wholly over the bed (#959): its own # level is the bed's, the line's voice is 30 dB above the tail From 5917752448209f72e3133c3afab4425652898970 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 00:52:09 -0500 Subject: [PATCH 032/105] fix(check-script): #960 - report audible untranscribed sound between two lines Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 4 ++-- dw/tasks/script_check.py | 30 ++++++++++++++++++++++++++---- tests/test_script_check.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 6 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index a5b4668a..bab630f9 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3070,7 +3070,7 @@ Markup stripped from an expected line: `[Language] ...`, ``, | `speech_where_silent` | `lines` is `[]` and words were heard above the floor and outside a repetition loop | Number of words heard / 0 | `{line: null, seconds, word}` - the first; `shot` when shots are known | | `speech_in_silent_shot` | Shots are known, at least one line names a shot, and words were heard above the floor and outside a repetition loop in a shot no line names. A word is in the shot its midpoint falls in; one a line in another shot matched (a word straddling the cut) is that line's, not counted | Number of words heard in the shot / 0 | `{line: null, seconds, word, shot}` - the shot's first counted word, `seconds` no earlier than the shot's start | | `unscripted_speech` | Shots are known and a line names the shot; words were heard inside it that align to no line (a line spoken twice, an ad-lib) | Number of unmatched words in the shot / 0 | `{line: null, seconds, end, word, shot, repeats_line}` - the first such word; `repeats_line` is true when those words repeat or contain one of the shot's lines | -| `speech_not_transcribed` | Between two words of one matched line there is a gap of `UNHEARD_MIN_SECONDS` or more, or one of its words lasts `UNHEARD_MIN_SECONDS` + 2 `UNHEARD_MARGIN_SECONDS` or more, and the interior (trimmed by `UNHEARD_MARGIN_SECONDS` each side) holds a window above `UNHEARD_FLOOR_DBFS`: audible speech the ASR left out of its text, which the line's timing absorbed (#923) | The gap's length in seconds / `UNHEARD_FLOOR_DBFS` | `{line, seconds, end, shot}` - `shot` when the line has one | +| `speech_not_transcribed` | Between two words of one matched line, or between one line's last word and the next line's first (#960; the finding sits on the next line, and a gap holding heard-but-unmatched words is `unscripted_speech`'s), there is a gap of `UNHEARD_MIN_SECONDS` or more, or one of its words lasts `UNHEARD_MIN_SECONDS` + 2 `UNHEARD_MARGIN_SECONDS` or more, and the interior (trimmed by `UNHEARD_MARGIN_SECONDS` each side) holds a window above `UNHEARD_FLOOR_DBFS`: audible speech the ASR left out of its text, which the line's timing absorbed (#923) | The gap's length in seconds / `UNHEARD_FLOOR_DBFS` | `{line, seconds, end, shot}` - `shot` when the line has one | Every finding is severity `warn`; `at.shot` is set whenever the shot is known. A rule that does not apply to the call is listed in `rules_skipped` with its reason: `speech_in_silent_shot` and `unscripted_speech` are skipped when no shots are known (`line_clipped_at_end` then looks at the file's end only), when the shots carry neither a sample span nor a frame span the take's frame rate can place, or when no line names a shot (no shot is known to be meant silent). @@ -3080,7 +3080,7 @@ The thresholds are module constants in `dw/tasks/script_check.py`, deliberately | -------- | ----- | --------------------- | | `DEFAULT_SIMILARITY` | 0.85 | A line below this similarity is a `line_mismatch` (the `similarity` argument overrides it) | | `GUARD_FLOOR_DBFS` | -65 dBFS (= `DEAD_AIR_FLOOR_DBFS`, `shot_dead_air`'s floor) | A heard word whose loudest window is at or below it is discarded as unheard; also the level a clipped tail must exceed | -| `UNHEARD_MIN_SECONDS` / `UNHEARD_MARGIN_SECONDS` / `UNHEARD_FLOOR_DBFS` | 1.0 s / 0.25 s / -40 dBFS | A gap in a line's words (or a stretched word) this long, with a window above the floor in its margin-trimmed interior, is a `speech_not_transcribed` | +| `UNHEARD_MIN_SECONDS` / `UNHEARD_MARGIN_SECONDS` / `UNHEARD_FLOOR_DBFS` | 1.0 s / 0.25 s / -40 dBFS | A gap in a line's words or between two lines (or a stretched word) this long, with a window above the floor in its margin-trimmed interior, is a `speech_not_transcribed` | | `GUARD_WINDOW_SECONDS` | 0.05 s (= `DEAD_AIR_WINDOW`, `shot_dead_air`'s window) | The window a heard word is measured in - its loudest one, so a span overhanging a pause does not average a real word away | | `CLIP_TAIL_SECONDS` | 0.25 s | The final stretch of the file a line's last word must end in to be `line_clipped_at_end` | | `CLIP_TAIL_BELOW_WORD_DB` | 15 dB | A tail more than this far under the last word's loudest window is not the word still sounding (a bed under a finished line), so not `line_clipped_at_end` | diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 4a993ffc..dea4fd73 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -149,8 +149,8 @@ def mismatch_rule(similarity): SPEECH_NOT_TRANSCRIBED = _rule( "speech_not_transcribed", UNHEARD_FLOOR_DBFS, - "audible sound lasting this many seconds sits between the line's words, or" - " inside one stretched word, with no word for it - the transcript left speech" + "audible sound lasting this many seconds sits between the line's words, between" + " two lines, or inside one stretched word, with no word for it - the transcript left speech" " out and the line's timing absorbed it; listen", ) SPEECH_WHERE_SILENT = _rule( @@ -477,7 +477,9 @@ def check( # another shot claimed (one straddling the cut) is that line's speech, # not speech in a silent shot claimed = {} + strays = [heard[j] for j, _ in unassigned] previous_end = 0.0 + heard_end = None # end of the last word any earlier line was heard to say for index, (line, words, indices) in enumerate( zip(parsed_lines, expected, assigned) ): @@ -526,6 +528,13 @@ def check( ) if own: + if heard_end is not None: + findings.extend( + _between_lines_findings( + mono, sample_rate, index, heard_end, start, shot, strays + ) + ) + heard_end = max(heard_end or 0.0, end) tails = ([(shot, span["end"])] if span else []) + [(None, duration)] findings.extend( _clipped_findings(mono, sample_rate, tails, index, own, shot) @@ -602,8 +611,8 @@ def _silent_take(heard, discarded, spans, shots_source): def _untranscribed_findings(mono, sample_rate, index, own, shot): """speech_not_transcribed: audible sound in a gap the line's words leave - between two of them - or inside one word stretched over - UNHEARD_MIN_SECONDS (#923). Not the gap before a line's first word: a pause - between shots may hold music or ambience above the floor.""" + UNHEARD_MIN_SECONDS (#923). The gap before a line's first word is + _between_lines_findings'.""" intervals = [(a["end"], b["start"]) for a, b in zip(own, own[1:])] findings = [] for start, end in intervals: @@ -623,6 +632,19 @@ def _untranscribed_findings(mono, sample_rate, index, own, shot): return [found for found in findings if found is not None] +def _between_lines_findings(mono, sample_rate, index, start, end, shot, strays): + """speech_not_transcribed for the stretch between one line's last word and + the next line's first (#960): audible sound with no word, which the + in-line check cannot see when the timing leaves it outside both lines. A + gap holding heard-but-unmatched words is unscripted_speech's.""" + if end - start < UNHEARD_MIN_SECONDS: + return [] + if any(word["start"] < end and word["end"] > start for word in strays): + return [] + found = _untranscribed(mono, sample_rate, index, start, end, shot) + return [found] if found is not None else [] + + def _untranscribed(mono, sample_rate, index, start, end, shot): lo, hi = start + UNHEARD_MARGIN_SECONDS, end - UNHEARD_MARGIN_SECONDS level = word_level(mono, sample_rate, lo, hi) diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 56a135a9..8b2defc9 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -593,6 +593,7 @@ def test_tagged_lines_carry_their_shot(self): answer = shot_run( [{"text": "alpha bravo", "shot": "A"}, {"text": "delta echo", "shot": "B"}], chunks, + mono=numpy.concatenate([voiced(0.9), silent(2.6), voiced(2.5)]), ) self.assertEqual([x["shot"] for x in answer["lines"]], ["A", "B"]) self.assertEqual(answer["findings"], []) @@ -690,6 +691,41 @@ def test_a_short_gap_is_not(self): answer = shot_run(self.LINES, chunks, mono=voiced(3.0)) self.assertEqual(rules(answer, "speech_not_transcribed"), []) + TWO = [ + {"text": "alpha bravo", "shot": "A"}, + {"text": "charlie delta", "shot": "B"}, + ] + + def test_sound_between_two_lines_is_a_finding_on_the_next_line(self): + # #960: the untranscribed stretch sits outside both lines' spans + chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) + answer = shot_run(self.TWO, chunks, mono=voiced(6.0)) + (found,) = rules(answer, "speech_not_transcribed") + self.assertEqual(found["at"]["line"], 1) + self.assertEqual(found["at"]["shot"], "B") + self.assertAlmostEqual(found["at"]["seconds"], 0.9, places=2) + self.assertAlmostEqual(found["at"]["end"], 4.0, places=2) + + def test_a_quiet_gap_between_lines_is_not(self): + chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) + mono = numpy.concatenate([voiced(0.9), silent(3.1), voiced(2.0)]) + answer = shot_run(self.TWO, chunks, mono=mono) + self.assertEqual(rules(answer, "speech_not_transcribed"), []) + + def test_a_short_gap_between_lines_is_not(self): + chunks = words("alpha bravo", 0.0) + words("charlie delta", 1.5) + answer = shot_run(self.TWO, chunks, mono=voiced(3.0)) + self.assertEqual(rules(answer, "speech_not_transcribed"), []) + + def test_a_gap_holding_unmatched_words_is_unscripted_speech_only(self): + chunks = ( + words("alpha bravo", 0.0) + + words("zulu yankee", 1.5) + + words("charlie delta", 4.0) + ) + answer = shot_run(self.TWO, chunks, mono=voiced(6.0)) + self.assertEqual(rules(answer, "speech_not_transcribed"), []) + class TestSpeechInSilentShot(unittest.TestCase): LINES = [{"text": "alpha bravo", "shot": "A"}] From 07f12f96cc8efbd921703d2e676cf2e78a306990 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 01:34:58 -0500 Subject: [PATCH 033/105] fix(check-script): #961 - keep a scripted loop's first take and flag transcript_loop Co-Authored-By: Claude Sonnet 5.5 --- dw/script_lines.py | 41 +++++++++++++++++++++++---- dw/tasks/script_check.py | 57 ++++++++++++++++++++++++++++++++++---- tests/test_script_check.py | 22 +++++++++++++++ 3 files changed, 108 insertions(+), 12 deletions(-) diff --git a/dw/script_lines.py b/dw/script_lines.py index 9354b570..5a5d12f5 100644 --- a/dw/script_lines.py +++ b/dw/script_lines.py @@ -55,11 +55,12 @@ def normalize_words(text): REPEAT_MAX_PERIOD = 4 -def repetition_loops(keys): - """The indices of `keys` (heard words) that sit in a repetition loop: a stretch that +def repetition_runs(keys): + """[(first, end, period)] for each stretch of `keys` (heard words) that repeats with a period of 1 to REPEAT_MAX_PERIOD and spans at least - REPEAT_RUN_MIN periods.""" - looped = set() + REPEAT_RUN_MIN periods; `keys[first:first + period]` is its first + occurrence.""" + runs = [] for period in range(1, REPEAT_MAX_PERIOD + 1): k = period while k < len(keys): @@ -70,7 +71,15 @@ def repetition_loops(keys): while k < len(keys) and keys[k] == keys[k - period]: k += 1 if k - first >= REPEAT_RUN_MIN * period: - looped.update(range(first, k)) + runs.append((first, k, period)) + return runs + + +def repetition_loops(keys): + """The indices of `keys` (heard words) that sit in a repetition loop.""" + looped = set() + for first, end, _ in repetition_runs(keys): + looped.update(range(first, end)) return looped @@ -78,12 +87,32 @@ def looped_texts(texts): """The indices of `texts` (transcript chunks' text, in order) holding a word of a repetition loop. Over words, not chunks: one chunk can hold a whole loop ('Pre-pre-pre'), and a chunk with no word is never in one.""" + return looped_texts_firsts(texts)[0] + + +def looped_texts_firsts(texts): + """(looped, firsts): `looped` as looped_texts; `firsts` the chunk indices + that are the first occurrence of a loop, every word of the chunk in it. A + period-1 loop of 'pre' has one first word; where runs of several periods + overlap, the shortest period's first occurrence is the one.""" sequence, owner = [], [] for index, text in enumerate(texts): words = normalize_words(text) owner.extend([index] * len(words)) sequence.extend(words) - return {owner[k] for k in repetition_loops(sequence)} + runs = sorted(repetition_runs(sequence), key=lambda run: run[2]) + looped, first_words = set(), set() + for first, end, period in runs: + if not set(range(first, end)) <= looped: + first_words.update(range(first, first + period)) + looped.update(range(first, end)) + chunks = {owner[k] for k in looped} + firsts = { + index + for index in chunks + if all(k in first_words for k in range(len(owner)) if owner[k] == index) + } + return chunks, firsts # transcribe_audio's `suspect` reasons whose chunk plan_cuts leaves out of the diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index dea4fd73..9df190bc 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -55,7 +55,7 @@ REPEAT_MAX_PERIOD, REPEAT_RUN_MIN, disambiguated, - looped_texts, + looped_texts_firsts, normalize_words, parse_lines, parse_shots, @@ -118,6 +118,7 @@ LINE_RULES = ( "line_mismatch", + "transcript_loop", "tag_spoken", "line_clipped_at_end", "speech_not_transcribed", @@ -153,6 +154,14 @@ def mismatch_rule(similarity): " two lines, or inside one stretched word, with no word for it - the transcript left speech" " out and the line's timing absorbed it; listen", ) +TRANSCRIPT_LOOP = _rule( + "transcript_loop", + REPEAT_RUN_MIN, + "the transcript repeated words of the script in a loop for this many chunks" + " (seen at 'seconds' to 'end') - Whisper stuck, so speech after it may be" + " missing from 'heard' and lines can read 0.0 without being wrong; listen," + " or check each shot on its own, before re-rolling", +) SPEECH_WHERE_SILENT = _rule( "speech_where_silent", 0, @@ -260,13 +269,17 @@ def _round(value, places=3): return None if value is None else round(float(value), places) -def guard_words(chunks, mono, sample_rate): +def guard_words(chunks, mono, sample_rate, script_words=None): """Split transcribe_audio's word chunks into (heard, discarded). `heard` is [{word, text, start, end}], one per normalized word - a chunk that normalizes to nothing ('♪', '...') is neither; `discarded` is [{text, start, end, level_dbfs, reason}] for a chunk in a repetition loop (`reason` "repetition") or at or below the floor ("below_floor"). + + A loop's first occurrence is kept as heard when `script_words` (the + words the script holds) contains all of its words and it has real + duration and speech level: Whisper loops on a line it did hear (#961). """ spoken = [] for chunk in chunks: @@ -275,13 +288,19 @@ def guard_words(chunks, mono, sample_rate): spoken.append((chunk, words)) # The loop detector is script_lines' (shared with transcribe_audio's # `suspect: "repetition"`); a chunk with no word is in no loop - looped = looped_texts([chunk.get("text", "") for chunk, _ in spoken]) + looped, firsts = looped_texts_firsts([chunk.get("text", "") for chunk, _ in spoken]) heard, discarded = [], [] for index, (chunk, words) in enumerate(spoken): start, end = chunk["start"], chunk["end"] level = word_level(mono, sample_rate, start, end) - if index in looped: + first_of_line = ( + index in firsts + and script_words is not None + and set(words) <= script_words + and end > start + ) + if index in looped and not first_of_line: reason = "repetition" elif level is None or level <= GUARD_FLOOR_DBFS: reason = "below_floor" @@ -460,9 +479,12 @@ def check( float waveform; `spans` the take's shots as shot_spans() places them, or None when none are known, and `shots_source` where they came from (resolve_shots: argument, artifact, manifest or none).""" - heard, discarded = guard_words(chunks, mono, sample_rate) + script_words = { + word for line in parsed_lines for word in normalize_words(line["text"]) + } + heard, discarded = guard_words(chunks, mono, sample_rate, script_words) duration = len(mono) / sample_rate if sample_rate else 0.0 - findings = [] + findings = _loop_findings(chunks, script_words) if parsed_lines else [] if not parsed_lines: return _silent_take(heard, discarded, spans, shots_source) @@ -574,6 +596,29 @@ def check( } +def _loop_findings(chunks, script_words): + """A `transcript_loop` finding for each repetition loop over the script's + own words: Whisper stuck on a line it heard, so what follows the loop was + never transcribed and its lines read as mismatches (#961).""" + spoken = [c for c in chunks if normalize_words(c.get("text", ""))] + looped, firsts = looped_texts_firsts([c.get("text", "") for c in spoken]) + found, run = [], [] + for index in sorted(looped) + [None]: + if run and (index is None or index != run[-1] + 1): + if any(i in firsts for i in run): + members = [spoken[i] for i in run] + if all( + set(normalize_words(m["text"])) <= script_words for m in members + ): + at = _at(None, seconds=members[0]["start"]) + at["end"] = _round(members[-1]["end"]) + found.append(finding(TRANSCRIPT_LOOP, len(members), at)) + run = [] + if index is not None: + run.append(index) + return found + + def _heard_shot(spans, word): """The shot a heard word's midpoint falls in, None when no shots are known.""" if spans is None: diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 8b2defc9..239d9fe1 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -321,6 +321,28 @@ def test_loop_inside_one_chunk_is_discarded(self): self.assertEqual(heard, []) self.assertEqual(discarded[0]["reason"], "repetition") + def test_loop_over_a_scripted_line_keeps_its_first_take(self): + # #961: Whisper looped on the line it heard; the first take is real + chunks = words("how", 0.0) + words( + " ".join(["the lid is gone"] * 8), 1.0, step=0.3, length=0.2 + ) + answer = run(["the lid is gone"], chunks, voiced(12.0)) + self.assertEqual(answer["lines"][0]["heard"], "the lid is gone") + self.assertEqual(answer["lines"][0]["similarity"], 1.0) + self.assertEqual(len(answer["discarded"]), 28) + loops = [f for f in answer["findings"] if f["rule"] == "transcript_loop"] + self.assertEqual(len(loops), 1) + self.assertEqual(loops[0]["value"], 32) + self.assertEqual(loops[0]["at"]["seconds"], 1.0) + + def test_loop_of_unscripted_words_keeps_nothing_and_is_no_finding(self): + chunks = words(" ".join(["pre"] * 10), 1.0, step=0.1, length=0.05) + answer = run(["hello"], chunks, voiced(4.0)) + self.assertEqual(answer["lines"][0]["heard"], "") + self.assertFalse( + [f for f in answer["findings"] if f["rule"] == "transcript_loop"] + ) + def test_below_floor_reason(self): mono = numpy.concatenate([voiced(1.0), silent(1.0)]) _, discarded = guard_words(words("real", 0.0) + words("ghost", 1.2), mono, RATE) From 2ff472f32d1f0890e7027ad1bc2c328fcf854f11 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 01:35:16 -0500 Subject: [PATCH 034/105] docs(check-script): #961 - transcript_loop rule Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/TASKS.md b/docs/TASKS.md index bab630f9..43777d9f 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3066,6 +3066,7 @@ Markup stripped from an expected line: `[Language] ...`, ``, | ---- | ---------- | --------------------- | ---- | | `line_mismatch` | A line's similarity is below `similarity`; a dropped line comes back with `heard: ""` | The line's similarity / `similarity` | `{line, seconds, word}` - the line's first heard word, or where it should have been | | `tag_spoken` | A word from the line's stripped markup was heard, and is not also a word of the line's dialogue | The word heard / none | `{line, seconds, word}` | +| `transcript_loop` | A repetition loop (`REPEAT_RUN_MIN` periods of up to `REPEAT_MAX_PERIOD` words) made only of words the script holds - Whisper stuck on a line it heard, so speech after it can be missing from `heard` and lines read 0.0 without being wrong (#961). The loop's first occurrence is kept as heard when it has real duration and speech level; the rest is under `discarded` | Chunks in the loop / `REPEAT_RUN_MIN` | `{line: null, seconds, end}` | | `line_clipped_at_end` | A line's last heard word ends inside the final `CLIP_TAIL_SECONDS` of its shot (when the line has one) or of the file, and that tail is still above `GUARD_FLOOR_DBFS` and within `CLIP_TAIL_BELOW_WORD_DB` of the last word's own loudest window (a quieter tail is a score bed or room tone, not the word - #953) | The tail's level in dBFS / `GUARD_FLOOR_DBFS` | `{line, seconds, word, shot}` - `shot` when the line has one | | `speech_where_silent` | `lines` is `[]` and words were heard above the floor and outside a repetition loop | Number of words heard / 0 | `{line: null, seconds, word}` - the first; `shot` when shots are known | | `speech_in_silent_shot` | Shots are known, at least one line names a shot, and words were heard above the floor and outside a repetition loop in a shot no line names. A word is in the shot its midpoint falls in; one a line in another shot matched (a word straddling the cut) is that line's, not counted | Number of words heard in the shot / 0 | `{line: null, seconds, word, shot}` - the shot's first counted word, `seconds` no earlier than the shot's start | From a5d7536fd6676d724bec93451cfdfce0183c5997 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 01:41:41 -0500 Subject: [PATCH 035/105] fix(check-script): #962 - collapse looped discarded runs, mark lines after a loop not transcribed, document unscripted_speech threshold Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 6 +++--- dw/tasks/script_check.py | 34 +++++++++++++++++++++++++++++----- tests/test_script_check.py | 32 ++++++++++++++++++++++++++------ 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 43777d9f..47765d6e 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3066,14 +3066,14 @@ Markup stripped from an expected line: `[Language] ...`, ``, | ---- | ---------- | --------------------- | ---- | | `line_mismatch` | A line's similarity is below `similarity`; a dropped line comes back with `heard: ""` | The line's similarity / `similarity` | `{line, seconds, word}` - the line's first heard word, or where it should have been | | `tag_spoken` | A word from the line's stripped markup was heard, and is not also a word of the line's dialogue | The word heard / none | `{line, seconds, word}` | -| `transcript_loop` | A repetition loop (`REPEAT_RUN_MIN` periods of up to `REPEAT_MAX_PERIOD` words) made only of words the script holds - Whisper stuck on a line it heard, so speech after it can be missing from `heard` and lines read 0.0 without being wrong (#961). The loop's first occurrence is kept as heard when it has real duration and speech level; the rest is under `discarded` | Chunks in the loop / `REPEAT_RUN_MIN` | `{line: null, seconds, end}` | +| `transcript_loop` | A repetition loop (`REPEAT_RUN_MIN` periods of up to `REPEAT_MAX_PERIOD` words) made only of words the script holds - Whisper stuck on a line it heard, so speech after it can be missing from `heard` and lines read 0.0 without being wrong (#961). The loop's first occurrence is kept as heard when it has real duration and speech level; the rest is under `discarded`, one entry per loop (`count`, `start`..`end`). A line heard nowhere, at or after the loop, reads `heard: null`, `similarity: null`, `not_transcribed: "transcript_loop"` and raises no `line_mismatch` | Chunks in the loop / `REPEAT_RUN_MIN` | `{line: null, seconds, end}` | | `line_clipped_at_end` | A line's last heard word ends inside the final `CLIP_TAIL_SECONDS` of its shot (when the line has one) or of the file, and that tail is still above `GUARD_FLOOR_DBFS` and within `CLIP_TAIL_BELOW_WORD_DB` of the last word's own loudest window (a quieter tail is a score bed or room tone, not the word - #953) | The tail's level in dBFS / `GUARD_FLOOR_DBFS` | `{line, seconds, word, shot}` - `shot` when the line has one | | `speech_where_silent` | `lines` is `[]` and words were heard above the floor and outside a repetition loop | Number of words heard / 0 | `{line: null, seconds, word}` - the first; `shot` when shots are known | | `speech_in_silent_shot` | Shots are known, at least one line names a shot, and words were heard above the floor and outside a repetition loop in a shot no line names. A word is in the shot its midpoint falls in; one a line in another shot matched (a word straddling the cut) is that line's, not counted | Number of words heard in the shot / 0 | `{line: null, seconds, word, shot}` - the shot's first counted word, `seconds` no earlier than the shot's start | | `unscripted_speech` | Shots are known and a line names the shot; words were heard inside it that align to no line (a line spoken twice, an ad-lib) | Number of unmatched words in the shot / 0 | `{line: null, seconds, end, word, shot, repeats_line}` - the first such word; `repeats_line` is true when those words repeat or contain one of the shot's lines | | `speech_not_transcribed` | Between two words of one matched line, or between one line's last word and the next line's first (#960; the finding sits on the next line, and a gap holding heard-but-unmatched words is `unscripted_speech`'s), there is a gap of `UNHEARD_MIN_SECONDS` or more, or one of its words lasts `UNHEARD_MIN_SECONDS` + 2 `UNHEARD_MARGIN_SECONDS` or more, and the interior (trimmed by `UNHEARD_MARGIN_SECONDS` each side) holds a window above `UNHEARD_FLOOR_DBFS`: audible speech the ASR left out of its text, which the line's timing absorbed (#923) | The gap's length in seconds / `UNHEARD_FLOOR_DBFS` | `{line, seconds, end, shot}` - `shot` when the line has one | -Every finding is severity `warn`; `at.shot` is set whenever the shot is known. A rule that does not apply to the call is listed in `rules_skipped` with its reason: `speech_in_silent_shot` and `unscripted_speech` are skipped when no shots are known (`line_clipped_at_end` then looks at the file's end only), when the shots carry neither a sample span nor a frame span the take's frame rate can place, or when no line names a shot (no shot is known to be meant silent). +Every finding is severity `warn`; `at.shot` is set whenever the shot is known. A rule that does not apply to the call is listed in `rules_skipped` with its reason: `speech_in_silent_shot` and `unscripted_speech` are skipped when no shots are known (`line_clipped_at_end` then looks at the file's end only), when the shots carry neither a sample span nor a frame span the take's frame rate can place, or when no line names a shot (no shot is known to be meant silent). A take checked alone, with no shot map (an `asset:` file with no manifest beside it), therefore never raises `unscripted_speech`, however many words are `unmatched`: read `unmatched` there, or pass `shots`. With shots known there is no minimum word count - one unmatched word inside a named shot fires it. The thresholds are module constants in `dw/tasks/script_check.py`, deliberately not in `RULES` (`dw/assessment_rules.py`): every rule there must fire on a synthetic file that holds no script. The answer echoes them under `thresholds`. @@ -3088,7 +3088,7 @@ The thresholds are module constants in `dw/tasks/script_check.py`, deliberately | `REPEAT_RUN_MIN` | 6 | Heard words repeating the same unit this many times over, back to back, are a repetition loop: every chunk holding one is discarded (`reason: "repetition"`), however loud | | `REPEAT_MAX_PERIOD` | 4 words | The longest repeating unit a loop is looked for in - `pre pre pre` has period 1, `thank you thank you` period 2 | -The result: `findings`, `lines[]` (`expected`, `heard`, `similarity`, `start`, `end`, `shot`), `discarded[]` (`text`, `start`, `end`, `level_dbfs`, `reason` - `below_floor` or `repetition`), `unmatched[]` (heard words aligned to no line), `transcript`, `model_name`, `rules_applied`, `rules_skipped[]` (`rule`, `reason`) and `thresholds`. A line's `shot` is the shot it names, else the shot its heard words overlap most (null when none is known). `shots` is the map as placed - `[{name, start, end}]` in seconds on the take, or null when none is known - and `shots_source` says where it came from: `argument`, `artifact`, `manifest` or `none`. +The result: `findings`, `lines[]` (`expected`, `heard`, `similarity`, `start`, `end`, `shot`), `discarded[]` (`text`, `start`, `end`, `level_dbfs`, `reason` - `below_floor` or `repetition`, `count` - a repetition loop is one entry spanning its chunks), `unmatched[]` (heard words aligned to no line), `transcript`, `model_name`, `rules_applied`, `rules_skipped[]` (`rule`, `reason`) and `thresholds`. A line's `shot` is the shot it names, else the shot its heard words overlap most (null when none is known). `shots` is the map as placed - `[{name, start, end}]` in seconds on the take, or null when none is known - and `shots_source` says where it came from: `argument`, `artifact`, `manifest` or `none`. How the alignment reads a take: diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 9df190bc..9dd48d74 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -274,8 +274,10 @@ def guard_words(chunks, mono, sample_rate, script_words=None): `heard` is [{word, text, start, end}], one per normalized word - a chunk that normalizes to nothing ('♪', '...') is neither; `discarded` is - [{text, start, end, level_dbfs, reason}] for a chunk in a repetition loop - (`reason` "repetition") or at or below the floor ("below_floor"). + [{text, start, end, level_dbfs, reason, count}] for a chunk at or below + the floor ("below_floor") or a repetition loop ("repetition": one entry + per run of consecutive looped chunks, `count` chunks, `text` the first's, + `start`/`end` the run's span). A loop's first occurrence is kept as heard when `script_words` (the words the script holds) contains all of its words and it has real @@ -291,6 +293,7 @@ def guard_words(chunks, mono, sample_rate, script_words=None): looped, firsts = looped_texts_firsts([chunk.get("text", "") for chunk, _ in spoken]) heard, discarded = [], [] + run_end = None # spoken index of the last chunk folded into discarded[-1] for index, (chunk, words) in enumerate(spoken): start, end = chunk["start"], chunk["end"] level = word_level(mono, sample_rate, start, end) @@ -306,6 +309,17 @@ def guard_words(chunks, mono, sample_rate, script_words=None): reason = "below_floor" else: reason = None + if reason == "repetition" and run_end == index - 1: + # A loop is one entry: its span, its chunk count, its loudest level + last = discarded[-1] + last["end"] = _round(max(last["end"], end)) + last["count"] += 1 + if level is not None and ( + last["level_dbfs"] is None or level > last["level_dbfs"] + ): + last["level_dbfs"] = _round(level, 2) + run_end = index + continue if reason: discarded.append( { @@ -314,9 +328,12 @@ def guard_words(chunks, mono, sample_rate, script_words=None): "end": _round(end), "level_dbfs": _round(level, 2), "reason": reason, + "count": 1, } ) + run_end = index if reason == "repetition" else None continue + run_end = None for word in words: heard.append( { @@ -485,6 +502,7 @@ def check( heard, discarded = guard_words(chunks, mono, sample_rate, script_words) duration = len(mono) / sample_rate if sample_rate else 0.0 findings = _loop_findings(chunks, script_words) if parsed_lines else [] + loop_starts = [found["at"]["seconds"] for found in findings] if not parsed_lines: return _silent_take(heard, discarded, spans, shots_source) @@ -520,17 +538,23 @@ def check( shot = overlapping_shot(spans, start, end) span = by_name.get(shot) claimed.update((j, shot) for j in indices) + # A line nothing was heard for, at or after a transcript_loop, was + # never transcribed: Whisper was stuck, so it is not a 0.0 mismatch + position = span["start"] if span else previous_end + looped = not own and any(at <= position for at in loop_starts) lines.append( { "expected": line["text"], - "heard": _joined(own), - "similarity": round(score, 4), + "heard": None if looped else _joined(own), + "similarity": None if looped else round(score, 4), "start": _round(start), "end": _round(end), "shot": shot, } ) - if score < similarity_threshold: + if looped: + lines[-1]["not_transcribed"] = "transcript_loop" + elif score < similarity_threshold: if own: at = _at(index, own[0], shot=shot) else: diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 239d9fe1..632f7368 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -282,14 +282,18 @@ def test_loud_loop_is_discarded_as_repetition(self): ] heard, discarded = guard_words(chunks, voiced(5.0), RATE) self.assertEqual(heard, []) - self.assertEqual(len(discarded), 40) - self.assertEqual({d["reason"] for d in discarded}, {"repetition"}) + self.assertEqual(len(discarded), 1) + self.assertEqual(discarded[0]["reason"], "repetition") + self.assertEqual(discarded[0]["count"], 40) + self.assertEqual(discarded[0]["start"], 0.0) + self.assertEqual(discarded[0]["end"], 3.9) def test_loop_over_music_is_not_speech_where_silent(self): chunks = words(" ".join(["pre"] * 30), step=0.1, length=0.0) answer = run([], chunks, voiced(5.0)) self.assertEqual(answer["findings"], []) - self.assertEqual(len(answer["discarded"]), 30) + self.assertEqual(len(answer["discarded"]), 1) + self.assertEqual(answer["discarded"][0]["count"], 30) def test_short_real_repetition_stays_heard(self): chunks = words("no no no no no stop") @@ -301,7 +305,8 @@ def test_period_two_loop_is_discarded(self): chunks = words(" ".join(["thank you"] * 6), step=0.3, length=0.2) heard, discarded = guard_words(chunks, voiced(5.0), RATE) self.assertEqual(heard, []) - self.assertEqual(len(discarded), 12) + self.assertEqual(len(discarded), 1) + self.assertEqual(discarded[0]["count"], 12) def test_loop_does_not_take_the_speech_around_it(self): chunks = ( @@ -313,7 +318,8 @@ def test_loop_does_not_take_the_speech_around_it(self): self.assertEqual( [h["word"] for h in heard], ["hello", "there", "good", "night"] ) - self.assertEqual(len(discarded), 10) + self.assertEqual(len(discarded), 1) + self.assertEqual(discarded[0]["count"], 10) def test_loop_inside_one_chunk_is_discarded(self): chunks = [{"start": 0.0, "end": 3.0, "text": " Pre-pre-pre-pre-pre-pre-pre"}] @@ -329,12 +335,26 @@ def test_loop_over_a_scripted_line_keeps_its_first_take(self): answer = run(["the lid is gone"], chunks, voiced(12.0)) self.assertEqual(answer["lines"][0]["heard"], "the lid is gone") self.assertEqual(answer["lines"][0]["similarity"], 1.0) - self.assertEqual(len(answer["discarded"]), 28) + self.assertEqual(len(answer["discarded"]), 1) + self.assertEqual(answer["discarded"][0]["count"], 28) loops = [f for f in answer["findings"] if f["rule"] == "transcript_loop"] self.assertEqual(len(loops), 1) self.assertEqual(loops[0]["value"], 32) self.assertEqual(loops[0]["at"]["seconds"], 1.0) + def test_line_after_a_loop_is_not_transcribed_not_a_zero(self): + chunks = words("how", 0.0) + words( + " ".join(["the lid is gone"] * 8), 1.0, step=0.3, length=0.2 + ) + answer = run(["the lid is gone", "i wore it as a hat"], chunks, voiced(12.0)) + line = answer["lines"][1] + self.assertIsNone(line["heard"]) + self.assertIsNone(line["similarity"]) + self.assertEqual(line["not_transcribed"], "transcript_loop") + rules = [f["rule"] for f in answer["findings"]] + self.assertNotIn("line_mismatch", rules) + self.assertIn("transcript_loop", rules) + def test_loop_of_unscripted_words_keeps_nothing_and_is_no_finding(self): chunks = words(" ".join(["pre"] * 10), 1.0, step=0.1, length=0.05) answer = run(["hello"], chunks, voiced(4.0)) From 222cc050ae032caa870e297ae357624d977c2e68 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 01:55:32 -0500 Subject: [PATCH 036/105] fix(check-script): #963 - re-transcribe a looped cut shot by shot when shots are known Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 2 +- dw/tasks/script_check.py | 52 ++++++++++++++++++++++++++++++++++- tests/test_script_check.py | 55 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 47765d6e..099299e1 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3066,7 +3066,7 @@ Markup stripped from an expected line: `[Language] ...`, ``, | ---- | ---------- | --------------------- | ---- | | `line_mismatch` | A line's similarity is below `similarity`; a dropped line comes back with `heard: ""` | The line's similarity / `similarity` | `{line, seconds, word}` - the line's first heard word, or where it should have been | | `tag_spoken` | A word from the line's stripped markup was heard, and is not also a word of the line's dialogue | The word heard / none | `{line, seconds, word}` | -| `transcript_loop` | A repetition loop (`REPEAT_RUN_MIN` periods of up to `REPEAT_MAX_PERIOD` words) made only of words the script holds - Whisper stuck on a line it heard, so speech after it can be missing from `heard` and lines read 0.0 without being wrong (#961). The loop's first occurrence is kept as heard when it has real duration and speech level; the rest is under `discarded`, one entry per loop (`count`, `start`..`end`). A line heard nowhere, at or after the loop, reads `heard: null`, `similarity: null`, `not_transcribed: "transcript_loop"` and raises no `line_mismatch` | Chunks in the loop / `REPEAT_RUN_MIN` | `{line: null, seconds, end}` | +| `transcript_loop` | A repetition loop (`REPEAT_RUN_MIN` periods of up to `REPEAT_MAX_PERIOD` words) made only of words the script holds - Whisper stuck on a line it heard, so speech after it can be missing from `heard` and lines read 0.0 without being wrong (#961). The loop's first occurrence is kept as heard when it has real duration and speech level; the rest is under `discarded`, one entry per loop (`count`, `start`..`end`). A line heard nowhere, at or after the loop, reads `heard: null`, `similarity: null`, `not_transcribed: "transcript_loop"` and raises no `line_mismatch`. When the shots are known the cut is then transcribed again one shot at a time and the findings and `lines` come from that pass (`transcription_pass: "per_shot"`, each line `pass: "per_shot"`, #963) | Chunks in the loop / `REPEAT_RUN_MIN` | `{line: null, seconds, end}` | | `line_clipped_at_end` | A line's last heard word ends inside the final `CLIP_TAIL_SECONDS` of its shot (when the line has one) or of the file, and that tail is still above `GUARD_FLOOR_DBFS` and within `CLIP_TAIL_BELOW_WORD_DB` of the last word's own loudest window (a quieter tail is a score bed or room tone, not the word - #953) | The tail's level in dBFS / `GUARD_FLOOR_DBFS` | `{line, seconds, word, shot}` - `shot` when the line has one | | `speech_where_silent` | `lines` is `[]` and words were heard above the floor and outside a repetition loop | Number of words heard / 0 | `{line: null, seconds, word}` - the first; `shot` when shots are known | | `speech_in_silent_shot` | Shots are known, at least one line names a shot, and words were heard above the floor and outside a repetition loop in a shot no line names. A word is in the shot its midpoint falls in; one a line in another shot matched (a word straddling the cut) is that line's, not counted | Number of words heard in the shot / 0 | `{line: null, seconds, word, shot}` - the shot's first counted word, `seconds` no earlier than the shot's start | diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 9dd48d74..420e1d9a 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -841,6 +841,27 @@ def _joined(words): return " ".join(texts) +def per_shot_chunks(transcribe, waveform, rate, spans): + """Word chunks from transcribing each shot's span on its own, timestamps + moved back onto the take's clock. `transcribe(leveled, rate)` is + transcribe_audio's chunks for one slice; each is levelled alone.""" + chunks = [] + for span in spans: + lo, hi = int(round(span["start"] * rate)), int(round(span["end"] * rate)) + if hi <= lo: + continue + piece, _gain = level_for_transcription(waveform[..., lo:hi]) + for chunk in transcribe(piece, rate): + chunks.append( + dict( + chunk, + start=chunk["start"] + span["start"], + end=chunk["end"] + span["start"], + ) + ) + return chunks + + def check_script( audio, lines, @@ -938,15 +959,44 @@ def check_script( fps = (probe_metadata(path) or {}).get("fps") spans = shot_spans(records, rate, fps, len(mono) / rate if rate else 0.0) + chunks, whole = transcript["chunks"], True + script_words = {w for line in parsed_lines for w in normalize_words(line["text"])} + if spans and parsed_lines and _loop_findings(chunks, script_words): + # Whisper looped on the whole cut and never transcribed what follows + # (#961): the shots are known, so transcribe each on its own (#963) + emit_log("check_script: transcript_loop on the cut; transcribing per shot") + chunks, whole = ( + per_shot_chunks( + lambda piece, sr: transcribe_audio( + piece, + device=device, + sample_rate=sr, + model_name=model_name, + timestamps="word", + )["chunks"], + waveform, + rate, + spans, + ), + False, + ) answer = check( parsed_lines, - transcript["chunks"], + chunks, mono, rate, float(similarity), spans=spans, shots_source=shots_source, ) + answer["transcription_pass"] = "whole_cut" if whole else "per_shot" + if not whole: + answer["transcription_note"] = ( + "the whole-cut transcript looped, so each shot was transcribed on" + " its own and 'lines' and 'findings' come from that pass" + ) + for line in answer["lines"]: + line["pass"] = "per_shot" answer["transcript"] = transcript["text"] answer["model_name"] = model_name answer["thresholds"] = dict(THRESHOLDS, similarity=float(similarity)) diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 632f7368..caf5d227 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -1253,3 +1253,58 @@ def test_transcription_input_is_level_independent(): assert float(numpy.abs(a).max()) <= 1.0 silent = numpy.zeros((1, 100), dtype=numpy.float32) assert level_for_transcription(silent) == (silent, 1.0) + + +class PerShotPassTest(unittest.TestCase): + """#963: a transcript_loop on a cut with known shots is re-transcribed per shot.""" + + def _run(self, whole, per_shot): + calls = [] + slice_calls = iter(per_shot) + + def fake(audio, **kwargs): + calls.append(len(audio[0]) / RATE) + if len(calls) == 1: + return {"text": "", "chunks": whole} + return {"text": "", "chunks": next(slice_calls)} + + records = [ + {"name": "shot@a", "start_sample": 0, "num_samples": 6 * RATE}, + {"name": "shot@b", "start_sample": 6 * RATE, "num_samples": 6 * RATE}, + ] + wave = voiced(12.0)[None, :] + with ( + patch("dw.tasks.audio_transcription.transcribe_audio", fake), + patch.object(script_check, "waveform_and_rate", return_value=(wave, RATE)), + patch.object( + script_check, "resolve_shots", return_value=(records, "manifest") + ), + ): + answer = script_check.check_script( + "x.mp4", + [ + {"text": "the lid is gone", "shot": "a"}, + {"text": "i wore it", "shot": "b"}, + ], + ) + return answer, calls + + def test_looped_cut_is_rechecked_shot_by_shot(self): + whole = words("the lid is gone", 1.0) + words( + " ".join(["the lid is gone"] * 8), 3.0, step=0.1, length=0.05 + ) + answer, calls = self._run( + whole, [words("the lid is gone", 1.0), words("i wore it", 1.0)] + ) + self.assertEqual(calls, [12.0, 6.0, 6.0]) + self.assertEqual(answer["transcription_pass"], "per_shot") + self.assertEqual([x["similarity"] for x in answer["lines"]], [1.0, 1.0]) + self.assertEqual(answer["lines"][1]["start"], 7.0) + self.assertEqual(answer["lines"][1]["pass"], "per_shot") + self.assertNotIn("not_transcribed", answer["lines"][1]) + + def test_clean_cut_is_one_pass(self): + whole = words("the lid is gone", 1.0) + words("i wore it", 7.0) + answer, calls = self._run(whole, []) + self.assertEqual(len(calls), 1) + self.assertEqual(answer["transcription_pass"], "whole_cut") From 40ae142fd840cb8e3457ea01af72e809d8fecb79 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 02:39:55 -0500 Subject: [PATCH 037/105] fix(check-script): #964 - judge the stretches a stray word leaves in a between-lines gap Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 25 +++++++++++++++++++------ tests/test_script_check.py | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 420e1d9a..afc10e9e 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -704,14 +704,27 @@ def _untranscribed_findings(mono, sample_rate, index, own, shot): def _between_lines_findings(mono, sample_rate, index, start, end, shot, strays): """speech_not_transcribed for the stretch between one line's last word and the next line's first (#960): audible sound with no word, which the - in-line check cannot see when the timing leaves it outside both lines. A - gap holding heard-but-unmatched words is unscripted_speech's.""" + in-line check cannot see when the timing leaves it outside both lines. + Heard-but-unmatched words in the gap are unscripted_speech's, but only the + stretches they cover: each stretch left between them is judged on its own + (#964).""" if end - start < UNHEARD_MIN_SECONDS: return [] - if any(word["start"] < end and word["end"] > start for word in strays): - return [] - found = _untranscribed(mono, sample_rate, index, start, end, shot) - return [found] if found is not None else [] + inside = sorted( + (w for w in strays if w["start"] < end and w["end"] > start), + key=lambda w: w["start"], + ) + stretches, cursor = [], start + for word in inside: + stretches.append((cursor, word["start"])) + cursor = max(cursor, word["end"]) + stretches.append((cursor, end)) + found = [ + _untranscribed(mono, sample_rate, index, lo, hi, shot) + for lo, hi in stretches + if hi - lo >= UNHEARD_MIN_SECONDS + ] + return [f for f in found if f is not None] def _untranscribed(mono, sample_rate, index, start, end, shot): diff --git a/tests/test_script_check.py b/tests/test_script_check.py index caf5d227..fb96e5bc 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -766,8 +766,31 @@ def test_a_gap_holding_unmatched_words_is_unscripted_speech_only(self): + words("charlie delta", 4.0) ) answer = shot_run(self.TWO, chunks, mono=voiced(6.0)) + # Stray words 1.5-2.4; the 2.4-4.0 stretch after them is audible + (found,) = rules(answer, "speech_not_transcribed") + self.assertAlmostEqual(found["at"]["seconds"], 2.4, places=2) + + def test_a_gap_fully_covered_by_unmatched_words_is_unscripted_only(self): + chunks = ( + words("alpha bravo", 0.0) + + words("zulu yankee xray whiskey", 0.9, step=0.5, length=0.5) + + words("charlie delta", 3.0) + ) + answer = shot_run(self.TWO, chunks, mono=voiced(5.0)) self.assertEqual(rules(answer, "speech_not_transcribed"), []) + def test_audible_stretch_before_a_stray_word_is_a_finding(self): + # #964: a stray word late in the gap must not hide the stretch before it + chunks = ( + words("alpha bravo", 0.0) + words("zulu", 4.0) + words("charlie delta", 6.0) + ) + answer = shot_run(self.TWO, chunks, mono=voiced(8.0)) + # the stretches on both sides of the stray word are findings + before, after = rules(answer, "speech_not_transcribed") + self.assertAlmostEqual(before["at"]["seconds"], 0.9, places=2) + self.assertAlmostEqual(before["at"]["end"], 4.0, places=2) + self.assertAlmostEqual(after["at"]["seconds"], 4.4, places=2) + class TestSpeechInSilentShot(unittest.TestCase): LINES = [{"text": "alpha bravo", "shot": "A"}] From ab22e152fbf512d366c4e865d9af9b48706ef635 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 02:47:40 -0500 Subject: [PATCH 038/105] fix(check-script): #964 - place a between-lines untranscribed stretch in the shot holding most of it Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 46 +++++++++++++++++++++++++++++++------- tests/test_script_check.py | 12 ++++++---- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index afc10e9e..0c908d99 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -577,7 +577,15 @@ def check( if heard_end is not None: findings.extend( _between_lines_findings( - mono, sample_rate, index, heard_end, start, shot, strays + mono, + sample_rate, + index, + heard_end, + start, + shot, + strays, + spans, + parsed_lines, ) ) heard_end = max(heard_end or 0.0, end) @@ -701,13 +709,17 @@ def _untranscribed_findings(mono, sample_rate, index, own, shot): return [found for found in findings if found is not None] -def _between_lines_findings(mono, sample_rate, index, start, end, shot, strays): +def _between_lines_findings( + mono, sample_rate, index, start, end, shot, strays, spans=None, parsed_lines=() +): """speech_not_transcribed for the stretch between one line's last word and the next line's first (#960): audible sound with no word, which the in-line check cannot see when the timing leaves it outside both lines. Heard-but-unmatched words in the gap are unscripted_speech's, but only the stretches they cover: each stretch left between them is judged on its own - (#964).""" + (#964). Each stretch is attributed to the shot holding most of it, and + to the line naming that shot (null when none does), not to the next + line's shot by default.""" if end - start < UNHEARD_MIN_SECONDS: return [] inside = sorted( @@ -719,14 +731,32 @@ def _between_lines_findings(mono, sample_rate, index, start, end, shot, strays): stretches.append((cursor, word["start"])) cursor = max(cursor, word["end"]) stretches.append((cursor, end)) - found = [ - _untranscribed(mono, sample_rate, index, lo, hi, shot) - for lo, hi in stretches - if hi - lo >= UNHEARD_MIN_SECONDS - ] + found = [] + for lo, hi in stretches: + if hi - lo < UNHEARD_MIN_SECONDS: + continue + where_line, where_shot = index, shot + holder = _majority_shot(spans, lo, hi) + if holder is not None and holder != shot: + where_shot = holder + where_line = next( + (i for i, line in enumerate(parsed_lines) if line["shot"] == holder), + None, + ) + found.append(_untranscribed(mono, sample_rate, where_line, lo, hi, where_shot)) return [f for f in found if f is not None] +def _majority_shot(spans, start, end): + """The shot overlapping [start, end] the most, None when no shots are known.""" + best, best_overlap = None, 0.0 + for span in spans or (): + overlap = min(end, span["end"]) - max(start, span["start"]) + if overlap > best_overlap: + best, best_overlap = span["name"], overlap + return best + + def _untranscribed(mono, sample_rate, index, start, end, shot): lo, hi = start + UNHEARD_MARGIN_SECONDS, end - UNHEARD_MARGIN_SECONDS level = word_level(mono, sample_rate, lo, hi) diff --git a/tests/test_script_check.py b/tests/test_script_check.py index fb96e5bc..05478c1c 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -738,13 +738,14 @@ def test_a_short_gap_is_not(self): {"text": "charlie delta", "shot": "B"}, ] - def test_sound_between_two_lines_is_a_finding_on_the_next_line(self): - # #960: the untranscribed stretch sits outside both lines' spans + def test_sound_between_two_lines_is_a_finding_in_its_shot(self): + # #960: the untranscribed stretch sits outside both lines' spans; + # #964: it is placed in the shot holding most of it (A), not the next line's chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) answer = shot_run(self.TWO, chunks, mono=voiced(6.0)) (found,) = rules(answer, "speech_not_transcribed") - self.assertEqual(found["at"]["line"], 1) - self.assertEqual(found["at"]["shot"], "B") + self.assertEqual(found["at"]["line"], 0) + self.assertEqual(found["at"]["shot"], "A") self.assertAlmostEqual(found["at"]["seconds"], 0.9, places=2) self.assertAlmostEqual(found["at"]["end"], 4.0, places=2) @@ -790,6 +791,9 @@ def test_audible_stretch_before_a_stray_word_is_a_finding(self): self.assertAlmostEqual(before["at"]["seconds"], 0.9, places=2) self.assertAlmostEqual(before["at"]["end"], 4.0, places=2) self.assertAlmostEqual(after["at"]["seconds"], 4.4, places=2) + # each is placed in the shot holding most of it, not the next line's + self.assertEqual((before["at"]["shot"], before["at"]["line"]), ("A", 0)) + self.assertEqual((after["at"]["shot"], after["at"]["line"]), ("B", 1)) class TestSpeechInSilentShot(unittest.TestCase): From 6f7fd3dea99ec691e9a62dd664b77a16bfdb7a0b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 02:58:54 -0500 Subject: [PATCH 039/105] refactor(check-script): #964 - place a between-lines stretch with overlapping_shot, drop the duplicate _majority_shot Co-Authored-By: Claude Opus 5.5 --- dw/tasks/script_check.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 0c908d99..28549939 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -736,7 +736,7 @@ def _between_lines_findings( if hi - lo < UNHEARD_MIN_SECONDS: continue where_line, where_shot = index, shot - holder = _majority_shot(spans, lo, hi) + holder = None if spans is None else overlapping_shot(spans, lo, hi) if holder is not None and holder != shot: where_shot = holder where_line = next( @@ -747,16 +747,6 @@ def _between_lines_findings( return [f for f in found if f is not None] -def _majority_shot(spans, start, end): - """The shot overlapping [start, end] the most, None when no shots are known.""" - best, best_overlap = None, 0.0 - for span in spans or (): - overlap = min(end, span["end"]) - max(start, span["start"]) - if overlap > best_overlap: - best, best_overlap = span["name"], overlap - return best - - def _untranscribed(mono, sample_rate, index, start, end, shot): lo, hi = start + UNHEARD_MARGIN_SECONDS, end - UNHEARD_MARGIN_SECONDS level = word_level(mono, sample_rate, lo, hi) From fda2ebfa83ec0f1cbdf3c2dcb0bb8cdb3ef06de6 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 05:12:19 -0500 Subject: [PATCH 040/105] fix(check-script): #965 - a second copy of a line inside its matched words is unscripted_speech, not absorbed into the line Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 30 ++++++++++++++++++++++++++++++ tests/test_script_check.py | 25 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 28549939..254ab065 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -390,6 +390,35 @@ def align(expected_lines, heard): return assigned, unassigned +def _release_repeats(expected, heard, assigned, unassigned): + """Take a second copy of a line out of the words align gave it. Align + keeps a heard word inserted inside a line with that line, so a doubled + line ("Then wear socks. Wear socks.") would read as one line at a lower + similarity - above the mismatch threshold, so no finding. A run of two or + more extra words (one for a one-word line) that is the line, or a stretch + of it, goes back to `unassigned` so unscripted_speech sees it.""" + for index, words in enumerate(expected): + own = assigned[index] + if len(own) <= len(words) or not words: + continue + heard_words = [heard[j]["word"] for j in own] + matcher = difflib.SequenceMatcher(None, words, heard_words, autojunk=False) + moved = set() + for op, _i1, _i2, j1, j2 in matcher.get_opcodes(): + run = heard_words[j1:j2] + if ( + op == "insert" + and len(run) >= min(2, len(words)) + and (_contains(words, run) or _contains(run, words)) + ): + moved.update(range(j1, j2)) + if not moved: + continue + unassigned.extend((own[k], (index,)) for k in sorted(moved)) + assigned[index] = [j for k, j in enumerate(own) if k not in moved] + unassigned.sort() + + def line_similarity(expected, heard): """How alike two word lists are, 0..1. Two empty lists are identical.""" if not expected and not heard: @@ -509,6 +538,7 @@ def check( expected = [normalize_words(line["text"]) for line in parsed_lines] assigned, unassigned = align(expected, heard) + _release_repeats(expected, heard, assigned, unassigned) mismatch = mismatch_rule(similarity_threshold) by_name = {span["name"]: span for span in spans or ()} diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 05478c1c..e6d73703 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -687,6 +687,31 @@ def test_a_line_spoken_twice_in_its_shot_is_a_finding(self): self.assertEqual(found["at"]["shot"], "A") self.assertTrue(found["at"]["repeats_line"]) + def test_a_doubled_line_inside_a_longer_cut_is_a_finding(self): + # align alone keeps the second copy inside the line (similarity 0.75) + chunks = ( + words("hal kettle froze", 0.0) + + words("then wear socks", 6.3) + + words("wear socks", 9.1) + + words("the kettle is whistling hal", 11.0) + ) + lines = [ + {"text": "Hal, the kettle froze.", "shot": "A"}, + {"text": "Then wear socks.", "shot": "B"}, + {"text": "The kettle is whistling, Hal.", "shot": "C"}, + ] + spans = [ + {"name": "A", "start": 0.0, "end": 5.0}, + {"name": "B", "start": 5.0, "end": 10.3}, + {"name": "C", "start": 10.3, "end": 16.0}, + ] + answer = shot_run(lines, chunks, mono=voiced(16.0), spans=spans) + self.assertEqual(answer["lines"][1]["similarity"], 1.0) + (found,) = rules(answer, "unscripted_speech") + self.assertEqual(found["at"]["shot"], "B") + self.assertTrue(found["at"]["repeats_line"]) + self.assertEqual([w["text"] for w in answer["unmatched"]], ["wear", "socks"]) + def test_an_ad_lib_in_a_scripted_shot_is_not_a_repeat(self): chunks = words("alpha bravo", 0.0) + words("zulu yankee", 1.0) answer = shot_run([{"text": "alpha bravo", "shot": "A"}], chunks) From e18d0a3c5198090d588bf51bb607cd615cb39717 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 20 Oct 2026 06:43:21 -0500 Subject: [PATCH 041/105] fix(check-script): #966 - a run scoring the mismatch threshold against a line in its shot is a repeat of it Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 16 +++++++++++++--- tests/test_script_check.py | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 254ab065..895c768e 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -640,7 +640,9 @@ def check( rules_applied.extend(SHOT_RULES) findings.extend(_silent_shot_findings(spans, parsed_lines, heard, claimed)) findings.extend( - _unscripted_findings(spans, parsed_lines, expected, heard, unassigned) + _unscripted_findings( + spans, parsed_lines, expected, heard, unassigned, similarity_threshold + ) ) else: rules_skipped.extend({"rule": rule, "reason": no_shot} for rule in SHOT_RULES) @@ -848,10 +850,13 @@ def _silent_shot_findings(spans, parsed_lines, heard, claimed): return findings -def _unscripted_findings(spans, parsed_lines, expected, heard, unassigned): +def _unscripted_findings( + spans, parsed_lines, expected, heard, unassigned, similarity_threshold +): """unscripted_speech: heard words align left to no line that sit inside a shot a line names. `repeats_line` is set when the shot's words, in order, - contain one of its lines' words again.""" + contain one of its lines' words again, or score at least the mismatch + threshold against one of them (a repeat misheard as the line was).""" findings = [] for span in spans: lines = [ @@ -874,6 +879,11 @@ def _unscripted_findings(spans, parsed_lines, expected, heard, unassigned): at["repeats_line"] = any( line and _contains(words, line) for line in lines ) or any(line and _contains(line, words) for line in lines) + if not at["repeats_line"]: + at["repeats_line"] = any( + line and line_similarity(line, words) >= similarity_threshold + for line in lines + ) findings.append(finding(UNSCRIPTED_SPEECH, len(inside), at)) return findings diff --git a/tests/test_script_check.py b/tests/test_script_check.py index e6d73703..3d86f9fc 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -712,6 +712,23 @@ def test_a_doubled_line_inside_a_longer_cut_is_a_finding(self): self.assertTrue(found["at"]["repeats_line"]) self.assertEqual([w["text"] for w in answer["unmatched"]], ["wear", "socks"]) + def test_a_misheard_repeat_scoring_as_the_line_does_is_a_repeat(self): + # "hal" heard as "how" both times: 0.75 for the line, so also a repeat + chunks = words("how kettle froze now", 0.0, 0.2, 0.15) + words( + "how kettle froze now", 1.0, 0.2, 0.15 + ) + answer = run( + [{"text": "hal kettle froze now", "shot": "A"}], + chunks, + voiced(6.0), + similarity=0.6, + spans=SHOTS, + shots_source="argument", + ) + self.assertEqual(answer["lines"][0]["similarity"], 0.75) + (found,) = rules(answer, "unscripted_speech") + self.assertTrue(found["at"]["repeats_line"]) + def test_an_ad_lib_in_a_scripted_shot_is_not_a_repeat(self): chunks = words("alpha bravo", 0.0) + words("zulu yankee", 1.0) answer = shot_run([{"text": "alpha bravo", "shot": "A"}], chunks) From a722ab919988c334f904ecc4371792900146b331 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 17:54:29 -0500 Subject: [PATCH 042/105] fix(check-script): #967 - a between-lines speech_not_transcribed counts and places only its audible part Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 31 +++++++++++++++++++++++++++++++ tests/test_script_check.py | 22 ++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 895c768e..bc799053 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -767,6 +767,10 @@ def _between_lines_findings( for lo, hi in stretches: if hi - lo < UNHEARD_MIN_SECONDS: continue + audible = _audible_extent(mono, sample_rate, lo, hi) + if audible is None: + continue + lo, hi = audible where_line, where_shot = index, shot holder = None if spans is None else overlapping_shot(spans, lo, hi) if holder is not None and holder != shot: @@ -779,6 +783,33 @@ def _between_lines_findings( return [f for f in found if f is not None] +def _audible_extent(mono, sample_rate, start, end): + """The span of the stretch [start, end] from its first to its last window + above UNHEARD_FLOOR_DBFS, or None when that is shorter than + UNHEARD_MIN_SECONDS (#967): sub-floor time at either edge is not "audible + sound", and must not size or place the finding. An edge that is itself + audible stays the stretch's own edge.""" + inner_start, inner_end = ( + start + UNHEARD_MARGIN_SECONDS, + end - UNHEARD_MARGIN_SECONDS, + ) + first = last = None + cursor = inner_start + while cursor < inner_end: + stop = min(cursor + GUARD_WINDOW_SECONDS, inner_end) + level = word_level(mono, sample_rate, cursor, stop) + if level is not None and level > UNHEARD_FLOOR_DBFS: + if first is None: + first = cursor + last = stop + cursor = stop + if first is None: + return None + first = start if first <= inner_start else first + last = end if last >= inner_end else last + return (first, last) if last - first >= UNHEARD_MIN_SECONDS else None + + def _untranscribed(mono, sample_rate, index, start, end, shot): lo, hi = start + UNHEARD_MARGIN_SECONDS, end - UNHEARD_MARGIN_SECONDS level = word_level(mono, sample_rate, lo, hi) diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 3d86f9fc..45c1e16a 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -837,6 +837,28 @@ def test_audible_stretch_before_a_stray_word_is_a_finding(self): self.assertEqual((before["at"]["shot"], before["at"]["line"]), ("A", 0)) self.assertEqual((after["at"]["shot"], after["at"]["line"]), ("B", 1)) + def test_only_the_audible_part_of_a_gap_sizes_and_places_the_finding(self): + # #967: the gap 0.9-5.0 is mostly silence in shot B; the audible 0.9-2.2 + # sits in A and is all the finding counts + spans = [ + {"name": "A", "start": 0.0, "end": 2.0}, + {"name": "B", "start": 2.0, "end": 6.0}, + ] + chunks = words("alpha bravo", 0.0) + words("charlie delta", 5.0) + mono = numpy.concatenate([voiced(2.2), silent(2.8), voiced(1.0)]) + answer = shot_run(self.TWO, chunks, mono=mono, spans=spans) + (found,) = rules(answer, "speech_not_transcribed") + self.assertEqual((found["at"]["shot"], found["at"]["line"]), ("A", 0)) + self.assertAlmostEqual(found["at"]["seconds"], 0.9, places=2) + self.assertLess(found["at"]["end"], 2.5) + self.assertLess(found["value"], 1.6) + + def test_under_a_second_of_audible_sound_in_a_long_gap_is_not(self): + chunks = words("alpha bravo", 0.0) + words("charlie delta", 5.0) + mono = numpy.concatenate([voiced(1.5), silent(3.5), voiced(1.0)]) + answer = shot_run(self.TWO, chunks, mono=mono) + self.assertEqual(rules(answer, "speech_not_transcribed"), []) + class TestSpeechInSilentShot(unittest.TestCase): LINES = [{"text": "alpha bravo", "shot": "A"}] From ab163e4d6140efb00684269b9e23164ec9e507a4 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 18:07:07 -0500 Subject: [PATCH 043/105] fix(check-script): #970 #971 - judge the tail after the last word like a between-lines stretch; a chunk repeating the previous span is discarded as repeated_span Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 42 ++++++++++++++++++++++++++--- tests/test_script_check.py | 54 +++++++++++++++++++++++++++++--------- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index bc799053..ba0b0165 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -275,7 +275,9 @@ def guard_words(chunks, mono, sample_rate, script_words=None): `heard` is [{word, text, start, end}], one per normalized word - a chunk that normalizes to nothing ('♪', '...') is neither; `discarded` is [{text, start, end, level_dbfs, reason, count}] for a chunk at or below - the floor ("below_floor") or a repetition loop ("repetition": one entry + the floor ("below_floor"), a chunk repeating the previous one's exact span + ("repeated_span": words cannot share a span, #971) or a repetition loop + ("repetition": one entry per run of consecutive looped chunks, `count` chunks, `text` the first's, `start`/`end` the run's span). @@ -297,19 +299,31 @@ def guard_words(chunks, mono, sample_rate, script_words=None): for index, (chunk, words) in enumerate(spoken): start, end = chunk["start"], chunk["end"] level = word_level(mono, sample_rate, start, end) + before = spoken[index - 1] if index else None + previous_span = ( + None + if before is None + else (before[0]["start"], before[0]["end"], before[1]) + ) first_of_line = ( index in firsts and script_words is not None and set(words) <= script_words and end > start ) - if index in looped and not first_of_line: + if (start, end, words) == previous_span: + reason = "repeated_span" + elif index in looped and not first_of_line: reason = "repetition" elif level is None or level <= GUARD_FLOOR_DBFS: reason = "below_floor" else: reason = None - if reason == "repetition" and run_end == index - 1: + if ( + reason in ("repetition", "repeated_span") + and run_end == index - 1 + and discarded[-1]["reason"] == reason + ): # A loop is one entry: its span, its chunk count, its loudest level last = discarded[-1] last["end"] = _round(max(last["end"], end)) @@ -331,7 +345,7 @@ def guard_words(chunks, mono, sample_rate, script_words=None): "count": 1, } ) - run_end = index if reason == "repetition" else None + run_end = index if reason in ("repetition", "repeated_span") else None continue run_end = None for word in words: @@ -618,6 +632,8 @@ def check( parsed_lines, ) ) + if heard_end is None or end >= heard_end: + last_line, last_shot = index, shot heard_end = max(heard_end or 0.0, end) tails = ([(shot, span["end"])] if span else []) + [(None, duration)] findings.extend( @@ -628,6 +644,24 @@ def check( ) previous_end = end + if heard_end is not None: + # The stretch after the last line heard, to the end of the take: the + # same rule as between two lines, so a shot checked alone agrees with + # the cut (#970) + findings.extend( + _between_lines_findings( + mono, + sample_rate, + last_line, + heard_end, + duration, + last_shot, + strays, + spans, + parsed_lines, + ) + ) + findings.extend(_unassigned_tags(heard, unassigned, parsed_lines, expected, spans)) rules_applied = list(LINE_RULES) diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 45c1e16a..c970c2a2 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -64,7 +64,7 @@ def words(text, start=0.0, step=0.5, length=0.4): def run(lines, chunks, mono=None, similarity=0.85, spans=None, shots_source="none"): if mono is None: end = max((c["end"] for c in chunks), default=1.0) - mono = voiced(end + 1.0) + mono = voiced(end + 0.5) return check( parse_lines(lines), chunks, @@ -586,7 +586,7 @@ def test_transcribes_with_word_timestamps_and_returns_the_answer(self): "text": " hello there", "chunks": words("hello there"), } - waveform = voiced(2.0)[numpy.newaxis, :] + waveform = voiced(1.4)[numpy.newaxis, :] with patch( "dw.tasks.audio_transcription.transcribe_audio", return_value=canned ) as transcribe: @@ -620,8 +620,6 @@ def test_transcribes_with_word_timestamps_and_returns_the_answer(self): def shot_run(lines, chunks, mono=None, spans=SHOTS, source="argument"): - if mono is None: - mono = voiced(6.0) return run(lines, chunks, mono, spans=spans, shots_source=source) @@ -635,7 +633,7 @@ def test_tagged_lines_carry_their_shot(self): answer = shot_run( [{"text": "alpha bravo", "shot": "A"}, {"text": "delta echo", "shot": "B"}], chunks, - mono=numpy.concatenate([voiced(0.9), silent(2.6), voiced(2.5)]), + mono=numpy.concatenate([voiced(0.9), silent(2.6), voiced(1.0)]), ) self.assertEqual([x["shot"] for x in answer["lines"]], ["A", "B"]) self.assertEqual(answer["findings"], []) @@ -746,7 +744,7 @@ class TestSpeechNotTranscribed(unittest.TestCase): def test_sound_in_a_gap_between_a_lines_words_is_a_finding(self): # The ASR dropped an ad-lib between "bravo" and "charlie" chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) - answer = shot_run(self.LINES, chunks, mono=voiced(6.0)) + answer = shot_run(self.LINES, chunks, mono=voiced(5.0)) (found,) = rules(answer, "speech_not_transcribed") self.assertEqual(found["at"]["line"], 0) self.assertEqual(found["at"]["shot"], "A") @@ -760,13 +758,13 @@ def test_a_stretched_word_over_sound_is_a_finding(self): {"start": 3.0, "end": 3.4, "text": " charlie"}, {"start": 3.5, "end": 3.9, "text": " delta"}, ] - answer = shot_run(self.LINES, chunks, mono=voiced(6.0)) + answer = shot_run(self.LINES, chunks, mono=voiced(4.0)) (found,) = rules(answer, "speech_not_transcribed") self.assertEqual(found["at"]["seconds"], 0.5) def test_a_quiet_gap_is_not(self): chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) - mono = numpy.concatenate([voiced(0.9), silent(3.1), voiced(2.0)]) + mono = numpy.concatenate([voiced(0.9), silent(3.1), voiced(1.0)]) answer = shot_run(self.LINES, chunks, mono=mono) self.assertEqual(rules(answer, "speech_not_transcribed"), []) @@ -784,7 +782,7 @@ def test_sound_between_two_lines_is_a_finding_in_its_shot(self): # #960: the untranscribed stretch sits outside both lines' spans; # #964: it is placed in the shot holding most of it (A), not the next line's chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) - answer = shot_run(self.TWO, chunks, mono=voiced(6.0)) + answer = shot_run(self.TWO, chunks, mono=voiced(5.0)) (found,) = rules(answer, "speech_not_transcribed") self.assertEqual(found["at"]["line"], 0) self.assertEqual(found["at"]["shot"], "A") @@ -793,7 +791,7 @@ def test_sound_between_two_lines_is_a_finding_in_its_shot(self): def test_a_quiet_gap_between_lines_is_not(self): chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) - mono = numpy.concatenate([voiced(0.9), silent(3.1), voiced(2.0)]) + mono = numpy.concatenate([voiced(0.9), silent(3.1), voiced(1.0)]) answer = shot_run(self.TWO, chunks, mono=mono) self.assertEqual(rules(answer, "speech_not_transcribed"), []) @@ -808,7 +806,7 @@ def test_a_gap_holding_unmatched_words_is_unscripted_speech_only(self): + words("zulu yankee", 1.5) + words("charlie delta", 4.0) ) - answer = shot_run(self.TWO, chunks, mono=voiced(6.0)) + answer = shot_run(self.TWO, chunks, mono=voiced(5.0)) # Stray words 1.5-2.4; the 2.4-4.0 stretch after them is audible (found,) = rules(answer, "speech_not_transcribed") self.assertAlmostEqual(found["at"]["seconds"], 2.4, places=2) @@ -819,7 +817,7 @@ def test_a_gap_fully_covered_by_unmatched_words_is_unscripted_only(self): + words("zulu yankee xray whiskey", 0.9, step=0.5, length=0.5) + words("charlie delta", 3.0) ) - answer = shot_run(self.TWO, chunks, mono=voiced(5.0)) + answer = shot_run(self.TWO, chunks, mono=voiced(4.0)) self.assertEqual(rules(answer, "speech_not_transcribed"), []) def test_audible_stretch_before_a_stray_word_is_a_finding(self): @@ -827,7 +825,7 @@ def test_audible_stretch_before_a_stray_word_is_a_finding(self): chunks = ( words("alpha bravo", 0.0) + words("zulu", 4.0) + words("charlie delta", 6.0) ) - answer = shot_run(self.TWO, chunks, mono=voiced(8.0)) + answer = shot_run(self.TWO, chunks, mono=voiced(7.0)) # the stretches on both sides of the stray word are findings before, after = rules(answer, "speech_not_transcribed") self.assertAlmostEqual(before["at"]["seconds"], 0.9, places=2) @@ -860,6 +858,36 @@ def test_under_a_second_of_audible_sound_in_a_long_gap_is_not(self): self.assertEqual(rules(answer, "speech_not_transcribed"), []) +class TestTailAndRepeatedSpan(unittest.TestCase): + LINES = [{"text": "alpha bravo", "shot": "A"}] + + def test_audible_tail_after_the_last_word_is_a_finding(self): + # #970: ~1.2 s of sound past the last word, then silence + chunks = words("alpha bravo", 0.0) + mono = numpy.concatenate([voiced(2.2), silent(2.8)]) + answer = run(self.LINES, chunks, mono) + (found,) = rules(answer, "speech_not_transcribed") + self.assertEqual(found["at"]["line"], 0) + self.assertLess(found["value"], 1.6) + + def test_quiet_tail_after_the_last_word_is_not(self): + chunks = words("alpha bravo", 0.0) + mono = numpy.concatenate([voiced(1.0), silent(4.0)]) + answer = run(self.LINES, chunks, mono) + self.assertEqual(rules(answer, "speech_not_transcribed"), []) + + def test_chunks_sharing_one_span_are_heard_once(self): + # #971: three words cannot share one span + chunks = words("alpha bravo", 0.0) + [ + {"start": 3.0, "end": 3.6, "text": " How?"} for _ in range(3) + ] + mono = voiced(4.0) + answer = run(self.LINES, chunks, mono) + self.assertEqual(len(answer["unmatched"]), 1) + (dup,) = [d for d in answer["discarded"] if d["reason"] == "repeated_span"] + self.assertEqual(dup["count"], 2) + + class TestSpeechInSilentShot(unittest.TestCase): LINES = [{"text": "alpha bravo", "shot": "A"}] From 2c50c2083f2dd13dc45d856e00724ca5c817c82c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 18:07:30 -0500 Subject: [PATCH 044/105] docs(check-script): #971 - repeated_span discard reason Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 099299e1..bca0b06f 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3088,7 +3088,7 @@ The thresholds are module constants in `dw/tasks/script_check.py`, deliberately | `REPEAT_RUN_MIN` | 6 | Heard words repeating the same unit this many times over, back to back, are a repetition loop: every chunk holding one is discarded (`reason: "repetition"`), however loud | | `REPEAT_MAX_PERIOD` | 4 words | The longest repeating unit a loop is looked for in - `pre pre pre` has period 1, `thank you thank you` period 2 | -The result: `findings`, `lines[]` (`expected`, `heard`, `similarity`, `start`, `end`, `shot`), `discarded[]` (`text`, `start`, `end`, `level_dbfs`, `reason` - `below_floor` or `repetition`, `count` - a repetition loop is one entry spanning its chunks), `unmatched[]` (heard words aligned to no line), `transcript`, `model_name`, `rules_applied`, `rules_skipped[]` (`rule`, `reason`) and `thresholds`. A line's `shot` is the shot it names, else the shot its heard words overlap most (null when none is known). `shots` is the map as placed - `[{name, start, end}]` in seconds on the take, or null when none is known - and `shots_source` says where it came from: `argument`, `artifact`, `manifest` or `none`. +The result: `findings`, `lines[]` (`expected`, `heard`, `similarity`, `start`, `end`, `shot`), `discarded[]` (`text`, `start`, `end`, `level_dbfs`, `reason` - `below_floor`, `repeated_span` (a chunk repeating the previous one's exact span) or `repetition`, `count` - a repetition loop is one entry spanning its chunks), `unmatched[]` (heard words aligned to no line), `transcript`, `model_name`, `rules_applied`, `rules_skipped[]` (`rule`, `reason`) and `thresholds`. A line's `shot` is the shot it names, else the shot its heard words overlap most (null when none is known). `shots` is the map as placed - `[{name, start, end}]` in seconds on the take, or null when none is known - and `shots_source` says where it came from: `argument`, `artifact`, `manifest` or `none`. How the alignment reads a take: From 2bbbc8506943735fb393e73155dd62cfcabed611 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 18:16:14 -0500 Subject: [PATCH 045/105] fix(admission): #972 - an unknown argument is reported as itself, not as a given argument being null Co-Authored-By: Claude Sonnet 5.5 --- dw/server/admission.py | 11 +++++++++++ tests/test_admission.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/dw/server/admission.py b/dw/server/admission.py index a8b1279a..5d23d377 100644 --- a/dw/server/admission.py +++ b/dw/server/admission.py @@ -189,6 +189,17 @@ def admit( capacity_label=capacity_label, ) admission.errors = candidate.validation_errors(context=context) + if admission.errors and supplied: + # A bad argument leaves every argument unfolded, so the + # document check reads each null-defaulted variable as null + # even where the caller supplied it (#972) - a finding about + # the arguments, not the document. Say the argument error + # instead of sending the caller to fix a correct one + bad = argument_errors(candidate.workflow_definition, arguments) + if bad: + admission.errors = [ + e for e in admission.errors if "variable" not in e + ] + bad except Exception as e: raise _validator_failure(e) from e admission.vram_need = _vram_need(candidate, context, arguments) diff --git a/tests/test_admission.py b/tests/test_admission.py index becc7387..b6c2e5a6 100644 --- a/tests/test_admission.py +++ b/tests/test_admission.py @@ -450,6 +450,42 @@ def admitted(arguments): assert not admitted({}).ok +def test_an_unknown_argument_does_not_make_a_given_one_read_as_null(root): + """A bad argument leaves the others unfolded, so the document check read + `audio` as null though the caller passed it (#972). The refusal names the + unknown argument instead.""" + definition = { + "id": "se-972", + "variables": {"audio": None}, + "steps": [ + { + "name": "n", + "task": { + "command": "normalize_audio", + "arguments": {"audio": "variable:audio"}, + }, + "result": {"content_type": "audio/wav"}, + } + ], + } + admission = admit( + workflow_path=None, + workflow=definition, + arguments={"audio": "asset:x.wav", "bogus": 1}, + base_dir=None, + workspace=root, + ceiling_index={}, + output_dir=root.outputs, + workflow_dir=root.workflows, + asset_library=library_path("assets", root), + prompt_library=library_path("prompts", root), + ) + assert not admission.ok + messages = " ".join(e["message"] for e in admission.errors) + assert "bogus" in messages + assert "which is null" not in messages + + def _a_crashing_select_check(monkeypatch): def boom(_context): raise RuntimeError("check exploded") From 273c00f1dd6e1ea9d1a8dc7f432ece3d10fdb7b1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 18:28:58 -0500 Subject: [PATCH 046/105] fix(validation): #972 - the null-variable rule lives in _task_errors; a supplied variable beside a bad argument is not read as null Co-Authored-By: Claude Sonnet 5.5 --- dw/server/admission.py | 11 ----------- dw/validation.py | 9 +++++++++ tests/test_admission.py | 26 +++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/dw/server/admission.py b/dw/server/admission.py index 5d23d377..a8b1279a 100644 --- a/dw/server/admission.py +++ b/dw/server/admission.py @@ -189,17 +189,6 @@ def admit( capacity_label=capacity_label, ) admission.errors = candidate.validation_errors(context=context) - if admission.errors and supplied: - # A bad argument leaves every argument unfolded, so the - # document check reads each null-defaulted variable as null - # even where the caller supplied it (#972) - a finding about - # the arguments, not the document. Say the argument error - # instead of sending the caller to fix a correct one - bad = argument_errors(candidate.workflow_definition, arguments) - if bad: - admission.errors = [ - e for e in admission.errors if "variable" not in e - ] + bad except Exception as e: raise _validator_failure(e) from e admission.vram_need = _vram_need(candidate, context, arguments) diff --git a/dw/validation.py b/dw/validation.py index ec31729a..bf081258 100644 --- a/dw/validation.py +++ b/dw/validation.py @@ -302,6 +302,15 @@ def _task_errors(context): ) if context.arguments is None: errors = [e for e in errors if "variable" not in e] + elif argument_errors(context.definition, context.arguments): + # A bad argument leaves every argument unfolded, so a variable the + # caller did supply still reads as null here (#972). The argument + # error is the one to say; drop the finding for a supplied name + errors = [ + e + for e in errors + if "variable" not in e or e["variable"] not in context.supplied + ] return errors diff --git a/tests/test_admission.py b/tests/test_admission.py index b6c2e5a6..08660869 100644 --- a/tests/test_admission.py +++ b/tests/test_admission.py @@ -21,7 +21,7 @@ from dw.server.app import create_app from dw.server.jobs import JobManager from dw.server.job_record import TERMINAL_STATES -from dw.workflow import Workflow +from dw.workflow import Workflow, workflow_from_definition from dw.workspace import Workspace from .test_prepare_pipeline import DEFINITION as prepare_definition @@ -486,6 +486,30 @@ def test_an_unknown_argument_does_not_make_a_given_one_read_as_null(root): assert "which is null" not in messages +def test_validation_errors_do_not_read_a_supplied_variable_as_null_beside_a_bad_argument(): + """The owner of the rule is validation_errors itself (#972): any caller, + not only admit(), gets no null finding for a name it supplied.""" + definition = { + "id": "ve-972", + "variables": {"audio": None}, + "steps": [ + { + "name": "n", + "task": { + "command": "normalize_audio", + "arguments": {"audio": "variable:audio"}, + }, + "result": {"content_type": "audio/wav"}, + } + ], + } + workflow = workflow_from_definition(definition, "/tmp", None, None) + bad = workflow.validation_errors(arguments={"audio": "a.wav", "bogus": 1}) + assert not any("variable" in e for e in bad) + left_null = workflow.validation_errors(arguments={"bogus": 1}) + assert any(e.get("variable") == "audio" for e in left_null) + + def _a_crashing_select_check(monkeypatch): def boom(_context): raise RuntimeError("check exploded") From 89e75acff103b6575f026789d55975f5b28b1c15 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 18:34:24 -0500 Subject: [PATCH 047/105] feat(tasks): #973 - remove_object, a prompt-free LaMa removal task Co-Authored-By: Claude Opus 5.5 --- dw/locations.py | 2 +- dw/tasks/model_handlers.py | 59 +++++++++++- dw/tasks/remove_object.py | 188 +++++++++++++++++++++++++++++++++++++ 3 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 dw/tasks/remove_object.py diff --git a/dw/locations.py b/dw/locations.py index 8062276b..48b04492 100644 --- a/dw/locations.py +++ b/dw/locations.py @@ -566,7 +566,7 @@ def validate_model_name(name, base_dir=None): # named bare (no subfolder: the task knows its default repo's layout). A # pickle format is refused by name rather than left to fail in the loader: # the refusal is the documented contract, not an accident of which loader runs -SAFETENSORS_ONLY_COMMANDS = ("upscale_h3_latents",) +SAFETENSORS_ONLY_COMMANDS = ("upscale_h3_latents", "remove_object") SAFETENSORS_SUFFIX = ".safetensors" diff --git a/dw/tasks/model_handlers.py b/dw/tasks/model_handlers.py index 4a73a9c0..da1e9f9f 100644 --- a/dw/tasks/model_handlers.py +++ b/dw/tasks/model_handlers.py @@ -17,7 +17,7 @@ from .image_ops import per_frame from .registry import register_command -from ..task_domains import POSITIVE, UNIT +from ..task_domains import NON_NEGATIVE, POSITIVE, UNIT from ..task_problems import script_lines_errors logger = logging.getLogger("dw") @@ -149,6 +149,63 @@ def _handle_segment(task, arguments, previous_pipelines): ) +@register_command( + "remove_object", + implementation="dw.tasks.remove_object.remove_object", + consumes_device=True, + summary=( + "Erase what a mask covers and fill the hole from the surrounding " + "texture, with no prompt (LaMa). Suits small to medium objects; an " + "image or video frames, one still mask applied to every frame." + ), + parameter_descriptions={ + "image": ( + "The image to clean, or video frames (e.g. from video_frames): the " + "one mask applies to every frame, and fps and audio are kept." + ), + "mask_image": ( + "White where to erase, black where to keep - segment's polarity, " + "so its mask chains with no invert. Binarized at 128 and resized " + "to the image if it differs. An all-black mask returns the image " + "unchanged, with a warning." + ), + "dilate": ( + "Pixels to grow the mask by before the fill, a whole number 0 or " + "above (default 8). LaMa fills cleanly only past the object's " + "edge; raise it for a soft or tight mask." + ), + "model_name": ( + "Hugging Face repo id holding the LaMa weights, never a path " + "(default michaelgold/big-lama, read at a pinned revision)." + ), + "weight_name": ( + "The safetensors file within model_name, a bare file name " + "(default big-lama.safetensors); a .pt/.ckpt is refused." + ), + }, + domains={"dilate": NON_NEGATIVE}, + whole_numbers=("dilate",), +) +def _handle_remove_object(task, arguments, previous_pipelines): + """Erase a masked region and fill it from the surroundings""" + logger.debug("Removing object") + image = arguments.pop("image") + mask_image = arguments.pop("mask_image") + from ..events import emit_warning + from .remove_object import EMPTY_MASK_WARNING, mask_is_empty, remove_object + + if mask_is_empty(mask_image): + # Checked once here, so a video warns once rather than per frame + emit_warning(EMPTY_MASK_WARNING) + return per_frame(image, lambda frame: frame) + + device = task.device_for(arguments) + return per_frame( + image, + lambda frame: remove_object(frame, mask_image, device=device, **arguments), + ) + + @register_command( "interpolate_frames", implementation="dw.tasks.interpolate_frames.interpolate_frames", diff --git a/dw/tasks/remove_object.py b/dw/tasks/remove_object.py new file mode 100644 index 00000000..c201c5bb --- /dev/null +++ b/dw/tasks/remove_object.py @@ -0,0 +1,188 @@ +""" +Prompt-free object removal with LaMa (#956). + +`remove_object` erases what a mask covers and fills the hole from the +surrounding texture - no prompt, no diffusion. The network is big-lama, loaded +through spandrel as a `MaskedImageModelDescriptor` from a safetensors state +dict at a pinned Hub revision. Only the masked region comes from the model: +everything outside the dilated mask is the input's own pixels. + +LaMa runs in fp32 on every device. It is ~51M parameters, so a lower +precision saves next to nothing, and spandrel marks it `supports_half=False`. +""" + +import logging + +import torch +from PIL import Image, ImageFilter + +from ..events import emit_warning +from ..task_domains import check_argument, whole_number +from .tensor_image import float_tensor_to_pil, pil_to_float_tensor + +logger = logging.getLogger("dw") + +DEFAULT_REMOVER_REPO = "michaelgold/big-lama" +DEFAULT_REMOVER_WEIGHTS = "big-lama.safetensors" +# The commit of the default repo the weights are read at. biali/big-lama holds +# a byte-identical big-lama.safetensors (same LFS hash): if michaelgold's repo +# disappears, point DEFAULT_REMOVER_REPO and this pin at it +REMOVER_REVISION = "98ab7cface89a594b381f32fc7a9f2cae3f91ccc" + +DEFAULT_DILATE = 8 +# A mask pixel at or above this reads as fill +MASK_THRESHOLD = 128 +EMPTY_MASK_WARNING = ( + "remove_object: the mask is all black, so there is nothing to remove; " + "the image is returned unchanged." +) + + +def binarize_mask(mask_image): + """The mask as an L image of 0 (keep) and 255 (fill), thresholded at 128.""" + if not isinstance(mask_image, Image.Image): + raise ValueError( + f"remove_object needs 'mask_image' as an image, got " + f"{type(mask_image).__name__}" + ) + return mask_image.convert("L").point(lambda p: 255 if p >= MASK_THRESHOLD else 0) + + +def mask_is_empty(mask_image): + """True when no pixel of the mask reads as fill - nothing to remove.""" + return binarize_mask(mask_image).getbbox() is None + + +def prepare_mask(mask_image, size, dilate=DEFAULT_DILATE): + """The fill mask for an image of `size`: binarized at 128, resized + (nearest) to the image if it differs, then grown by `dilate` pixels. + + LaMa fills cleanly only when the mask overshoots the object's edge, and a + `segment` mask is tight to it, hence the dilation. + """ + mask = binarize_mask(mask_image) + if mask.size != tuple(size): + mask = mask.resize(tuple(size), Image.NEAREST) + if dilate: + mask = mask.filter(ImageFilter.MaxFilter(2 * dilate + 1)) + return mask + + +def _check_remover_source(model_name, weight_name): + """Refuse a remover source that is not a Hub repo and a safetensors file + in it - the run-time half of the SAFETENSORS_ONLY_COMMANDS gate, for a + value that arrives through a variable or a run that skipped validation.""" + from ..locations import ( + SAFETENSORS_SUFFIX, + validate_hub_repo_id, + validate_weight_name, + ) + + validate_hub_repo_id(model_name) + validate_weight_name(weight_name, (SAFETENSORS_SUFFIX,), subfolders=False) + + +def _remover_revision(model_name, weight_name): + # The pin names a commit of the default file; another repo is read at its main + if (model_name, weight_name) == (DEFAULT_REMOVER_REPO, DEFAULT_REMOVER_WEIGHTS): + return REMOVER_REVISION + return None + + +def _descriptor_from_state_dict(state_dict): + """The spandrel descriptor for a state dict, refused unless it is a masked + (inpainting) model.""" + from spandrel import MaskedImageModelDescriptor, ModelLoader + + descriptor = ModelLoader().load_from_state_dict(state_dict) + if not isinstance(descriptor, MaskedImageModelDescriptor): + raise ValueError( + f"remove_object needs an inpainting model such as LaMa; these " + f"weights load as {descriptor.architecture.name} " + f"({type(descriptor).__name__}), not a masked image model." + ) + return descriptor + + +def _load_remover(device, model_name, weight_name): + from huggingface_hub import hf_hub_download + + from .model_cache import cached_model + + revision = _remover_revision(model_name, weight_name) + + def load_descriptor(): + from safetensors.torch import load_file + + path = hf_hub_download( + repo_id=model_name, filename=weight_name, revision=revision + ) + logger.info(f"Loading object remover {model_name}/{weight_name} to {device}") + descriptor = _descriptor_from_state_dict(load_file(path)) + descriptor.to(torch.device(device)) + descriptor.float() + descriptor.eval() + return descriptor + + return cached_model( + ("remove_object", model_name, weight_name, revision, str(device)), + load_descriptor, + ) + + +def remove_object( + image, + mask_image, + dilate=DEFAULT_DILATE, + device="cpu", + model_name=DEFAULT_REMOVER_REPO, + weight_name=DEFAULT_REMOVER_WEIGHTS, +): + """Erase what a mask covers and fill it from the surroundings, with no prompt. + + Args: + image: The image (or, through the command, video frames) to clean + mask_image: White where to erase, black where to keep - the polarity + `segment` returns, so its mask chains with no `invert`. Converted + to grayscale, binarized at 128 and resized to the image if it + differs + dilate: Pixels to grow the mask by before the fill, a whole number 0 + or above; LaMa fills cleanly only past the object's edge + device: Device the network runs on + model_name: Hugging Face repo holding the LaMa weights (default: + michaelgold/big-lama, read at a pinned revision) + weight_name: The safetensors file within it, a bare file name + (default: big-lama.safetensors) + + Returns: + PIL Image the size of the input; pixels outside the dilated mask are + the input's own + """ + dilate = whole_number(dilate, "dilate", "remove_object") + dilate = DEFAULT_DILATE if dilate is None else dilate + check_argument("remove_object", "dilate", dilate) + model_name = model_name or DEFAULT_REMOVER_REPO + weight_name = weight_name or DEFAULT_REMOVER_WEIGHTS + _check_remover_source(model_name, weight_name) + + if mask_is_empty(mask_image): + emit_warning(EMPTY_MASK_WARNING) + return image + + source = image.convert("RGB") + mask = prepare_mask(mask_image, source.size, dilate) + descriptor = _load_remover(device, model_name, weight_name) + + image_tensor = pil_to_float_tensor(source, device, dtype=torch.float32) + mask_tensor = ( + pil_to_float_tensor(mask, device, dtype=torch.float32)[:, :1] + .gt(0.5) + .to(torch.float32) + ) + with torch.inference_mode(): + output = descriptor(image_tensor, mask_tensor) + filled = float_tensor_to_pil(output) + + logger.info(f"Removed the masked region of a {source.width}x{source.height} image") + # Only the masked region comes from the model + return Image.composite(filled, source, mask) From 8bc5b8841e54e29c25be79026769915670ca14cc Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 18:40:23 -0500 Subject: [PATCH 048/105] feat(tasks): #973 - remove_object tests and TASKS.md section Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 44 ++++- tests/test_remove_object.py | 342 ++++++++++++++++++++++++++++++++++++ 2 files changed, 384 insertions(+), 2 deletions(-) create mode 100644 tests/test_remove_object.py diff --git a/docs/TASKS.md b/docs/TASKS.md index bca0b06f..0206dc54 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -2529,8 +2529,8 @@ Pass through arguments directly. Useful for organizing data flow. ## Videos in image tasks -Every image command - the upscalers, face restoration, segmentation and the -image processors - takes a video where it takes an image: an `AudioVideo` from +Every image command - the upscalers, face restoration, segmentation, object +removal and the image processors - takes a video where it takes an image: an `AudioVideo` from a generation, `concat_videos` or `dissolve_videos` step, or a frame array from `video_frames`. The command runs over the frames one at a time and returns one video artifact, its soundtrack carried through untouched, so a generated clip @@ -2736,6 +2736,46 @@ Returns a grayscale PIL Image (mode "L") — white (255) for detected objects, b - [segment.json](../workflows/templates/segment.json) — Segment an object from an image - [segment-and-inpaint.json](../workflows/templates/segment-and-inpaint.json) — Segment, then inpaint the masked region +## Object Removal + +Erase what a mask covers and fill the hole from the surrounding texture, with no prompt (#956). `remove_object` runs [LaMa](https://github.com/advimman/lama) (big-lama) through spandrel: no diffusion, so a warm run takes well under a second on a GPU, and it runs on CUDA, MPS and CPU. + +```json +{ + "task": { + "command": "remove_object", + "arguments": { + "image": "asset:photo.png", + "mask_image": "previous_result:mask", + "dilate": 8 + } + }, + "result": { "content_type": "image/png" } +} +``` + +| Argument | Required | Description | +| -------- | -------- | ----------- | +| `image` | Yes | PIL Image or reference - a video runs frame by frame, see [Videos in image tasks](#videos-in-image-tasks) | +| `mask_image` | Yes | White where to erase, black where to keep | +| `dilate` | No | Pixels to grow the mask by before the fill, a whole number 0 or above (default: 8) | +| `model_name` | No | Hugging Face repo id of the weights, never a local path (default: `michaelgold/big-lama`, at a pinned revision) | +| `weight_name` | No | The `.safetensors` file in that repo (default: `big-lama.safetensors`) | + +**The mask.** White means fill, the polarity [`segment`](#object-segmentation) returns, so `segment` → `remove_object` chains with no `invert`. The mask is converted to grayscale and binarized at 128 (at or above is fill). A mask of a different size from the image is resized to it (nearest). An all-black mask, as `segment` returns when it finds nothing, gives the image back unchanged with a warning in the job's events, and loads no model. + +**`dilate`.** LaMa fills cleanly only when the mask overshoots the object's edge, and a `segment` mask is tight to it. The default 8 px suits a segmented object. Raise it (16 to 24) for a soft edge, a shadow or a glow around the object. Use 0 only for a hand-drawn mask that already has a margin. + +**The output** is the input's size. Only the region inside the dilated mask comes from the model: every pixel outside it is the input's own, unchanged. + +**Video.** Frames (e.g. from `video_frames`) are cleaned one at a time with the same still mask, and come back at the same count and fps, with the audio kept. There is no temporal model, so a fill can flicker from frame to frame, and the mask cannot follow a moving object. + +**Limits.** LaMa suits small to medium objects: a cup, a sign, a watermark, a passer-by in the distance. A large region, such as a person filling a third of the frame, comes back smeared. For that, a prompted inpainter whose prompt names the background is the remaining route, and it may paint a patch that does not match (#946). + +**Weights.** `michaelgold/big-lama`, file `big-lama.safetensors` (about 200 MB), read at revision `98ab7cface89a594b381f32fc7a9f2cae3f91ccc`. `biali/big-lama` holds a byte-identical copy and is the fallback if that repo disappears; it is not tried automatically. LaMa is by advimman (Suvorov et al., *Resolution-robust Large Mask Inpainting with Fourier Convolutions*) and is licensed **Apache-2.0**. The Hub repo's card says MIT, but the upstream licence is the one that applies. The task reads safetensors only: `validate_workflow` refuses a `weight_name` that is not `.safetensors` (the common `big-lama.pt` is a TorchScript file) and a `model_name` that is a local path, and the loader refuses both again at run time. Weights that do not load as an inpainting model are refused too. The network runs in fp32 on every device. + +**First run.** The weights download on the first run, about 200 MB. `validate_workflow`'s `plan.downloads_required` does not list task weights yet, so nothing warns of the download beforehand. + ## Image Captioning Generate text captions from images using a vision-language model. diff --git a/tests/test_remove_object.py b/tests/test_remove_object.py new file mode 100644 index 00000000..1d0485bf --- /dev/null +++ b/tests/test_remove_object.py @@ -0,0 +1,342 @@ +"""remove_object (#956): prompt-free LaMa removal. + +The mask is binarized, resized and dilated here, an all-black mask returns the +input without loading a model, and only the masked region comes from the +network. The weights are a Hub repo and a safetensors file, refused otherwise +at validation and again in the loader, and they must load as a masked image +model. No test downloads the real weights: a stub descriptor stands in for +LaMa, and the wrong-architecture case uses a tiny real spandrel network.""" + +from unittest import mock + +import numpy as np +import pytest +import torch +from PIL import Image, ImageDraw + +from dw.locations import location_errors +from dw.security import InvalidInputError, PathTraversalError +from dw.task_domains import task_argument_errors +from dw.tasks import model_cache, remove_object as remover +from dw.tasks.remove_object import ( + DEFAULT_REMOVER_REPO, + DEFAULT_REMOVER_WEIGHTS, + EMPTY_MASK_WARNING, + REMOVER_REVISION, + prepare_mask, + remove_object, +) +from dw.tasks.task import Task +from dw.media_types import AudioVideo + +PICKLES = ["big-lama.pt", "big-lama.ckpt", "big-lama.bin", "big-lama.pth"] +LOCAL_PATHS = ["/tmp/lama", "./weights", "../lama", "~/lama"] + + +def photo(size=(64, 48)): + """A two-tone image with a red square in the middle - the 'object'.""" + image = Image.new("RGB", size, (40, 120, 200)) + draw = ImageDraw.Draw(image) + draw.rectangle((0, size[1] * 3 // 4, size[0], size[1]), fill=(200, 180, 90)) + w, h = size + draw.rectangle((w // 2 - 4, h // 2 - 4, w // 2 + 4, h // 2 + 4), fill=(255, 0, 0)) + return image + + +def square_mask(size=(64, 48), box=None, value=255): + w, h = size + mask = Image.new("L", size, 0) + ImageDraw.Draw(mask).rectangle( + box or (w // 2 - 5, h // 2 - 5, w // 2 + 5, h // 2 + 5), fill=value + ) + return mask + + +class StubDescriptor: + """Stands in for spandrel's LaMa descriptor: records what it is handed and + returns solid green, so any changed pixel is visibly the model's.""" + + def __init__(self): + self.calls = [] + + def __call__(self, image, mask): + self.calls.append((image, mask)) + out = torch.zeros_like(image) + out[:, 1] = 1.0 + return out + + +@pytest.fixture +def stub(): + descriptor = StubDescriptor() + with mock.patch.object(remover, "_load_remover", return_value=descriptor) as load: + descriptor.load = load + yield descriptor + + +def step(arguments, command="remove_object"): + return { + "name": "erase", + "task": { + "command": command, + "arguments": { + "image": "previous_result:photo", + "mask_image": "previous_result:mask", + **arguments, + }, + }, + } + + +class TestTheMask: + def test_binarized_at_128(self): + mask = Image.new("L", (4, 1)) + mask.putdata([0, 127, 128, 255]) + assert list(prepare_mask(mask, (4, 1), dilate=0).get_flattened_data()) == [ + 0, + 0, + 255, + 255, + ] + + def test_an_rgb_mask_is_read_as_grayscale(self): + mask = Image.new("RGB", (4, 4), (255, 255, 255)) + assert prepare_mask(mask, (4, 4), dilate=0).mode == "L" + assert set(prepare_mask(mask, (4, 4), dilate=0).get_flattened_data()) == {255} + + def test_resized_to_the_image_nearest(self): + mask = square_mask((32, 24), box=(16, 0, 31, 23)) + out = prepare_mask(mask, (64, 48), dilate=0) + assert out.size == (64, 48) + assert set(out.get_flattened_data()) <= {0, 255} + arr = np.array(out) + assert arr[:, :32].max() == 0 and arr[:, 32:].min() == 255 + + def test_dilated_by_n_pixels(self): + mask = Image.new("L", (21, 21), 0) + mask.putpixel((10, 10), 255) + arr = np.array(prepare_mask(mask, (21, 21), dilate=3)) > 0 + ys, xs = np.nonzero(arr) + assert (xs.min(), xs.max(), ys.min(), ys.max()) == (7, 13, 7, 13) + + def test_dilate_zero_leaves_it_alone(self): + mask = square_mask() + assert np.array_equal( + np.array(prepare_mask(mask, mask.size, dilate=0)), np.array(mask) + ) + + +class TestAllBlack: + def test_returns_the_input_with_no_model_load(self): + image = photo() + with ( + mock.patch.object(remover, "_load_remover") as load, + mock.patch.object(remover, "emit_warning") as warn, + ): + assert remove_object(image, Image.new("L", image.size, 0)) is image + load.assert_not_called() + warn.assert_called_once_with(EMPTY_MASK_WARNING) + + def test_a_mask_below_the_threshold_is_all_black(self): + image = photo() + with mock.patch.object(remover, "_load_remover") as load: + assert remove_object(image, square_mask(value=127)) is image + load.assert_not_called() + + def test_the_command_warns_once_for_a_video(self): + frames = AudioVideo([photo() for _ in range(3)], None, None, fps=12) + task = Task({"command": "remove_object", "arguments": {}}, "cpu") + with ( + mock.patch.object(remover, "_load_remover") as load, + mock.patch("dw.events.emit_warning") as warn, + ): + out = task.run({"image": frames, "mask_image": Image.new("L", (64, 48))}) + load.assert_not_called() + warn.assert_called_once_with(EMPTY_MASK_WARNING) + assert len(out.frames) == 3 and out.fps == 12 + + +class TestTheFill: + def test_outside_the_dilated_mask_is_unchanged(self, stub): + image = photo() + mask = square_mask() + out = remove_object(image, mask, dilate=2) + assert out.size == image.size + grown = np.array(prepare_mask(mask, image.size, dilate=2)) > 0 + before, after = np.array(image), np.array(out) + assert np.array_equal(before[~grown], after[~grown]) + # and inside it is the model's + assert (after[grown] == [0, 255, 0]).all() + + def test_a_mismatched_mask_still_gives_the_images_size(self, stub): + image = photo((64, 48)) + out = remove_object(image, square_mask((128, 96))) + assert out.size == (64, 48) + image_tensor, mask_tensor = stub.calls[0] + assert mask_tensor.shape == (1, 1, 48, 64) + assert image_tensor.shape == (1, 3, 48, 64) + + def test_the_mask_sent_is_zero_or_one(self, stub): + remove_object(photo(), square_mask()) + _, mask_tensor = stub.calls[0] + assert set(mask_tensor.unique().tolist()) == {0.0, 1.0} + + @pytest.mark.parametrize("device", ["cpu", "cuda", "mps"]) + def test_tensors_sent_are_fp32_on_every_device(self, stub, device): + moved = [] + + def fake_to_tensor(image, target, dtype=None): + moved.append((target, dtype)) + arr = np.array(image.convert("RGB")).astype(np.float32) / 255.0 + return torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0).to(dtype) + + with mock.patch.object(remover, "pil_to_float_tensor", fake_to_tensor): + remove_object(photo(), square_mask(), device=device) + assert moved == [(device, torch.float32), (device, torch.float32)] + image_tensor, mask_tensor = stub.calls[0] + assert image_tensor.dtype == torch.float32 + assert mask_tensor.dtype == torch.float32 + assert stub.load.call_args.args[0] == device + + def test_the_loaded_descriptor_is_made_fp32(self): + descriptor = mock.MagicMock() + model_cache.clear_model_cache() + with ( + mock.patch("huggingface_hub.hf_hub_download", return_value="w"), + mock.patch("safetensors.torch.load_file", return_value={}), + mock.patch.object( + remover, "_descriptor_from_state_dict", return_value=descriptor + ), + ): + remover._load_remover("cpu", DEFAULT_REMOVER_REPO, DEFAULT_REMOVER_WEIGHTS) + model_cache.clear_model_cache() + descriptor.float.assert_called_once() + descriptor.half.assert_not_called() + + def test_video_frames_keep_count_and_fps(self, stub): + frames = AudioVideo([photo() for _ in range(4)], None, None, fps=8) + task = Task({"command": "remove_object", "arguments": {}}, "cpu") + out = task.run({"image": frames, "mask_image": square_mask()}) + assert len(out.frames) == 4 and out.fps == 8 + assert len(stub.calls) == 4 + + def test_the_command_takes_the_tasks_device(self, stub): + task = Task({"command": "remove_object", "arguments": {}}, "mps") + task.run({"image": photo(), "mask_image": square_mask()}) + assert stub.load.call_args.args[0] == "mps" + + +class TestTheWeights: + def test_the_pin_is_for_the_default_file_alone(self): + assert ( + remover._remover_revision(DEFAULT_REMOVER_REPO, DEFAULT_REMOVER_WEIGHTS) + == REMOVER_REVISION + ) + assert ( + remover._remover_revision("biali/big-lama", DEFAULT_REMOVER_WEIGHTS) is None + ) + + def test_the_default_downloads_at_the_pin_and_is_cached(self): + descriptor = mock.MagicMock() + model_cache.clear_model_cache() + with ( + mock.patch("huggingface_hub.hf_hub_download", return_value="w") as download, + mock.patch("safetensors.torch.load_file", return_value={}), + mock.patch.object( + remover, "_descriptor_from_state_dict", return_value=descriptor + ), + ): + for _ in range(2): + remover._load_remover( + "cpu", DEFAULT_REMOVER_REPO, DEFAULT_REMOVER_WEIGHTS + ) + assert download.call_count == 1 + assert download.call_args.kwargs == { + "repo_id": DEFAULT_REMOVER_REPO, + "filename": DEFAULT_REMOVER_WEIGHTS, + "revision": REMOVER_REVISION, + } + assert model_cache._cache + model_cache.clear_model_cache() + assert not model_cache._cache + + def test_wrong_architecture_is_refused(self): + from spandrel.architectures.Compact.__arch.SRVGG import SRVGGNetCompact + + net = SRVGGNetCompact(num_feat=8, num_conv=2, upscale=2) + with pytest.raises(ValueError, match="inpainting model"): + remover._descriptor_from_state_dict(net.state_dict()) + + @pytest.mark.parametrize("name", PICKLES) + def test_a_pickle_weight_name_is_refused_in_the_loader(self, name): + with ( + mock.patch("huggingface_hub.hf_hub_download") as download, + mock.patch.object(remover, "_load_remover") as load, + ): + with pytest.raises(InvalidInputError, match="safetensors"): + remove_object(photo(), square_mask(), weight_name=name) + download.assert_not_called() + load.assert_not_called() + + @pytest.mark.parametrize("repo", LOCAL_PATHS) + def test_a_local_model_name_is_refused_in_the_loader(self, repo): + with mock.patch.object(remover, "_load_remover") as load: + with pytest.raises((InvalidInputError, PathTraversalError)): + remove_object(photo(), square_mask(), model_name=repo) + load.assert_not_called() + + +class TestAtValidation: + def test_the_defaults_are_clean(self): + definition = {"steps": [step({})]} + assert location_errors(definition) == [] + assert task_argument_errors(definition) == [] + + def test_named_defaults_are_clean(self): + definition = { + "steps": [ + step( + { + "model_name": DEFAULT_REMOVER_REPO, + "weight_name": DEFAULT_REMOVER_WEIGHTS, + "dilate": 0, + } + ) + ] + } + assert location_errors(definition) == [] + assert task_argument_errors(definition) == [] + + @pytest.mark.parametrize("name", PICKLES) + def test_a_pickle_weight_name_is_refused(self, name): + errors = location_errors({"steps": [step({"weight_name": name})]}) + assert [e["path"] for e in errors] == ["steps[0].task.arguments.weight_name"] + + @pytest.mark.parametrize("repo", LOCAL_PATHS) + def test_a_local_model_name_is_refused(self, repo): + errors = location_errors({"steps": [step({"model_name": repo})]}) + assert [e["path"] for e in errors] == ["steps[0].task.arguments.model_name"] + + @pytest.mark.parametrize("dilate", [-1, 2.5]) + def test_a_bad_dilate_is_refused(self, dilate): + errors = task_argument_errors({"steps": [step({"dilate": dilate})]}) + assert [e["path"] for e in errors] == ["steps[0].task.arguments.dilate"] + + def test_a_traversing_mask_is_refused(self): + errors = location_errors( + {"steps": [step({"mask_image": "../../etc/mask.png"})]} + ) + assert [e["path"] for e in errors] == ["steps[0].task.arguments.mask_image"] + + +class TestRegistration: + def test_get_task_lists_the_arguments_and_defaults(self): + from dw.introspection import describe_task + + params = {p["name"]: p for p in describe_task("remove_object")["parameters"]} + assert {"image", "mask_image", "dilate", "model_name", "weight_name"} <= set( + params + ) + assert params["dilate"]["default"] == 8 + assert params["model_name"]["default"] == DEFAULT_REMOVER_REPO + assert params["weight_name"]["default"] == DEFAULT_REMOVER_WEIGHTS From 3ac72b5e183255d4f282a16e99d6a7a480807668 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 19:02:38 -0500 Subject: [PATCH 049/105] fix(admission): #976 - a document error no longer hides an unknown argument in the same call Co-Authored-By: Claude Sonnet 5.5 --- dw/server/admission.py | 12 ++++++++++++ tests/test_admission.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/dw/server/admission.py b/dw/server/admission.py index a8b1279a..f8e8db28 100644 --- a/dw/server/admission.py +++ b/dw/server/admission.py @@ -193,6 +193,18 @@ def admit( raise _validator_failure(e) from e admission.vram_need = _vram_need(candidate, context, arguments) if admission.errors: + # Every error comes back at once (#976): the caller's own bad + # arguments (an unknown name) are reported beside the + # document's errors, not only after they are fixed + try: + known = {(e.get("path"), e.get("message")) for e in admission.errors} + admission.errors += [ + e + for e in argument_errors(candidate.workflow_definition, arguments) + if (e.get("path"), e.get("message")) not in known + ] + except Exception: + logger.debug("argument_errors skipped beside a failed document") # The warnings walk the steps array, which a definition failing # the schema may not have admission.schema_errors = True diff --git a/tests/test_admission.py b/tests/test_admission.py index 08660869..6c3b6d65 100644 --- a/tests/test_admission.py +++ b/tests/test_admission.py @@ -486,6 +486,41 @@ def test_an_unknown_argument_does_not_make_a_given_one_read_as_null(root): assert "which is null" not in messages +def test_a_null_variable_and_an_unknown_argument_are_both_reported(root): + """Every error comes back at once (#976): the null `audio` no longer + hides the unknown `bogus`.""" + definition = { + "id": "se-976", + "variables": {"audio": None}, + "steps": [ + { + "name": "n", + "task": { + "command": "normalize_audio", + "arguments": {"audio": "variable:audio"}, + }, + "result": {"content_type": "audio/wav"}, + } + ], + } + admission = admit( + workflow_path=None, + workflow=definition, + arguments={"bogus": 1}, + base_dir=None, + workspace=root, + ceiling_index={}, + output_dir=root.outputs, + workflow_dir=root.workflows, + asset_library=library_path("assets", root), + prompt_library=library_path("prompts", root), + ) + assert not admission.ok + messages = [e["message"] for e in admission.errors] + assert any("which is null" in m for m in messages) + assert any("bogus" in m and "Unknown variable" in m for m in messages) + + def test_validation_errors_do_not_read_a_supplied_variable_as_null_beside_a_bad_argument(): """The owner of the rule is validation_errors itself (#972): any caller, not only admit(), gets no null finding for a name it supplied.""" From 4844c1f7796bb3b36aeb13c8485ac1567da60c47 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 19:13:53 -0500 Subject: [PATCH 050/105] feat(catalog): #974 - remove-object and segment-and-remove templates, erase-vs-repaint routing Two task-only templates declared image-edit, each with a measured cost. The routing rule goes in WORKFLOW_GUIDE (Erasing versus repainting a masked region) and TASKS.md Object Segmentation/Removal; z-image-inpaint no longer names FLUX.1 Fill for removal, and inpaint/segment-and-inpaint say they repaint with a prompt and are not the removal route. Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 10 ++++- docs/WORKFLOW_GUIDE.md | 19 ++++++++ tests/test_catalog_structure.py | 12 ++++- tests/test_observed_cost.py | 4 ++ tests/test_step_cache.py | 2 + workflows/models/z-image-inpaint.json | 2 +- workflows/templates/inpaint.json | 2 +- workflows/templates/remove-object.json | 31 +++++++++++++ workflows/templates/segment-and-inpaint.json | 2 +- workflows/templates/segment-and-remove.json | 46 ++++++++++++++++++++ 10 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 workflows/templates/remove-object.json create mode 100644 workflows/templates/segment-and-remove.json diff --git a/docs/TASKS.md b/docs/TASKS.md index 0206dc54..d6aba58a 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -2729,12 +2729,13 @@ Detect and segment objects using text prompts via GroundingDINO + SAM2. Returns | `threshold` | No | Detection confidence threshold (default: 0.3) | | `invert` | No | Invert the output mask (default: false) | -Returns a grayscale PIL Image (mode "L") — white (255) for detected objects, black (0) for background. Use with inpainting pipelines like FluxFillPipeline. +Returns a grayscale PIL Image (mode "L") — white (255) for detected objects, black (0) for background. To erase what it found, pass the mask to [`remove_object`](#object-removal), which chains with no `invert`. To paint something new in its place, pass it to a prompted inpainter such as FluxFillPipeline. **Examples:** - [segment.json](../workflows/templates/segment.json) — Segment an object from an image -- [segment-and-inpaint.json](../workflows/templates/segment-and-inpaint.json) — Segment, then inpaint the masked region +- [segment-and-remove.json](../workflows/templates/segment-and-remove.json) — Segment, then erase the object with `remove_object` (no prompt) +- [segment-and-inpaint.json](../workflows/templates/segment-and-inpaint.json) — Segment, then repaint the masked region with a prompt ## Object Removal @@ -2776,6 +2777,11 @@ Erase what a mask covers and fill the hole from the surrounding texture, with no **First run.** The weights download on the first run, about 200 MB. `validate_workflow`'s `plan.downloads_required` does not list task weights yet, so nothing warns of the download beforehand. +**Examples:** + +- [remove-object.json](../workflows/templates/remove-object.json) — Erase what a supplied mask covers +- [segment-and-remove.json](../workflows/templates/segment-and-remove.json) — Find an object by name, then erase it + ## Image Captioning Generate text captions from images using a vision-language model. diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index b1998b19..6ab290f0 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -827,6 +827,25 @@ workflow. `reference-to-video` and `dialogue-short` templates). The `identity-referenced` trait in the listing marks the workflows that take one. +### Erasing versus repainting a masked region + +A request to *erase or remove* an object and a request to *put something +new* in its place are different operations, and only one of them takes a +prompt. + +- **Erase or remove an object** → the `remove_object` task: prompt-free, it + fills the hole from the surrounding scene, and suits small to medium + objects (a cup, a sign, a watermark). `templates/remove-object.json` takes + a mask you already have; `templates/segment-and-remove.json` masks the + object by name first. A large region, such as a person filling a third of + the frame, comes back smeared. +- **Put something new inside a mask** → a prompted inpainter, + `models/z-image-inpaint` or `templates/inpaint.json`. The prompt names what + to paint. Do not use one to erase: given a prompt for "empty background" it + paints a new object or a patch that does not match (#946). + +`docs/TASKS.md` *Object Removal* has the task's arguments and its mask rules. + ### A spoken scene breaking into a song A musical number: spoken shots, each carrying its own dialogue audio, then diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index ecc670f1..c38c3beb 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -96,6 +96,13 @@ def load(path): "image-edit", ["needs-input-media"], ), + # Task-only, so the rules read them as utilities; they declare image-edit + # because an erase is an edit of the image it was handed (#974) + "workflows/templates/remove-object.json": ("image-edit", ["needs-input-media"]), + "workflows/templates/segment-and-remove.json": ( + "image-edit", + ["needs-input-media"], + ), "workflows/templates/describe-and-regenerate.json": ( "image-set", ["composes-workflows", "needs-input-media"], @@ -487,7 +494,10 @@ def test_no_stale_entry_in_the_allowlist(): # Then to 11_400 for the five FLUX 3 Klein templates (#930, 2026-10-15), # measured at 11_256: a new family, its summaries already trimmed to the idea # each template adds, with about 85 tokens held for its Task 9 cost blocks. -COMPACT_BUDGET = 11_400 +# Then to 11_650 for the `remove-object` and `segment-and-remove` templates +# (#956, stage #974, 2026-10-22), measured at 11_592: two new entries, the +# catalog's only prompt-free removal route, each with a cost block. +COMPACT_BUDGET = 11_650 FILTERED_BUDGET = 1_500 diff --git a/tests/test_observed_cost.py b/tests/test_observed_cost.py index b08ed037..6445b0d1 100644 --- a/tests/test_observed_cost.py +++ b/tests/test_observed_cost.py @@ -648,6 +648,10 @@ def test_orphaning_removes_only_that_workspace_and_name(self, tmp_path): "workflows/templates/upscale-spandrel.json": "cost follows the input image", # Three prompt edits over one image: nothing numeric moves the run. "workflows/templates/consistent-set.json": "no variable moves the cost", + # One LaMa pass (after one segment pass) over the image handed to it: + # `dilate` grows the mask, not the work, so nothing moves the run (#974). + "workflows/templates/remove-object.json": "cost follows the input image", + "workflows/templates/segment-and-remove.json": "cost follows the input image", } diff --git a/tests/test_step_cache.py b/tests/test_step_cache.py index c2c1620d..57b47366 100644 --- a/tests/test_step_cache.py +++ b/tests/test_step_cache.py @@ -880,12 +880,14 @@ def test_normalized_downstream_false_when_nothing_references_it(): }, "recenter-crop.json": {}, "relevel-clip.json": {}, + "remove-object.json": {}, "restore-faces.json": { "generate": "8655c1f1635a7f9a1d3ce02a9cbf6f0f9e8b27130052d6e2758fcb7838270c14" }, "segment-and-inpaint.json": { "inpaint": "8ece64de576884be947f5af07c5713b92d3a1e641035434ef805629f12ae8c77" }, + "segment-and-remove.json": {}, "segment.json": {}, "step-caching.json": { "txt2img": "3e3ea6a2cb1d2d1b1abaec9de159758b137ac0bcd3197a1bac0f7bbcead7ac24" diff --git a/workflows/models/z-image-inpaint.json b/workflows/models/z-image-inpaint.json index 2444666f..ee2236ba 100644 --- a/workflows/models/z-image-inpaint.json +++ b/workflows/models/z-image-inpaint.json @@ -12,7 +12,7 @@ }, "seed": "variable:seed", "id": "ZImageInpaint", - "description": "Masked add or replace with Z-Image Turbo (ZImageInpaintPipeline): white mask pixels are repainted with what the prompt names, the rest is kept. NOT for object removal: with a prompt it paints a new object or a mismatched patch rather than continuing the surrounding scene (#946); use templates/inpaint (FLUX.1 Fill) to remove an object or outpaint. Uses the same Tongyi-MAI/Z-Image-Turbo checkpoint as models/z-image, so a server that has one has the other, and fits a 24 GB card with offload where FLUX.1 Fill needs a 54 GB download. Prompt for the thing to put in the mask.", + "description": "Masked add or replace with Z-Image Turbo (ZImageInpaintPipeline): white mask pixels are repainted with what the prompt names, the rest is kept. NOT for object removal: with a prompt it paints a new object or a mismatched patch rather than continuing the surrounding scene (#946); to erase an object use templates/remove-object (a mask to hand) or templates/segment-and-remove (the object by name), which run the prompt-free remove_object task, and to extend past the frame use templates/outpaint. Uses the same Tongyi-MAI/Z-Image-Turbo checkpoint as models/z-image, so a server that has one has the other, and fits a 24 GB card with offload where FLUX.1 Fill needs a 54 GB download. Prompt for the thing to put in the mask.", "vram_estimate": { "base_gb": 11.8, "bytes_per_voxel": 1031, diff --git a/workflows/templates/inpaint.json b/workflows/templates/inpaint.json index 4eea1907..f1aaa728 100644 --- a/workflows/templates/inpaint.json +++ b/workflows/templates/inpaint.json @@ -5,7 +5,7 @@ "seed": null }, "id": "FluxFill", - "description": "Inpaints the masked region of an image with FLUX.1 Fill dev.", + "description": "Inpaints the masked region of an image with FLUX.1 Fill dev. It repaints the mask with what the prompt names and is not the removal route: to erase an object, use templates/remove-object (the prompt-free remove_object task).", "seed": "variable:seed", "steps": [ { diff --git a/workflows/templates/remove-object.json b/workflows/templates/remove-object.json new file mode 100644 index 00000000..01c3e4c6 --- /dev/null +++ b/workflows/templates/remove-object.json @@ -0,0 +1,31 @@ +{ + "id": "RemoveObject", + "description": "Erases what a mask covers with no prompt, filling from the surroundings (remove_object, LaMa). This is the removal route: white mask pixels are erased, the rest comes back unchanged. Suits small to medium objects - a cup, a sign, a watermark; a large region comes back smeared. Raise dilate (16 to 24) for a soft edge or a shadow; use segment-and-remove to mask the object by name instead of drawing one. To put something new in the mask, use a prompted inpainter (models/z-image-inpaint or templates/inpaint) instead.", + "shape": "image-edit", + "cost": [ + {"device": "mps", "vram_gb": 1, "minutes": 0.2} + ], + "variables": { + "image": "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png", + "mask_image": "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup_mask.png", + "dilate": 8 + }, + "steps": [ + { + "name": "erase", + "task": { + "command": "remove_object", + "arguments": { + "image": "variable:image", + "mask_image": "variable:mask_image", + "dilate": "variable:dilate" + } + }, + "result": { + "content_type": "image/png", + "embed_metadata": true, + "subfolder": "final" + } + } + ] +} diff --git a/workflows/templates/segment-and-inpaint.json b/workflows/templates/segment-and-inpaint.json index 6689610f..f0b91d3d 100644 --- a/workflows/templates/segment-and-inpaint.json +++ b/workflows/templates/segment-and-inpaint.json @@ -1,6 +1,6 @@ { "id": "SegmentAndInpaint", - "description": "Segments an input image, then inpaints the selected region with FLUX.1 Fill dev.", + "description": "Segments an input image, then inpaints the selected region with FLUX.1 Fill dev. It repaints the region with what the prompt names and is not the removal route: to erase the object, use templates/segment-and-remove (the prompt-free remove_object task).", "variables": { "image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png", "segment_prompt": "cat", diff --git a/workflows/templates/segment-and-remove.json b/workflows/templates/segment-and-remove.json new file mode 100644 index 00000000..45aa36e5 --- /dev/null +++ b/workflows/templates/segment-and-remove.json @@ -0,0 +1,46 @@ +{ + "id": "SegmentAndRemove", + "description": "Finds an object by name and erases it with no prompt, filling from the surroundings (segment, then remove_object). This is the removal route when there is no mask to hand: segment's mask is white on the object, which is what remove_object erases. Suits small to medium objects; a large region comes back smeared. Raise dilate (16 to 24) when the object has a soft edge or casts a shadow. If segment finds nothing, the image comes back unchanged with a warning. To replace the object with something else, use segment-and-inpaint instead.", + "shape": "image-edit", + "cost": [ + {"device": "cuda", "name": "RTX 3090", "vram_gb": 4, "minutes": 1.2} + ], + "variables": { + "image": "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png", + "object": "cup", + "dilate": 8 + }, + "steps": [ + { + "name": "mask", + "task": { + "command": "segment", + "arguments": { + "image": "variable:image", + "prompt": "variable:object" + } + }, + "result": { + "content_type": "image/png", + "embed_metadata": true, + "subfolder": "intermediate" + } + }, + { + "name": "erase", + "task": { + "command": "remove_object", + "arguments": { + "image": "variable:image", + "mask_image": "previous_result:mask", + "dilate": "variable:dilate" + } + }, + "result": { + "content_type": "image/png", + "embed_metadata": true, + "subfolder": "final" + } + } + ] +} From bfb4e222473a17e13aa8bd0d9c1f47a82243bd48 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 19:25:03 -0500 Subject: [PATCH 051/105] fix(segment): #974 - take each box's best-rated SAM2 mask, not the first SAM2's first candidate on the docs cup was the body alone, without the lid and with a hole at the logo, so segment-and-remove left the cup. Adds the whole-cup mask segment now returns, as remove-object's default mask. Co-Authored-By: Claude Opus 5.5 --- docs/img/cup-object-mask.png | Bin 0 -> 1731 bytes dw/tasks/segment.py | 7 +++- tests/test_segment.py | 61 +++++++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 docs/img/cup-object-mask.png diff --git a/docs/img/cup-object-mask.png b/docs/img/cup-object-mask.png new file mode 100644 index 0000000000000000000000000000000000000000..2d419eaa031d8d9ac41044e5532a8c82628de4ac GIT binary patch literal 1731 zcmcgs{X3Iu9ABr7&WVvmc?l<-LvhrlvK66|A!#KqqeqfnpIPH%S(ebLtB|uKqNd0; z(a9`%jJ%f$M`GBsJY$t!EX>PZTDEYOAI=|eemLJB?(6z~ug}}}zOMUD57_OCUShEX zg+ihIcHje1s72icu5(%ymrLdzN$UEpP2L^jznq4auo3t6FnR=Jmd5{*U#$gtj(_ zFjbVZfzu(3KWby+|Kx@bzfutGH-40tmB^8*p3L2-AHSo zb9kr=Fg@p)lyrI|1rD|9iX`-}8coOz3^b7Pf^NBTTox;b3MwaKrs495EiJjpQ}DQD zO$@LVu~E;wseWb;J`&YTR?tqCYB!iJR?7b89;^YOPA|{1 zO&}zrhR)`OTJMdwJw;tVK3ChYON=~wu%A5WJOabjbw&eDFyb}Z=a-j*X1lAa5H$Go zK`_*j=d^-qC>77J?HiMe;BsqT)Q-e5cPd_MC&JrDBrBa9>6HPH4D)dyCd*v&YMF(x zF)sfXAhD<6qt;tb>8Jlb-kK~}mQzy}-P zV~ypb3XcTD72(sN`{v`J;nkl#VxiBt#0la93uPypa%^*jire*LUA?}Lue#FQYqPqc zpF@$|cFe>WX56pmMp$(3iwqziH|F*-K?+gb$CVjK&GF(746IoZ4K&Yh;quAbhON$17+&rsSuCQ1sFov(u` zE9zVS%zu=`aUZq_5a2Nc=-3{WWw#;?OQJ-CTC*42R@m8|Bnu@u@+c_EweT7aagK$& zB%rol;4#d=a@={>>y5@&Dhfn~baqm^B_$h+P$VwOa0ytb@Dbl-5akDWRtg`tSBj++ zq2TVuSW4_R?~Ehb8o(_RWW#t2r5m`_@fuxT0u?%vE;T*@y~)EEO5}_nWuXJG>CB}F zGRp;7(*w+c*)VKM(cNy=l>d2XIsfobA1;licj~jcoT+C#HHK>?6ChA&ZVX_!!t+Hg z0#>I_7Tn{T57_~nuktu%ou<6+w311;y4_rAIA?WhCBjepevK~+s99Oo<6dAB`E zEp`zQ&E_*wg96DF(piLUxp**+D|-leca&LyPrH{B<>+7F!94g zcs={zepe)5x?OcaLq;bbHP@R$)TCF@MA^BEjCvnqkM(Kw z^4c{eP1`+(UrI%3orGL3b#K=Si8hb1)AJom`Qhm6?}c4oo(gY7yW9v&#!~SE)7k!c z4oFwzOQI%tA>eR80Uz+HHYJwc+&{OOOK1nQ#8^Ijg%&=6{z1qSVd?9en}Wr}(P&@o z$z!7rtBCy9yjA`j>P%NUTRfaWn>z2_b916JX7v!SO=EK6wiUZWBZ%QQE&lKSfAgpc YSDLTwI&!m^{7uErdpEv(`@!@708aoukpKVy literal 0 HcmV?d00001 diff --git a/dw/tasks/segment.py b/dw/tasks/segment.py index afef6bee..dedf7c9e 100644 --- a/dw/tasks/segment.py +++ b/dw/tasks/segment.py @@ -120,7 +120,12 @@ def load_sam(): sam_outputs.pred_masks, sam_inputs["original_sizes"] ) - combined = masks[0][:, 0].sum(dim=0).clamp(0, 1) + # SAM2 answers each box with three candidate masks and rates each in + # iou_scores. Take each box's best-rated one: the first is often a part of + # the object - on the docs cup it was the body alone, missing the lid and + # with a hole where the logo is, so remove_object left the cup (#974) + best = sam_outputs.iou_scores[0].argmax(dim=-1).cpu() + combined = masks[0][torch.arange(len(best)), best].sum(dim=0).clamp(0, 1) mask_array = (combined.cpu().numpy() * 255).astype(np.uint8) mask_image = Image.fromarray(mask_array, mode="L") diff --git a/tests/test_segment.py b/tests/test_segment.py index 4f38c9d6..e8272e4c 100644 --- a/tests/test_segment.py +++ b/tests/test_segment.py @@ -64,7 +64,9 @@ def test_returns_pil_image_mode_l( mock_sam_model_instance.to.return_value = mock_sam_model_instance mask_tensor = torch.zeros(1, 1, 3, 480, 640) mask_tensor[0, 0, 0, 100:300, 100:300] = 1.0 - mock_sam_model_instance.return_value = MagicMock(pred_masks=mask_tensor) + mock_sam_model_instance.return_value = MagicMock( + pred_masks=mask_tensor, iou_scores=torch.ones(1, 1, 1) + ) post_mask = torch.zeros(1, 1, 480, 640) post_mask[0, 0, 100:300, 100:300] = 1.0 @@ -198,7 +200,9 @@ def test_masks_come_back_at_the_image_size( # SAM2's low-res mask logits: (batch, boxes, masks, 256, 256) logits = torch.full((1, 1, 3, 256, 256), -10.0) logits[..., 64:128, 64:128] = 10.0 - sam_model.return_value = MagicMock(pred_masks=logits) + sam_model.return_value = MagicMock( + pred_masks=logits, iou_scores=torch.tensor([[[0.7, 0.9, 0.8]]]) + ) result = segment_image(_make_test_image(), "dog") @@ -207,6 +211,59 @@ def test_masks_come_back_at_the_image_size( assert np.array(result).max() == 255 + @patch("dw.tasks.segment.Sam2Model") + @patch("dw.tasks.segment.Sam2Processor") + @patch("dw.tasks.segment.AutoModelForZeroShotObjectDetection") + @patch("dw.tasks.segment.AutoProcessor") + def test_each_box_takes_its_best_rated_candidate( + self, mock_auto_proc, mock_auto_model, mock_sam_proc, mock_sam_model + ): + """SAM2 returns three candidates per box; the first is often a part of + the object (the docs cup's body without its lid, #974). The mask is + the union of each box's highest-iou candidate.""" + from dw.tasks.segment import segment_image + + dino = MagicMock() + mock_auto_proc.from_pretrained.return_value = dino + dino.return_value = _make_batch_encoding({"input_ids": torch.zeros(1, 10)}) + dino.post_process_grounded_object_detection.return_value = [ + { + "boxes": torch.tensor([[0.0, 0.0, 100.0, 100.0]] * 2), + "scores": torch.tensor([0.9, 0.8]), + "labels": ["cup", "cup"], + } + ] + dino_model = MagicMock() + mock_auto_model.from_pretrained.return_value = dino_model + dino_model.to.return_value = dino_model + + sam_proc = MagicMock() + mock_sam_proc.from_pretrained.return_value = sam_proc + sam_proc.return_value = _make_batch_encoding( + {"original_sizes": torch.tensor([[480, 640]])} + ) + sam_model = MagicMock() + mock_sam_model.from_pretrained.return_value = sam_model + sam_model.to.return_value = sam_model + sam_model.return_value = MagicMock( + pred_masks=torch.zeros(1, 2, 3, 256, 256), + iou_scores=torch.tensor([[[0.74, 0.99, 0.98], [0.5, 0.6, 0.9]]]), + ) + # Each candidate marks its own column band, so the result names which + # candidates were taken + post = torch.zeros(2, 3, 480, 640) + for box in range(2): + for k in range(3): + col = (box * 3 + k) * 100 + post[box, k, :, col : col + 100] = 1.0 + sam_proc.post_process_masks.return_value = [post] + + mask = np.array(segment_image(_make_test_image(), "cup")) + + taken = [band for band in range(6) if mask[0, band * 100] == 255] + assert taken == [1, 5] + + class TestGroundingDinoQuery: """GroundingDINO only scores a query written its way - lowercase, each phrase ending in a period. 'cat' found nothing in a photo of a cat (max From 941b5fd754756ff049577784df959cce8d6b61ce Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 19:28:20 -0500 Subject: [PATCH 052/105] feat(catalog): #974 - remove-object defaults to a mask of the cup The docs cup_mask.png is the region the diffusers docs paint a cup into, so the shipped defaults erased empty sand. TASKS.md says segment keeps SAM2's best-rated mask. Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 2 +- tests/test_segment.py | 1 - workflows/templates/remove-object.json | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index d6aba58a..b1bd8db6 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -2729,7 +2729,7 @@ Detect and segment objects using text prompts via GroundingDINO + SAM2. Returns | `threshold` | No | Detection confidence threshold (default: 0.3) | | `invert` | No | Invert the output mask (default: false) | -Returns a grayscale PIL Image (mode "L") — white (255) for detected objects, black (0) for background. To erase what it found, pass the mask to [`remove_object`](#object-removal), which chains with no `invert`. To paint something new in its place, pass it to a prompted inpainter such as FluxFillPipeline. +Returns a grayscale PIL Image (mode "L") — white (255) for detected objects, black (0) for background. SAM2 proposes three masks for each detected box, and segment keeps the one SAM2 rates best. That is usually the whole object: on the docs cup photo the first proposal was the body alone, without the lid and with a hole at the logo. To erase what it found, pass the mask to [`remove_object`](#object-removal), which chains with no `invert`. To paint something new in its place, pass it to a prompted inpainter such as FluxFillPipeline. **Examples:** diff --git a/tests/test_segment.py b/tests/test_segment.py index e8272e4c..e447c610 100644 --- a/tests/test_segment.py +++ b/tests/test_segment.py @@ -210,7 +210,6 @@ def test_masks_come_back_at_the_image_size( assert result.size == (640, 480) assert np.array(result).max() == 255 - @patch("dw.tasks.segment.Sam2Model") @patch("dw.tasks.segment.Sam2Processor") @patch("dw.tasks.segment.AutoModelForZeroShotObjectDetection") diff --git a/workflows/templates/remove-object.json b/workflows/templates/remove-object.json index 01c3e4c6..75267a22 100644 --- a/workflows/templates/remove-object.json +++ b/workflows/templates/remove-object.json @@ -7,7 +7,7 @@ ], "variables": { "image": "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png", - "mask_image": "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup_mask.png", + "mask_image": "https://raw.githubusercontent.com/dkackman/diffusers-workflow/bfb4e222473a17e13aa8bd0d9c1f47a82243bd48/docs/img/cup-object-mask.png", "dilate": 8 }, "steps": [ From 98edd7c254820458d73c2a542101e9ddb9af1943 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 22 Oct 2026 19:35:40 -0500 Subject: [PATCH 053/105] docs(proposals): #956 close-out - remove_object design record Co-Authored-By: Claude Opus 5.5 --- .../complete/remove-object-complete.md | 154 ++++++++++++++++++ docs/proposals/todo.md | 9 + 2 files changed, 163 insertions(+) create mode 100644 docs/proposals/complete/remove-object-complete.md diff --git a/docs/proposals/complete/remove-object-complete.md b/docs/proposals/complete/remove-object-complete.md new file mode 100644 index 00000000..bf1556df --- /dev/null +++ b/docs/proposals/complete/remove-object-complete.md @@ -0,0 +1,154 @@ +# `remove_object`: prompt-free LaMa object removal (#956) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out session, +2026-10-22). The idea came from #946: on lem an agent asked to erase an +object hand-composited it in PIL, and `models/z-image-inpaint` failed +removal in all four runs. Plan v1 was designed on #956. Don's review made +v2, written after #946 closed `wontfix`, and Don approved v2. It was built +as two stages, #973 and #974. The plan comment on #956 stays the full +design. This doc records what shipped. + +## The problem + +dw had no "erase this and fill it from the surroundings" operation. Every +masked route was a prompted diffusion pipeline, and those add or replace +content. With #946 declined, FLUX.1 Fill was never verified for removal on +lem, so dw had no working removal route at all. + +## Verdict: build smaller + +Two small stages: the task, then the routing docs and templates. No new +dependency was needed, because `spandrel` (already a core dependency) +loads LaMa as a `MaskedImageModelDescriptor`. + +Cut, with what would bring each back: +- **A temporally consistent video removal model** (ProPainter-style). + *Trigger:* a field report where per-frame LaMa flickers visibly. +- **A per-frame mask video.** It needs a two-input `per_frame`. *Trigger:* + a field report that needs a moving mask. The task applies one still mask + to every frame. +- **A general mask utility task** (dilate/feather/union). *Trigger:* a + second consumer. The task dilates internally. +- **A second, larger removal model for big regions.** *Trigger:* a field + report where LaMa smears a large removal. + +Non-goals stayed out: prompting, outpainting, temporal consistency, +per-frame masks, and any change to the Fill templates beyond their removal +wording. + +## Plan history + +- **v1:** Fill stayed the prompted fallback for large regions and the + outpaint route. The signature had no `weight_name`. It used bf16 where + the device supported it, and left the weights source open. +- **v2 (approved):** #946 is `wontfix`, so `remove_object` is dw's only + removal route. Fill was dropped from the fallback and overlap, and + `templates/outpaint` was left alone. `weight_name` was added, so the + safetensors-only gate has a key to check. fp32 runs on every device + (LaMa is ~51M params, and bf16 is an untested quality risk). Q1, ruled + by Don: pin `michaelgold/big-lama@98ab7cface89a594b381f32fc7a9f2cae3f91ccc`, + file `big-lama.safetensors` (204 MB), with the byte-identical + `biali/big-lama` as the named fallback. LaMa (advimman) is credited as + Apache-2.0, not under the repo card's MIT. Q2 kept the 8 px `dilate` + default. Q3 accepted the `downloads_required` gap and called for a + follow-up issue. + +Corrections the plan made to the issue: the dependency question went away +(spandrel); the common `big-lama.pt` is TorchScript and must not be loaded; +the H3 backdrop leak in `minimax-h3`'s `references/staging.md` is fixed by +`subject_definitions`, not by a mask; `WORKFLOW_GUIDE.md` had no +masked-editing section to edit, so one was added. + +## What was built + +### Stage A (#973): the `remove_object` task + +Commits 89e75acf (the task) and 8bc5b884 (tests and TASKS.md), merged as +c7a694db. +- `remove_object(image, mask_image, dilate=8, model_name="michaelgold/big-lama", + weight_name="big-lama.safetensors")`. The implementation is in + `dw/tasks/remove_object.py`, and the handler is in + `dw/tasks/model_handlers.py` (lazy import, `per_frame`, + `consumes_device=True`). +- **Mask:** converted to L, binarized at 128 (white means fill, the same as + `segment`'s output, so no `invert`), resized to the image with nearest if + it differs, then dilated. The model's output is composited only inside + the dilated mask, so pixels outside it are the input's own. +- **All-black mask:** the input comes back unchanged with a job `warning` + event, and no model is loaded. For frames the check runs once, so a video + warns once. +- **Weights:** `REMOVER_REVISION` is applied only to the default repo and + file. The loader runs `validate_hub_repo_id` and `validate_weight_name`, + then `hf_hub_download`, `safetensors.torch.load_file`, and spandrel + `load_from_state_dict`. It refuses anything that isn't a + `MaskedImageModelDescriptor`. The model is cached through `cached_model` + under `(repo, file, revision, device)`. +- **fp32 everywhere:** no `.half()` and no device branch. +- **Trust gate:** `remove_object` is in `SAFETENSORS_ONLY_COMMANDS`, so a + `.pt`/`.ckpt`/`.bin` `weight_name` or a local `model_name` is refused at + validate, and again in the loader. +- `tests/test_remove_object.py` (42 cases). `docs/TASKS.md` gained an + *Object Removal* section: parameters, polarity, `dilate` advice, the + weights source and fallback, the Apache-2.0 credit, the large-region + limit and the first-run download. +- **Deviations:** `image`/`mask_image` are not declared as + `media_arguments`, because the media-key naming convention already covers + both and a test pins the traversal refusal. With frames and an all-black + mask, the frames come back re-wrapped at the same count and fps. +- Measured on mps with the real weights: about 10 s cold including the + download, and 0.08 s warm. + +### Stage B (#974): routing and templates + +Commit 4844c1f7 (merged as d1141a66), and the bounce fix in bfb4e222 and +941b5fd7 (merged as 1a6d8b5c). +- `workflows/templates/remove-object.json` (`image`, `mask_image`, + `dilate`) and `workflows/templates/segment-and-remove.json` (`image`, + `object`, `dilate`). Both declare `"shape": "image-edit"`, because the + shape rules would otherwise read a task-only workflow as `utility`. Their + default image is the diffusers docs `cup.png`. `remove-object`'s default + mask is `docs/img/cup-object-mask.png`, served from this repo pinned at + bfb4e222. +- **Routing rule** ("erase → `remove_object`; put something new in a mask → + a prompted inpainter"): a new *Erasing versus repainting a masked region* + section in `WORKFLOW_GUIDE.md`'s *Authoring a workflow from an agent*, + and TASKS.md *Object Segmentation*. The `models/z-image-inpaint` + description was rewritten so it no longer names Fill for removal. The + `templates/inpaint` and `segment-and-inpaint` descriptions each gained + one clause. +- The compact listing budget in `tests/test_catalog_structure.py` went from + 11_400 to 11_650. +- **Outside the plan:** `dw/tasks/segment.py` now keeps each box's + best-rated SAM2 candidate (`iou_scores` argmax), not the first. On + `cup.png` the first candidate was the cup's body without its lid, with a + hole at the logo. This changes the mask every `segment` caller gets + (including `segment-and-inpaint`). It is a bug fix with no change to the + MCP shape. + +## Verification and bounces + +| Stage | Bounces | Cases | +|---|---|---| +| A (#973) | 0 | C-F510–C-F515, SE-F050 passed on lem first time | +| B (#974) | 1 | C-F517 failed: the cup survived. Fixed by the `segment` best-mask change and the new default mask. C-F516–C-F518 then passed | + +On lem the warm `segment` → `remove_object` job took 1.6 s, and its +`remove_object` phase 0.3 s. The stage comments name no `usage:` figures, +so cost per stage is not recorded here. The plan estimated $3–5 for A and +$2–3 for B. + +## Found along the way + +- **segment's mask quality** (above): every `segment` caller used to get + SAM2's first proposal, which can be a partial object. +- In C-F511, `segment("picture frame")` covered only part of the frame's + matte, so a faint remnant was left. That is a segment setup shortfall, + not a `remove_object` finding. + +## Follow-up + +- **#975:** `plan.downloads_required` doesn't see task weights (upscale, + restore_faces, remove_object). The first `remove_object` run downloads + about 200 MB without a warning, and TASKS.md says so. +- The templates' `vram_gb` figures (1 and 4) are estimates from model + sizes, not measurements. Replace them when one is measured. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 9b1ebd68..29353059 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -334,6 +334,15 @@ remaining deferred fix is recorded in that hold layered voices (a Music 3 chorus). Record, including what was deferred (the voice/pitch conversion task, structured spans, the check in `attribute_voices`): `complete/stacked-vocals-warning-complete.md`. +- **`remove_object`, prompt-free LaMa object removal** (#956, stages + #973-#974), shipped 2026-10-22. A task that fills a white-means-fill mask + from the surrounding image, with no prompt and fp32 on any device, using + pinned `big-lama.safetensors` loaded through spandrel. It comes with the + `remove-object` and `segment-and-remove` templates and an + erase-versus-repaint routing rule. `segment` now keeps SAM2's best-rated + mask. Record, including what was deferred (temporal video removal, + per-frame masks, a larger model for big regions, #975's download + warning): `complete/remove-object-complete.md`. ## Declined From 36305b36e7a947276aeca27a798ce879a0a4d580 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Fri, 23 Oct 2026 10:25:50 -0500 Subject: [PATCH 054/105] feat(plan): #977 - task steps declare required_weights; the plan lists their repos Co-Authored-By: Claude Opus 5.5 --- dw/plan.py | 54 +++++++++++++++++++++++++++++++-- dw/tasks/audio_transcription.py | 13 +++++++- dw/tasks/background_remover.py | 7 +++++ dw/tasks/depth_estimator.py | 15 ++++++++- dw/tasks/diffusion_upscale.py | 20 ++++++++++-- dw/tasks/image_to_text.py | 1 + dw/tasks/image_utils.py | 46 ++++++++++++++++++++++------ dw/tasks/judge.py | 1 + dw/tasks/model_handlers.py | 13 +++++++- dw/tasks/registry.py | 29 ++++++++++++++++++ dw/tasks/segment.py | 18 +++++++++-- dw/tasks/speech_generation.py | 18 ++++++++++- dw/tasks/text_generation.py | 29 ++++++++++++++---- dw/tasks/voice_attribution.py | 8 +++++ dw/tasks/weights.py | 50 ++++++++++++++++++++++++++++++ 15 files changed, 297 insertions(+), 25 deletions(-) create mode 100644 dw/tasks/weights.py diff --git a/dw/plan.py b/dw/plan.py index 6f31e713..327795a4 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -899,7 +899,10 @@ def downloads_required(expanded, base_dir, workflow_dir, cache_dir, lookup_sizes composed child) that `scan_models` does not find, plus every one it does find but whose cached copy is left over from an interrupted pull (#382) - a component the load needs is missing, or a blob is still - `.incomplete` - plus every `from_single_file` that is a URL. Sizes come + `.incomplete` - plus every `from_single_file` that is a URL, and every + repo a task step declares through its command's `required_weights` hook + (#975, dw/tasks/weights.py), listed once when a pipeline step loads it + too. Sizes come from the hub when asked and are None whenever it does not answer - an offline box is a state, not an error, so nothing here raises or logs above debug. @@ -908,14 +911,17 @@ def downloads_required(expanded, base_dir, workflow_dir, cache_dir, lookup_sizes variants = {} urls = [] _collect_sources(expanded, names, variants, urls) + _collect_task_weights(expanded, names, urls) for path, _arguments in _sub_workflow_paths(expanded): raw = read_sub_workflow(path, base_dir, workflow_dir) if raw is None: continue try: - _collect_sources(json.loads(raw), names, variants, urls) + child = json.loads(raw) except ValueError: continue + _collect_sources(child, names, variants, urls) + _collect_task_weights(child, names, urls) present = {repo.get("repo_id") for repo in scan_models(cache_dir).get("repos", [])} required = [] for name in names: @@ -981,6 +987,50 @@ def _collect_sources(tree, names, variants, urls): _collect_sources(value, names, variants, urls) +def _collect_task_weights(tree, names, urls): + """Every repo (or URL) a task step in a tree declares, appended after the + pipeline sources and deduplicated against them. Each `task` block's + command is looked up by name and its hook called on the block's + arguments - expanded at the top level; a composed child's are its raw + definition, so a `variable:` value there is no repo and gives no entry. + + A hook that raises costs its entries, never the validate: the plan + answers what it can name, and the run still loads what it loads. + """ + if isinstance(tree, dict): + task = tree.get("task") + if isinstance(task, dict) and isinstance(task.get("command"), str): + arguments = task.get("arguments") + for entry in _task_weights( + task["command"], arguments if isinstance(arguments, dict) else {} + ): + repo, url = entry.get("repo"), entry.get("url") + if isinstance(repo, str) and repo not in names: + names.append(repo) + elif repo is None and isinstance(url, str) and _is_url(url): + if url not in urls: + urls.append(url) + for value in tree.values(): + _collect_task_weights(value, names, urls) + elif isinstance(tree, list): + for value in tree: + _collect_task_weights(value, names, urls) + + +def _task_weights(command, arguments): + # Imported here: the registry loads task modules only when a workflow has + # a task step, so a pipeline-only validate pays nothing for it + from .tasks.registry import required_weights_hook + + try: + hook = required_weights_hook(command) + entries = hook(copy.deepcopy(arguments)) if hook is not None else [] + except Exception as error: + logger.debug(f"required_weights for task {command!r} failed: {error}") + return [] + return [entry for entry in entries or [] if isinstance(entry, dict)] + + def gate_warnings(downloads_required): """One line per required download this box's token is blocked from - the pre-flight signal #186 asked for, so a gated repo is a diff --git a/dw/tasks/audio_transcription.py b/dw/tasks/audio_transcription.py index d72410e2..89e7c385 100644 --- a/dw/tasks/audio_transcription.py +++ b/dw/tasks/audio_transcription.py @@ -21,6 +21,7 @@ from .. import preferred_task_dtype from ..events import emit_warning from .model_cache import cached_model, hf_pipeline_placement +from .weights import argument_or_default, repo_entries from ..assessment_rules import DEAD_AIR_FLOOR_DBFS from ..dsp import loudest_window_dbfs, resample_waveform from ..script_lines import looped_texts, normalize_words @@ -38,6 +39,16 @@ TIMESTAMP_KINDS = ("segment", "word") +def model_source(arguments): + """The ASR repo a transcribe_audio (or check_script) step loads.""" + return argument_or_default(arguments, "model_name", WHISPER_DEFAULT_MODEL) + + +def required_weights(arguments): + """The ASR model's repo, for the plan (dw/tasks/weights.py).""" + return repo_entries(model_source(arguments)) + + def _downmixed_mono(waveform): """Average a (channels, samples) waveform down to one channel. @@ -92,7 +103,7 @@ def transcribe_audio(audio, device="cpu", sample_rate=None, **kwargs): 0 ] - model_name = kwargs.get("model_name", WHISPER_DEFAULT_MODEL) + model_name = model_source(kwargs) dtype = preferred_task_dtype(device) def load_pipe(): diff --git a/dw/tasks/background_remover.py b/dw/tasks/background_remover.py index bc2fa9e8..c52ffed8 100644 --- a/dw/tasks/background_remover.py +++ b/dw/tasks/background_remover.py @@ -4,10 +4,17 @@ from transformers import AutoModelForImageSegmentation from .model_cache import cached_model +from .weights import repo_entries _MODEL_NAME = "briaai/RMBG-2.0" +def required_weights(arguments): + """The one fixed repo remove_background loads, for the plan + (dw/tasks/weights.py) - it takes no model argument.""" + return repo_entries(_MODEL_NAME) + + def remove_background(image: Image, device) -> Image: # Model settings def load_model(): diff --git a/dw/tasks/depth_estimator.py b/dw/tasks/depth_estimator.py index aeab0542..2c988d79 100644 --- a/dw/tasks/depth_estimator.py +++ b/dw/tasks/depth_estimator.py @@ -5,6 +5,17 @@ from .. import preferred_task_dtype from .model_cache import cached_model +from .weights import repo_entries + +# transformers' own default for "depth-estimation", named so the plan can list +# it and a transformers upgrade cannot move it silently +_MODEL_NAME = "Intel/dpt-large" +_REVISION = "bc15f29" + + +def required_weights(arguments): + """The depth model's repo, for the plan (dw/tasks/weights.py).""" + return repo_entries(_MODEL_NAME) def make_hint_tensor(image, device, dtype=None): @@ -21,7 +32,9 @@ def make_hint_tensor(image, device, dtype=None): """ depth_estimator = cached_model( ("depth_estimator", str(device)), - lambda: pipeline("depth-estimation", device=device), + lambda: pipeline( + "depth-estimation", model=_MODEL_NAME, revision=_REVISION, device=device + ), ) image = depth_estimator(image)["depth"] diff --git a/dw/tasks/diffusion_upscale.py b/dw/tasks/diffusion_upscale.py index 56066681..2546b19d 100644 --- a/dw/tasks/diffusion_upscale.py +++ b/dw/tasks/diffusion_upscale.py @@ -14,6 +14,7 @@ import diffusers from .. import preferred_task_dtype from .model_cache import cached_model +from .weights import argument_or_default, repo_entries logger = logging.getLogger("dw") @@ -29,6 +30,21 @@ } +def model_source(arguments): + """The repo a diffusion_upscale step loads - `model_name` over the + selected mode's default - or None for a mode that is not one of _MODELS + (refused when the step runs).""" + config = _MODELS.get(argument_or_default(arguments, "mode", "x4")) + if config is None: + return None + return argument_or_default(arguments, "model_name", config["model_name"]) + + +def required_weights(arguments): + """The upscale pipeline's repo, for the plan (dw/tasks/weights.py).""" + return repo_entries(model_source(arguments)) + + def diffusion_upscale(image, device="cpu", **kwargs): """Upscale an image using a Stable Diffusion upscale pipeline. @@ -47,12 +63,12 @@ def diffusion_upscale(image, device="cpu", **kwargs): Returns: PIL Image (upscaled). """ - mode = kwargs.get("mode", "x4") + mode = argument_or_default(kwargs, "mode", "x4") if mode not in _MODELS: raise ValueError(f"mode must be one of {sorted(_MODELS.keys())}, got '{mode}'") config = _MODELS[mode] - model_name = kwargs.get("model_name", config["model_name"]) + model_name = model_source(kwargs) prompt = kwargs.get("prompt", "") negative_prompt = kwargs.get("negative_prompt", None) num_inference_steps = int(kwargs.get("num_inference_steps", 25)) diff --git a/dw/tasks/image_to_text.py b/dw/tasks/image_to_text.py index 9107b3d6..f9c41943 100644 --- a/dw/tasks/image_to_text.py +++ b/dw/tasks/image_to_text.py @@ -10,6 +10,7 @@ import logging from .text_generation import generate_text, DEFAULT_VISION_MODEL +from .text_generation import vision_required_weights as required_weights # noqa: F401 logger = logging.getLogger("dw") diff --git a/dw/tasks/image_utils.py b/dw/tasks/image_utils.py index fec31ead..738b3930 100644 --- a/dw/tasks/image_utils.py +++ b/dw/tasks/image_utils.py @@ -174,14 +174,17 @@ def image_to_canny(image, low_threshold=100, high_threshold=200): return Image.fromarray(image) +_DEPTH_MODEL = "Intel/dpt-hybrid-midas" +_SEGMENTATION_MODEL = "openmmlab/upernet-convnext-small" +_SAM_REPO = "ybelkada/segment-anything" + + def image_to_depth(image, device, height=1024, width=1024): from transformers import DPTForDepthEstimation, DPTImageProcessor size = (width, height) - depth_estimator = DPTForDepthEstimation.from_pretrained( - "Intel/dpt-hybrid-midas" - ).to(device) - feature_extractor = DPTImageProcessor.from_pretrained("Intel/dpt-hybrid-midas") + depth_estimator = DPTForDepthEstimation.from_pretrained(_DEPTH_MODEL).to(device) + feature_extractor = DPTImageProcessor.from_pretrained(_DEPTH_MODEL) image = feature_extractor(images=image, return_tensors="pt").pixel_values.to(device) # MPS doesn't support autocast, so use 'cpu' for autocast when on MPS @@ -216,11 +219,9 @@ def image_to_segmentation(image): """Semantic segmentation map from the UperNet ConvNeXt model, colored by class.""" from transformers import AutoImageProcessor, UperNetForSemanticSegmentation - image_processor = AutoImageProcessor.from_pretrained( - "openmmlab/upernet-convnext-small" - ) + image_processor = AutoImageProcessor.from_pretrained(_SEGMENTATION_MODEL) image_segmentor = UperNetForSemanticSegmentation.from_pretrained( - "openmmlab/upernet-convnext-small" + _SEGMENTATION_MODEL ) pixel_values = image_processor(image, return_tensors="pt").pixel_values with torch.no_grad(): @@ -554,7 +555,7 @@ def add_watermark( # the pre-refactor behavior, which called it straight off from_pretrained(). _PROCESSORS["sam"] = _make_pretrained_handler( "SamDetector", - ("ybelkada/segment-anything",), + (_SAM_REPO,), {"subfolder": "checkpoints"}, {}, to_device=False, @@ -568,6 +569,33 @@ def add_watermark( del _name, _attr, _repo_args, _repo_kwargs, _call_kwargs +def required_weights(processor, arguments): + """The repos an image processor step loads, for the plan + (dw/tasks/weights.py). Unlike a registered command's hook it takes the + processor name too, since one table serves every processor; one that loads + nothing from the Hub (or, like zoe_depth and dw_pose, fetches outside the + Hugging Face cache) gives none.""" + from .weights import repo_entries + + if processor in _PRETRAINED_DETECTOR_SPECS: + return repo_entries(_PRETRAINED_DETECTOR_SPECS[processor][1][0]) + if processor == "sam": + return repo_entries(_SAM_REPO) + if processor == "depth": + return repo_entries(_DEPTH_MODEL) + if processor == "segmentation": + return repo_entries(_SEGMENTATION_MODEL) + if processor == "remove_background": + from .background_remover import required_weights as background_weights + + return background_weights(arguments) + if processor in ("depth_estimator", "depth_estimator_tensor"): + from .depth_estimator import required_weights as depth_weights + + return depth_weights(arguments) + return [] + + def available_processors(): """Return the sorted list of processor names process_image accepts. diff --git a/dw/tasks/judge.py b/dw/tasks/judge.py index 8f7a1ed8..9809ba15 100644 --- a/dw/tasks/judge.py +++ b/dw/tasks/judge.py @@ -18,6 +18,7 @@ import re from .text_generation import generate_text, DEFAULT_VISION_MODEL +from .text_generation import vision_required_weights as required_weights # noqa: F401 logger = logging.getLogger("dw") diff --git a/dw/tasks/model_handlers.py b/dw/tasks/model_handlers.py index da1e9f9f..2bfc0704 100644 --- a/dw/tasks/model_handlers.py +++ b/dw/tasks/model_handlers.py @@ -33,6 +33,7 @@ def _voices_argument_errors(arguments): @register_command( "attribute_voices", implementation="dw.tasks.voice_attribution.attribute_voices", + required_weights="dw.tasks.voice_attribution.required_weights", consumes_device=True, returns="json", domains={"window_seconds": POSITIVE, "min_reference_seconds": POSITIVE}, @@ -48,6 +49,7 @@ def _handle_attribute_voices(task, arguments, previous_pipelines): @register_command( "check_script", implementation="dw.tasks.script_check.check_script", + required_weights="dw.tasks.audio_transcription.required_weights", consumes_device=True, returns="json", domains={"similarity": UNIT}, @@ -92,6 +94,7 @@ def _handle_upscale(task, arguments, previous_pipelines): @register_command( "diffusion_upscale", implementation="dw.tasks.diffusion_upscale.diffusion_upscale", + required_weights="dw.tasks.diffusion_upscale.required_weights", consumes_device=True, ) def _handle_diffusion_upscale(task, arguments, previous_pipelines): @@ -134,7 +137,10 @@ def _handle_restore_faces(task, arguments, previous_pipelines): @register_command( - "segment", implementation="dw.tasks.segment.segment_image", consumes_device=True + "segment", + implementation="dw.tasks.segment.segment_image", + required_weights="dw.tasks.segment.required_weights", + consumes_device=True, ) def _handle_segment(task, arguments, previous_pipelines): """Segment objects in an image using text prompt""" @@ -298,6 +304,7 @@ def _handle_decode_h3_latents(task, arguments, previous_pipelines): @register_command( "image_to_text", implementation="dw.tasks.image_to_text.image_to_text", + required_weights="dw.tasks.image_to_text.required_weights", consumes_device=True, ) def _handle_image_to_text(task, arguments, previous_pipelines): @@ -312,6 +319,7 @@ def _handle_image_to_text(task, arguments, previous_pipelines): @register_command( "judge", implementation="dw.tasks.judge.judge", + required_weights="dw.tasks.judge.required_weights", consumes_device=True, returns="scalar", ) @@ -327,6 +335,7 @@ def _handle_judge(task, arguments, previous_pipelines): @register_command( "transcribe_audio", implementation="dw.tasks.audio_transcription.transcribe_audio", + required_weights="dw.tasks.audio_transcription.required_weights", consumes_device=True, ) def _handle_transcribe_audio(task, arguments, previous_pipelines): @@ -341,6 +350,7 @@ def _handle_transcribe_audio(task, arguments, previous_pipelines): @register_command( "text_generation", implementation="dw.tasks.text_generation.generate_text", + required_weights="dw.tasks.text_generation.required_weights", consumes_device=True, ) def _handle_text_generation(task, arguments, previous_pipelines): @@ -355,6 +365,7 @@ def _handle_text_generation(task, arguments, previous_pipelines): @register_command( "generate_speech", implementation="dw.tasks.speech_generation.generate_speech", + required_weights="dw.tasks.speech_generation.required_weights", consumes_device=True, ) def _handle_speech_generation(task, arguments, previous_pipelines): diff --git a/dw/tasks/registry.py b/dw/tasks/registry.py index 8e88a317..cebc3c99 100644 --- a/dw/tasks/registry.py +++ b/dw/tasks/registry.py @@ -96,6 +96,7 @@ def register_command( static_check=None, media_arguments=(), whole_numbers=(), + required_weights=None, ): """ Decorator to register a command handler function. @@ -158,6 +159,11 @@ def register_command( reads as whole numbers (`task_domains.whole_number`): 3, 3.0, "3" and "3.0" are 3, and 3.5 is refused - at validation as at run time. Every other argument with a domain is a real number + required_weights: Dotted path to the task module's + `required_weights(arguments)` - the hub repos the command fetches, + listed in the plan's `downloads_required` (`dw/tasks/weights.py`, + #975). A string, so the plan path imports the module only for a + workflow that has the command Returns: Decorator function @@ -198,8 +204,31 @@ def handler(task, arguments, previous_pipelines, _func=func): rules["media_arguments"] = tuple(media_arguments) if whole_numbers: rules["whole_numbers"] = tuple(whole_numbers) + if required_weights: + rules["required_weights"] = required_weights _COMMAND_RULES[command_name] = rules logger.debug(f"Registered command handler: {command_name}") return func return decorator + + +def required_weights_hook(command): + """The `required_weights(arguments)` callable for `command`, or None when + it fetches nothing the plan lists - a registered command's declared hook, + or an image processor's (`process_image`'s names are commands of their + own, dispatched by `dw/tasks/image_utils.py` rather than registered).""" + if not isinstance(command, str): + return None + importlib.import_module(_REGISTERING_MODULE) + if command in _COMMAND_RULES: + path = _COMMAND_RULES[command].get("required_weights") + if path is None: + return None + module_path, _, name = path.rpartition(".") + return getattr(importlib.import_module(module_path), name) + from .image_utils import available_processors, required_weights + + if command.lower() in available_processors(): + return lambda arguments: required_weights(command.lower(), arguments) + return None diff --git a/dw/tasks/segment.py b/dw/tasks/segment.py index dedf7c9e..eb3aa017 100644 --- a/dw/tasks/segment.py +++ b/dw/tasks/segment.py @@ -16,6 +16,7 @@ Sam2Processor, ) from .model_cache import cached_model +from .weights import argument_or_default, repo_entries logger = logging.getLogger("dw") @@ -23,6 +24,20 @@ _DEFAULT_SAM_MODEL = "facebook/sam2-hiera-large" +def model_sources(arguments): + """The (detector, segmenter) repos a segment step loads - `model_name` + and `sam_model_name` over the defaults.""" + return ( + argument_or_default(arguments, "model_name", _DEFAULT_DINO_MODEL), + argument_or_default(arguments, "sam_model_name", _DEFAULT_SAM_MODEL), + ) + + +def required_weights(arguments): + """The detector and segmenter repos, for the plan (dw/tasks/weights.py).""" + return repo_entries(*model_sources(arguments)) + + def grounding_query(prompt): """A prompt in the form GroundingDINO scores: lowercase, each phrase ending in a period. 'cat' scored 0.20 against a photo of a cat - under the 0.3 @@ -47,8 +62,7 @@ def segment_image(image, prompt, device="cpu", **kwargs): Returns: PIL Image in mode "L" — white (255) for detected objects, black (0) for background. """ - model_name = kwargs.get("model_name", _DEFAULT_DINO_MODEL) - sam_model_name = kwargs.get("sam_model_name", _DEFAULT_SAM_MODEL) + model_name, sam_model_name = model_sources(kwargs) threshold = kwargs.get("threshold", 0.3) invert = kwargs.get("invert", False) diff --git a/dw/tasks/speech_generation.py b/dw/tasks/speech_generation.py index 0034e98a..6cd5c5f8 100644 --- a/dw/tasks/speech_generation.py +++ b/dw/tasks/speech_generation.py @@ -28,6 +28,7 @@ from .audio_utils import load_audio from ..trust import require_trusted_generate_arguments from .model_cache import cached_model, hf_pipeline_placement +from .weights import argument_or_default, repo_entries logger = logging.getLogger("dw") @@ -47,6 +48,21 @@ _SPEAKER_ENCODER_SAMPLE_RATE = 16000 +def model_source(arguments): + """The text-to-speech repo a generate_speech step loads.""" + return argument_or_default(arguments, "model_name", _DEFAULT_MODEL) + + +def required_weights(arguments): + """The speech model's repo, plus speechbrain's speaker encoder when the + step clones a voice from a `speaker_embedding` clip, for the plan + (dw/tasks/weights.py).""" + encoder = ( + _SPEAKER_ENCODER_MODEL if (arguments or {}).get("speaker_embedding") else None + ) + return repo_entries(model_source(arguments), encoder) + + def _speaker_embedding_tensor(location, device, dtype): """The x-vector speechbrain's spkrec-xvect-voxceleb extracts from a reference audio file - what SpeechT5 conditions its voice on. @@ -195,7 +211,7 @@ def generate_speech(text=None, device="cpu", seed=None, **kwargs): kwargs.get("generate_kwargs"), "generate_kwargs" ) - model_name = kwargs.get("model_name", _DEFAULT_MODEL) + model_name = model_source(kwargs) voice_preset = kwargs.get("voice_preset", None) speaker_embedding = kwargs.get("speaker_embedding", None) dtype = preferred_task_dtype(device) diff --git a/dw/tasks/text_generation.py b/dw/tasks/text_generation.py index 03a81063..17fa5dc8 100644 --- a/dw/tasks/text_generation.py +++ b/dw/tasks/text_generation.py @@ -22,6 +22,7 @@ from ..locations import validate_hub_repo_id, validate_weight_name from ..trust import require_trusted_generate_arguments from .model_cache import cached_model, hf_pipeline_placement +from .weights import argument_or_default, repo_entries logger = logging.getLogger("dw") @@ -328,6 +329,26 @@ def _validated_generate_kwargs(value): return value +def model_source(arguments): + """The repo a text_generation step loads - `model_name` over the text + default, or over the vision default when the step gives an image (an + image_to_text or judge step always does).""" + default = DEFAULT_VISION_MODEL if (arguments or {}).get("image") else _DEFAULT_MODEL + return argument_or_default(arguments, "model_name", default) + + +def required_weights(arguments): + """The language model's repo, for the plan (dw/tasks/weights.py). A + `system_prompt.from_hub` file is a single-file fetch, not listed.""" + return repo_entries(model_source(arguments)) + + +def vision_required_weights(arguments): + """required_weights for a task that always hands generate_text an image + (image_to_text, judge).""" + return required_weights({**(arguments or {}), "image": True}) + + def generate_text(prompt, device="cpu", **kwargs): """Generate text from a prompt using a local language model. @@ -372,12 +393,8 @@ def generate_text(prompt, device="cpu", **kwargs): ) generate_kwargs = _validated_generate_kwargs(kwargs.get("generate_kwargs")) - if images is None: - pipeline_task = "text-generation" - model_name = kwargs.get("model_name", _DEFAULT_MODEL) - else: - pipeline_task = "image-text-to-text" - model_name = kwargs.get("model_name", DEFAULT_VISION_MODEL) + pipeline_task = "text-generation" if images is None else "image-text-to-text" + model_name = model_source({**kwargs, "image": images}) dtype = preferred_task_dtype(device) diff --git a/dw/tasks/voice_attribution.py b/dw/tasks/voice_attribution.py index 60a9ad7f..d976e3ab 100644 --- a/dw/tasks/voice_attribution.py +++ b/dw/tasks/voice_attribution.py @@ -36,6 +36,7 @@ from ..dsp import resample_waveform from .audio_utils import waveform_and_rate, load_audio from .model_cache import cached_model +from .weights import repo_entries logger = logging.getLogger("dw") @@ -131,6 +132,13 @@ STACK_SPANS_LISTED = 8 +def required_weights(arguments): + """speechbrain's speaker encoder, for the plan (dw/tasks/weights.py). + The demucs separator downloads outside the Hugging Face cache and is not + listed (docs/TASKS.md).""" + return repo_entries(_EMBEDDER_MODEL) + + def _missing(package, error): return RuntimeError( f"{COMMAND} needs the '{package}' package, which is not installed " diff --git a/dw/tasks/weights.py b/dw/tasks/weights.py new file mode 100644 index 00000000..0be7e17a --- /dev/null +++ b/dw/tasks/weights.py @@ -0,0 +1,50 @@ +"""What a task step fetches from the Hub: the `required_weights` hook's helpers. + +Every model-backed task module exports `required_weights(arguments) -> +list[dict]` - each dict `{"repo": }` for a whole repo - and its loader +reads the repo it loads from the same resolution, so the plan's +`downloads_required` (dw/plan.py) and the run cannot disagree about which +argument wins over which default (#975). A command names its hook on its +registration (`register_command(required_weights=...)`, a dotted path, so the +plan path imports a task module only when a workflow has that task). + +A hook reads the step's *expanded* arguments, which at validate time can still +hold a reference (`previous_result:mask`) or a local checkout path. Neither is +a download it can name, so it gives no entry and never raises: whether a name +is a hub id is decided by its shape, never by touching the disk, as the +pipeline walk decides it. + +Torch-free on purpose: `dw/plan.py` imports it on every validate. +""" + +from huggingface_hub.utils import HFValidationError, validate_repo_id + + +def argument_or_default(arguments, name, default): + """The value the task will load for `name`: the step's own argument when + it gives one, else the task's default - the one rule both the hook and the + loader read, so an override and a default cannot be resolved two ways.""" + value = (arguments or {}).get(name) + return default if value is None else value + + +def is_hub_repo(value): + """Whether `value` is shaped like a hub repo id - a reference + (`previous_result:x`), a path (`/models/x`, `./x`) or a non-string is not.""" + if not isinstance(value, str): + return False + try: + validate_repo_id(value) + return True + except HFValidationError: + return False + + +def repo_entries(*names): + """`[{"repo": name}, ...]` for each name shaped like a hub repo id, in + order and once each; every other value is dropped silently.""" + entries = [] + for name in names: + if is_hub_repo(name) and {"repo": name} not in entries: + entries.append({"repo": name}) + return entries From b0d0d857e578f7fb46aee3b1fae46bcffe72dcdc Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Fri, 23 Oct 2026 10:31:14 -0500 Subject: [PATCH 055/105] feat(plan): #977 - tests and docs for task required_weights Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 1 + docs/MCP.md | 4 +- docs/RELEASING.md | 9 + docs/SERVER.md | 4 +- docs/TASKS.md | 29 ++- tests/test_admission.py | 85 +++++++ tests/test_plan.py | 144 +++++++++++ tests/test_task_weights.py | 481 +++++++++++++++++++++++++++++++++++++ 8 files changed, 752 insertions(+), 5 deletions(-) create mode 100644 tests/test_task_weights.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f5f7d19f..21a2ac9a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -97,6 +97,7 @@ one to open. | Variable constraints | `dw/variable_constraints.py`: `apply_constraints`; the grid arithmetic `aligned` / `aligned_down` | A model's rule about a value is declared in the workflow's `variable_constraints`, and it is checked at validation and again before anything loads. `plan_cuts` (and its static check) rounds render lengths through `aligned` / `aligned_down`, never its own copy, so a planned `num_frames` is one the template's constraint accepts; `crop_face_track` pads its crops to the step's `modulus`/`remainder` grid through `aligned` too (`padding_to_grid`). | `tests/test_variable_constraints.py::TestTheGrid::test_without_snap_an_off_grid_value_is_refused_not_rounded`, `tests/test_variable_constraints.py::TestTheGrid::test_plan_cuts_rounds_through_the_constraint_owner`, `tests/test_face_track.py::TestPadding::test_padding_to_grid_rounds_through_the_constraint_owner`, `tests/test_variable_constraints.py::TestTheRunTimePass::test_an_illegal_value_is_refused_before_anything_loads` | | Bound acknowledgement | `dw/plan.py`: `fingerprint`, `dw/server/admission.py`: `check_bound_acknowledgement` | A bound `acknowledged_cost` is refused with 409 when the plan's fingerprint or its required downloads changed. The fingerprint is keyed only by what can change the output: `DOCUMENTATION_KEYS` (`cost`, `cost_drivers`, `vram_estimate`, descriptions, catalog shape/traits), a constraint's `reason` and a result's role, subfolder and file name are left out, of the workflow and of each composed child; every schema key is sorted into left out or kept. A number digests by value, not spelling (`12.0` as `12`), and the seed stays out (#85). | `tests/test_plan.py::TestFingerprintKeys::test_the_exclusion_lists_are_pinned`, `tests/test_plan.py::TestTheCatalogH3Template::test_a_client_round_trip_keeps_it`, `tests/test_plan.py::TestFingerprintKeys::test_every_top_level_schema_key_is_sorted`, `tests/test_server.py::TestBoundRerun::test_a_rerun_bound_to_a_stale_plan_is_refused`, `tests/test_server.py::TestBoundAcknowledgement::test_a_matching_fingerprint_queues` | | Per-repo gating | `dw/plan.py`: `build_plan` | Each gated repo, including a `loras` entry's repo, is probed and reported on its own. | `tests/test_plan.py::TestDownloadsRequired::test_a_gated_repo_this_token_lacks_access_to_is_blocked` | +| Task weights | `dw/tasks/weights.py`: `argument_or_default`; `dw/tasks/registry.py`: `required_weights_hook`; `dw/plan.py`: `_collect_task_weights` | A model-backed task declares `required_weights(arguments)` on its `register_command`, and `downloads_required` lists those repos beside the pipeline `model_name`s (a shared repo once; a reference or local-path argument none). The declaration and the loader resolve the repo through `argument_or_default`, so they cannot differ. | `tests/test_task_weights.py` | | Observed cost | `dw/server/observed_cost.py`: `declared_drivers`, `ObservedCosts._on_this_card`, `ObservedCosts.card` | `observed` comes only from this box's finished jobs on the named card (the one `route` names, for `plan.estimate`), else the server's own card; `ObservedCosts.card` owns a card's (kind, name, default). Rows are matched by card name whatever index they ran on (a job from before jobs carried a `device` counts only while the server is on the default card), and never writes `cost`. Cold and warm runs are reported separately. | `tests/test_observed_cost.py::TestColdIsNotWarm::test_the_two_are_reported_separately_each_with_its_runs`, `tests/test_observed_cost.py::TestByCard::test_only_rows_from_this_card_count_whichever_index_they_had`, `tests/test_observed_cost.py::TestByCard::test_rows_from_before_jobs_carried_a_device_do_not_count_off_the_default_card` | | The estimate | `dw/plan.py`: `estimate`, `_tempered`; the driver scaling is `dw/cost_scaling.py`: `scale_curated`, `_driver_scaling` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. A `cost` entry is measured at the workflow's variable defaults; a run moving `num_inference_steps`, `width`/`height` or `num_frames` (per list entry, summed) scales it in proportion (`scale_curated`, which `dw/server/deps.py` reuses for an inherited figure), basis `derived` with `scaled` naming each axis, times the list-length re-price, and that scaled figure is what a thin observed one blends toward (#866). | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure`, `tests/test_plan.py::TestDriverScaling::test_steps_canvas_and_frames_multiply`, `tests/test_plan.py::TestScalingAccuracy::test_within_thirty_percent` | | Cost drivers | `dw/server/observed_cost.py`: `declared_drivers` | A cost driver that names no declared variable is dropped. | `tests/test_observed_cost.py::TestComparability::test_a_driver_naming_no_variable_is_dropped`, `tests/test_observed_cost.py::TestTheCatalogsDriversAreReal::test_every_declared_driver_is_a_variable_of_its_workflow` | diff --git a/docs/MCP.md b/docs/MCP.md index f614b32a..72916857 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -277,7 +277,7 @@ tools) rather than relying on the session pin. | Tool | Arguments | Purpose | | --- | --- | --- | -| `validate_workflow(workflow=None, name=None, workspace=None, arguments=None)` | exactly one of `workflow` (inline definition) or `name` (a stored workflow, as `list_workflows` reports it), optional `workspace`, optional `arguments` | Check a workflow against the schema and against real pipeline signatures. Free and instant. Validating by name uses the workflow file's own directory as the base directory, so it sees what a run would. Returns every schema violation in `errors`, each with the JSON path it sits at, so a draft is fixed in one pass, and a `previous_result:` that names no earlier step is one of them. `warnings` covers what still runs but is probably wrong - a signature mismatch, and, for a list-driven variable, an entry key no step reads, at the entry's path. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's. Pass the same `arguments` you will pass to `run_workflow` and they are checked too - an undeclared or renamed variable name, a value that will not coerce to the declared type, and an `asset:`, `prompt:` or `output:` reference that names nothing this workspace can reach, each reported at `arguments.`. `checked_arguments` lists what was covered, so a `valid: true` about the stored defaults cannot be mistaken for one about your values. A reference set the model would refuse - too many images, videos or audio clips, or, for MiniMax-H3, audio as the only reference - is an error here too, rather than a failure minutes into a run you acknowledged. `run_workflow` makes the same check and refuses a bad argument rather than queuing a job that fails on its first step. A valid answer carries `plan` - the fingerprint, step count, list lengths, `downloads_required`, `estimate` (with `basis`), `host_memory` (`{projected_mb, basis, runs}`, the projected host-RAM peak; `basis` is `observed` from same-key successful runs on this machine or `fit`; absent when there is no figure) and `elided_steps` for the arguments given; quote from it. `steps` counts what will run: a step nothing reads and which saves no file does not run, and is named in `elided_steps` instead | +| `validate_workflow(workflow=None, name=None, workspace=None, arguments=None)` | exactly one of `workflow` (inline definition) or `name` (a stored workflow, as `list_workflows` reports it), optional `workspace`, optional `arguments` | Check a workflow against the schema and against real pipeline signatures. Free and instant. Validating by name uses the workflow file's own directory as the base directory, so it sees what a run would. Returns every schema violation in `errors`, each with the JSON path it sits at, so a draft is fixed in one pass, and a `previous_result:` that names no earlier step is one of them. `warnings` covers what still runs but is probably wrong - a signature mismatch, and, for a list-driven variable, an entry key no step reads, at the entry's path. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's. Pass the same `arguments` you will pass to `run_workflow` and they are checked too - an undeclared or renamed variable name, a value that will not coerce to the declared type, and an `asset:`, `prompt:` or `output:` reference that names nothing this workspace can reach, each reported at `arguments.`. `checked_arguments` lists what was covered, so a `valid: true` about the stored defaults cannot be mistaken for one about your values. A reference set the model would refuse - too many images, videos or audio clips, or, for MiniMax-H3, audio as the only reference - is an error here too, rather than a failure minutes into a run you acknowledged. `run_workflow` makes the same check and refuses a bad argument rather than queuing a job that fails on its first step. A valid answer carries `plan` - the fingerprint, step count, list lengths, `downloads_required` (pipeline `model_name`s and the repos task steps fetch), `estimate` (with `basis`), `host_memory` (`{projected_mb, basis, runs}`, the projected host-RAM peak; `basis` is `observed` from same-key successful runs on this machine or `fit`; absent when there is no figure) and `elided_steps` for the arguments given; quote from it. `steps` counts what will run: a step nothing reads and which saves no file does not run, and is named in `elided_steps` instead | | `list_workspaces()` | — | The server's workspaces and which one this session is using. Each has its own workflows, assets and outputs; the prompt library is shared by all of them. Compact entries are `name`/`default`/`usage` plus any `note` (clipped, `note_truncated` when cut); `detail=true` adds each entry's folder paths | | `use_workspace(name)` | `name` | Work in that workspace for the rest of the session - every later call reads and writes there. This is how to keep your work out of another agent's namespace rather than sharing the default one. Checked against the server, so a typo fails here rather than scoping every later call to nothing. On a `dw.serve --mcp` endpoint the pin is shared by every connected client (#298, see the note above the table) - a `warning` field appears when this call overwrote a pin already pointed away from `default` | | `create_workspace(name, use=False)` | `name`, `use` | Create a workspace. Pass use=true to switch this session to it as well; otherwise the session stays where it was and the result says so. The result names the workspace (`name`, `default`, `current`, `next`), not its folders - `list_workspaces(detail=true)` is the opt-in for those. `use=true` carries the same mounted-server caveat and `warning` field as `use_workspace` | @@ -388,7 +388,7 @@ The intended loop: verdict covers the stored definition and its stock defaults, not the values you wrote, and its `plan` is the number to say out loud: `estimate.minutes` with its `basis`, plus each `downloads_required` - entry as a line item of its own + entry as a line item of its own (task steps' repos are included) 2. `run_workflow` with `acknowledged_cost` bound to the plan (`{fingerprint, minutes, downloads}`), or `true` when there was no plan — pass a name straight from `list_workflows` as `workflow_path`; queues the job and returns diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 98e3cf5f..f2b08027 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -7,6 +7,15 @@ notes from commits at tag time (see below). This section is a scratch pad for items a branch's author wants the next release note to name; clear it when a release ships. +- A validate's `downloads_required` now also lists the Hugging Face repos + task steps fetch (`segment`, `remove_background`, `diffusion_upscale`, + `transcribe_audio`, `check_script`, `generate_speech`, `attribute_voices`, + `text_generation`, `image_to_text`, `judge`, the image processors; #977). + A bound `acknowledged_cost` taken from a validate made before this deploy + is answered 409 `downloads` after it when the workflow has one of these + tasks and its repo is not cached: re-validate and resend the new + `acknowledge`. + ### 0.15.0 diff --git a/docs/SERVER.md b/docs/SERVER.md index 08b4672d..f6a3a8ed 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -200,7 +200,7 @@ running: | Route | What it does | | --- | --- | -| `POST /api/jobs` | Queue a run: `{"workflow_path": ...}` or an inline `{"workflow": {...}, "base_dir": ...}`, plus `arguments` for variable overrides. `workflow_path` accepts a stored workflow name as listed by `/api/workflows` (with or without `.json`, nested names included), or a relative/absolute path that still resolves under `--workflow-dir` - confined the same way the `/api/workflows` CRUD routes are; a path that names a real file outside that directory is rejected with 400, not opened. Answers with argument warnings from signature checking. Takes an optional `acknowledged_cost`: `true` is recorded as `acknowledged: boolean`; the object `{fingerprint, minutes, downloads}` from a validate answer's `plan` is `bound` - the server re-plans the run for the arguments given and answers **409** when the fingerprint differs or a repo in `downloads_required` is not in `downloads` (a download that has since vanished is not a refusal); the body is `{"detail": {message, reason: "fingerprint" \| "downloads" \| "unplannable", acknowledged, plan, acknowledge}}` with the current plan, so the caller re-quotes from it, and `acknowledge` - the `{fingerprint, minutes, downloads}` to resend - whenever there is a plan. A `null` in `downloads` (a URL with no repo) is ignored. `minutes` is recorded, never compared. Nothing is required: the web UI and every caller that sends nothing are `acknowledged: none`, and every job answer and history row carries `acknowledged` (and `acknowledged_cost` when bound). `POST /api/jobs/{id}/rerun` takes the same field and checks against the stored spec; a fresh seed does not change a fingerprint. | +| `POST /api/jobs` | Queue a run: `{"workflow_path": ...}` or an inline `{"workflow": {...}, "base_dir": ...}`, plus `arguments` for variable overrides. `workflow_path` accepts a stored workflow name as listed by `/api/workflows` (with or without `.json`, nested names included), or a relative/absolute path that still resolves under `--workflow-dir` - confined the same way the `/api/workflows` CRUD routes are; a path that names a real file outside that directory is rejected with 400, not opened. Answers with argument warnings from signature checking. Takes an optional `acknowledged_cost`: `true` is recorded as `acknowledged: boolean`; the object `{fingerprint, minutes, downloads}` from a validate answer's `plan` is `bound` - the server re-plans the run for the arguments given and answers **409** when the fingerprint differs or a repo in `downloads_required` (pipeline `model_name`s and the repos task steps fetch) is not in `downloads` (a download that has since vanished is not a refusal); the body is `{"detail": {message, reason: "fingerprint" \| "downloads" \| "unplannable", acknowledged, plan, acknowledge}}` with the current plan, so the caller re-quotes from it, and `acknowledge` - the `{fingerprint, minutes, downloads}` to resend - whenever there is a plan. A `null` in `downloads` (a URL with no repo) is ignored. `minutes` is recorded, never compared. Nothing is required: the web UI and every caller that sends nothing are `acknowledged: none`, and every job answer and history row carries `acknowledged` (and `acknowledged_cost` when bound). `POST /api/jobs/{id}/rerun` takes the same field and checks against the stored spec; a fresh seed does not change a fingerprint. | | `GET /api/jobs?workspace=&status=&limit=` | Queue + history summaries, oldest first, with `total` beside them. `status` narrows to one state or a comma-separated set (`queued`, `running`, `succeeded`, `failed`, `cancelled`; anything else is a 400); `limit` keeps the newest N, and `total` still reports how many matched, so a bounded answer cannot be mistaken for a complete one. Each summary carries `device`, the card the job ran on (`"cuda:1 NVIDIA GeForce RTX 3090"`; null while queued and for older jobs), and `rerun_of`, present on a job started by rerun: the id of the job it reran; absent for a fresh submit. No parameters means every job, which is what the web UI polls | | `GET /api/jobs/{id}` | Full detail: spec, events, manifest, error. A manifest entry for a step served from the step cache carries `reused: true`. Every entry carries `subfolder` - the in-run subfolder the step's `result.subfolder` chose, `''` for none. A `for_each` step appears in the manifest as its members (`shot@wide_open`, `shot@closeup`), because the manifest records what ran; the run's `workflow.json` keeps the `for_each` form, because it records what was asked Carries `output_kinds`: each manifest file mapped to its media kind (`image`, `video`, `audio`, `text`, or null), and each file-carrying run event (`step_end`) carries the same map for its own files. The entry of a pipeline step that ran carries `timing` (see [Step timing](WORKSPACES.md#step-timing)). Carries `rerun_of` - present on a job started by rerun: the id of the job it reran; absent for a fresh submit. | | `DELETE /api/jobs/{id}/run` | Delete the run directory a finished job wrote, whole, from the output root the job ran against (no workspace selector applies): `{job_id, run_dir, deleted, run_swept}`. 404 for an unknown job or one with no run directory, 409 for one still queued or running | @@ -426,7 +426,7 @@ The editor's forms come from these; they are just as usable from scripts: when the worker is busy or did not answer - a workflow with no `seed` also gets a warning saying so, since `0` alone does not distinguish a disabled cache from an empty one); - `downloads_required`, each `model_name` the hub cache does not hold as + `downloads_required`, each `model_name` the hub cache does not hold, and each repo a task step fetches (`dw/tasks/weights.py`; a repo shared with a pipeline step is listed once, a reference or local-path argument gives none) as `{repo, gb, gated, access_blocked}` (`gb` from the hub, `null` when it could not be asked - `?sizes=false` skips the hub, and then `gated` and `access_blocked` are `null` too); `gated` is the hub's own field for the diff --git a/docs/TASKS.md b/docs/TASKS.md index b1bd8db6..008b6d3a 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -107,6 +107,15 @@ A new task is: automatically, so a module it does not import registers nothing; 3. a section in this file. +A task that loads Hub weights also declares `required_weights(arguments)`, +named on the decorator as +`register_command(required_weights="dw.tasks..required_weights")`. It +returns `[{"repo": ...}]` for the step's arguments (an empty list when the +repo is not known yet, e.g. a `previous_result:` `model_name`), and resolves +each repo with `argument_or_default` from `dw/tasks/weights.py`, the same call +the loader makes, so the declaration and the load cannot name different +repos. A validate's `downloads_required` lists those repos; `tests/test_task_weights.py` pins it. + ## Image Processing ### ControlNet Preprocessors @@ -139,6 +148,8 @@ Generate control images for ControlNet pipelines: | `depth_estimator` | Depth hint generation | | `depth_estimator_tensor` | Depth hint as tensor | +A validate's `downloads_required` lists the repo each detector fetches when it is not cached: `depth` `Intel/dpt-hybrid-midas`; `depth_estimator` and `depth_estimator_tensor` `Intel/dpt-large`; the `controlnet_aux` detectors `lllyasviel/Annotators` (`teed` `fal-ai/teed`, `anyline` `TheMistoAI/MistoLine`, `sam` `ybelkada/segment-anything`); `segmentation` `openmmlab/upernet-convnext-small`. `zoe_depth` and `dw_pose` fetch outside the Hugging Face cache and are not listed. + All accept an `image` argument with processing parameters: ```json @@ -175,6 +186,8 @@ All accept an `image` argument with processing parameters: | `add_watermark` | Add visible text watermark | `text`, `position`, `opacity`, `font_size`, `color`, `margin` | | `get_image_size` | Return `{width, height}` dict | | +A validate's `downloads_required` lists `briaai/RMBG-2.0` for `remove_background` when it is not cached. + ### EXIF Stripping Remove all EXIF metadata, GPS coordinates, camera info, and timestamps from images for privacy-safe preprocessing: @@ -2282,6 +2295,8 @@ word never flag it. The thresholds are the `STACK_*` constants in ## Voice attribution +A validate's `downloads_required` lists `speechbrain/spkrec-ecapa-voxceleb` only. The demucs separator (and `separate_stems`) downloads outside the Hugging Face cache and is not listed. + Which reference voice sings each line of a song, by timbre - staging lip-sync shots for a generated song needs its section-to-singer map, and a generation model (MiniMax Music3 included) does not hand one back. Pitch @@ -2596,6 +2611,8 @@ Two modes are available: - **x4** (default): `StableDiffusionUpscalePipeline` — 4x upscale via `stabilityai/stable-diffusion-x4-upscaler` - **x2**: `StableDiffusionLatentUpscalePipeline` — 2x upscale via `stabilityai/sd-x2-latent-upscaler` +A validate's `downloads_required` lists the repo for the `mode` given; `model_name` overrides it. + ```json { "task": { @@ -2707,6 +2724,8 @@ This gives the best results: the super-resolution model handles background detai Detect and segment objects using text prompts via GroundingDINO + SAM2. Returns a binary mask image suitable for inpainting workflows. +A validate's `downloads_required` lists `IDEA-Research/grounding-dino-base` and `facebook/sam2-hiera-large`; `model_name` and `sam_model_name` override them. + ```json { "task": { @@ -2788,6 +2807,8 @@ Generate text captions from images using a vision-language model. Transformers 5 removed the dedicated `image-to-text` pipeline this task used to build, along with the BLIP/ViT-GPT2/GIT captioning models that ran on it. Captioning now goes through the same `image-text-to-text` pipeline as any other VLM, so `model_name` needs a vision-language model (SmolVLM, Qwen2.5-VL, LLaVA, etc.) and `prompt` is a question put to the model rather than a text fragment to continue. +A validate's `downloads_required` lists `HuggingFaceTB/SmolVLM-256M-Instruct` for `image_to_text` and `judge`; `model_name` overrides it. + ```json { "task": { @@ -2897,6 +2918,8 @@ The built-in `h3_context_ir` workflow applies this to its own output, so a workf Generate or expand text using a local language model. Useful for expanding short prompts into detailed image generation prompts, rewriting text, or other text-to-text tasks. +A validate's `downloads_required` lists `Qwen/Qwen2.5-1.5B-Instruct`, or `HuggingFaceTB/SmolVLM-256M-Instruct` when an `image` is given; `model_name` overrides it. A `system_prompt.from_hub` file is not listed. + ```json { "task": { @@ -2981,7 +3004,7 @@ It is merged after everything else, so it can override `repetition_penalty` and ## Speech Generation -Speak a line of text with a local text-to-speech model. Any transformers text-to-speech checkpoint can be named as `model_name`; three families are wired for their voice controls: +Speak a line of text with a local text-to-speech model. A validate's `downloads_required` lists `suno/bark-small` (`model_name` overrides it), plus `speechbrain/spkrec-xvect-voxceleb` when `speaker_embedding` is given. Any transformers text-to-speech checkpoint can be named as `model_name`; three families are wired for their voice controls: | Family | Checkpoint | Voice | |--------|-----------|-------| @@ -3044,6 +3067,8 @@ A speech model is worth releasing before a video model loads — set `release_mo ## Speech Transcription +A validate's `downloads_required` lists `openai/whisper-base`; `model_name` overrides it. `check_script` uses the same model. + Transcribe spoken audio to text with a local Whisper-class model. The word-correctness of a TTS deliverable — a dropped line, a mid-sentence truncation — can only be inferred from duration and timing arithmetic without this; `transcribe_audio` checks it directly against the text the deliverable was supposed to speak. ```json @@ -3077,6 +3102,8 @@ To compare a take against the lines it was meant to speak, use [`check_script`]( ## Script check +A validate's `downloads_required` lists `openai/whisper-base` for its transcription; `model_name` overrides it. + Whether a dialogue take speaks its script. Confirming it used to mean transcribing the take and comparing the transcript to the script by eye, which missed a word Whisper dropped, an H3 tag read aloud, and a last word cut off by the end of the file. `check_script` does the comparison and reports where to look. It transcribes the take with [`transcribe_audio`](#speech-transcription) word timestamps - the same code and model cache, no second ASR path. Every heard word is measured, and one at or below the dead-air floor is discarded as unheard, because Whisper invents words over silence. So is every word of a repetition loop - the same words over and over (`Pre-pre-pre-...`), which Whisper invents over music or room tone loud enough to pass the floor. H3 markup is stripped from each expected line, and the heard words are aligned to the expected words in order (`difflib`). Each line scores the similarity of its own expected and heard words. It decides nothing: findings are places to look, not verdicts. It is not an assessment probe (`assessment=True`); like [`attribute_voices`](#voice-attribution) it answers a JSON document. diff --git a/tests/test_admission.py b/tests/test_admission.py index 6c3b6d65..cc8ff4b9 100644 --- a/tests/test_admission.py +++ b/tests/test_admission.py @@ -186,6 +186,91 @@ def test_a_bound_submit_expands_once(server, folds): assert len(folds) == 1 +@pytest.fixture +def empty_hub_cache(monkeypatch): + """No repo cached, whatever this machine's own Hugging Face cache holds, + so every task step's repo is a download to acknowledge.""" + import dw.plan + + monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) + + +def segment_workflow(): + """A task-only workflow whose one model is a task step's: the plan lists + the segment step's two repos, and no pipeline step loads either.""" + return { + "id": "segmented", + "variables": {"image": "asset:present.png"}, + "steps": [ + { + "name": "mask", + "task": { + "command": "segment", + "arguments": {"image": "variable:image", "prompt": "a cup"}, + }, + "result": {"content_type": "image/png"}, + } + ], + } + + +def test_a_bound_acknowledgement_missing_a_task_steps_repo_is_refused( + server, empty_hub_cache +): + """#975: a task step's weights are in downloads_required, so an + acknowledgement that names none of them is a download the caller never + saw - answered 409 "downloads" like a pipeline's.""" + body = {"workflow": segment_workflow()} + with server() as client: + plan = client.post("/api/validate?sizes=false", json=body).json()["plan"] + assert plan is not None + repos = [entry["repo"] for entry in plan["downloads_required"]] + assert "IDEA-Research/grounding-dino-base" in repos + + response = client.post( + "/api/jobs", + json={ + **body, + "acknowledged_cost": { + "fingerprint": plan["fingerprint"], + "minutes": plan["estimate"]["minutes"], + "downloads": [], + }, + }, + ) + + assert response.status_code == 409, response.text + detail = response.json()["detail"] + assert detail["reason"] == "downloads" + assert set(repos) <= set(detail["acknowledge"]["downloads"]) + + +def test_a_bound_acknowledgement_naming_a_task_steps_repos_is_admitted( + server, empty_hub_cache +): + body = {"workflow": segment_workflow()} + with server() as client: + plan = client.post("/api/validate?sizes=false", json=body).json()["plan"] + assert plan is not None + repos = [entry["repo"] for entry in plan["downloads_required"]] + assert repos + + response = client.post( + "/api/jobs", + json={ + **body, + "acknowledged_cost": { + "fingerprint": plan["fingerprint"], + "minutes": plan["estimate"]["minutes"], + "downloads": repos, + }, + }, + ) + + assert response.status_code == 201, response.text + assert response.json()["acknowledged"] == "bound" + + def test_a_rerun_expands_once(server, folds): with server() as client: submitted = client.post( diff --git a/tests/test_plan.py b/tests/test_plan.py index 9c1c6dc1..a5416727 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -1791,6 +1791,150 @@ def test_components_and_children_are_scanned_and_deduplicated(self, plan, tmp_pa "org/vae", ] + def test_task_steps_list_the_repos_their_commands_declare(self, plan): + """#975: an image processor (remove_background) and a registered + command (segment) both name their weights, after the pipeline's.""" + spec = definition() + spec["steps"] += [ + { + "name": "cutout", + "task": {"command": "remove_background", "arguments": {}}, + "result": {"content_type": "image/png"}, + }, + { + "name": "mask", + "task": {"command": "segment", "arguments": {"prompt": "a cup"}}, + "result": {"content_type": "image/png"}, + }, + ] + assert [d["repo"] for d in plan(spec)["downloads_required"]] == [ + "org/still-model", + "briaai/RMBG-2.0", + "IDEA-Research/grounding-dino-base", + "facebook/sam2-hiera-large", + ] + + def test_a_task_override_replaces_the_default_repo(self, plan): + spec = definition() + spec["steps"].append( + { + "name": "mask", + "task": { + "command": "segment", + "arguments": {"model_name": "org/dino", "sam_model_name": None}, + }, + } + ) + assert [d["repo"] for d in plan(spec)["downloads_required"]] == [ + "org/still-model", + "org/dino", + "facebook/sam2-hiera-large", + ] + + def test_a_repo_a_pipeline_and_a_task_both_load_is_listed_once(self, plan): + spec = definition() + spec["steps"][0]["pipeline"]["from_pretrained_arguments"]["model_name"] = ( + "openai/whisper-base" + ) + spec["steps"].append( + { + "name": "words", + "task": {"command": "transcribe_audio", "arguments": {}}, + } + ) + assert [d["repo"] for d in plan(spec)["downloads_required"]] == [ + "openai/whisper-base" + ] + + def test_two_task_steps_naming_one_repo_list_it_once(self, plan): + spec = definition() + spec["steps"] += [ + {"name": "a", "task": {"command": "transcribe_audio", "arguments": {}}}, + {"name": "b", "task": {"command": "check_script", "arguments": {}}}, + ] + repos = [d["repo"] for d in plan(spec)["downloads_required"]] + assert repos == ["org/still-model", "openai/whisper-base"] + + def test_a_task_argument_that_is_a_reference_or_a_path_is_not_listed(self, plan): + for local in ("previous_result:detector", "/models/dino", "./models/dino"): + spec = definition() + spec["steps"].append( + { + "name": "mask", + "task": { + "command": "segment", + "arguments": {"model_name": local, "sam_model_name": local}, + }, + } + ) + assert [d["repo"] for d in plan(spec)["downloads_required"]] == [ + "org/still-model" + ], local + + def test_a_cached_task_repo_is_not_required(self, plan, monkeypatch): + import dw.plan + + monkeypatch.setattr( + dw.plan, + "scan_models", + lambda cache_dir=None: { + "repos": [ + {"repo_id": "org/still-model"}, + {"repo_id": "briaai/RMBG-2.0"}, + ] + }, + ) + monkeypatch.setattr(dw.plan, "repo_download_incomplete", lambda *a, **k: False) + spec = definition() + spec["steps"].append( + {"name": "cutout", "task": {"command": "remove_background"}} + ) + assert plan(spec)["downloads_required"] == [] + + def test_a_hook_that_raises_costs_its_entries_and_not_the_plan( + self, plan, monkeypatch + ): + import dw.tasks.registry + + def exploding(command): + def hook(arguments): + raise RuntimeError("hook broke") + + return hook + + monkeypatch.setattr(dw.tasks.registry, "required_weights_hook", exploding) + spec = definition() + spec["steps"].append( + {"name": "mask", "task": {"command": "segment", "arguments": {}}} + ) + assert [d["repo"] for d in plan(spec)["downloads_required"]] == [ + "org/still-model" + ] + + def test_a_composed_childs_task_step_contributes_its_repo(self, plan, tmp_path): + spec = definition() + child = { + "id": "child", + "steps": [ + { + "name": "c", + "task": { + "command": "segment", + "arguments": {"model_name": "org/child-dino"}, + }, + } + ], + } + (tmp_path / "child.json").write_text(json.dumps(child)) + spec["steps"].append( + {"name": "sub", "workflow": {"path": "child.json", "arguments": {}}} + ) + assert [d["repo"] for d in plan(spec)["downloads_required"]] == [ + "org/still-model", + "org/child-dino", + "facebook/sam2-hiera-large", + ] + def test_sizes_come_from_the_hub_in_gib(self, plan, monkeypatch): import dw.plan diff --git a/tests/test_task_weights.py b/tests/test_task_weights.py new file mode 100644 index 00000000..f6245ed1 --- /dev/null +++ b/tests/test_task_weights.py @@ -0,0 +1,481 @@ +"""What a task step fetches from the Hub (#975): each model-backed command's +`required_weights` hook, that the loader resolves its repo through the same +helper, and the pin that a task module touching the Hub declares one.""" + +import ast +import glob +import importlib +import os +import re +from unittest.mock import MagicMock + +import numpy +import pytest +from PIL import Image + +from dw.tasks.registry import required_weights_hook + + +def declared(command, arguments=None): + """The repos `command`'s hook names for `arguments`, in order.""" + hook = required_weights_hook(command) + assert hook is not None, command + return [entry["repo"] for entry in hook(arguments or {})] + + +class TestSegment: + def test_the_defaults_are_the_detector_and_the_segmenter(self): + assert declared("segment") == [ + "IDEA-Research/grounding-dino-base", + "facebook/sam2-hiera-large", + ] + + def test_model_name_overrides_the_detector(self): + assert declared("segment", {"model_name": "org/dino"}) == [ + "org/dino", + "facebook/sam2-hiera-large", + ] + + def test_sam_model_name_overrides_the_segmenter(self): + assert declared("segment", {"sam_model_name": "org/sam"}) == [ + "IDEA-Research/grounding-dino-base", + "org/sam", + ] + + def test_a_null_override_is_the_default(self): + assert declared("segment", {"model_name": None}) == [ + "IDEA-Research/grounding-dino-base", + "facebook/sam2-hiera-large", + ] + + def test_a_reference_or_a_local_path_gives_no_entry_for_its_slot(self): + assert declared("segment", {"model_name": "previous_result:dino"}) == [ + "facebook/sam2-hiera-large" + ] + assert declared("segment", {"sam_model_name": "/models/sam"}) == [ + "IDEA-Research/grounding-dino-base" + ] + assert ( + declared( + "segment", + {"model_name": "./models/dino", "sam_model_name": "/models/sam"}, + ) + == [] + ) + + +class TestRemoveBackground: + def test_is_the_one_fixed_repo(self): + assert declared("remove_background") == ["briaai/RMBG-2.0"] + + def test_the_command_name_is_not_case_sensitive(self): + assert declared("Remove_Background") == ["briaai/RMBG-2.0"] + + +class TestDiffusionUpscale: + def test_the_default_mode_is_x4(self): + assert declared("diffusion_upscale") == [ + "stabilityai/stable-diffusion-x4-upscaler" + ] + + def test_x2_mode_names_the_latent_upscaler(self): + assert declared("diffusion_upscale", {"mode": "x2"}) == [ + "stabilityai/sd-x2-latent-upscaler" + ] + + def test_model_name_overrides_the_mode_default(self): + assert declared( + "diffusion_upscale", {"mode": "x2", "model_name": "org/up"} + ) == ["org/up"] + + def test_an_unknown_mode_names_nothing(self): + assert declared("diffusion_upscale", {"mode": "x8"}) == [] + + def test_a_reference_mode_names_nothing(self): + assert declared("diffusion_upscale", {"mode": "previous_result:m"}) == [] + + def test_a_path_model_name_names_nothing(self): + assert declared("diffusion_upscale", {"model_name": "/models/up"}) == [] + + +class TestAudioTranscription: + @pytest.mark.parametrize("command", ["transcribe_audio", "check_script"]) + def test_the_default_is_whisper_base(self, command): + assert declared(command) == ["openai/whisper-base"] + + @pytest.mark.parametrize("command", ["transcribe_audio", "check_script"]) + def test_model_name_overrides_it(self, command): + assert declared(command, {"model_name": "openai/whisper-small"}) == [ + "openai/whisper-small" + ] + + def test_a_reference_names_nothing(self): + assert declared("transcribe_audio", {"model_name": "previous_result:m"}) == [] + + +class TestSpeechGeneration: + def test_the_default_is_bark_small(self): + assert declared("generate_speech") == ["suno/bark-small"] + + def test_model_name_overrides_it(self): + assert declared("generate_speech", {"model_name": "org/tts"}) == ["org/tts"] + + def test_a_speaker_embedding_adds_the_speaker_encoder(self): + assert declared( + "generate_speech", + {"model_name": "microsoft/speecht5_tts", "speaker_embedding": "me.wav"}, + ) == ["microsoft/speecht5_tts", "speechbrain/spkrec-xvect-voxceleb"] + + def test_a_local_model_still_lists_the_speaker_encoder(self): + assert declared( + "generate_speech", + {"model_name": "/models/tts", "speaker_embedding": "me.wav"}, + ) == ["speechbrain/spkrec-xvect-voxceleb"] + + +class TestVoiceAttribution: + def test_is_only_the_speaker_embedder(self): + assert declared("attribute_voices") == ["speechbrain/spkrec-ecapa-voxceleb"] + + def test_the_stem_separator_is_not_listed(self): + assert required_weights_hook("separate_stems") is None + + +class TestTextGeneration: + def test_the_default_is_a_text_model(self): + assert declared("text_generation") == ["Qwen/Qwen2.5-1.5B-Instruct"] + + def test_an_image_selects_the_vision_default(self): + assert declared("text_generation", {"image": "cup.png"}) == [ + "HuggingFaceTB/SmolVLM-256M-Instruct" + ] + + def test_an_empty_image_is_the_text_default(self): + assert declared("text_generation", {"image": None}) == [ + "Qwen/Qwen2.5-1.5B-Instruct" + ] + + def test_model_name_overrides_either_default(self): + assert declared("text_generation", {"model_name": "org/lm"}) == ["org/lm"] + assert declared("text_generation", {"model_name": "org/vlm", "image": "x"}) == [ + "org/vlm" + ] + + def test_a_local_path_names_nothing(self): + assert declared("text_generation", {"model_name": "/models/lm"}) == [] + + +class TestVisionTasks: + @pytest.mark.parametrize("command", ["image_to_text", "judge"]) + def test_the_default_is_the_vision_model(self, command): + assert declared(command) == ["HuggingFaceTB/SmolVLM-256M-Instruct"] + + @pytest.mark.parametrize("command", ["image_to_text", "judge"]) + def test_model_name_overrides_it(self, command): + assert declared(command, {"model_name": "org/vlm"}) == ["org/vlm"] + + @pytest.mark.parametrize("command", ["image_to_text", "judge"]) + def test_a_reference_names_nothing(self, command): + assert declared(command, {"model_name": "previous_result:m"}) == [] + + +class TestImageProcessors: + @pytest.mark.parametrize( + "processor, repo", + [ + ("depth", "Intel/dpt-hybrid-midas"), + ("depth_estimator", "Intel/dpt-large"), + ("depth_estimator_tensor", "Intel/dpt-large"), + ("openpose", "lllyasviel/Annotators"), + ("mlsd", "lllyasviel/Annotators"), + ("hed", "lllyasviel/Annotators"), + ("lineart", "lllyasviel/Annotators"), + ("teed", "fal-ai/teed"), + ("anyline", "TheMistoAI/MistoLine"), + ("sam", "ybelkada/segment-anything"), + ("segmentation", "openmmlab/upernet-convnext-small"), + ], + ) + def test_a_model_backed_processor_names_its_repo(self, processor, repo): + assert declared(processor) == [repo] + + @pytest.mark.parametrize( + "processor", + ["canny_cv", "resize_rescale", "crop_square", "dw_pose", "zoe_depth"], + ) + def test_a_processor_that_fetches_nothing_from_the_hub_names_nothing( + self, processor + ): + assert declared(processor) == [] + + +class TestNoEntry: + @pytest.mark.parametrize("command", ["separate_stems", "no_such_command"]) + def test_a_command_without_a_hook_has_none(self, command): + assert required_weights_hook(command) is None + + def test_a_non_string_command_has_none(self): + assert required_weights_hook(None) is None + + @pytest.mark.parametrize( + "name", ["previous_result:x", "variable:m", "/models/dino", "./models/x", 3] + ) + @pytest.mark.parametrize( + "command, remaining", + [ + ("segment", ["facebook/sam2-hiera-large"]), + ("text_generation", []), + ("transcribe_audio", []), + ("diffusion_upscale", []), + ("generate_speech", []), + ], + ) + def test_a_name_that_is_not_a_hub_id_never_raises_and_gives_no_entry( + self, command, remaining, name + ): + assert declared(command, {"model_name": name}) == remaining + + def test_missing_arguments_are_the_defaults(self): + assert required_weights_hook("transcribe_audio")(None) == [ + {"repo": "openai/whisper-base"} + ] + + +class Reached(Exception): + """Raised by a recording stand-in once it has what the test wants.""" + + +def recording_cache(monkeypatch, module, stop_at=None): + """Replace `module.cached_model` with a recorder of the cache keys it was + given - the key carries the repo the loader resolved. It raises Reached on + the first key whose first element is `stop_at` (or on the first, when None) + without running the factory, so no model loads.""" + keys = [] + + def fake(key, factory): + keys.append(key) + if stop_at is None or key[0] == stop_at: + raise Reached + return (None, None) + + monkeypatch.setattr(module, "cached_model", fake) + return keys + + +class TestLoaderAgreesWithDeclaration: + """The hook and the loader read one resolution helper, so what the plan + lists is what the run loads, defaults and overrides alike.""" + + @pytest.mark.parametrize( + "arguments", + [ + {}, + {"model_name": "org/dino"}, + {"sam_model_name": "org/sam"}, + {"model_name": "org/dino", "sam_model_name": "org/sam"}, + ], + ) + def test_segment_loads_the_repos_it_declares(self, monkeypatch, arguments): + from dw.tasks import segment + + keys = [] + boxes = MagicMock() + boxes.__len__.return_value = 1 + processor = MagicMock() + processor.post_process_grounded_object_detection.return_value = [ + {"boxes": boxes, "scores": MagicMock(), "labels": []} + ] + + def fake(key, factory): + keys.append(key) + if key[0] == "segment_sam": + raise Reached + return processor, MagicMock() + + monkeypatch.setattr(segment, "cached_model", fake) + with pytest.raises(Reached): + segment.segment_image(Image.new("RGB", (8, 8)), "cup", **arguments) + + assert [key[1] for key in keys] == declared("segment", arguments) + + @pytest.mark.parametrize( + "arguments", + [ + {}, + {"mode": "x2"}, + {"model_name": "org/up"}, + {"mode": "x2", "model_name": "org/up"}, + ], + ) + def test_diffusion_upscale_loads_the_repo_it_declares(self, monkeypatch, arguments): + from dw.tasks import diffusion_upscale + + keys = recording_cache(monkeypatch, diffusion_upscale) + with pytest.raises(Reached): + diffusion_upscale.diffusion_upscale(Image.new("RGB", (8, 8)), **arguments) + + assert [key[2] for key in keys] == declared("diffusion_upscale", arguments) + + @pytest.mark.parametrize("arguments", [{}, {"model_name": "org/asr"}]) + def test_transcribe_audio_loads_the_repo_it_declares(self, monkeypatch, arguments): + from dw.tasks import audio_transcription + + keys = recording_cache(monkeypatch, audio_transcription) + with pytest.raises(Reached): + audio_transcription.transcribe_audio( + numpy.zeros(1600, dtype=numpy.float32), sample_rate=16000, **arguments + ) + + assert [key[1] for key in keys] == declared("transcribe_audio", arguments) + + @pytest.mark.parametrize("arguments", [{}, {"model_name": "org/tts"}]) + def test_generate_speech_loads_the_repo_it_declares(self, monkeypatch, arguments): + from dw.tasks import speech_generation + + keys = recording_cache(monkeypatch, speech_generation) + with pytest.raises(Reached): + speech_generation.generate_speech("hello", **arguments) + + assert [key[1] for key in keys] == declared("generate_speech", arguments) + + @pytest.mark.parametrize( + "arguments", + [ + {}, + {"model_name": "org/lm"}, + {"image": Image.new("RGB", (8, 8))}, + {"image": Image.new("RGB", (8, 8)), "model_name": "org/vlm"}, + ], + ) + def test_generate_text_loads_the_repo_it_declares(self, monkeypatch, arguments): + from dw.tasks import text_generation + + keys = recording_cache(monkeypatch, text_generation) + with pytest.raises(Reached): + text_generation.generate_text("hi", **arguments) + + assert [key[2] for key in keys] == declared("text_generation", arguments) + + +TASKS = os.path.join(os.path.dirname(__file__), "..", "dw", "tasks") + +# A module reaching the Hub by any of these loads weights the plan has to name +HUB_LOADS = re.compile( + r"hf_hub_download|from_pretrained|from_hparams|hf_pipeline\(|pipeline\(\s*[\"']" +) + +# Stage B' of #975: these modules load Hub weights and have no hook yet. Delete +# a name here when its module gains `required_weights` - the pin below fails +# for a name that has one, so the list only shrinks. +# (restore_faces reaches the Hub through upscale.resolve_model_path, so the +# text scan does not see it; it is listed because it loads weights all the same.) +NOT_YET_DECLARED = { + "upscale", + "restore_faces", + "remove_object", + "h3_latent_upscale", + "face_track", + "interpolate_frames", +} + +# Modules that fetch only outside the Hugging Face cache would be listed here +# with the reason; none do - voice_attribution's demucs download shares a module +# with a speechbrain load, which it declares. +OUTSIDE_THE_HF_CACHE = {} + + +def task_modules(): + for path in sorted(glob.glob(os.path.join(TASKS, "*.py"))): + yield os.path.basename(path)[:-3], path + + +def declares_required_weights(path): + """Whether the module defines `required_weights` or imports one under that + name, at module level.""" + with open(path, encoding="utf-8") as file: + tree = ast.parse(file.read()) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "required_weights": + return True + if isinstance(node, ast.ImportFrom): + for alias in node.names: + if (alias.asname or alias.name) == "required_weights": + return True + return False + + +def hub_loading_modules(): + found = set() + for name, path in task_modules(): + with open(path, encoding="utf-8") as file: + if HUB_LOADS.search(file.read()): + found.add(name) + return found + + +class TestEveryHubLoadIsDeclared: + def test_the_scan_finds_the_modules_that_load_from_the_hub(self): + found = hub_loading_modules() + assert { + "segment", + "diffusion_upscale", + "audio_transcription", + "speech_generation", + "text_generation", + "depth_estimator", + "image_utils", + } <= found + + def test_a_module_that_loads_from_the_hub_declares_required_weights(self): + undeclared = sorted( + name + for name, path in task_modules() + if name in hub_loading_modules() + and name not in NOT_YET_DECLARED + and name not in OUTSIDE_THE_HF_CACHE + and not declares_required_weights(path) + ) + assert not undeclared, ( + f"{undeclared} load from the Hub with no required_weights, so the " + "plan cannot list what they download (#975)" + ) + + def test_the_allow_list_names_only_modules_still_without_a_hook(self): + stale = sorted( + name + for name in NOT_YET_DECLARED + if declares_required_weights(os.path.join(TASKS, f"{name}.py")) + ) + assert not stale, f"{stale} are declared now: drop them from NOT_YET_DECLARED" + + def test_every_registered_command_in_a_declaring_module_names_its_hook(self): + from dw.tasks import registry + + importlib.import_module("dw.tasks.model_handlers") + declaring = { + name + for name, path in task_modules() + if name in hub_loading_modules() + and name not in NOT_YET_DECLARED + and declares_required_weights(path) + } + unnamed = [] + for command, rules in registry._COMMAND_RULES.items(): + module = (rules.get("implementation") or "").split(".") + if len(module) < 4 or module[:2] != ["dw", "tasks"]: + continue + # demucs downloads outside the Hugging Face cache, so the stem + # separator in voice_attribution's module names no hook + if command == "separate_stems": + continue + if module[2] in declaring and not rules.get("required_weights"): + unnamed.append(command) + assert not unnamed, f"{unnamed} register without required_weights" + + def test_every_named_hook_resolves_to_a_callable(self): + from dw.tasks import registry + + importlib.import_module("dw.tasks.model_handlers") + for command, rules in registry._COMMAND_RULES.items(): + if rules.get("required_weights"): + assert callable(required_weights_hook(command)), command From 138bbc42196233ed0dece501542f8dc3799d2652 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Fri, 23 Oct 2026 10:48:04 -0500 Subject: [PATCH 056/105] feat(admission): #977 - the validate plan passed verbatim is a bound acknowledgement With no `downloads` key, its downloads_required repos and estimate.minutes stand in; an explicit downloads list still wins. Bounce C-F525 step 5. Co-Authored-By: Claude Opus 5.5 --- docs/MCP.md | 4 +++- docs/SERVER.md | 2 +- dw/server/admission.py | 28 +++++++++++++++++++++++++++- tests/test_admission.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/MCP.md b/docs/MCP.md index 72916857..5bad1deb 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -372,7 +372,9 @@ answers with a `plan`; pass `acknowledged_cost={"fingerprint": plan.fingerprint, "minutes": plan.estimate.minutes, "downloads": [...]}` and the server refuses with 409 if the run's shape changed between the quote and the call - a longer list, a stored prompt edited meanwhile, weights that now -have to be downloaded - naming the new plan so the agent re-quotes. Bare +have to be downloaded - naming the new plan so the agent re-quotes. The +`plan` itself, passed unedited, binds the same way: with no `downloads` key, +its `downloads_required` repos and `estimate.minutes` stand in. Bare `true` still works and is for a `plan` that came back null; the job records which form it got (`acknowledged: none | boolean | bound`). diff --git a/docs/SERVER.md b/docs/SERVER.md index f6a3a8ed..dc3443e2 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -200,7 +200,7 @@ running: | Route | What it does | | --- | --- | -| `POST /api/jobs` | Queue a run: `{"workflow_path": ...}` or an inline `{"workflow": {...}, "base_dir": ...}`, plus `arguments` for variable overrides. `workflow_path` accepts a stored workflow name as listed by `/api/workflows` (with or without `.json`, nested names included), or a relative/absolute path that still resolves under `--workflow-dir` - confined the same way the `/api/workflows` CRUD routes are; a path that names a real file outside that directory is rejected with 400, not opened. Answers with argument warnings from signature checking. Takes an optional `acknowledged_cost`: `true` is recorded as `acknowledged: boolean`; the object `{fingerprint, minutes, downloads}` from a validate answer's `plan` is `bound` - the server re-plans the run for the arguments given and answers **409** when the fingerprint differs or a repo in `downloads_required` (pipeline `model_name`s and the repos task steps fetch) is not in `downloads` (a download that has since vanished is not a refusal); the body is `{"detail": {message, reason: "fingerprint" \| "downloads" \| "unplannable", acknowledged, plan, acknowledge}}` with the current plan, so the caller re-quotes from it, and `acknowledge` - the `{fingerprint, minutes, downloads}` to resend - whenever there is a plan. A `null` in `downloads` (a URL with no repo) is ignored. `minutes` is recorded, never compared. Nothing is required: the web UI and every caller that sends nothing are `acknowledged: none`, and every job answer and history row carries `acknowledged` (and `acknowledged_cost` when bound). `POST /api/jobs/{id}/rerun` takes the same field and checks against the stored spec; a fresh seed does not change a fingerprint. | +| `POST /api/jobs` | Queue a run: `{"workflow_path": ...}` or an inline `{"workflow": {...}, "base_dir": ...}`, plus `arguments` for variable overrides. `workflow_path` accepts a stored workflow name as listed by `/api/workflows` (with or without `.json`, nested names included), or a relative/absolute path that still resolves under `--workflow-dir` - confined the same way the `/api/workflows` CRUD routes are; a path that names a real file outside that directory is rejected with 400, not opened. Answers with argument warnings from signature checking. Takes an optional `acknowledged_cost`: `true` is recorded as `acknowledged: boolean`; the object `{fingerprint, minutes, downloads}` from a validate answer's `plan` is `bound` - the server re-plans the run for the arguments given and answers **409** when the fingerprint differs or a repo in `downloads_required` (pipeline `model_name`s and the repos task steps fetch) is not in `downloads` (a download that has since vanished is not a refusal); the body is `{"detail": {message, reason: "fingerprint" \| "downloads" \| "unplannable", acknowledged, plan, acknowledge}}` with the current plan, so the caller re-quotes from it, and `acknowledge` - the `{fingerprint, minutes, downloads}` to resend - whenever there is a plan. A `null` in `downloads` (a URL with no repo) is ignored. The validate `plan` sent unedited is bound too: when `downloads` is absent, its `downloads_required` repos (and `estimate.minutes`) are what was acknowledged; an explicit `downloads`, even `[]`, wins. `minutes` is recorded, never compared. Nothing is required: the web UI and every caller that sends nothing are `acknowledged: none`, and every job answer and history row carries `acknowledged` (and `acknowledged_cost` when bound). `POST /api/jobs/{id}/rerun` takes the same field and checks against the stored spec; a fresh seed does not change a fingerprint. | | `GET /api/jobs?workspace=&status=&limit=` | Queue + history summaries, oldest first, with `total` beside them. `status` narrows to one state or a comma-separated set (`queued`, `running`, `succeeded`, `failed`, `cancelled`; anything else is a 400); `limit` keeps the newest N, and `total` still reports how many matched, so a bounded answer cannot be mistaken for a complete one. Each summary carries `device`, the card the job ran on (`"cuda:1 NVIDIA GeForce RTX 3090"`; null while queued and for older jobs), and `rerun_of`, present on a job started by rerun: the id of the job it reran; absent for a fresh submit. No parameters means every job, which is what the web UI polls | | `GET /api/jobs/{id}` | Full detail: spec, events, manifest, error. A manifest entry for a step served from the step cache carries `reused: true`. Every entry carries `subfolder` - the in-run subfolder the step's `result.subfolder` chose, `''` for none. A `for_each` step appears in the manifest as its members (`shot@wide_open`, `shot@closeup`), because the manifest records what ran; the run's `workflow.json` keeps the `for_each` form, because it records what was asked Carries `output_kinds`: each manifest file mapped to its media kind (`image`, `video`, `audio`, `text`, or null), and each file-carrying run event (`step_end`) carries the same map for its own files. The entry of a pipeline step that ran carries `timing` (see [Step timing](WORKSPACES.md#step-timing)). Carries `rerun_of` - present on a job started by rerun: the id of the job it reran; absent for a fresh submit. | | `DELETE /api/jobs/{id}/run` | Delete the run directory a finished job wrote, whole, from the output root the job ran against (no workspace selector applies): `{job_id, run_dir, deleted, run_swept}`. 404 for an unknown job or one with no run directory, 409 for one still queued or running | diff --git a/dw/server/admission.py b/dw/server/admission.py index f8e8db28..6e38fa39 100644 --- a/dw/server/admission.py +++ b/dw/server/admission.py @@ -17,7 +17,7 @@ from typing import Any, Dict, List, Optional, Union from fastapi import HTTPException -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator from ..assets import ( activate_asset_dir, @@ -384,6 +384,32 @@ class AcknowledgedCost(BaseModel): description="The repos in plan.downloads_required that were acknowledged", ) + @model_validator(mode="before") + @classmethod + def _from_plan(cls, value): + # The plan validate answered with, passed as-is (#975): its + # downloads_required and estimate.minutes are what was quoted. Only + # when `downloads` is absent - an explicit list, even an empty one, + # is what the caller acknowledged + if ( + not isinstance(value, dict) + or "downloads" in value + or not isinstance(value.get("downloads_required"), list) + ): + return value + bound = { + **value, + "downloads": [ + entry.get("repo") + for entry in value["downloads_required"] + if isinstance(entry, dict) + ], + } + estimate = value.get("estimate") + if "minutes" not in value and isinstance(estimate, dict): + bound["minutes"] = estimate.get("minutes") + return bound + @field_validator("downloads", mode="after") @classmethod def _drop_unnamed(cls, downloads): diff --git a/tests/test_admission.py b/tests/test_admission.py index cc8ff4b9..bf51461c 100644 --- a/tests/test_admission.py +++ b/tests/test_admission.py @@ -271,6 +271,39 @@ def test_a_bound_acknowledgement_naming_a_task_steps_repos_is_admitted( assert response.json()["acknowledged"] == "bound" +def test_the_plan_passed_verbatim_is_a_bound_acknowledgement(server, empty_hub_cache): + """#975: a client that hands back the validate plan unedited has + acknowledged its downloads_required - the refusal's own example (C-F525 + step 5).""" + body = {"workflow": segment_workflow()} + with server() as client: + plan = client.post("/api/validate?sizes=false", json=body).json()["plan"] + assert plan is not None + assert plan["downloads_required"] + + response = client.post("/api/jobs", json={**body, "acknowledged_cost": plan}) + + assert response.status_code == 201, response.text + assert response.json()["acknowledged"] == "bound" + + +def test_a_plan_with_its_downloads_emptied_is_still_refused(server, empty_hub_cache): + """An explicit `downloads` is what was acknowledged, even beside the + plan's own downloads_required.""" + body = {"workflow": segment_workflow()} + with server() as client: + plan = client.post("/api/validate?sizes=false", json=body).json()["plan"] + assert plan is not None + + response = client.post( + "/api/jobs", + json={**body, "acknowledged_cost": {**plan, "downloads": []}}, + ) + + assert response.status_code == 409, response.text + assert response.json()["detail"]["reason"] == "downloads" + + def test_a_rerun_expands_once(server, folds): with server() as client: submitted = client.post( From 4a21d4140b277fa6cff43a1e4ef3f97b420f5134 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Fri, 23 Oct 2026 10:58:57 -0500 Subject: [PATCH 057/105] test(server): #977 - finished_job waits for the history row The job's status turns terminal before _finish records it, so a read of history right after could find nothing under parallel load. Co-Authored-By: Claude Opus 5.5 --- tests/test_server_jobs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_server_jobs.py b/tests/test_server_jobs.py index f140d249..b8f1f249 100644 --- a/tests/test_server_jobs.py +++ b/tests/test_server_jobs.py @@ -66,6 +66,9 @@ def finished_job(manager): while job.status not in TERMINAL_STATES and time.time() < deadline: time.sleep(0.01) assert job.status == "succeeded", job.error + # The status turns terminal a moment before _finish records the row + while manager.history.get(job.id) is None and time.time() < deadline: + time.sleep(0.01) return job From 9da24cc31bdc885882cbcfd5d8227df0a9a23f89 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Fri, 23 Oct 2026 11:19:45 -0500 Subject: [PATCH 058/105] feat(plan): #978 - file-level download entries, single-file task and facexlib declarations Task steps that fetch one file of a repo (upscale with a filename, restore_faces, remove_object, upscale_h3_latents, crop_face_track, interpolate_frames) declare {repo, filename, revision}; downloads_required checks each file in the cache at its revision, sizes it by a HEAD of the file, and lets a whole-repo entry subsume it. restore_faces lists facexlib's weights as URL entries while missing from its package's weights dir, found without importing facexlib. RequiredDownload gains filename and revision. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/DEPENDENCIES.md | 2 +- docs/MCP.md | 8 +- docs/RELEASING.md | 7 +- docs/SERVER.md | 16 +- docs/TASKS.md | 10 +- docs/WORKFLOW_GUIDE.md | 8 +- dw/plan.py | 72 ++++++- dw/server/api_models.py | 5 + dw/tasks/face_track.py | 19 +- dw/tasks/h3_latent_upscale.py | 41 +++- dw/tasks/interpolate_frames.py | 20 +- dw/tasks/model_handlers.py | 9 +- dw/tasks/remove_object.py | 32 ++- dw/tasks/restore_faces.py | 41 +++- dw/tasks/upscale.py | 20 +- dw/tasks/video_handlers.py | 1 + dw/tasks/weights.py | 37 +++- dw_mcp/client.py | 6 +- tests/test_mcp_client.py | 17 +- tests/test_remove_object.py | 13 +- tests/test_task_file_weights.py | 333 +++++++++++++++++++++++++++++ tests/test_task_weights.py | 43 ++-- ui/src/lib/generated/api-schema.ts | 10 + ui/src/lib/generated/openapi.json | 17 ++ 25 files changed, 704 insertions(+), 85 deletions(-) create mode 100644 tests/test_task_file_weights.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 21a2ac9a..bd1e8e4b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -97,7 +97,7 @@ one to open. | Variable constraints | `dw/variable_constraints.py`: `apply_constraints`; the grid arithmetic `aligned` / `aligned_down` | A model's rule about a value is declared in the workflow's `variable_constraints`, and it is checked at validation and again before anything loads. `plan_cuts` (and its static check) rounds render lengths through `aligned` / `aligned_down`, never its own copy, so a planned `num_frames` is one the template's constraint accepts; `crop_face_track` pads its crops to the step's `modulus`/`remainder` grid through `aligned` too (`padding_to_grid`). | `tests/test_variable_constraints.py::TestTheGrid::test_without_snap_an_off_grid_value_is_refused_not_rounded`, `tests/test_variable_constraints.py::TestTheGrid::test_plan_cuts_rounds_through_the_constraint_owner`, `tests/test_face_track.py::TestPadding::test_padding_to_grid_rounds_through_the_constraint_owner`, `tests/test_variable_constraints.py::TestTheRunTimePass::test_an_illegal_value_is_refused_before_anything_loads` | | Bound acknowledgement | `dw/plan.py`: `fingerprint`, `dw/server/admission.py`: `check_bound_acknowledgement` | A bound `acknowledged_cost` is refused with 409 when the plan's fingerprint or its required downloads changed. The fingerprint is keyed only by what can change the output: `DOCUMENTATION_KEYS` (`cost`, `cost_drivers`, `vram_estimate`, descriptions, catalog shape/traits), a constraint's `reason` and a result's role, subfolder and file name are left out, of the workflow and of each composed child; every schema key is sorted into left out or kept. A number digests by value, not spelling (`12.0` as `12`), and the seed stays out (#85). | `tests/test_plan.py::TestFingerprintKeys::test_the_exclusion_lists_are_pinned`, `tests/test_plan.py::TestTheCatalogH3Template::test_a_client_round_trip_keeps_it`, `tests/test_plan.py::TestFingerprintKeys::test_every_top_level_schema_key_is_sorted`, `tests/test_server.py::TestBoundRerun::test_a_rerun_bound_to_a_stale_plan_is_refused`, `tests/test_server.py::TestBoundAcknowledgement::test_a_matching_fingerprint_queues` | | Per-repo gating | `dw/plan.py`: `build_plan` | Each gated repo, including a `loras` entry's repo, is probed and reported on its own. | `tests/test_plan.py::TestDownloadsRequired::test_a_gated_repo_this_token_lacks_access_to_is_blocked` | -| Task weights | `dw/tasks/weights.py`: `argument_or_default`; `dw/tasks/registry.py`: `required_weights_hook`; `dw/plan.py`: `_collect_task_weights` | A model-backed task declares `required_weights(arguments)` on its `register_command`, and `downloads_required` lists those repos beside the pipeline `model_name`s (a shared repo once; a reference or local-path argument none). The declaration and the loader resolve the repo through `argument_or_default`, so they cannot differ. | `tests/test_task_weights.py` | +| Task weights | `dw/tasks/weights.py`: `argument_or_default`; `dw/tasks/registry.py`: `required_weights_hook`; `dw/plan.py`: `_collect_task_weights` | A model-backed task declares `required_weights(arguments)` on its `register_command`, and `downloads_required` lists those repos beside the pipeline `model_name`s (a shared repo once; a reference or local-path argument none). A single-file task lists `{repo, filename, revision}` entries, checked per file against the cache; `restore_faces` also lists facexlib's weights as URL entries. The declaration and the loader resolve the repo through `argument_or_default`, so they cannot differ. | `tests/test_task_weights.py` | | Observed cost | `dw/server/observed_cost.py`: `declared_drivers`, `ObservedCosts._on_this_card`, `ObservedCosts.card` | `observed` comes only from this box's finished jobs on the named card (the one `route` names, for `plan.estimate`), else the server's own card; `ObservedCosts.card` owns a card's (kind, name, default). Rows are matched by card name whatever index they ran on (a job from before jobs carried a `device` counts only while the server is on the default card), and never writes `cost`. Cold and warm runs are reported separately. | `tests/test_observed_cost.py::TestColdIsNotWarm::test_the_two_are_reported_separately_each_with_its_runs`, `tests/test_observed_cost.py::TestByCard::test_only_rows_from_this_card_count_whichever_index_they_had`, `tests/test_observed_cost.py::TestByCard::test_rows_from_before_jobs_carried_a_device_do_not_count_off_the_default_card` | | The estimate | `dw/plan.py`: `estimate`, `_tempered`; the driver scaling is `dw/cost_scaling.py`: `scale_curated`, `_driver_scaling` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. A `cost` entry is measured at the workflow's variable defaults; a run moving `num_inference_steps`, `width`/`height` or `num_frames` (per list entry, summed) scales it in proportion (`scale_curated`, which `dw/server/deps.py` reuses for an inherited figure), basis `derived` with `scaled` naming each axis, times the list-length re-price, and that scaled figure is what a thin observed one blends toward (#866). | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure`, `tests/test_plan.py::TestDriverScaling::test_steps_canvas_and_frames_multiply`, `tests/test_plan.py::TestScalingAccuracy::test_within_thirty_percent` | | Cost drivers | `dw/server/observed_cost.py`: `declared_drivers` | A cost driver that names no declared variable is dropped. | `tests/test_observed_cost.py::TestComparability::test_a_driver_naming_no_variable_is_dropped`, `tests/test_observed_cost.py::TestTheCatalogsDriversAreReal::test_every_declared_driver_is_a_variable_of_its_workflow` | diff --git a/docs/DEPENDENCIES.md b/docs/DEPENDENCIES.md index d015a611..088443bc 100644 --- a/docs/DEPENDENCIES.md +++ b/docs/DEPENDENCIES.md @@ -47,7 +47,7 @@ pyproject; install.ps1 installs it explicitly on Windows. ## Platform-Specific Dependencies -**All platforms (core ML):** peft, transformers, accelerate, safetensors, controlnet_aux, sentencepiece, torchsde, torchao, optimum-quanto, gguf, kornia, ftfy, sdnq, spandrel, facexlib (spandrel + facexlib back the `upscale` and `restore_faces` tasks). On first use, `restore_faces` has facexlib download its face detection and parsing weights (about 110 MB) from facexlib's own GitHub releases into the facexlib package directory. It is the one model download that bypasses the Hugging Face cache, dw's download manager and the `plan.downloads_required` estimate. +**All platforms (core ML):** peft, transformers, accelerate, safetensors, controlnet_aux, sentencepiece, torchsde, torchao, optimum-quanto, gguf, kornia, ftfy, sdnq, spandrel, facexlib (spandrel + facexlib back the `upscale` and `restore_faces` tasks). On first use, `restore_faces` has facexlib download its face detection and parsing weights (about 110 MB) from facexlib's own GitHub releases into the facexlib package directory. It is the one model download that bypasses the Hugging Face cache and dw's download manager; a validate's `plan.downloads_required` lists the two weights as URL entries while they are missing from that directory. **All platforms (sparse attention):** comfy-kitchen (Sol-Attn sparse attention for MiniMax H3's generated-video rows, #931). Its compiled CUDA/HIP kernels ship for Linux and Windows; elsewhere it installs an eager-only build. diff --git a/docs/MCP.md b/docs/MCP.md index 5bad1deb..cbda3203 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -277,7 +277,7 @@ tools) rather than relying on the session pin. | Tool | Arguments | Purpose | | --- | --- | --- | -| `validate_workflow(workflow=None, name=None, workspace=None, arguments=None)` | exactly one of `workflow` (inline definition) or `name` (a stored workflow, as `list_workflows` reports it), optional `workspace`, optional `arguments` | Check a workflow against the schema and against real pipeline signatures. Free and instant. Validating by name uses the workflow file's own directory as the base directory, so it sees what a run would. Returns every schema violation in `errors`, each with the JSON path it sits at, so a draft is fixed in one pass, and a `previous_result:` that names no earlier step is one of them. `warnings` covers what still runs but is probably wrong - a signature mismatch, and, for a list-driven variable, an entry key no step reads, at the entry's path. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's. Pass the same `arguments` you will pass to `run_workflow` and they are checked too - an undeclared or renamed variable name, a value that will not coerce to the declared type, and an `asset:`, `prompt:` or `output:` reference that names nothing this workspace can reach, each reported at `arguments.`. `checked_arguments` lists what was covered, so a `valid: true` about the stored defaults cannot be mistaken for one about your values. A reference set the model would refuse - too many images, videos or audio clips, or, for MiniMax-H3, audio as the only reference - is an error here too, rather than a failure minutes into a run you acknowledged. `run_workflow` makes the same check and refuses a bad argument rather than queuing a job that fails on its first step. A valid answer carries `plan` - the fingerprint, step count, list lengths, `downloads_required` (pipeline `model_name`s and the repos task steps fetch), `estimate` (with `basis`), `host_memory` (`{projected_mb, basis, runs}`, the projected host-RAM peak; `basis` is `observed` from same-key successful runs on this machine or `fit`; absent when there is no figure) and `elided_steps` for the arguments given; quote from it. `steps` counts what will run: a step nothing reads and which saves no file does not run, and is named in `elided_steps` instead | +| `validate_workflow(workflow=None, name=None, workspace=None, arguments=None)` | exactly one of `workflow` (inline definition) or `name` (a stored workflow, as `list_workflows` reports it), optional `workspace`, optional `arguments` | Check a workflow against the schema and against real pipeline signatures. Free and instant. Validating by name uses the workflow file's own directory as the base directory, so it sees what a run would. Returns every schema violation in `errors`, each with the JSON path it sits at, so a draft is fixed in one pass, and a `previous_result:` that names no earlier step is one of them. `warnings` covers what still runs but is probably wrong - a signature mismatch, and, for a list-driven variable, an entry key no step reads, at the entry's path. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's. Pass the same `arguments` you will pass to `run_workflow` and they are checked too - an undeclared or renamed variable name, a value that will not coerce to the declared type, and an `asset:`, `prompt:` or `output:` reference that names nothing this workspace can reach, each reported at `arguments.`. `checked_arguments` lists what was covered, so a `valid: true` about the stored defaults cannot be mistaken for one about your values. A reference set the model would refuse - too many images, videos or audio clips, or, for MiniMax-H3, audio as the only reference - is an error here too, rather than a failure minutes into a run you acknowledged. `run_workflow` makes the same check and refuses a bad argument rather than queuing a job that fails on its first step. A valid answer carries `plan` - the fingerprint, step count, list lengths, `downloads_required` (pipeline `model_name`s and the repos and single files task steps fetch), `estimate` (with `basis`), `host_memory` (`{projected_mb, basis, runs}`, the projected host-RAM peak; `basis` is `observed` from same-key successful runs on this machine or `fit`; absent when there is no figure) and `elided_steps` for the arguments given; quote from it. `steps` counts what will run: a step nothing reads and which saves no file does not run, and is named in `elided_steps` instead | | `list_workspaces()` | — | The server's workspaces and which one this session is using. Each has its own workflows, assets and outputs; the prompt library is shared by all of them. Compact entries are `name`/`default`/`usage` plus any `note` (clipped, `note_truncated` when cut); `detail=true` adds each entry's folder paths | | `use_workspace(name)` | `name` | Work in that workspace for the rest of the session - every later call reads and writes there. This is how to keep your work out of another agent's namespace rather than sharing the default one. Checked against the server, so a typo fails here rather than scoping every later call to nothing. On a `dw.serve --mcp` endpoint the pin is shared by every connected client (#298, see the note above the table) - a `warning` field appears when this call overwrote a pin already pointed away from `default` | | `create_workspace(name, use=False)` | `name`, `use` | Create a workspace. Pass use=true to switch this session to it as well; otherwise the session stays where it was and the result says so. The result names the workspace (`name`, `default`, `current`, `next`), not its folders - `list_workspaces(detail=true)` is the opt-in for those. `use=true` carries the same mounted-server caveat and `warning` field as `use_workspace` | @@ -331,7 +331,7 @@ references written in the same session. | Tool | Arguments | Purpose | | --- | --- | --- | | `list_models()` | — | (Catalog) List what the Hugging Face cache holds, largest first | -| `download_model(repo_id, acknowledged_cost=False)` | `repo_id`, `acknowledged_cost` | Fetch a model repo into the cache. Costs disk and bandwidth, so it passes the gate. Returns as soon as the download starts | +| `download_model(repo_id, acknowledged_cost=False)` | `repo_id`, `acknowledged_cost` | Fetch a model repo into the cache. Costs disk and bandwidth, so it passes the gate. Returns as soon as the download starts. It fetches the whole repo at `main`, so it does not pre-fetch a single pinned file a task entry names; the task fetches that on its first run | | `list_downloads()` | — | List downloads the server is running or recently ran | | `cancel_download(download_id)` | `download_id` | Stop a running download; the record shows `cancel_requested: true` until it is `cancelled`. Partial files stay cached and resume on a retry | | `delete_model(repo, acknowledged_cost=False)` | `repo`, `acknowledged_cost` | Delete every cached revision of one repo. Not recoverable locally | @@ -374,7 +374,7 @@ the server refuses with 409 if the run's shape changed between the quote and the call - a longer list, a stored prompt edited meanwhile, weights that now have to be downloaded - naming the new plan so the agent re-quotes. The `plan` itself, passed unedited, binds the same way: with no `downloads` key, -its `downloads_required` repos and `estimate.minutes` stand in. Bare +its `downloads_required` repos (a file entry by its repo) and `estimate.minutes` stand in. Bare `true` still works and is for a `plan` that came back null; the job records which form it got (`acknowledged: none | boolean | bound`). @@ -390,7 +390,7 @@ The intended loop: verdict covers the stored definition and its stock defaults, not the values you wrote, and its `plan` is the number to say out loud: `estimate.minutes` with its `basis`, plus each `downloads_required` - entry as a line item of its own (task steps' repos are included) + entry as a line item of its own (task steps' repos and single files are included) 2. `run_workflow` with `acknowledged_cost` bound to the plan (`{fingerprint, minutes, downloads}`), or `true` when there was no plan — pass a name straight from `list_workflows` as `workflow_path`; queues the job and returns diff --git a/docs/RELEASING.md b/docs/RELEASING.md index f2b08027..c82c6290 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -14,7 +14,12 @@ when a release ships. A bound `acknowledged_cost` taken from a validate made before this deploy is answered 409 `downloads` after it when the workflow has one of these tasks and its repo is not cached: re-validate and resend the new - `acknowledge`. + `acknowledge`. The same holds for the tasks that fetch single files + (`upscale`, `restore_faces`, `remove_object`, `upscale_h3_latents`, + `crop_face_track`, `interpolate_frames`), and `restore_faces` also lists + facexlib's two weights as URL entries. A `downloads_required` entry gains + the optional `filename` and `revision` fields; the OpenAPI document and + the UI's types carry them. ### 0.15.0 diff --git a/docs/SERVER.md b/docs/SERVER.md index dc3443e2..a45690ed 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -445,7 +445,21 @@ The editor's forms come from these; they are just as usable from scripts: offline) - "unknown" is not "not blocked". `validate_workflow`'s `warnings` carries one line per entry with `access_blocked: true`. Each `from_single_file` URL is `{repo: null, url, gb: null, gated: null, - access_blocked: null}`, since a direct file URL is never gated; and + access_blocked: null}`, since a direct file URL is never gated. + A task that fetches one file rather than a whole repo is listed as a file + entry, `{repo, filename, revision, gb, gated, access_blocked}`: `revision` + is a 40-character commit pin or `null` for `main`; `gb` is that one file's + size, from a HEAD of the file (`null` offline or with `?sizes=false`); + `gated` is `null`, since the HEAD that sizes it does not say; and + `access_blocked` is `true` when that HEAD is refused by the gate, `false` + when it succeeds, `null` when unknown. It is listed when that file at that + revision is not in the hub cache (checked per file, so another cached file + of the same repo does not count), and a whole-repo entry for the same repo + subsumes it. Acknowledgements still compare by `repo`. `restore_faces` + also lists facexlib's weights as URL entries (`detection_Resnet50_Final.pth`, + and `parsing_parsenet.pth` unless `use_parse` is false) while they are + missing from facexlib's weights directory; they download outside the + Hugging Face cache. And `estimate`, `{minutes, basis, device, measured_on, partial, runs}` from this box's own history when it has one and otherwise from the workflow's `cost` block - diff --git a/docs/TASKS.md b/docs/TASKS.md index 008b6d3a..0a455a4e 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -1112,6 +1112,8 @@ face-detail pass that needs the face large and still: | `detector_file` | No | The detector file in that repo, a bare `.onnx` name. Default `face_detection_yunet_2023mar.onnx` | | `device` | No | Where detection runs | +A validate's `downloads_required` lists the detector file as a file entry when it is not cached, pinned at its revision for the default repo and file. + `modulus`, `remainder` and `multiple` follow `plan_cuts`: the model's grid is the workflow's to declare. The defaults (8, 1, 32) are LTX's 8n+1 frames and 32-pixel steps, so a workflow that names none gets the output it always did; @@ -2597,6 +2599,8 @@ Upscale images using spandrel-compatible super-resolution models (ESRGAN, SwinIR | `tile_size` | No | Tile size for large images (default: 512) | | `tile_overlap` | No | Overlap between tiles in pixels (default: 32) | +A validate's `downloads_required` lists the `model_name` / `filename` file when it is not cached, or the whole repo when no `filename` is given (such a repo must hold exactly one weight file); a `previous_result:` or local-path model gives none. + Large images are automatically tiled to avoid GPU memory issues. Models can be loaded from HuggingFace Hub repos or local `.pth`/`.safetensors` files. **Examples:** @@ -2672,6 +2676,8 @@ Restore and enhance faces in images using spandrel-compatible face restoration m | `eye_dist_threshold` | No | Skip faces with eye distance below this (default: 5) | | `upsample_img` | No | Pre-upscaled background image (e.g., from a prior upscale step) | +A validate's `downloads_required` lists the `model_name` / `filename` file when it is not cached (the whole repo when no `filename` is given), and facexlib's two weights as URL entries while they are missing from facexlib's weights directory: `detection_Resnet50_Final.pth` always and `parsing_parsenet.pth` unless `use_parse` is false. Those still download outside the Hugging Face cache. + Models are loaded via spandrel, so any `.pth`/`.safetensors` face restoration weights work. CodeFormer requires `pip install spandrel-extra-arches` (non-commercial license). The detector and parser are loaded once per settings/device and reused across frames; `release_models` drops them with the restorer. **Example:** [restore-faces.json](../workflows/templates/restore-faces.json) — Generate a portrait, then restore faces with GFPGAN v1.4. @@ -2794,7 +2800,7 @@ Erase what a mask covers and fill the hole from the surrounding texture, with no **Weights.** `michaelgold/big-lama`, file `big-lama.safetensors` (about 200 MB), read at revision `98ab7cface89a594b381f32fc7a9f2cae3f91ccc`. `biali/big-lama` holds a byte-identical copy and is the fallback if that repo disappears; it is not tried automatically. LaMa is by advimman (Suvorov et al., *Resolution-robust Large Mask Inpainting with Fourier Convolutions*) and is licensed **Apache-2.0**. The Hub repo's card says MIT, but the upstream licence is the one that applies. The task reads safetensors only: `validate_workflow` refuses a `weight_name` that is not `.safetensors` (the common `big-lama.pt` is a TorchScript file) and a `model_name` that is a local path, and the loader refuses both again at run time. Weights that do not load as an inpainting model are refused too. The network runs in fp32 on every device. -**First run.** The weights download on the first run, about 200 MB. `validate_workflow`'s `plan.downloads_required` does not list task weights yet, so nothing warns of the download beforehand. +**First run.** The weights download on the first run, about 200 MB. A validate's `downloads_required` lists `michaelgold/big-lama` / `big-lama.safetensors` as a file entry (pinned at that revision for the default pair only; another `model_name` or `weight_name` is read at `main`) while it is not cached, so the download is quoted beforehand. **Examples:** @@ -3199,7 +3205,7 @@ Increase video frame rate using RIFE (Real-Time Intermediate Flow Estimation). T | `model_name` | No | HuggingFace repo with RIFE v4.13 weights (default: `imaginairy/rife-interpolation`) | | `filename` | No | Weights filename within the repo (default: `rife-flownet-4.13.2.safetensors`) | -Uses vendored IFNet v4.13 architecture. Weights are downloaded from HuggingFace Hub on first use. +Uses vendored IFNet v4.13 architecture. Weights are downloaded from HuggingFace Hub on first use. A validate's `downloads_required` lists the `model_name` / `filename` file when it is not cached. **Example:** [interpolate-frames.json](../workflows/templates/interpolate-frames.json) — Generate video with Mochi, then 2x interpolate from 30fps to 60fps. diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 6ab290f0..69277ae7 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -1921,10 +1921,10 @@ upscaling only makes sense for a take chosen from something already reviewed, so The upscaler weights (~691 MB, `LBH-123-AI/Minimax_h3_latent_Upscaler`, Apache-2.0, read at a pinned revision) and the H3 VAE download on first use, the same as any other -model. Neither is counted in `plan.downloads_required`, since that walk collects -`from_pretrained_arguments` sources on pipeline steps and does not see a task -argument naming a Hugging Face repo - a box that has run the base pass before but never -this task can still stall mid-run pulling the upscaler. +model. A validate's `plan.downloads_required` lists the upscaler file as a file entry +(`{repo, filename, revision, ...}`) while it is not cached, and the VAE is covered by +the pipeline step's whole `MiniMaxAI/MiniMax-H3` entry, so a box that has run the base +pass before but never this task is told of the upscaler download beforehand. ```json { diff --git a/dw/plan.py b/dw/plan.py index 327795a4..10f36b99 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -37,6 +37,7 @@ ) from .elision import elide_definition from .hub_cache import repo_download_incomplete, scan_models +from .quantized_cache import _cached_file from . import references from .realize import read_sub_workflow, realize_workflow from .security import validate_url @@ -902,7 +903,11 @@ def downloads_required(expanded, base_dir, workflow_dir, cache_dir, lookup_sizes `.incomplete` - plus every `from_single_file` that is a URL, and every repo a task step declares through its command's `required_weights` hook (#975, dw/tasks/weights.py), listed once when a pipeline step loads it - too. Sizes come + too. A task that fetches one file of a repo declares the file, keyed by + `(repo, filename, revision)`: it is checked in the cache file by file + (`_cached_file` - a pinned commit needs no `refs/main`, and another + cached file of the repo does not stand in for it), sized by a HEAD of + that file, and dropped when the same walk needs the repo whole. Sizes come from the hub when asked and are None whenever it does not answer - an offline box is a state, not an error, so nothing here raises or logs above debug. @@ -910,8 +915,9 @@ def downloads_required(expanded, base_dir, workflow_dir, cache_dir, lookup_sizes names = [] variants = {} urls = [] + files = [] _collect_sources(expanded, names, variants, urls) - _collect_task_weights(expanded, names, urls) + _collect_task_weights(expanded, names, files, urls) for path, _arguments in _sub_workflow_paths(expanded): raw = read_sub_workflow(path, base_dir, workflow_dir) if raw is None: @@ -921,7 +927,7 @@ def downloads_required(expanded, base_dir, workflow_dir, cache_dir, lookup_sizes except ValueError: continue _collect_sources(child, names, variants, urls) - _collect_task_weights(child, names, urls) + _collect_task_weights(child, names, files, urls) present = {repo.get("repo_id") for repo in scan_models(cache_dir).get("repos", [])} required = [] for name in names: @@ -938,6 +944,23 @@ def downloads_required(expanded, base_dir, workflow_dir, cache_dir, lookup_sizes if lookup_sizes: entry["gb"], entry["gated"], entry["access_blocked"] = _model_info(name) required.append(entry) + for repo, filename, revision in files: + # A whole-repo entry for the same repo already covers the file + if repo in names or not _is_repo_id(repo): + continue + if _cached_file(repo, filename, revision or "main", cache_dir) is not None: + continue + entry = { + "repo": repo, + "filename": filename, + "revision": revision, + "gb": None, + "gated": None, + "access_blocked": None, + } + if lookup_sizes: + entry["gb"], entry["access_blocked"] = _file_info(repo, filename, revision) + required.append(entry) for url in urls: required.append( { @@ -987,9 +1010,10 @@ def _collect_sources(tree, names, variants, urls): _collect_sources(value, names, variants, urls) -def _collect_task_weights(tree, names, urls): - """Every repo (or URL) a task step in a tree declares, appended after the - pipeline sources and deduplicated against them. Each `task` block's +def _collect_task_weights(tree, names, files, urls): + """Every repo, file or URL a task step in a tree declares, appended after + the pipeline sources and deduplicated against them - a file as a + `(repo, filename, revision)` key in `files`. Each `task` block's command is looked up by name and its hook called on the block's arguments - expanded at the top level; a composed child's are its raw definition, so a `variable:` value there is no repo and gives no entry. @@ -1005,16 +1029,21 @@ def _collect_task_weights(tree, names, urls): task["command"], arguments if isinstance(arguments, dict) else {} ): repo, url = entry.get("repo"), entry.get("url") - if isinstance(repo, str) and repo not in names: + filename = entry.get("filename") + if isinstance(repo, str) and isinstance(filename, str): + key = (repo, filename, entry.get("revision")) + if key not in files: + files.append(key) + elif isinstance(repo, str) and repo not in names: names.append(repo) elif repo is None and isinstance(url, str) and _is_url(url): if url not in urls: urls.append(url) for value in tree.values(): - _collect_task_weights(value, names, urls) + _collect_task_weights(value, names, files, urls) elif isinstance(tree, list): for value in tree: - _collect_task_weights(value, names, urls) + _collect_task_weights(value, names, files, urls) def _task_weights(command, arguments): @@ -1041,7 +1070,8 @@ def gate_warnings(downloads_required): "Hugging Face token - accept its license at " f"https://huggingface.co/{entry['repo']} before running this workflow" for entry in downloads_required - if entry.get("access_blocked") + # A URL entry has no repo to send anyone to + if entry.get("access_blocked") and entry.get("repo") ] @@ -1097,6 +1127,28 @@ def _model_info(name): return gb, gated, access_blocked +def _file_info(repo, filename, revision): + """One file's size in GiB and whether this box's token is blocked from it + - a `(gb, access_blocked)` pair from a HEAD of the file itself, the call + `_probe_gate_blocked` makes. `_model_info` would size the whole repo at + main, overstating one file of a multi-file repo. A tiny file reads 0.01 + rather than 0, which would look like nothing to fetch.""" + try: + metadata = get_hf_file_metadata( + hf_hub_url(repo, filename, revision=revision), + timeout=SIZE_LOOKUP_TIMEOUT, + ) + except GatedRepoError as e: + logger.debug(f"Gate not accepted for {repo}/{filename}: {e}") + return None, True + except Exception as e: + logger.debug(f"No size for {repo}/{filename}: {e}") + return None, None + size = getattr(metadata, "size", None) + gb = max(round(size / GIB, 2), 0.01) if size else None + return gb, False + + def _probe_gate_blocked(name, siblings): """HEAD one real file of a gated repo to see whether this box's token is actually accepted - `model_info` succeeding says nothing either way diff --git a/dw/server/api_models.py b/dw/server/api_models.py index 664c3139..790eadf2 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -608,6 +608,11 @@ class ElidedStep(ApiModel): class RequiredDownload(ApiModel): repo: str | None url: str = sometimes("A from_single_file URL, which has no repo.") + filename: str = sometimes( + "A single weight file within `repo`, for a task that fetches one file " + "rather than the whole repo." + ) + revision: str | None = sometimes("The commit the file is read at; null means main.") gb: int | float | None gated: bool | Literal["auto", "manual"] | None = Field( description="The hub's own `gated` field: false, or how access is granted." diff --git a/dw/tasks/face_track.py b/dw/tasks/face_track.py index 3536f75c..654986cb 100644 --- a/dw/tasks/face_track.py +++ b/dw/tasks/face_track.py @@ -48,6 +48,7 @@ ) from ..variable_constraints import aligned from .video_utils import frames_as_pil_list, load_audio_video +from .weights import argument_or_default, file_entries logger = logging.getLogger("dw") @@ -345,6 +346,20 @@ def pad_frames(crops, before, after): return [crops[i] for i in indices] +def detector_source(arguments): + """The (repo, file, revision) a crop_face_track step loads YuNet from - + `detector_repo` and `detector_file` over the defaults, the default repo + read at its pinned commit and any other at its main.""" + repo = argument_or_default(arguments, "detector_repo", DEFAULT_DETECTOR_REPO) + filename = argument_or_default(arguments, "detector_file", DEFAULT_DETECTOR_FILE) + return repo, filename, DETECTOR_REVISION if repo == DEFAULT_DETECTOR_REPO else None + + +def required_weights(arguments): + """The face detector's weight file, for the plan (dw/tasks/weights.py).""" + return file_entries(*detector_source(arguments)) + + def _detector(repo, filename, min_confidence, device): """A callable running YuNet on an RGB array, loaded once per device.""" import cv2 @@ -352,8 +367,10 @@ def _detector(repo, filename, min_confidence, device): from .. import get_device_type from .model_cache import cached_model + repo, filename, revision = detector_source( + {"detector_repo": repo, "detector_file": filename} + ) check_face_detector_source(repo, filename) - revision = DETECTOR_REVISION if repo == DEFAULT_DETECTOR_REPO else None use_cuda = ( get_device_type(device) == "cuda" diff --git a/dw/tasks/h3_latent_upscale.py b/dw/tasks/h3_latent_upscale.py index bfde1733..d9aae2dc 100644 --- a/dw/tasks/h3_latent_upscale.py +++ b/dw/tasks/h3_latent_upscale.py @@ -23,6 +23,7 @@ import torch from ..result import AudioVideo +from .weights import file_entries logger = logging.getLogger("dw") @@ -207,18 +208,41 @@ def _check_upscaler_source(model_name, weight_name): validate_weight_name(weight_name, (SAFETENSORS_SUFFIX,), subfolders=False) +def upscaler_source(arguments): + """The (repo, file, subfolder, revision) an upscale_h3_latents step + loads - `model_name` and `weight_name` over the defaults. The pin names a + commit of the default repo; a caller naming another repo gets that + repo's main.""" + arguments = arguments or {} + model_name = arguments.get("model_name") or DEFAULT_UPSCALER_REPO + weight_name = arguments.get("weight_name") or DEFAULT_UPSCALER_WEIGHTS + source = KNOWN_UPSCALER_SOURCES.get( + (model_name, weight_name) + ) or KNOWN_UPSCALER_SOURCES.get((model_name, None)) + subfolder, revision = source if source else (None, None) + return model_name, weight_name, subfolder, revision + + +def required_weights(arguments): + """The upscaler's weight file, for the plan (dw/tasks/weights.py). The + H3 VAE that decodes its latents is not listed: the H3 pipeline step that + makes them lists MiniMaxAI/MiniMax-H3 whole (#975 Q4).""" + model_name, weight_name, subfolder, revision = upscaler_source(arguments) + if not isinstance(weight_name, str): + return [] + filename = f"{subfolder}/{weight_name}" if subfolder else weight_name + return file_entries(model_name, filename, revision) + + def _load_upscaler(device, model_name, weight_name): from huggingface_hub import hf_hub_download from .h3_latent_upscaler_model import build_upscaler from .model_cache import cached_model - # The pin names a commit of the default repo; a caller naming another - # repo gets that repo's main - source = KNOWN_UPSCALER_SOURCES.get( - (model_name, weight_name) - ) or KNOWN_UPSCALER_SOURCES.get((model_name, None)) - subfolder, revision = source if source else (None, None) + model_name, weight_name, subfolder, revision = upscaler_source( + {"model_name": model_name, "weight_name": weight_name} + ) def load_net(): from safetensors.torch import load_file @@ -268,8 +292,9 @@ def upscale_h3_latents( input's dtype and on the input's device """ size, (scale_h, scale_w) = target_latent_size(latents, width, height) - model_name = model_name or DEFAULT_UPSCALER_REPO - weight_name = weight_name or DEFAULT_UPSCALER_WEIGHTS + model_name, weight_name, _, _ = upscaler_source( + {"model_name": model_name, "weight_name": weight_name} + ) # Before anything downloads: a value arriving through a variable or an # earlier step was never in the document validation checked _check_upscaler_source(model_name, weight_name) diff --git a/dw/tasks/interpolate_frames.py b/dw/tasks/interpolate_frames.py index c6c47a6c..ed87f25f 100644 --- a/dw/tasks/interpolate_frames.py +++ b/dw/tasks/interpolate_frames.py @@ -14,6 +14,7 @@ from ..shots import rescaled_shots from .tensor_image import pil_to_float_tensor as _pil_to_tensor, float_tensor_to_pil from .video_utils import frames_as_pil_list +from .weights import argument_or_default, file_entries logger = logging.getLogger("dw") @@ -101,6 +102,20 @@ def _interpolate_2x(frames, model): _DEFAULT_RIFE_FILENAME = "rife-flownet-4.13.2.safetensors" +def rife_source(arguments): + """The (repo, file) an interpolate_frames step loads RIFE from - + `model_name` and `filename` over the defaults.""" + return ( + argument_or_default(arguments, "model_name", _DEFAULT_RIFE_REPO), + argument_or_default(arguments, "filename", _DEFAULT_RIFE_FILENAME), + ) + + +def required_weights(arguments): + """The RIFE weight file, for the plan (dw/tasks/weights.py).""" + return file_entries(*rife_source(arguments)) + + def _pad_tensor(t, ph, pw): """Zero-pad a (1, 3, H, W) tensor on the bottom/right to (ph, pw).""" h, w = t.shape[2], t.shape[3] @@ -151,8 +166,9 @@ def _load_rife_model(device, model_name=None, filename=None): from huggingface_hub import hf_hub_download from .model_cache import cached_model - repo_id = model_name if model_name is not None else _DEFAULT_RIFE_REPO - weights_file = filename if filename is not None else _DEFAULT_RIFE_FILENAME + repo_id, weights_file = rife_source( + {"model_name": model_name, "filename": filename} + ) def load_net(): model_path = hf_hub_download(repo_id=repo_id, filename=weights_file) diff --git a/dw/tasks/model_handlers.py b/dw/tasks/model_handlers.py index 2bfc0704..3bbc74c4 100644 --- a/dw/tasks/model_handlers.py +++ b/dw/tasks/model_handlers.py @@ -75,7 +75,10 @@ def _handle_separate_stems(task, arguments, previous_pipelines): @register_command( - "upscale", implementation="dw.tasks.upscale.upscale_image", consumes_device=True + "upscale", + implementation="dw.tasks.upscale.upscale_image", + consumes_device=True, + required_weights="dw.tasks.upscale.required_weights", ) def _handle_upscale(task, arguments, previous_pipelines): """Upscale an image using a spandrel-compatible super-resolution model""" @@ -113,6 +116,7 @@ def _handle_diffusion_upscale(task, arguments, previous_pipelines): "restore_faces", implementation="dw.tasks.restore_faces.restore_faces", consumes_device=True, + required_weights="dw.tasks.restore_faces.required_weights", ) def _handle_restore_faces(task, arguments, previous_pipelines): """Restore faces in an image using a spandrel-compatible face restoration model""" @@ -159,6 +163,7 @@ def _handle_segment(task, arguments, previous_pipelines): "remove_object", implementation="dw.tasks.remove_object.remove_object", consumes_device=True, + required_weights="dw.tasks.remove_object.required_weights", summary=( "Erase what a mask covers and fill the hole from the surrounding " "texture, with no prompt (LaMa). Suits small to medium objects; an " @@ -216,6 +221,7 @@ def _handle_remove_object(task, arguments, previous_pipelines): "interpolate_frames", implementation="dw.tasks.interpolate_frames.interpolate_frames", consumes_device=True, + required_weights="dw.tasks.interpolate_frames.required_weights", ) def _handle_interpolate_frames(task, arguments, previous_pipelines): """Interpolate video frames to increase frame rate""" @@ -230,6 +236,7 @@ def _handle_interpolate_frames(task, arguments, previous_pipelines): "upscale_h3_latents", implementation="dw.tasks.h3_latent_upscale.upscale_h3_latents", consumes_device=True, + required_weights="dw.tasks.h3_latent_upscale.required_weights", summary=( "Resize MiniMax-H3 video latents to a larger canvas (e.g. a 960x544 " "take to 1344x768) without denoising again; decode_h3_latents turns " diff --git a/dw/tasks/remove_object.py b/dw/tasks/remove_object.py index c201c5bb..67e18d93 100644 --- a/dw/tasks/remove_object.py +++ b/dw/tasks/remove_object.py @@ -19,6 +19,7 @@ from ..events import emit_warning from ..task_domains import check_argument, whole_number from .tensor_image import float_tensor_to_pil, pil_to_float_tensor +from .weights import file_entries logger = logging.getLogger("dw") @@ -82,11 +83,23 @@ def _check_remover_source(model_name, weight_name): validate_weight_name(weight_name, (SAFETENSORS_SUFFIX,), subfolders=False) -def _remover_revision(model_name, weight_name): - # The pin names a commit of the default file; another repo is read at its main - if (model_name, weight_name) == (DEFAULT_REMOVER_REPO, DEFAULT_REMOVER_WEIGHTS): - return REMOVER_REVISION - return None +def remover_source(arguments): + """The (repo, file, revision) a remove_object step loads - `model_name` + and `weight_name` over the defaults. The pin names a commit of the + default file; another source is read at its main.""" + arguments = arguments or {} + model_name = arguments.get("model_name") or DEFAULT_REMOVER_REPO + weight_name = arguments.get("weight_name") or DEFAULT_REMOVER_WEIGHTS + pinned = (model_name, weight_name) == ( + DEFAULT_REMOVER_REPO, + DEFAULT_REMOVER_WEIGHTS, + ) + return model_name, weight_name, REMOVER_REVISION if pinned else None + + +def required_weights(arguments): + """The remover's weight file, for the plan (dw/tasks/weights.py).""" + return file_entries(*remover_source(arguments)) def _descriptor_from_state_dict(state_dict): @@ -109,7 +122,9 @@ def _load_remover(device, model_name, weight_name): from .model_cache import cached_model - revision = _remover_revision(model_name, weight_name) + model_name, weight_name, revision = remover_source( + {"model_name": model_name, "weight_name": weight_name} + ) def load_descriptor(): from safetensors.torch import load_file @@ -161,8 +176,9 @@ def remove_object( dilate = whole_number(dilate, "dilate", "remove_object") dilate = DEFAULT_DILATE if dilate is None else dilate check_argument("remove_object", "dilate", dilate) - model_name = model_name or DEFAULT_REMOVER_REPO - weight_name = weight_name or DEFAULT_REMOVER_WEIGHTS + model_name, weight_name, _ = remover_source( + {"model_name": model_name, "weight_name": weight_name} + ) _check_remover_source(model_name, weight_name) if mask_is_empty(mask_image): diff --git a/dw/tasks/restore_faces.py b/dw/tasks/restore_faces.py index 7adb161c..0237b958 100644 --- a/dw/tasks/restore_faces.py +++ b/dw/tasks/restore_faces.py @@ -12,10 +12,47 @@ from PIL import Image from .model_cache import cached_model +from .upscale import required_weights as spandrel_weights, spandrel_source +from .weights import package_dir logger = logging.getLogger("dw") +# The weights facexlib's FaceRestoreHelper fetches itself, into its own +# package's weights dir rather than the Hub cache (det_model +# "retinaface_resnet50"; ParseNet only with use_parse) +FACEXLIB_DETECTION_URL = "https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_Resnet50_Final.pth" +FACEXLIB_PARSING_URL = ( + "https://github.com/xinntao/facexlib/releases/download/v0.2.2/parsing_parsenet.pth" +) + + +def uses_parse(arguments): + """Whether the helper loads ParseNet - `use_parse`, default True.""" + return (arguments or {}).get("use_parse", True) + + +def facexlib_weights(use_parse): + """`{"url"}` for each facexlib weight not yet in facexlib's weights dir + (`/weights`, facexlib's default `model_rootpath`), found without + importing facexlib. Not installed, every weight is listed: over-warning, + never under.""" + package = package_dir("facexlib") + weights = package / "weights" if package else None + urls = [FACEXLIB_DETECTION_URL] + ([FACEXLIB_PARSING_URL] if use_parse else []) + return [ + {"url": url} + for url in urls + if weights is None or not (weights / url.rsplit("/", 1)[1]).is_file() + ] + + +def required_weights(arguments): + """The spandrel face model (as `upscale` declares it) and the facexlib + weights not yet fetched, for the plan (dw/tasks/weights.py).""" + return spandrel_weights(arguments) + facexlib_weights(uses_parse(arguments)) + + def restore_faces(image, model_name, device="cpu", **kwargs): """Restore faces in an image using a spandrel-compatible face restoration model. @@ -45,10 +82,10 @@ def restore_faces(image, model_name, device="cpu", **kwargs): from .upscale import resolve_model_path - filename = kwargs.get("filename", None) + model_name, filename = spandrel_source({**kwargs, "model_name": model_name}) upscale_factor = kwargs.get("upscale_factor", 1) face_size = kwargs.get("face_size", 512) - use_parse = kwargs.get("use_parse", True) + use_parse = uses_parse(kwargs) only_center_face = kwargs.get("only_center_face", False) detection_resize = kwargs.get("detection_resize", 640) eye_dist_threshold = kwargs.get("eye_dist_threshold", 5) diff --git a/dw/tasks/upscale.py b/dw/tasks/upscale.py index c40c3e8b..6d83a7f6 100644 --- a/dw/tasks/upscale.py +++ b/dw/tasks/upscale.py @@ -12,6 +12,7 @@ from .model_cache import cached_model from .tensor_image import pil_to_float_tensor, float_tensor_to_pil +from .weights import file_entries, repo_entries logger = logging.getLogger("dw") @@ -19,6 +20,23 @@ _MAX_PIXELS_NO_TILE = 512 * 512 +def spandrel_source(arguments): + """The (model_name, filename) an upscale or restore_faces step loads - + filename None reads the repo's one weight file (`resolve_model_path`).""" + arguments = arguments or {} + return arguments.get("model_name"), arguments.get("filename") + + +def required_weights(arguments): + """The weight file, or with no `filename` the repo whole - the task needs + such a repo to hold exactly one weight file, so the repo's size is the + file's - for the plan (dw/tasks/weights.py).""" + model_name, filename = spandrel_source(arguments) + if filename is None: + return repo_entries(model_name) + return file_entries(model_name, filename) + + def upscale_image(image, model_name, device="cpu", **kwargs): """Upscale an image using a spandrel-compatible super-resolution model. @@ -42,7 +60,7 @@ def upscale_image(image, model_name, device="cpu", **kwargs): "spandrel is required for upscaling. Install with: pip install spandrel" ) - filename = kwargs.get("filename", None) + model_name, filename = spandrel_source({**kwargs, "model_name": model_name}) tile_size = kwargs.get("tile_size", 512) tile_overlap = kwargs.get("tile_overlap", 32) diff --git a/dw/tasks/video_handlers.py b/dw/tasks/video_handlers.py index 5f2c60fa..814fd5e9 100644 --- a/dw/tasks/video_handlers.py +++ b/dw/tasks/video_handlers.py @@ -286,6 +286,7 @@ def _handle_stabilize_video(task, arguments, previous_pipelines): "crop_face_track", implementation="dw.tasks.face_track.crop_face_track", consumes_device=True, + required_weights="dw.tasks.face_track.required_weights", summary=( "Follow one face through a clip and crop a steady square around it, " "with a per-frame record of the box and a face-detail strength." diff --git a/dw/tasks/weights.py b/dw/tasks/weights.py index 0be7e17a..7c3ea3fd 100644 --- a/dw/tasks/weights.py +++ b/dw/tasks/weights.py @@ -1,7 +1,9 @@ """What a task step fetches from the Hub: the `required_weights` hook's helpers. Every model-backed task module exports `required_weights(arguments) -> -list[dict]` - each dict `{"repo": }` for a whole repo - and its loader +list[dict]` - each dict `{"repo": }` for a whole repo, `{"repo", +"filename", "revision"}` for one file of it (revision None is main), or +`{"url"}` for a fetch from outside the Hub - and its loader reads the repo it loads from the same resolution, so the plan's `downloads_required` (dw/plan.py) and the run cannot disagree about which argument wins over which default (#975). A command names its hook on its @@ -17,6 +19,9 @@ Torch-free on purpose: `dw/plan.py` imports it on every validate. """ +import importlib.util +from pathlib import Path + from huggingface_hub.utils import HFValidationError, validate_repo_id @@ -48,3 +53,33 @@ def repo_entries(*names): if is_hub_repo(name) and {"repo": name} not in entries: entries.append({"repo": name}) return entries + + +def file_entries(repo, filename, revision=None): + """`[{"repo", "filename", "revision"}]` for one file of a hub repo, or + `[]` when the repo is not shaped like a hub id or the file name is not a + plain relative name inside it - the plan checks the cache by joining the + name onto it, so a name climbing out would make that check a probe of + the disk.""" + from ..locations import validate_weight_name + from ..security import SecurityError + + if not is_hub_repo(repo): + return [] + try: + validate_weight_name(filename) + except SecurityError: + return [] + return [{"repo": repo, "filename": filename, "revision": revision}] + + +def package_dir(name): + """The directory a package is installed in, found without importing it, + or None when it is not installed - facexlib's import loads cv2, which a + validate should not pay for.""" + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): + return None + locations = list(spec.submodule_search_locations or []) if spec else [] + return Path(locations[0]) if locations else None diff --git a/dw_mcp/client.py b/dw_mcp/client.py index 3463533f..c83f3c63 100644 --- a/dw_mcp/client.py +++ b/dw_mcp/client.py @@ -215,7 +215,11 @@ def _cost_gate_sentence(plan, acknowledge): f"(basis {estimate.get('basis')})" ) downloads = [ - entry.get("repo") or entry.get("url") + ( + f"{entry['repo']}/{entry['filename']}" + if entry.get("repo") and entry.get("filename") + else entry.get("repo") or entry.get("url") + ) for entry in plan.get("downloads_required") or [] ] if downloads: diff --git a/tests/test_mcp_client.py b/tests/test_mcp_client.py index a8264e32..f1bf1353 100644 --- a/tests/test_mcp_client.py +++ b/tests/test_mcp_client.py @@ -444,7 +444,7 @@ def handler(request): ACKNOWLEDGE = {"fingerprint": "sha256:abc", "minutes": None, "downloads": ["org/model"]} -def _cost_gate_message(minutes, acknowledge=ACKNOWLEDGE): +def _cost_gate_message(minutes, acknowledge=ACKNOWLEDGE, downloads=None): """The 409 the cost gate raises, formatted the way an MCP caller reads it. `acknowledge` is the ready-made body the server sends (None: an older server that sends none).""" @@ -453,7 +453,7 @@ def _cost_gate_message(minutes, acknowledge=ACKNOWLEDGE): "plan": { "fingerprint": "sha256:abc", "estimate": {"minutes": minutes, "basis": "none"}, - "downloads_required": [{"repo": "org/model"}], + "downloads_required": downloads or [{"repo": "org/model"}], }, } if acknowledge is not None: @@ -485,6 +485,19 @@ def test_an_older_server_without_an_acknowledgement_gets_no_resend_sentence(): assert "12.5 minutes" in message +def test_a_file_entry_renders_as_repo_slash_filename(): + downloads = [ + {"repo": "org/whole", "gb": 3.5}, + {"repo": "org/model", "filename": "sub/w.safetensors", "revision": None}, + {"repo": None, "url": "https://example.com/w.pth"}, + ] + message = _cost_gate_message(None, downloads=downloads) + assert ( + "would download org/whole, org/model/sub/w.safetensors, " + "https://example.com/w.pth first" + ) in message + + class TrackingStream(httpx.SyncByteStream): def __init__(self, chunks): self.chunks = chunks diff --git a/tests/test_remove_object.py b/tests/test_remove_object.py index 1d0485bf..fc8989de 100644 --- a/tests/test_remove_object.py +++ b/tests/test_remove_object.py @@ -228,12 +228,15 @@ def test_the_command_takes_the_tasks_device(self, stub): class TestTheWeights: def test_the_pin_is_for_the_default_file_alone(self): - assert ( - remover._remover_revision(DEFAULT_REMOVER_REPO, DEFAULT_REMOVER_WEIGHTS) - == REMOVER_REVISION + assert remover.remover_source({}) == ( + DEFAULT_REMOVER_REPO, + DEFAULT_REMOVER_WEIGHTS, + REMOVER_REVISION, ) - assert ( - remover._remover_revision("biali/big-lama", DEFAULT_REMOVER_WEIGHTS) is None + assert remover.remover_source({"model_name": "biali/big-lama"}) == ( + "biali/big-lama", + DEFAULT_REMOVER_WEIGHTS, + None, ) def test_the_default_downloads_at_the_pin_and_is_cached(self): diff --git a/tests/test_task_file_weights.py b/tests/test_task_file_weights.py new file mode 100644 index 00000000..e3304813 --- /dev/null +++ b/tests/test_task_file_weights.py @@ -0,0 +1,333 @@ +"""#978: tasks that fetch one file of a repo declare the file, the plan checks +the cache for that file, and facexlib's own downloads are listed as URLs.""" + +import copy +import subprocess +import sys +import textwrap + +import pytest + +import dw.plan +from dw.plan import build_plan, downloads_required, gate_warnings +from dw.tasks import face_track, h3_latent_upscale, interpolate_frames +from dw.tasks import remove_object, restore_faces, upscale +from dw.workflow import workflow_from_definition + +PIN = "a" * 40 + + +def workflow(*task_steps): + return { + "id": "file_weights", + "description": "docs only", + "summary": "docs only", + "steps": [ + { + "name": f"t{i}", + "task": {"command": command, "arguments": arguments}, + "result": {"content_type": "image/png"}, + } + for i, (command, arguments) in enumerate(task_steps) + ], + } + + +@pytest.fixture +def hub(tmp_path): + return tmp_path / "hub" + + +@pytest.fixture +def plan(tmp_path, hub, monkeypatch): + """build_plan against an empty fake hub cache, offline, with facexlib's + weights all present so only the hub entries are in play.""" + monkeypatch.setattr(dw.plan, "scan_models", lambda cache_dir=None: {"repos": []}) + weights = tmp_path / "facexlib" / "weights" + weights.mkdir(parents=True) + for url in ( + restore_faces.FACEXLIB_DETECTION_URL, + restore_faces.FACEXLIB_PARSING_URL, + ): + (weights / url.rsplit("/", 1)[1]).write_bytes(b"w") + monkeypatch.setattr(restore_faces, "package_dir", lambda name: weights.parent) + + def make(spec): + candidate = workflow_from_definition( + copy.deepcopy(spec), str(tmp_path), str(tmp_path), str(tmp_path) + ) + return build_plan( + candidate, {}, device="cuda", cache_dir=str(hub), lookup_sizes=False + )["downloads_required"] + + return make + + +def cache(hub, repo, filename, commit=PIN, ref=None): + """A file in the fake hub cache at `commit`, with `refs/` naming it + only when `ref` is given.""" + root = hub / f"models--{repo.replace('/', '--')}" + path = root / "snapshots" / commit / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"w") + if ref: + (root / "refs").mkdir(exist_ok=True) + (root / "refs" / ref).write_text(commit) + + +class TestDeclarations: + def test_upscale_with_a_filename_declares_the_file(self): + assert upscale.required_weights( + {"model_name": "org/up", "filename": "x.pth"} + ) == [{"repo": "org/up", "filename": "x.pth", "revision": None}] + + def test_upscale_without_a_filename_declares_the_repo(self): + assert upscale.required_weights({"model_name": "org/up"}) == [ + {"repo": "org/up"} + ] + + def test_restore_faces_declares_its_spandrel_file(self, monkeypatch, tmp_path): + monkeypatch.setattr(restore_faces, "package_dir", lambda name: tmp_path) + (tmp_path / "weights").mkdir() + for url in ( + restore_faces.FACEXLIB_DETECTION_URL, + restore_faces.FACEXLIB_PARSING_URL, + ): + (tmp_path / "weights" / url.rsplit("/", 1)[1]).write_bytes(b"w") + assert restore_faces.required_weights( + {"model_name": "org/gfp", "filename": "g.pth"} + ) == [{"repo": "org/gfp", "filename": "g.pth", "revision": None}] + + def test_remove_object_default_is_pinned(self): + assert remove_object.required_weights({}) == [ + { + "repo": remove_object.DEFAULT_REMOVER_REPO, + "filename": remove_object.DEFAULT_REMOVER_WEIGHTS, + "revision": remove_object.REMOVER_REVISION, + } + ] + + def test_remove_object_override_reads_main(self): + assert remove_object.required_weights( + {"model_name": "org/lama", "weight_name": "l.safetensors"} + ) == [{"repo": "org/lama", "filename": "l.safetensors", "revision": None}] + + def test_h3_upscaler_default_is_its_subfolder_file_at_the_pin(self): + assert h3_latent_upscale.required_weights({}) == [ + { + "repo": h3_latent_upscale.DEFAULT_UPSCALER_REPO, + "filename": f"{h3_latent_upscale.DEFAULT_UPSCALER_SUBFOLDER}/" + f"{h3_latent_upscale.DEFAULT_UPSCALER_WEIGHTS}", + "revision": h3_latent_upscale.UPSCALER_REVISION, + } + ] + + def test_h3_upscaler_known_source_has_its_folder_and_pin(self): + repo = h3_latent_upscale.LMS_UPSCALER_REPO + weight = "h3_upscaler_lms_v0.1.safetensors" + folder, commit = h3_latent_upscale.KNOWN_UPSCALER_SOURCES[(repo, weight)] + assert h3_latent_upscale.required_weights( + {"model_name": repo, "weight_name": weight} + ) == [{"repo": repo, "filename": f"{folder}/{weight}", "revision": commit}] + + def test_h3_upscaler_unknown_source_reads_its_root_at_main(self): + assert h3_latent_upscale.required_weights( + {"model_name": "org/h3up", "weight_name": "u.safetensors"} + ) == [{"repo": "org/h3up", "filename": "u.safetensors", "revision": None}] + + def test_face_track_default_is_pinned(self): + assert face_track.required_weights({}) == [ + { + "repo": face_track.DEFAULT_DETECTOR_REPO, + "filename": face_track.DEFAULT_DETECTOR_FILE, + "revision": face_track.DETECTOR_REVISION, + } + ] + + def test_face_track_pin_is_for_the_default_repo_alone(self): + assert face_track.required_weights( + {"detector_repo": "org/yunet", "detector_file": "y.onnx"} + ) == [{"repo": "org/yunet", "filename": "y.onnx", "revision": None}] + + def test_interpolate_frames_default_and_override(self): + assert interpolate_frames.required_weights({}) == [ + { + "repo": "imaginairy/rife-interpolation", + "filename": "rife-flownet-4.13.2.safetensors", + "revision": None, + } + ] + assert interpolate_frames.required_weights( + {"model_name": "org/rife", "filename": "r.safetensors"} + ) == [{"repo": "org/rife", "filename": "r.safetensors", "revision": None}] + + @pytest.mark.parametrize( + "arguments", + [ + {"model_name": "previous_result:m", "filename": "r.safetensors"}, + {"model_name": "/models/rife", "filename": "r.safetensors"}, + {"model_name": "org/rife", "filename": "../../etc/passwd"}, + {"model_name": "org/rife", "filename": "/abs/r.safetensors"}, + ], + ) + def test_a_reference_path_or_climbing_name_gives_no_entry(self, arguments): + assert interpolate_frames.required_weights(arguments) == [] + + +class TestFacexlib: + def test_not_installed_lists_both_urls(self, monkeypatch): + monkeypatch.setattr(restore_faces, "package_dir", lambda name: None) + assert restore_faces.required_weights({}) == [ + {"url": restore_faces.FACEXLIB_DETECTION_URL}, + {"url": restore_faces.FACEXLIB_PARSING_URL}, + ] + + def test_without_parse_only_detection_is_listed(self, monkeypatch): + monkeypatch.setattr(restore_faces, "package_dir", lambda name: None) + assert restore_faces.required_weights({"use_parse": False}) == [ + {"url": restore_faces.FACEXLIB_DETECTION_URL} + ] + + def test_a_weight_already_fetched_is_omitted(self, monkeypatch, tmp_path): + (tmp_path / "weights").mkdir() + (tmp_path / "weights" / "detection_Resnet50_Final.pth").write_bytes(b"w") + monkeypatch.setattr(restore_faces, "package_dir", lambda name: tmp_path) + assert restore_faces.required_weights({}) == [ + {"url": restore_faces.FACEXLIB_PARSING_URL} + ] + + def test_the_plan_lists_them_as_url_entries(self, tmp_path, monkeypatch): + monkeypatch.setattr(restore_faces, "package_dir", lambda name: None) + monkeypatch.setattr( + dw.plan, "scan_models", lambda cache_dir=None: {"repos": []} + ) + spec = workflow(("restore_faces", {"use_parse": False})) + required = downloads_required( + spec, str(tmp_path), str(tmp_path), str(tmp_path / "hub"), False + ) + assert [(d["repo"], d.get("url")) for d in required] == [ + (None, restore_faces.FACEXLIB_DETECTION_URL) + ] + + def test_the_hook_does_not_import_facexlib_or_cv2(self): + """torch is not asserted: importing dw loads it already.""" + script = textwrap.dedent( + """ + import sys + from dw.tasks.registry import required_weights_hook + import dw.tasks.model_handlers + required_weights_hook("restore_faces")({"model_name": "org/gfp"}) + print(sorted({"facexlib", "cv2"} & set(sys.modules))) + """ + ) + out = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=True + ) + assert out.stdout.strip().splitlines()[-1] == "[]" + + +class TestThePlanChecksTheFile: + def test_an_uncached_file_is_listed_with_its_key(self, plan): + assert plan(workflow(("interpolate_frames", {}))) == [ + { + "repo": "imaginairy/rife-interpolation", + "filename": "rife-flownet-4.13.2.safetensors", + "revision": None, + "gb": None, + "gated": None, + "access_blocked": None, + } + ] + + def test_a_pinned_file_is_cached_without_refs_main(self, plan, hub): + cache( + hub, + face_track.DEFAULT_DETECTOR_REPO, + face_track.DEFAULT_DETECTOR_FILE, + commit=face_track.DETECTOR_REVISION, + ) + assert plan(workflow(("crop_face_track", {}))) == [] + + def test_a_file_at_main_is_cached_through_refs_main(self, plan, hub): + cache(hub, "org/rife", "r.safetensors", ref="main") + spec = workflow( + ( + "interpolate_frames", + {"model_name": "org/rife", "filename": "r.safetensors"}, + ) + ) + assert plan(spec) == [] + + def test_another_file_of_the_repo_does_not_stand_in(self, plan, hub): + cache(hub, "org/rife", "other.safetensors", ref="main") + spec = workflow( + ( + "interpolate_frames", + {"model_name": "org/rife", "filename": "r.safetensors"}, + ) + ) + assert [(d["repo"], d["filename"]) for d in plan(spec)] == [ + ("org/rife", "r.safetensors") + ] + + def test_a_whole_repo_entry_subsumes_the_file(self, plan): + spec = workflow( + ("upscale", {"model_name": "org/up"}), + ("upscale", {"model_name": "org/up", "filename": "x.pth"}), + ) + assert [(d["repo"], d.get("filename")) for d in plan(spec)] == [ + ("org/up", None) + ] + + def test_two_steps_naming_one_file_list_it_once(self, plan): + spec = workflow(("interpolate_frames", {}), ("interpolate_frames", {})) + assert len(plan(spec)) == 1 + + +class TestFileInfo: + def test_sizes_the_file_from_its_head(self, monkeypatch): + seen = {} + + class Meta: + size = 3 * 1024**3 + + def head(url, timeout=None): + seen["url"] = url + return Meta() + + monkeypatch.setattr(dw.plan, "get_hf_file_metadata", head) + assert dw.plan._file_info("org/r", "sub/w.safetensors", PIN) == (3.0, False) + assert PIN in seen["url"] and seen["url"].endswith("sub/w.safetensors") + + def test_a_tiny_file_is_not_zero(self, monkeypatch): + class Meta: + size = 1000 + + monkeypatch.setattr(dw.plan, "get_hf_file_metadata", lambda *a, **k: Meta()) + assert dw.plan._file_info("org/r", "w.onnx", None) == (0.01, False) + + def test_a_gated_file_is_blocked(self, monkeypatch): + import httpx + from huggingface_hub.utils import GatedRepoError + + response = httpx.Response(403, request=httpx.Request("GET", "https://hf.co/x")) + + def gated(*a, **k): + raise GatedRepoError("no access", response=response) + + monkeypatch.setattr(dw.plan, "get_hf_file_metadata", gated) + assert dw.plan._file_info("org/r", "w.onnx", None) == (None, True) + + def test_offline_is_unknown(self, monkeypatch): + def offline(*a, **k): + raise OSError("offline") + + monkeypatch.setattr(dw.plan, "get_hf_file_metadata", offline) + assert dw.plan._file_info("org/r", "w.onnx", None) == (None, None) + + +def test_gate_warnings_skip_an_entry_with_no_repo(): + assert ( + gate_warnings([{"repo": None, "url": "https://x/y", "access_blocked": True}]) + == [] + ) diff --git a/tests/test_task_weights.py b/tests/test_task_weights.py index f6245ed1..771ce7c4 100644 --- a/tests/test_task_weights.py +++ b/tests/test_task_weights.py @@ -364,20 +364,6 @@ def test_generate_text_loads_the_repo_it_declares(self, monkeypatch, arguments): r"hf_hub_download|from_pretrained|from_hparams|hf_pipeline\(|pipeline\(\s*[\"']" ) -# Stage B' of #975: these modules load Hub weights and have no hook yet. Delete -# a name here when its module gains `required_weights` - the pin below fails -# for a name that has one, so the list only shrinks. -# (restore_faces reaches the Hub through upscale.resolve_model_path, so the -# text scan does not see it; it is listed because it loads weights all the same.) -NOT_YET_DECLARED = { - "upscale", - "restore_faces", - "remove_object", - "h3_latent_upscale", - "face_track", - "interpolate_frames", -} - # Modules that fetch only outside the Hugging Face cache would be listed here # with the reason; none do - voice_attribution's demucs download shares a module # with a speechbrain load, which it declares. @@ -413,6 +399,18 @@ def hub_loading_modules(): return found +# Commands that share a module with a declared hook but name none, and why +LOADS_NOTHING_THE_PLAN_LISTS = { + # demucs downloads outside the Hugging Face cache + "separate_stems": "voice_attribution's demucs", + # Decodes with MiniMaxAI/MiniMax-H3's VAE, which the H3 pipeline step + # that made the latents lists whole (#975 Q4) + "decode_h3_latents": "the H3 repo, listed by its pipeline step", + # Pastes crops back; loads no weights + "paste_face_track": "nothing", +} + + class TestEveryHubLoadIsDeclared: def test_the_scan_finds_the_modules_that_load_from_the_hub(self): found = hub_loading_modules() @@ -431,7 +429,6 @@ def test_a_module_that_loads_from_the_hub_declares_required_weights(self): name for name, path in task_modules() if name in hub_loading_modules() - and name not in NOT_YET_DECLARED and name not in OUTSIDE_THE_HF_CACHE and not declares_required_weights(path) ) @@ -440,14 +437,6 @@ def test_a_module_that_loads_from_the_hub_declares_required_weights(self): "plan cannot list what they download (#975)" ) - def test_the_allow_list_names_only_modules_still_without_a_hook(self): - stale = sorted( - name - for name in NOT_YET_DECLARED - if declares_required_weights(os.path.join(TASKS, f"{name}.py")) - ) - assert not stale, f"{stale} are declared now: drop them from NOT_YET_DECLARED" - def test_every_registered_command_in_a_declaring_module_names_its_hook(self): from dw.tasks import registry @@ -455,18 +444,14 @@ def test_every_registered_command_in_a_declaring_module_names_its_hook(self): declaring = { name for name, path in task_modules() - if name in hub_loading_modules() - and name not in NOT_YET_DECLARED - and declares_required_weights(path) + if name in hub_loading_modules() and declares_required_weights(path) } unnamed = [] for command, rules in registry._COMMAND_RULES.items(): module = (rules.get("implementation") or "").split(".") if len(module) < 4 or module[:2] != ["dw", "tasks"]: continue - # demucs downloads outside the Hugging Face cache, so the stem - # separator in voice_attribution's module names no hook - if command == "separate_stems": + if command in LOADS_NOTHING_THE_PLAN_LISTS: continue if module[2] in declaring and not rules.get("required_weights"): unnamed.append(command) diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 8a36dd5a..a544f312 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -3442,6 +3442,11 @@ export interface components { RequiredDownload: { /** Access Blocked */ access_blocked: boolean | null; + /** + * Filename + * @description A single weight file within `repo`, for a task that fetches one file rather than the whole repo. + */ + filename?: string; /** * Gated * @description The hub's own `gated` field: false, or how access is granted. @@ -3451,6 +3456,11 @@ export interface components { gb: number | null; /** Repo */ repo: string | null; + /** + * Revision + * @description The commit the file is read at; null means main. + */ + revision?: string | null; /** * Url * @description A from_single_file URL, which has no repo. diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index b1e5cb53..d2801ffe 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -3883,6 +3883,11 @@ ], "title": "Access Blocked" }, + "filename": { + "description": "A single weight file within `repo`, for a task that fetches one file rather than the whole repo.", + "title": "Filename", + "type": "string" + }, "gated": { "anyOf": [ { @@ -3927,6 +3932,18 @@ ], "title": "Repo" }, + "revision": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The commit the file is read at; null means main.", + "title": "Revision" + }, "url": { "description": "A from_single_file URL, which has no repo.", "title": "Url", From b72ca04a5fdfaccd64907f762a363200556e56b2 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Fri, 23 Oct 2026 11:29:38 -0500 Subject: [PATCH 059/105] docs(architecture): #978 - task weights row names each task's source resolution The arch review found the row credited argument_or_default with keeping declaration and loader in step; single-file tasks share a *_source function instead, and remover_source/upscaler_source treat any falsy value as absent. Names file_entries, package_dir and _file_info too. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index bd1e8e4b..84c2b1d4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -97,7 +97,7 @@ one to open. | Variable constraints | `dw/variable_constraints.py`: `apply_constraints`; the grid arithmetic `aligned` / `aligned_down` | A model's rule about a value is declared in the workflow's `variable_constraints`, and it is checked at validation and again before anything loads. `plan_cuts` (and its static check) rounds render lengths through `aligned` / `aligned_down`, never its own copy, so a planned `num_frames` is one the template's constraint accepts; `crop_face_track` pads its crops to the step's `modulus`/`remainder` grid through `aligned` too (`padding_to_grid`). | `tests/test_variable_constraints.py::TestTheGrid::test_without_snap_an_off_grid_value_is_refused_not_rounded`, `tests/test_variable_constraints.py::TestTheGrid::test_plan_cuts_rounds_through_the_constraint_owner`, `tests/test_face_track.py::TestPadding::test_padding_to_grid_rounds_through_the_constraint_owner`, `tests/test_variable_constraints.py::TestTheRunTimePass::test_an_illegal_value_is_refused_before_anything_loads` | | Bound acknowledgement | `dw/plan.py`: `fingerprint`, `dw/server/admission.py`: `check_bound_acknowledgement` | A bound `acknowledged_cost` is refused with 409 when the plan's fingerprint or its required downloads changed. The fingerprint is keyed only by what can change the output: `DOCUMENTATION_KEYS` (`cost`, `cost_drivers`, `vram_estimate`, descriptions, catalog shape/traits), a constraint's `reason` and a result's role, subfolder and file name are left out, of the workflow and of each composed child; every schema key is sorted into left out or kept. A number digests by value, not spelling (`12.0` as `12`), and the seed stays out (#85). | `tests/test_plan.py::TestFingerprintKeys::test_the_exclusion_lists_are_pinned`, `tests/test_plan.py::TestTheCatalogH3Template::test_a_client_round_trip_keeps_it`, `tests/test_plan.py::TestFingerprintKeys::test_every_top_level_schema_key_is_sorted`, `tests/test_server.py::TestBoundRerun::test_a_rerun_bound_to_a_stale_plan_is_refused`, `tests/test_server.py::TestBoundAcknowledgement::test_a_matching_fingerprint_queues` | | Per-repo gating | `dw/plan.py`: `build_plan` | Each gated repo, including a `loras` entry's repo, is probed and reported on its own. | `tests/test_plan.py::TestDownloadsRequired::test_a_gated_repo_this_token_lacks_access_to_is_blocked` | -| Task weights | `dw/tasks/weights.py`: `argument_or_default`; `dw/tasks/registry.py`: `required_weights_hook`; `dw/plan.py`: `_collect_task_weights` | A model-backed task declares `required_weights(arguments)` on its `register_command`, and `downloads_required` lists those repos beside the pipeline `model_name`s (a shared repo once; a reference or local-path argument none). A single-file task lists `{repo, filename, revision}` entries, checked per file against the cache; `restore_faces` also lists facexlib's weights as URL entries. The declaration and the loader resolve the repo through `argument_or_default`, so they cannot differ. | `tests/test_task_weights.py` | +| Task weights | each fetching task's `*_source(arguments)` (e.g. `dw/tasks/remove_object.py`: `remover_source`); `dw/tasks/weights.py`: `argument_or_default`, `file_entries`, `package_dir`; `dw/tasks/registry.py`: `required_weights_hook`; `dw/plan.py`: `_collect_task_weights`, `_file_info` | A model-backed task declares `required_weights(arguments)` on its `register_command`, and `downloads_required` lists those repos beside the pipeline `model_name`s (a shared repo once; a reference or local-path argument none). A single-file task lists `{repo, filename, revision}` entries, checked per file against the cache; `restore_faces` also lists facexlib's weights as URL entries. A task's declaration and its loader read the same resolution of which argument wins (a single-file task's `*_source(arguments)`, an overridable whole-repo task's `argument_or_default` call, a fixed repo's module constant), so they cannot differ; `argument_or_default` is the shared helper only where `None` means the default (`remover_source` and `upscaler_source` treat any falsy value as absent; `spandrel_source` has no default). `file_entries` builds a file entry, `package_dir` finds facexlib's weights dir without importing it, and `_file_info` sizes one file. | `tests/test_task_weights.py` | | Observed cost | `dw/server/observed_cost.py`: `declared_drivers`, `ObservedCosts._on_this_card`, `ObservedCosts.card` | `observed` comes only from this box's finished jobs on the named card (the one `route` names, for `plan.estimate`), else the server's own card; `ObservedCosts.card` owns a card's (kind, name, default). Rows are matched by card name whatever index they ran on (a job from before jobs carried a `device` counts only while the server is on the default card), and never writes `cost`. Cold and warm runs are reported separately. | `tests/test_observed_cost.py::TestColdIsNotWarm::test_the_two_are_reported_separately_each_with_its_runs`, `tests/test_observed_cost.py::TestByCard::test_only_rows_from_this_card_count_whichever_index_they_had`, `tests/test_observed_cost.py::TestByCard::test_rows_from_before_jobs_carried_a_device_do_not_count_off_the_default_card` | | The estimate | `dw/plan.py`: `estimate`, `_tempered`; the driver scaling is `dw/cost_scaling.py`: `scale_curated`, `_driver_scaling` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. A `cost` entry is measured at the workflow's variable defaults; a run moving `num_inference_steps`, `width`/`height` or `num_frames` (per list entry, summed) scales it in proportion (`scale_curated`, which `dw/server/deps.py` reuses for an inherited figure), basis `derived` with `scaled` naming each axis, times the list-length re-price, and that scaled figure is what a thin observed one blends toward (#866). | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure`, `tests/test_plan.py::TestDriverScaling::test_steps_canvas_and_frames_multiply`, `tests/test_plan.py::TestScalingAccuracy::test_within_thirty_percent` | | Cost drivers | `dw/server/observed_cost.py`: `declared_drivers` | A cost driver that names no declared variable is dropped. | `tests/test_observed_cost.py::TestComparability::test_a_driver_naming_no_variable_is_dropped`, `tests/test_observed_cost.py::TestTheCatalogsDriversAreReal::test_every_declared_driver_is_a_variable_of_its_workflow` | From 3b983fc0ac2691f611f413ecbca88b9264e35006 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Fri, 23 Oct 2026 11:41:33 -0500 Subject: [PATCH 060/105] docs(proposals): #975 design record - task weights in downloads_required Co-Authored-By: Claude Opus 5.5 --- .../complete/remove-object-complete.md | 6 +- .../task-weights-downloads-complete.md | 156 ++++++++++++++++++ docs/proposals/todo.md | 8 + 3 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 docs/proposals/complete/task-weights-downloads-complete.md diff --git a/docs/proposals/complete/remove-object-complete.md b/docs/proposals/complete/remove-object-complete.md index bf1556df..2036635e 100644 --- a/docs/proposals/complete/remove-object-complete.md +++ b/docs/proposals/complete/remove-object-complete.md @@ -147,8 +147,8 @@ $2–3 for B. ## Follow-up -- **#975:** `plan.downloads_required` doesn't see task weights (upscale, - restore_faces, remove_object). The first `remove_object` run downloads - about 200 MB without a warning, and TASKS.md says so. +- **#975** (shipped 2026-10-23): `plan.downloads_required` now lists task + weights, including `remove_object`'s pinned big-lama file. Record: + `task-weights-downloads-complete.md`. - The templates' `vram_gb` figures (1 and 4) are estimates from model sizes, not measurements. Replace them when one is measured. diff --git a/docs/proposals/complete/task-weights-downloads-complete.md b/docs/proposals/complete/task-weights-downloads-complete.md new file mode 100644 index 00000000..d68287db --- /dev/null +++ b/docs/proposals/complete/task-weights-downloads-complete.md @@ -0,0 +1,156 @@ +# Task weights in `plan.downloads_required` (#975) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out session, +2026-10-23). Filed from #973 (stage A of #956) after `remove_object`'s first +run downloaded about 200 MB that nobody quoted. Plan v1 was designed on +#975. Don's review made v2, and Don approved v2. It was built as two stages, +#977 and #978. The plan comment on #975 stays the full design. This doc +records what shipped. + +## The problem + +`validate_workflow`'s `plan.downloads_required` walked only pipeline weights +(`from_pretrained_arguments`, `loras`, URL `from_single_file`). Every task +that fetched weights did so with nothing quoted, and nothing in the bound +acknowledgement (#85) covered it. The issue named `upscale`, +`restore_faces` and `remove_object` (0.06–0.2 GB files). The design sweep +found bigger unlisted ones: `remove_background` (RMBG-2.0, ~1 GB), +`segment` (~1.8 GB), `text_generation` (Qwen2.5-1.5B, ~3 GB), +`diffusion_upscale` (16 GB), speech and transcription. + +## Verdict: build both stages + +Demand was thin (no field report), but the field is a contract: it is what +a bound acknowledgement checks, so a list that silently leaves things out +is worse than none (Don, review). The field had been wrong three times +before in the same way (#382, #894, #151). + +Non-goals stayed out: no task's download behaviour changed; facexlib is +reported, not mirrored (#616); `download_model` still fetches whole repos +at `main`; no UI change; zoe_depth, dw_pose and demucs (fetched outside the +HF cache) stay unlisted and documented. + +## Plan history + +- **v1:** one hook per task, a per-file cache check and HEAD sizing, and + `(repo, filename, revision)` keying. Stage A was single-file tasks, B the + whole-repo ones. +- **v2 (approved), Don's four changes:** + 1. facexlib's weights dir is derived with `importlib.util.find_spec`, never + imported (its `misc.py` imports `cv2` and `torch`). + 2. Each loader reads its source from the same resolution as its + declaration, so the two can't disagree. + 3. Q4: no H3 VAE subfolder entry (every H3 box lists `MiniMaxAI/MiniMax-H3` + whole); `subfolder` dropped from the shape. + 4. Q1: re-split. A′ is the hook plus the multi-GB whole-repo tasks; B′ is the + file-level detail, so if either were cut it would be the smaller one. + +Corrections the plan made to the issue's premise: the repo-level cache +check is wrong for single files (a pinned `hf_hub_download` writes no +`refs/main`, and any other cached file of the repo would mask the wanted +one); `_model_info` sizes the whole repo, not one file; dedup was by repo +name only. + +## What was built + +### Stage A′ (#977): the hook, the plan walk, whole-repo declarations + +Commits 36305b36 (code) and b0d0d857 (tests, docs), merged as c6ff498a. The +bounce fix 138bbc42 was merged as a27b61be. +- **Hook:** `required_weights(arguments)`, named on `register_command` and + looked up by `dw/tasks/registry.py:required_weights_hook`. Helpers are in + `dw/tasks/weights.py`. `dw/plan.py:_collect_task_weights` walks the + expanded definition and each composed child. Task repos join the same + list as pipeline repos, so they share its shape filter, cache test and + sizing. A shared repo is listed once. A reference or local-path argument + gives no entry. A hook that raises gives no entry and never fails the + validate. +- **Declarations:** segment (`model_name` detector, `sam_model_name` + segmenter), remove_background, diffusion_upscale, transcribe_audio (and + check_script through it), generate_speech (plus speechbrain x-vector with + `speaker_embedding`), attribute_voices (ECAPA only), text_generation, + image_to_text, judge, and the image processors (depth, depth_estimator, + controlnet_aux detectors, sam, segmentation). +- **Pin:** `tests/test_task_weights.py::TestEveryHubLoadIsDeclared`. +- **Bound acknowledgement fix (the bounce):** `AcknowledgedCost._from_plan` + in `dw/server/admission.py` reads a validate `plan` sent unedited as the + acknowledgement. Its `downloads_required` and `estimate.minutes` become + `downloads` and `minutes`. An explicit `downloads`, even `[]`, wins. +- **Deviations:** judge and image_to_text default to SmolVLM-256M (what + they load), and Qwen applies only to text_generation without an `image`. + depth_estimator now passes transformers' default (`Intel/dpt-large` at a + pin) explicitly, which loads the same weights as before. A + `system_prompt.from_hub` file is not listed. The first validate of a + workflow with a task step imports that task's module, which costs a few + seconds once per server process. +- **Test-only:** `tests/test_server_jobs.py::finished_job` waits for the + history row (a parallel-load flake that tripped the hand-off gate). + +### Stage B′ (#978): file-level entries, single-file tasks, facexlib + +Commit 9da24cc3, merged as d5bb9f8b. The map-row fix b72ca04a was merged as +b01f1238. +- **Entries** keyed by `(repo, filename, revision)`. Each file is checked on + its own with `_cached_file` at `revision or "main"` and sized by a HEAD of + that file (`_file_info`). A whole-repo entry for the same repo subsumes it. + `RequiredDownload` gained optional `filename`/`revision`, and the + OpenAPI and UI types were regenerated. The MCP cost-gate line renders + `repo/filename`. +- **Declarations** (hook and loader share each task's `*_source`): + `upscale` (file, or whole repo with no `filename`), `restore_faces` + (spandrel entry plus facexlib URLs), `remove_object` (pinned only for the + default pair), `upscale_h3_latents` (the upscaler file at its pin, no VAE + entry), `crop_face_track` (YuNet, pinned for the default repo), + `interpolate_frames` (RIFE, no pin). +- **facexlib:** `detection_Resnet50_Final.pth` always, and + `parsing_parsenet.pth` unless `use_parse` is false. Each is listed as + `{repo: null, url}` while missing from `/weights`. The + directory is found with `find_spec`. If facexlib isn't installed, both are + listed. +- The pin's B′ allow-list is gone. Three commands are exempt with reasons in + the test: `separate_stems`, `decode_h3_latents` and `paste_face_track`. + `tests/test_task_file_weights.py` covers the rest. +- **Deviations:** `gated` is always `null` on a file entry (the file HEAD + doesn't report the gating mode). A file under ~10 MB reads `gb: 0.01`. + A filename that fails `validate_weight_name` gives no entry. The + no-heavy-import test checks `facexlib`/`cv2` only, because `import dw` + already loads torch. `remove_object` and `upscale_h3_latents` keep + treating any falsy argument as unset. + +Docs updated across both stages: TASKS, SERVER, MCP (including the +`download_model`-fetches-whole-repo note), WORKFLOW_GUIDE, DEPENDENCIES, +ARCHITECTURE (the "Task weights" row), and the RELEASING "Unreleased" note +that a bound acknowledgement from a pre-deploy validate gets 409 +`downloads` when the workflow has one of these tasks uncached. + +## Verification and bounces + +| Stage | Bounces | Cause | +|---|---|---| +| A′ (#977) | 1 (tester) | C-F525 step 5: an unedited validate `plan` sent as the acknowledgement was refused, against the plan's "clients that copy the plan verbatim are unaffected". Fixed by `_from_plan`. C-F519–C-F527 and SE-F051 then passed | +| B′ (#978) | 1 (architecture review) | The "Task weights" map row named `argument_or_default` as the one resolution rule, but three of the six new `*_source` functions don't use it. The row was amended and no code changed. All 11 `pending: #978` cases (C-F528–C-F536, M-F157, SE-F052) then passed | + +The stage comments name no `usage:` figures, so this record leaves out cost +per stage. The plan estimated $4–6 for each stage. + +Some arms couldn't run on lem because the defaults were already cached: +the real-repo arm of C-F530, the uncached pinned big-lama arm of C-F531, and +the default-file arms of C-F533. The offline arm of C-F536 couldn't run +either. The bogus-repo override arms covered each declaration instead. + +## Open after close + +- `pending: #977` is still on C-F519, C-F520, C-F522, C-F523, C-F525, + C-F526 and SE-F051. They pass in substance, but as written they use + argument names or expectations the plan didn't match. The amendment + is with Don on dkackman/harnest#139. + +## Deferred + +- zoe_depth, dw_pose and demucs/`separate_stems` (fetched outside the HF + cache) and text_generation's `system_prompt.from_hub` file are still + unlisted. Each is documented in TASKS.md. *Trigger:* a field report of an + unquoted download from one of them. +- `download_model` can't pre-fetch a pinned single file. *Trigger:* an + offline box that needs one. +- `gated` on file entries. *Trigger:* a gated single-file task source. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 29353059..b7aa72fb 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -343,6 +343,14 @@ remaining deferred fix is recorded in mask. Record, including what was deferred (temporal video removal, per-frame masks, a larger model for big regions, #975's download warning): `complete/remove-object-complete.md`. +- **Task weights in `plan.downloads_required`** (#975, stages #977-#978), + shipped 2026-10-23. Each model-backed task declares + `required_weights(arguments)`. Its loader reads the same resolution, and + the plan lists those repos, plus single files at their pinned revision + and facexlib's URLs, so a bound acknowledgement covers them. Record, + including what was deferred (fetchers outside the HF cache, pinned-file + `download_model`, `gated` on file entries): + `complete/task-weights-downloads-complete.md`. ## Declined From b80dec772ae289e466a84b5d6257c24cfe81c087 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sat, 24 Oct 2026 23:09:53 -0500 Subject: [PATCH 061/105] fix(plan): #981 - downloads_required lists load_components repo overrides Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 1 + dw/plan.py | 17 +++++++++++++++++ tests/test_plan.py | 23 +++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 84c2b1d4..fcf11628 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -97,6 +97,7 @@ one to open. | Variable constraints | `dw/variable_constraints.py`: `apply_constraints`; the grid arithmetic `aligned` / `aligned_down` | A model's rule about a value is declared in the workflow's `variable_constraints`, and it is checked at validation and again before anything loads. `plan_cuts` (and its static check) rounds render lengths through `aligned` / `aligned_down`, never its own copy, so a planned `num_frames` is one the template's constraint accepts; `crop_face_track` pads its crops to the step's `modulus`/`remainder` grid through `aligned` too (`padding_to_grid`). | `tests/test_variable_constraints.py::TestTheGrid::test_without_snap_an_off_grid_value_is_refused_not_rounded`, `tests/test_variable_constraints.py::TestTheGrid::test_plan_cuts_rounds_through_the_constraint_owner`, `tests/test_face_track.py::TestPadding::test_padding_to_grid_rounds_through_the_constraint_owner`, `tests/test_variable_constraints.py::TestTheRunTimePass::test_an_illegal_value_is_refused_before_anything_loads` | | Bound acknowledgement | `dw/plan.py`: `fingerprint`, `dw/server/admission.py`: `check_bound_acknowledgement` | A bound `acknowledged_cost` is refused with 409 when the plan's fingerprint or its required downloads changed. The fingerprint is keyed only by what can change the output: `DOCUMENTATION_KEYS` (`cost`, `cost_drivers`, `vram_estimate`, descriptions, catalog shape/traits), a constraint's `reason` and a result's role, subfolder and file name are left out, of the workflow and of each composed child; every schema key is sorted into left out or kept. A number digests by value, not spelling (`12.0` as `12`), and the seed stays out (#85). | `tests/test_plan.py::TestFingerprintKeys::test_the_exclusion_lists_are_pinned`, `tests/test_plan.py::TestTheCatalogH3Template::test_a_client_round_trip_keeps_it`, `tests/test_plan.py::TestFingerprintKeys::test_every_top_level_schema_key_is_sorted`, `tests/test_server.py::TestBoundRerun::test_a_rerun_bound_to_a_stale_plan_is_refused`, `tests/test_server.py::TestBoundAcknowledgement::test_a_matching_fingerprint_queues` | | Per-repo gating | `dw/plan.py`: `build_plan` | Each gated repo, including a `loras` entry's repo, is probed and reported on its own. | `tests/test_plan.py::TestDownloadsRequired::test_a_gated_repo_this_token_lacks_access_to_is_blocked` | +| Download sources | `dw/plan.py`: `_collect_sources`, `_collect_component_overrides` | `downloads_required` lists every `model_name`, `loras` repo, single-file URL and `load_components.pretrained_model_name_or_path` override (string, per-component dict, `default`), once each. The base repo stays listed whole: an override's replaced subfolder is not subtracted, so the list is a floor. | `tests/test_plan.py::TestDownloadsRequired::test_a_load_components_repo_override_is_listed` | | Task weights | each fetching task's `*_source(arguments)` (e.g. `dw/tasks/remove_object.py`: `remover_source`); `dw/tasks/weights.py`: `argument_or_default`, `file_entries`, `package_dir`; `dw/tasks/registry.py`: `required_weights_hook`; `dw/plan.py`: `_collect_task_weights`, `_file_info` | A model-backed task declares `required_weights(arguments)` on its `register_command`, and `downloads_required` lists those repos beside the pipeline `model_name`s (a shared repo once; a reference or local-path argument none). A single-file task lists `{repo, filename, revision}` entries, checked per file against the cache; `restore_faces` also lists facexlib's weights as URL entries. A task's declaration and its loader read the same resolution of which argument wins (a single-file task's `*_source(arguments)`, an overridable whole-repo task's `argument_or_default` call, a fixed repo's module constant), so they cannot differ; `argument_or_default` is the shared helper only where `None` means the default (`remover_source` and `upscaler_source` treat any falsy value as absent; `spandrel_source` has no default). `file_entries` builds a file entry, `package_dir` finds facexlib's weights dir without importing it, and `_file_info` sizes one file. | `tests/test_task_weights.py` | | Observed cost | `dw/server/observed_cost.py`: `declared_drivers`, `ObservedCosts._on_this_card`, `ObservedCosts.card` | `observed` comes only from this box's finished jobs on the named card (the one `route` names, for `plan.estimate`), else the server's own card; `ObservedCosts.card` owns a card's (kind, name, default). Rows are matched by card name whatever index they ran on (a job from before jobs carried a `device` counts only while the server is on the default card), and never writes `cost`. Cold and warm runs are reported separately. | `tests/test_observed_cost.py::TestColdIsNotWarm::test_the_two_are_reported_separately_each_with_its_runs`, `tests/test_observed_cost.py::TestByCard::test_only_rows_from_this_card_count_whichever_index_they_had`, `tests/test_observed_cost.py::TestByCard::test_rows_from_before_jobs_carried_a_device_do_not_count_off_the_default_card` | | The estimate | `dw/plan.py`: `estimate`, `_tempered`; the driver scaling is `dw/cost_scaling.py`: `scale_curated`, `_driver_scaling` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. A `cost` entry is measured at the workflow's variable defaults; a run moving `num_inference_steps`, `width`/`height` or `num_frames` (per list entry, summed) scales it in proportion (`scale_curated`, which `dw/server/deps.py` reuses for an inherited figure), basis `derived` with `scaled` naming each axis, times the list-length re-price, and that scaled figure is what a thin observed one blends toward (#866). | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure`, `tests/test_plan.py::TestDriverScaling::test_steps_canvas_and_frames_multiply`, `tests/test_plan.py::TestScalingAccuracy::test_within_thirty_percent` | diff --git a/dw/plan.py b/dw/plan.py index 10f36b99..90be65ee 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -892,6 +892,8 @@ def _repriced(minutes, list_entries, measured_entries): LORAS_KEY = "loras" SINGLE_FILE_KEY = "from_single_file" VARIANT_KEY = "variant" +LOAD_COMPONENTS_KEY = "load_components" +OVERRIDE_KEY = "pretrained_model_name_or_path" def downloads_required(expanded, base_dir, workflow_dir, cache_dir, lookup_sizes): @@ -1003,6 +1005,7 @@ def _collect_sources(tree, names, variants, urls): single = source.get(SINGLE_FILE_KEY) if isinstance(single, str) and _is_url(single) and single not in urls: urls.append(single) + _collect_component_overrides(tree.get(LOAD_COMPONENTS_KEY), names) for value in tree.values(): _collect_sources(value, names, variants, urls) elif isinstance(tree, list): @@ -1010,6 +1013,20 @@ def _collect_sources(tree, names, variants, urls): _collect_sources(value, names, variants, urls) +def _collect_component_overrides(arguments, names): + """The repos a modular step's `load_components.pretrained_model_name_or_path` + names: a string for every component, or a dict keyed by component name + with an optional `default`. Diffusers merges it over each component's + spec, so the repo is pulled mid-run (#981). The base repo is still listed + whole - the subfolder an override replaces is not subtracted.""" + if not isinstance(arguments, dict): + return + value = arguments.get(OVERRIDE_KEY) + for name in value.values() if isinstance(value, dict) else [value]: + if isinstance(name, str) and name not in names: + names.append(name) + + def _collect_task_weights(tree, names, files, urls): """Every repo, file or URL a task step in a tree declares, appended after the pipeline sources and deduplicated against them - a file as a diff --git a/tests/test_plan.py b/tests/test_plan.py index a5416727..64cb2582 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -1791,6 +1791,29 @@ def test_components_and_children_are_scanned_and_deduplicated(self, plan, tmp_pa "org/vae", ] + @pytest.mark.parametrize( + "override, expected", + [ + ("org/other", ["org/still-model", "org/other"]), + ( + {"transformer_ref": "org/other", "vae": "org/vae"}, + ["org/still-model", "org/other", "org/vae"], + ), + ({"default": "org/other"}, ["org/still-model", "org/other"]), + ({"default": "org/still-model"}, ["org/still-model"]), + (None, ["org/still-model"]), + ], + ) + def test_a_load_components_repo_override_is_listed(self, plan, override, expected): + """#981: the string, per-component and default forms, deduplicated + against model_name, and unchanged when the key is absent.""" + spec = definition() + if override is not None: + spec["steps"][0]["pipeline"]["load_components"] = { + "pretrained_model_name_or_path": override + } + assert [d["repo"] for d in plan(spec)["downloads_required"]] == expected + def test_task_steps_list_the_repos_their_commands_declare(self, plan): """#975: an image processor (remove_background) and a registered command (segment) both name their weights, after the pipeline's.""" From a92c42541c3ba45fba1cd92f495eca266e10ef1e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 25 Oct 2026 11:17:16 -0500 Subject: [PATCH 062/105] docs(audits): #988 - WanGP probes for claims 1 and 4 Claim 1 (no for held-track lip-sync) refuted on both tracks and seeds; claim 4 (FL2VA with a held soundtrack) confirmed: frame 0 kept pixel for pixel and the mouth follows the track on both seeds. int8, turbo LoRA on, 960x544, 124 frames, workspace qa-h3-wangp-probes. Co-Authored-By: Claude Opus 5.5 --- .../audits/2026-10-h3-wangp-probes.md | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 docs/proposals/audits/2026-10-h3-wangp-probes.md diff --git a/docs/proposals/audits/2026-10-h3-wangp-probes.md b/docs/proposals/audits/2026-10-h3-wangp-probes.md new file mode 100644 index 00000000..8ee7e33a --- /dev/null +++ b/docs/proposals/audits/2026-10-h3-wangp-probes.md @@ -0,0 +1,181 @@ +# MiniMax-H3 claims from WanGP's field notes: record-only probes (#985 stage A, #988) + +Probe date: 2026-10-25. Server `lem`, develop @ b80dec77. +Agent: claude-opus-5-5 (anthropic). Workspace `qa-h3-wangp-probes`. +No engine or skill file was changed. This document only records results. + +Pinned configuration (Don, at plan approval), the same for every cited job: +- `weights_dtype` **int8**, with the turbo LoRA **on**. Each job's `note` + states both. +- Ref2VA (P1, P4 arm b): `transformer_ref`, LoRA + `minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors`, 9 steps, video + shift 12, audio shift 3, `sparse` attention. +- FL2VA (P4 arm a): `transformer`, LoRA + `minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors`, 5 steps, shift 6. +- 960x544, 124 frames (5.17 s at 24 fps). Every shot holds its source slice + with `hold_audio`. + +Each arm ran on seeds 1001 and 2002. A claim is **confirmed** only when both +seeds of the claim-broken arm show the effect and neither seed of the +claim-followed arm does. It is **refuted** when the broken arm doesn't show +the effect, and **inconclusive** when the seeds disagree. Claim 4 has no +broken arm (it asks whether a route works, not whether a rule matters). It is +confirmed when both FL2VA seeds keep frame 0 and their mouths follow the +track, and refuted when either fails. + +With `hold_audio`, the output soundtrack *is* the source slice. So the +transcripts and `measure_sync` can't tell the arms apart: they measure audio +against audio, not lips. The mouth frames decide every verdict here. + +## Verdicts + +| # | Claim (as stated on #985) | Verdict | Jobs | Observation | +| --- | --- | --- | --- | --- | +| 1 | "Lip-sync without ``": for lip-sync to supplied audio, leave the words out of `` and ask for natural lip movements synchronized to the track. | **refuted** | sung: 3d9a4e7377fa, 6d7f6faf8e60 (a); d2384deebb50, f27b7a7af223 (b). spoken: c029d63a15e8, c0763cc81d6b (a); 8451e9fd9289, e0790f00b4c2 (b) | With the track held, the words in `` did no harm. On both tracks and both seeds, arm a's mouth opens and shuts at the same instants as arm b's, and the identity holds. See *Probe 1*. | +| 4 | "FL2VA with a held soundtrack" lip-syncs. Our skill says "FL2VA takes no audio" (SKILL.md:31). | **confirmed** | 52195bad071b, f4fe60dcca4d (FL2VA); 422f22a7df71, 8e42ec21cf9a (Ref2VA keyframe) | FL2VA plus `hold_audio` keeps the input picture pixel for pixel at frame 0. On both seeds its mouth follows the held song with the same open and shut pattern as the audio-referenced Ref2VA shots. #852's Ref2VA keyframe route also lip-syncs, but frame 0 keeps only the framing: the arms hang instead of being folded, and the shot is wider. See *Probe 4*. | + +Not run: none. Claims 2 and 3 were dropped and deferred by the plan, not +probed. + +### `measure_sync` figures + +| Track | Jobs | `lag_seconds` | `confidence` | `reference_seconds` | Findings | +| --- | --- | --- | --- | --- | --- | +| sung (la-vela) | all 4 P1 sung, all 4 P4 | 0.0 | 0.175 | 118.751 | `sync_low_confidence` (warn, threshold 0.3) | +| spoken (hal-voice) | all 4 P1 spoken | 0.0 | 0.756 | 2.499 | none | + +Lag 0.0 is expected, because the output audio is the held slice. The sung +figure looks unreliable: `audio_seconds` is 5.1667 but `reference_seconds` +is 118.751. The step's `reference` is `previous_result:slice` (a 5.17 s +slice), so `measure_sync` seems to have read something song-length, possibly +the whole source track. The spoken reference length of 2.499 s is about the +speech span, which suggests the trailing silence was trimmed. This is a +possible `measure_sync` or reference-resolution bug, recorded here for +follow-up. It doesn't affect a verdict. + +## Probe 1: lip-sync with and without `` + +Workflow `probes/wangp-p1-ref2va` has four steps: +1. `slice_audio`: the track at `start_seconds`, `num_frames` 124 at 24 fps. +2. Ref2VA shot: portrait `asset:cast/tomas.jpg` as `` and the + slice as `