From 921fa64d1019ed6ccfa4c60e8dcd53a948f6d499 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Wed, 21 Oct 2026 22:47:02 +0300 Subject: [PATCH 1/8] ci: update release notes --- .github/workflows/main.yml | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bbdff52..0907902 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -220,6 +220,62 @@ jobs: packages: write steps: *build-image + release-notes: + name: Update GitHub release notes + needs: docker + if: github.server_url == 'https://github.com' && vars.REGISTRY == '' && startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write + steps: + - name: Append release commits + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const tag = context.ref.replace('refs/tags/', ''); + const repo = context.repo; + const { data: release } = await github.rest.repos.getReleaseByTag({ ...repo, tag }); + const releases = await github.paginate(github.rest.repos.listReleases, { + ...repo, + per_page: 100, + }); + const published = releases.filter(r => !r.draft); + const baseTag = published[published.findIndex(r => r.id === release.id) + 1]?.tag_name; + const commits = baseTag + ? await github.paginate(github.rest.repos.compareCommitsWithBasehead, { + ...repo, + basehead: `${baseTag}...${tag}`, + per_page: 100, + }) + : await github.paginate(github.rest.repos.listCommits, { + ...repo, + sha: tag, + per_page: 100, + }); + const changelog = commits + .filter(c => c.parents.length <= 1) + .map(c => `- ${c.sha.substring(0, 7)} ${c.commit.message.split('\n')[0]}`) + .join('\n'); + + if (!changelog) { + core.setFailed('No commits found for the release notes.'); + return; + } + + const heading = baseTag ? `## Commits since ${baseTag}` : '## Commits'; + const existingBody = (release.body || '').trim(); + const body = existingBody + ? `${existingBody}\n\n---\n\n${heading}\n\n${changelog}` + : `${heading}\n\n${changelog}`; + + await github.rest.repos.updateRelease({ + ...repo, + release_id: release.id, + body, + }); + dockerhub-sync-readme: name: Sync Docker Hub README needs: test From 3de345a28088e1905e8885c2aa206c6cd42925e1 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Wed, 21 Oct 2026 23:18:09 +0300 Subject: [PATCH 2/8] tests: improve tests for CI reliability --- compose.yml | 3 + docs/installation.md | 2 +- install.sh | 17 +- tests/conftest.py | 2 +- tests/test_api.py | 28 +- tests/test_changelog.py | 9 +- tests/test_config.py | 10 +- tests/test_executor_safety.py | 4 + tests/test_finalization.py | 6 +- tests/test_hub.py | 634 +++++++++++++++------------- tests/test_job_service.py | 74 ++-- tests/test_job_store.py | 6 +- tests/test_library_deletion.py | 10 +- tests/test_lifecycle_safety.py | 9 +- tests/test_notification_delivery.py | 4 +- tests/test_queue_staging_limits.py | 7 +- tests/test_scanner.py | 2 +- tests/test_staging.py | 5 + tests/test_staging_safety.py | 16 +- tests/test_workspace_recheck.py | 6 +- ui/tests/batchDryRun.test.ts | 18 +- ui/tests/filesActions.test.ts | 26 +- ui/tests/jobRequeue.test.ts | 18 +- ui/tests/liveUpdates.test.ts | 223 ++++++---- ui/tests/logsLifecycle.test.ts | 120 +++--- ui/tests/sharedFeedback.test.ts | 50 ++- ui/tests/stateHelpers.test.ts | 2 +- 27 files changed, 784 insertions(+), 527 deletions(-) diff --git a/compose.yml b/compose.yml index a7be609..f80bd34 100644 --- a/compose.yml +++ b/compose.yml @@ -2,6 +2,7 @@ services: postgres: image: postgres:18 + container_name: millwright-postgres environment: POSTGRES_USER: millwright POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} @@ -16,6 +17,7 @@ services: restart: unless-stopped server: + container_name: millwright-server image: ghcr.io/arabcoders/millwright:latest user: "${UID:-1000}:${GID:-1000}" depends_on: @@ -37,6 +39,7 @@ services: restart: unless-stopped node: + container_name: millwright-node image: ghcr.io/arabcoders/millwright:latest user: "${UID:-1000}:${GID:-1000}" depends_on: diff --git a/docs/installation.md b/docs/installation.md index 411b7c1..811f17c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -11,7 +11,7 @@ Run this in the directory where you want to keep Millwright: bash <(curl -fsSL https://raw.githubusercontent.com/arabcoders/millwright/HEAD/install.sh) ``` -Enter your media folder when prompted. The installer starts PostgreSQL, the server, and a node. +Enter your media folder when prompted. The installer asks before starting compose file. Press Enter to skip startup. Rerunning it keeps your existing configuration and credentials. Open `http://YOUR_SERVER_IP:8000` and create your account. Use your server's IP address, or diff --git a/install.sh b/install.sh index 97c8aa3..527269f 100755 --- a/install.sh +++ b/install.sh @@ -45,9 +45,14 @@ for name in api_key node_enrollment_token; do fi done -docker compose -f compose.yml up -d --pull always --wait - -printf '\nMillwright is ready on port 8000.\n' -printf 'Open http://:8000\n' -printf 'Installation directory: %s\n' "$PWD" -printf 'Create your account, then add /media as a library.\n' +printf '\nInstallation directory: %s\n' "$PWD" +if IFS= read -r -p 'Start Millwright containers now? [y/N] ' answer && [[ "$answer" =~ ^[Yy]([Ee][Ss])?$ ]]; then + docker compose -f compose.yml up -d --pull always --wait + printf '\nMillwright is ready on port 8000.\n' + printf 'Open http://:8000\n' + printf 'Create your account, then add /media as a library.\n' +else + printf '\nContainers were not started.\n' + printf 'To start them, run this from the installation directory:\n' + printf 'docker compose -f compose.yml up -d --pull always --wait\n' +fi diff --git a/tests/conftest.py b/tests/conftest.py index 20edc11..73b7c24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -147,7 +147,7 @@ def pytest_configure(config) -> None: if getattr(config.option, "basetemp", None) is None or hasattr(config, "workerinput"): config.option.basetemp = str(run_root / "pytest") - # Keep eager plugin-dir mkdirs under the per-run root, not the real XDG state home. + os.environ["MW_CONFIG_PATH"] = str(run_root / "config") os.environ["MW_STATE_PATH"] = str(run_root / "state") os.environ["MW_CACHE_PATH"] = str(run_root / "cache") diff --git a/tests/test_api.py b/tests/test_api.py index e3b5a6b..b15b5ab 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from datetime import UTC, datetime, timedelta from pathlib import Path +from types import SimpleNamespace from typing import TYPE_CHECKING, Literal from unittest.mock import AsyncMock @@ -101,7 +102,8 @@ class ApiEnv: @asynccontextmanager -async def api_context(*, background: bool = False) -> AsyncIterator[ApiEnv]: +async def api_context(monkeypatch: pytest.MonkeyPatch, *, background: bool = False) -> AsyncIterator[ApiEnv]: + monkeypatch.setattr("server.staging.shutil.disk_usage", lambda _: SimpleNamespace(free=20_000_000_000)) lib_dir = make_test_temp_dir("api-lib") async with database_context("api") as factory: app = create_app( @@ -174,9 +176,9 @@ async def drain_services() -> None: @pytest_asyncio.fixture -async def api_env(request: pytest.FixtureRequest) -> AsyncIterator[ApiEnv]: +async def api_env(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[ApiEnv]: # Only background integration tests start service loops. - async with api_context(background=getattr(request, "param", False)) as env: + async with api_context(monkeypatch, background=getattr(request, "param", False)) as env: yield env @@ -234,7 +236,7 @@ async def requeue_outcome(env: ApiEnv, job_id: int, *, event_type: str = "requeu @pytest.mark.asyncio @pytest.mark.parametrize("outcome", ["complete", "failure", "cancel"]) async def test_cleanup_joins_scans(monkeypatch: pytest.MonkeyPatch, outcome: str) -> None: - context = api_context() + context = api_context(monkeypatch) env = await context.__aenter__() started, closing_started, release = asyncio.Event(), asyncio.Event(), asyncio.Event() order: list[str] = [] @@ -442,7 +444,7 @@ async def test_queue_rotation(api_env: ApiEnv, monkeypatch) -> None: ) h.entries[entry.name] = entry async with api_env.factory() as session: - before_jobs = [dict(row) for row in (await session.execute(select(Job.__table__))).mappings()] + before_jobs = [dict(row) for row in (await session.execute(select(Job.__table__).order_by(Job.id))).mappings()] before_node = dict((await session.execute(select(NodeRow.__table__))).mappings().one()) before_events = await session.scalar(select(func.count(JobEvent.id))) h._wake.clear() @@ -471,7 +473,9 @@ def track(_conn, _cursor, statement, _parameters, _context, _many): assert "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY" in statements assert not any(statement.startswith(("INSERT", "UPDATE", "DELETE")) for statement in statements) async with api_env.factory() as session: - assert [dict(row) for row in (await session.execute(select(Job.__table__))).mappings()] == before_jobs + assert [ + dict(row) for row in (await session.execute(select(Job.__table__).order_by(Job.id))).mappings() + ] == before_jobs assert dict((await session.execute(select(NodeRow.__table__))).mappings().one()) == before_node assert await session.scalar(select(func.count(JobEvent.id))) == before_events @@ -3589,7 +3593,13 @@ async def test_attempt_history_and(api_env: ApiEnv) -> None: ] ) await session.flush() - attempts = list((await session.execute(select(JobAttempt).where(JobAttempt.job_id == job.id))).scalars()) + attempts = list( + ( + await session.execute( + select(JobAttempt).where(JobAttempt.job_id == job.id).order_by(JobAttempt.attempt_number) + ) + ).scalars() + ) session.add(JobEvent(job_id=job.id, attempt_id=attempts[1].id, type="finished", payload={})) await session.commit() job_id, first_id, second_id, third_id = job.id, attempts[0].id, attempts[1].id, attempts[2].id @@ -3844,7 +3854,9 @@ async def test_system_timezone(api_env: ApiEnv, tmp_path: Path) -> None: @pytest.mark.asyncio async def test_stats_overview_scope(api_env: ApiEnv) -> None: - now = datetime.now(UTC) + now = T0 + api_env.app.state.stats.clock = lambda: now + api_env.app.state.cache.clock = lambda: now async with api_env.factory() as session: libraries = [Library(name=f"overview-{uuid.uuid4().hex}", roots=[str(api_env.lib_dir)]) for _ in range(3)] session.add_all(libraries) diff --git a/tests/test_changelog.py b/tests/test_changelog.py index 3907aa3..5629c7a 100644 --- a/tests/test_changelog.py +++ b/tests/test_changelog.py @@ -16,6 +16,7 @@ def git(repo: Path, *args: str, date: str = "2026-01-01T12:00:00+00:00") -> str: return subprocess.check_output( ["git", "-C", str(repo), *args], text=True, + timeout=5, env={ **os.environ, "GIT_AUTHOR_NAME": "History Author", @@ -29,7 +30,9 @@ def git(repo: Path, *args: str, date: str = "2026-01-01T12:00:00+00:00") -> str: @pytest.fixture -def repo(tmp_path: Path) -> Path: +def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) git(tmp_path, "init", "--initial-branch=main") return tmp_path @@ -41,7 +44,9 @@ def commit(repo: Path, message: str, date: str = "2026-01-01T12:00:00+00:00") -> def generate(repo: Path, *args: str) -> list[dict]: output = repo / "exported/CHANGELOG.json" - subprocess.run([sys.executable, str(SCRIPT), "--repo", str(repo), "--output", str(output), *args], check=True) + subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo), "--output", str(output), *args], check=True, timeout=10 + ) return json.loads(output.read_text()) diff --git a/tests/test_config.py b/tests/test_config.py index af8be6c..c7f3ad6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,12 +8,20 @@ from common.config import apply_env_overrides from common.enums import JobStatus from server.api.app import create_app -from server.core.config import ServerConfig, ensure_config, is_oidc_ready, load_config +from server.core.config import ENV_OVERRIDES, ServerConfig, ensure_config, is_oidc_ready, load_config from server.settings import OperationalSettings from tests.helpers import database_context, make_test_temp_dir from tests.test_staging import add_job, get_job +@pytest.fixture(autouse=True) +def clean_env(monkeypatch: pytest.MonkeyPatch) -> None: + for variable in ENV_OVERRIDES.values(): + if variable != "MW_DATABASE_URL": + monkeypatch.delenv(variable, raising=False) + monkeypatch.setenv("TZ", "UTC") + + @pytest.mark.parametrize("field", ["max_probes", "max_mkv_stats"]) def test_metadata_limits(field: str) -> None: with pytest.raises(ValidationError) as raised: diff --git a/tests/test_executor_safety.py b/tests/test_executor_safety.py index b12e478..dd83e2f 100644 --- a/tests/test_executor_safety.py +++ b/tests/test_executor_safety.py @@ -699,13 +699,16 @@ async def test_log_fsync_preserves(tmp_path: Path, monkeypatch: pytest.MonkeyPat job = spec(tmp_path, min_free_gb=0) logs: list[_Log] = [] opening, syncing = _Log.__init__, os.fsync + fsync_failed = False def open_log(log, path): opening(log, path) logs.append(log) def failed_fsync(fd): + nonlocal fsync_failed if logs and not logs[0].file.closed and fd == logs[0].file.fileno(): + fsync_failed = True raise OSError("NFS log fsync rejected") return syncing(fd) @@ -737,5 +740,6 @@ async def probe(*_args): assert result.output_path == job.dst and result.output_probe is not None elif outcome == "aborted": assert result.error and "src.mkv" in result.error + assert fsync_failed assert logs and logs[0].file.closed assert Path(job.dst).exists() == (outcome in {"succeeded", "unreaped"}) diff --git a/tests/test_finalization.py b/tests/test_finalization.py index 2cefab6..aedd08c 100644 --- a/tests/test_finalization.py +++ b/tests/test_finalization.py @@ -1542,7 +1542,11 @@ async def test_recovery_batches(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, assert [job.status for job in jobs] == ["finalizing", *(["failed"] * 4)] journals = (await session.scalars(select(FinalizationJournal).order_by(FinalizationJournal.id))).all() assert [row.state for row in journals] == ["discard_prepared", *(["discard_rolled_back"] * 4)] - events = (await session.scalars(select(JobEvent).where(JobEvent.type == "finalization_rolled_back"))).all() + events = ( + await session.scalars( + select(JobEvent).where(JobEvent.type == "finalization_rolled_back").order_by(JobEvent.job_id) + ) + ).all() assert [event.job_id for event in events] == job_ids[1:] assert all((tmp_path / f"source-{index}.mkv").read_bytes() == b"original" for index in range(5)) diff --git a/tests/test_hub.py b/tests/test_hub.py index e4008b3..dd9faab 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -267,7 +267,8 @@ async def test_foreign_action_report(env): @pytest_asyncio.fixture -async def env(): +async def env(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr("server.staging.shutil.disk_usage", lambda _: SimpleNamespace(free=20_000_000_000)) async with database_context("hub") as factory: async with factory() as session: await register_node(session, "node-a", "tok") @@ -282,7 +283,8 @@ async def env(): @pytest_asyncio.fixture -async def writer_env(tmp_path: Path): +async def writer_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr("server.staging.shutil.disk_usage", lambda _: SimpleNamespace(free=20_000_000_000)) library = tmp_path / "library" library.mkdir() async with database_context("hub-writer") as factory: @@ -701,13 +703,18 @@ async def wait_for(pred, event, timeout=3): async def serve(h, conn, *, enroll=False): task = asyncio.create_task(h.serve_connection(conn)) - await wait_for(lambda: conn.messages("workspace_challenge"), conn.changed) - answer_challenge(h, conn) - if enroll: - await wait_for(lambda: conn.messages("enrolled"), conn.changed) - conn.push({"v": 1, "type": "enrollment_saved", "node_id": conn.messages("enrolled")[0]["node_id"]}) - await wait_for(lambda: conn.messages("welcome"), conn.changed) - await asyncio.wait_for(conn.ready.wait(), 3) + try: + await wait_for(lambda: conn.messages("workspace_challenge"), conn.changed) + answer_challenge(h, conn) + if enroll: + await wait_for(lambda: conn.messages("enrolled"), conn.changed) + conn.push({"v": 1, "type": "enrollment_saved", "node_id": conn.messages("enrolled")[0]["node_id"]}) + await wait_for(lambda: conn.messages("welcome"), conn.changed) + await asyncio.wait_for(conn.ready.wait(), 3) + except BaseException: + task.cancel() + await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 3) + raise return task @@ -4234,10 +4241,12 @@ async def test_handshake_ready(env): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - n = await node(env) - assert n.state == "online" and n.workspace_ready and n.workspace_free_bytes == 123456 - c.disconnect() - await asyncio.wait_for(task, 2) + try: + n = await node(env) + assert n.state == "online" and n.workspace_ready and n.workspace_free_bytes == 123456 + finally: + c.disconnect() + await asyncio.wait_for(task, 2) n = await node(env) assert n.state == "offline" and not n.workspace_ready and n.workspace_free_bytes is None @@ -4425,18 +4434,24 @@ async def test_enrollment_workspace_failure(env): h = hub(env, enrollment_token="fleet-secret") c = FakeConn([hello(name=None, token="fleet-secret")]) task = asyncio.create_task(h.serve_connection(c)) - await wait_for(lambda: c.messages("workspace_challenge"), c.changed) - challenge = c.messages("workspace_challenge")[0] - c.push( - WorkspaceResponse( - challenge_id=challenge["challenge_id"], - response_path=challenge["response_path"], - workspace_ready=False, - workspace_free_bytes=0, - ).model_dump(mode="json") - ) - await asyncio.wait_for(task, 2) - assert c.messages("error")[0]["code"] == "workspace" + try: + await wait_for(lambda: c.messages("workspace_challenge"), c.changed) + challenge = c.messages("workspace_challenge")[0] + c.push( + WorkspaceResponse( + challenge_id=challenge["challenge_id"], + response_path=challenge["response_path"], + workspace_ready=False, + workspace_free_bytes=0, + ).model_dump(mode="json") + ) + await asyncio.wait_for(task, 2) + assert c.messages("error")[0]["code"] == "workspace" + finally: + c.disconnect() + if not task.done(): + task.cancel() + await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 2) @pytest.mark.asyncio @@ -4555,17 +4570,19 @@ async def test_dispatch_paths(env): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - assert await h.dispatch_once() == 1 - offer = c.messages("job")[0]["job"] - assert offer["min_free_gb"] == h.staging.min_free_gb == 1 - assert offer["attempt_id"] > 0 and all( - not p.startswith("/") for p in (offer["source_path"], offer["output_path"], offer["log_path"]) - ) - assert str(env.library) not in json.dumps(offer) - a = (await attempt(env, jid))[0] - assert a.source_path == offer["source_path"] and a.output_path == offer["output_path"] - c.disconnect() - await asyncio.wait_for(task, 2) + try: + assert await h.dispatch_once() == 1 + offer = c.messages("job")[0]["job"] + assert offer["min_free_gb"] == h.staging.min_free_gb == 1 + assert offer["attempt_id"] > 0 and all( + not p.startswith("/") for p in (offer["source_path"], offer["output_path"], offer["log_path"]) + ) + assert str(env.library) not in json.dumps(offer) + a = (await attempt(env, jid))[0] + assert a.source_path == offer["source_path"] and a.output_path == offer["output_path"] + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -4751,16 +4768,18 @@ async def test_paused_skips_dispatch(env): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - h.set_paused(h.entries["node-a"].node_db_id, True) - assert await h.dispatch_once() == 0 - j = await job(env, jid) - assert j.status == JobStatus.QUEUED - assert j.warning is not None - h.set_paused(h.entries["node-a"].node_db_id, False) - assert await h.dispatch_once() == 1 - assert c.messages("job")[0]["job"]["job_id"] == jid - c.disconnect() - await asyncio.wait_for(task, 2) + try: + h.set_paused(h.entries["node-a"].node_db_id, True) + assert await h.dispatch_once() == 0 + j = await job(env, jid) + assert j.status == JobStatus.QUEUED + assert j.warning is not None + h.set_paused(h.entries["node-a"].node_db_id, False) + assert await h.dispatch_once() == 1 + assert c.messages("job")[0]["job"]["job_id"] == jid + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -4786,20 +4805,24 @@ async def test_waiting_warning_clears(env): h = hub(env) c = FakeConn([hello(encoders=("hevc_nvenc",))]) task = await serve(h, c) - assert await h.dispatch_once() == 0 - assert await h.dispatch_once() == 0 - waiting = await job(env, jid) - assert waiting.status == JobStatus.QUEUED and waiting.warning is not None - assert (await events(env, jid)).count("waiting") == 1 - c.disconnect() - await asyncio.wait_for(task, 2) + try: + assert await h.dispatch_once() == 0 + assert await h.dispatch_once() == 0 + waiting = await job(env, jid) + assert waiting.status == JobStatus.QUEUED and waiting.warning is not None + assert (await events(env, jid)).count("waiting") == 1 + finally: + c.disconnect() + await asyncio.wait_for(task, 2) c2 = FakeConn([hello()]) task2 = await serve(h, c2) - await h.dispatch_once() - j = await job(env, jid) - assert j.warning is None and j.status == JobStatus.CLAIMED - c2.disconnect() - await asyncio.wait_for(task2, 2) + try: + await h.dispatch_once() + j = await job(env, jid) + assert j.warning is None and j.status == JobStatus.CLAIMED + finally: + c2.disconnect() + await asyncio.wait_for(task2, 2) @pytest.mark.asyncio @@ -4849,12 +4872,14 @@ async def test_cancel_attempt(env): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - assert await h.cancel_job(jid) - msg = c.messages("cancel")[0] - assert msg["attempt_id"] == (await attempt(env, jid))[0].id - c.disconnect() - await asyncio.wait_for(task, 2) + try: + await h.dispatch_once() + assert await h.cancel_job(jid) + msg = c.messages("cancel")[0] + assert msg["attempt_id"] == (await attempt(env, jid))[0].id + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -5173,12 +5198,14 @@ async def test_stall_cancel(env): h = hub(env, stall_timeout_s=1) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - env.clock.advance(2) - await h.sweep_once() - assert c.messages("cancel")[0]["attempt_id"] > 0 - c.disconnect() - await asyncio.wait_for(task, 2) + try: + await h.dispatch_once() + env.clock.advance(2) + await h.sweep_once() + assert c.messages("cancel")[0]["attempt_id"] > 0 + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -5187,20 +5214,24 @@ async def test_disconnect_retains_attempt(env): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - first = (await attempt(env, jid))[0].id - c.disconnect() - await asyncio.wait_for(task, 2) + try: + await h.dispatch_once() + first = (await attempt(env, jid))[0].id + finally: + c.disconnect() + await asyncio.wait_for(task, 2) assert (await job(env, jid)).status == JobStatus.CLAIMED assert jid in h.entries["node-a"].in_flight c2 = FakeConn([hello(active_attempts=[{"job_id": jid, "attempt_id": first}])]) task = await serve(h, c2) - assert await h.dispatch_once() == 0 - rows = await attempt(env, jid) - assert len(rows) == 1 and rows[0].id == first and rows[0].result is None - assert jid in h.entries["node-a"].in_flight and not c2.messages("job") - c2.disconnect() - await asyncio.wait_for(task, 2) + try: + assert await h.dispatch_once() == 0 + rows = await attempt(env, jid) + assert len(rows) == 1 and rows[0].id == first and rows[0].result is None + assert jid in h.entries["node-a"].in_flight and not c2.messages("job") + finally: + c2.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -5210,15 +5241,17 @@ async def test_finished_failure(env, result): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - o = c.messages("job")[0]["job"] - await h._handle_finished(h.entries["node-a"], finished(jid, o["attempt_id"], o["log_path"], result)) - status = (await job(env, jid)).status - assert status == JobStatus.FAILED - assert not (h.workspace_dir / f"jobs/{jid}").exists() - assert (env.library / "movie.avi").read_bytes() == b"source" - c.disconnect() - await asyncio.wait_for(task, 2) + try: + await h.dispatch_once() + o = c.messages("job")[0]["job"] + await h._handle_finished(h.entries["node-a"], finished(jid, o["attempt_id"], o["log_path"], result)) + status = (await job(env, jid)).status + assert status == JobStatus.FAILED + assert not (h.workspace_dir / f"jobs/{jid}").exists() + assert (env.library / "movie.avi").read_bytes() == b"source" + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -5592,21 +5625,23 @@ async def test_finishes_cancelled(env): h = hub(env, max_attempts=3) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - offer = c.messages("job")[0]["job"] + try: + await h.dispatch_once() + offer = c.messages("job")[0]["job"] - assert await h.cancel_job(jid) - await h._handle_finished( - h.entries["node-a"], - finished(jid, offer["attempt_id"], offer["log_path"], "cancelled"), - ) + assert await h.cancel_job(jid) + await h._handle_finished( + h.entries["node-a"], + finished(jid, offer["attempt_id"], offer["log_path"], "cancelled"), + ) - assert (await job(env, jid)).status == JobStatus.CANCELLED - assert (await attempt(env, jid))[0].status == JobStatus.CANCELLED - assert await h.dispatch_once() == 0 - assert len(await attempt(env, jid)) == 1 - c.disconnect() - await asyncio.wait_for(task, 2) + assert (await job(env, jid)).status == JobStatus.CANCELLED + assert (await attempt(env, jid))[0].status == JobStatus.CANCELLED + assert await h.dispatch_once() == 0 + assert len(await attempt(env, jid)) == 1 + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -5615,33 +5650,37 @@ async def test_duplicate_finished(env, monkeypatch): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - offer = c.messages("job")[0]["job"] - await h._handle_finished(h.entries["node-a"], finished(jid, offer["attempt_id"], offer["log_path"], "cancelled")) - calls = 0 + try: + await h.dispatch_once() + offer = c.messages("job")[0]["job"] + await h._handle_finished( + h.entries["node-a"], finished(jid, offer["attempt_id"], offer["log_path"], "cancelled") + ) + calls = 0 - async def probe(*args, **kwargs): - nonlocal calls - calls += 1 - raise AssertionError("duplicate was probed") + async def probe(*args, **kwargs): + nonlocal calls + calls += 1 + raise AssertionError("duplicate was probed") - monkeypatch.setattr("server.nodehub.hub.run_ffprobe", probe) - duplicate = finished( - jid, - offer["attempt_id"], - offer["log_path"], - "succeeded", - output_path=offer["output_path"], - output_probe=ProbeData.model_validate( - {"file_path": offer["output_path"], "duration_s": 1, "size_bytes": 1, "streams": []} - ), - ) - await h._handle_finished(h.entries["node-a"], duplicate) - assert calls == 0 and (await job(env, jid)).status == JobStatus.CANCELLED - assert "foreign_finished" in await events(env, jid) - assert (await events(env, jid)).count("cancelled") == 1 - c.disconnect() - await asyncio.wait_for(task, 2) + monkeypatch.setattr("server.nodehub.hub.run_ffprobe", probe) + duplicate = finished( + jid, + offer["attempt_id"], + offer["log_path"], + "succeeded", + output_path=offer["output_path"], + output_probe=ProbeData.model_validate( + {"file_path": offer["output_path"], "duration_s": 1, "size_bytes": 1, "streams": []} + ), + ) + await h._handle_finished(h.entries["node-a"], duplicate) + assert calls == 0 and (await job(env, jid)).status == JobStatus.CANCELLED + assert "foreign_finished" in await events(env, jid) + assert (await events(env, jid)).count("cancelled") == 1 + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -5650,17 +5689,23 @@ async def test_reconnect_keeps_locks(env): h = hub(env) old = FakeConn([hello()]) old_task = await serve(h, old) - await h.dispatch_once() - offer = old.messages("job")[0]["job"] - new = FakeConn([hello(active_attempts=[{"job_id": jid, "attempt_id": offer["attempt_id"]}])]) - new_task = await serve(h, new) - old.disconnect() - await asyncio.wait_for(old_task, 2) - assert h.entries["node-a"].conn is new and jid in h.entries["node-a"].in_flight - assert (await job(env, jid)).status == JobStatus.CLAIMED - assert "retried" not in await events(env, jid) - new.disconnect() - await asyncio.wait_for(new_task, 2) + try: + await h.dispatch_once() + offer = old.messages("job")[0]["job"] + new = FakeConn([hello(active_attempts=[{"job_id": jid, "attempt_id": offer["attempt_id"]}])]) + new_task = await serve(h, new) + try: + old.disconnect() + await asyncio.wait_for(old_task, 2) + assert h.entries["node-a"].conn is new and jid in h.entries["node-a"].in_flight + assert (await job(env, jid)).status == JobStatus.CLAIMED + assert "retried" not in await events(env, jid) + finally: + new.disconnect() + await asyncio.wait_for(new_task, 2) + finally: + old.disconnect() + await asyncio.wait_for(old_task, 2) @pytest.mark.asyncio @@ -5711,22 +5756,24 @@ async def test_heartbeat_filters_locks(env): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - entry = h.entries["node-a"] - before = await job(env, jid) - env.clock.advance(1) - await h._handle_heartbeat(entry, Heartbeat(cpu_percent=1, memory_percent=1, gpus=[])) - renewed = await job(env, jid) - assert renewed.lease_expires_at > before.lease_expires_at - async with env.factory() as session: - row = await session.get(Job, jid) - row.status = JobStatus.QUEUED.value - row.node_id = None - await session.commit() - await h._handle_heartbeat(entry, Heartbeat(cpu_percent=1, memory_percent=1, gpus=[])) - assert jid not in entry.in_flight and jid not in entry.offered_at and jid not in entry.last_progress_at - c.disconnect() - await asyncio.wait_for(task, 2) + try: + await h.dispatch_once() + entry = h.entries["node-a"] + before = await job(env, jid) + env.clock.advance(1) + await h._handle_heartbeat(entry, Heartbeat(cpu_percent=1, memory_percent=1, gpus=[])) + renewed = await job(env, jid) + assert renewed.lease_expires_at > before.lease_expires_at + async with env.factory() as session: + row = await session.get(Job, jid) + row.status = JobStatus.QUEUED.value + row.node_id = None + await session.commit() + await h._handle_heartbeat(entry, Heartbeat(cpu_percent=1, memory_percent=1, gpus=[])) + assert jid not in entry.in_flight and jid not in entry.offered_at and jid not in entry.last_progress_at + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -6007,34 +6054,36 @@ async def test_probe_failure(env, monkeypatch): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - offer = c.messages("job")[0]["job"] - materialize_offer(h, offer) + try: + await h.dispatch_once() + offer = c.messages("job")[0]["job"] + materialize_offer(h, offer) - async def fail_probe(*args, **kwargs): - raise RuntimeError("bad artifact") + async def fail_probe(*args, **kwargs): + raise RuntimeError("bad artifact") - monkeypatch.setattr("server.nodehub.hub.run_ffprobe", fail_probe) - node_probe = ProbeData.model_validate( - {"file_path": offer["output_path"], "duration_s": 60, "size_bytes": 6, "streams": []} - ) - await h._handle_finished( - h.entries["node-a"], - finished( - jid, - offer["attempt_id"], - offer["log_path"], - "succeeded", - output_path=offer["output_path"], - output_probe=node_probe, - ), - ) - row = await job(env, jid) - assert row.status == JobStatus.FAILED and row.error is not None - assert not (h.workspace_dir / offer["output_path"]).exists() - assert (env.library / "movie.avi").exists() - c.disconnect() - await asyncio.wait_for(task, 2) + monkeypatch.setattr("server.nodehub.hub.run_ffprobe", fail_probe) + node_probe = ProbeData.model_validate( + {"file_path": offer["output_path"], "duration_s": 60, "size_bytes": 6, "streams": []} + ) + await h._handle_finished( + h.entries["node-a"], + finished( + jid, + offer["attempt_id"], + offer["log_path"], + "succeeded", + output_path=offer["output_path"], + output_probe=node_probe, + ), + ) + row = await job(env, jid) + assert row.status == JobStatus.FAILED and row.error is not None + assert not (h.workspace_dir / offer["output_path"]).exists() + assert (env.library / "movie.avi").exists() + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -6043,33 +6092,35 @@ async def test_post_failure(env, monkeypatch): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - offer = c.messages("job")[0]["job"] - output = materialize_offer(h, offer) + try: + await h.dispatch_once() + offer = c.messages("job")[0]["job"] + output = materialize_offer(h, offer) - async def fail_gate(*args, **kwargs): - raise RuntimeError("plugin exploded") + async def fail_gate(*args, **kwargs): + raise RuntimeError("plugin exploded") - monkeypatch.setattr(h, "_post_gate", fail_gate) - node_probe = ProbeData.model_validate( - {"file_path": offer["output_path"], "duration_s": 60, "size_bytes": 6, "streams": []} - ) - await h._handle_finished( - h.entries["node-a"], - finished( - jid, - offer["attempt_id"], - offer["log_path"], - "succeeded", - output_path=offer["output_path"], - output_probe=node_probe, - ), - ) - row = await job(env, jid) - assert row.status == JobStatus.FAILED and row.error is not None - assert not output.exists() - c.disconnect() - await asyncio.wait_for(task, 2) + monkeypatch.setattr(h, "_post_gate", fail_gate) + node_probe = ProbeData.model_validate( + {"file_path": offer["output_path"], "duration_s": 60, "size_bytes": 6, "streams": []} + ) + await h._handle_finished( + h.entries["node-a"], + finished( + jid, + offer["attempt_id"], + offer["log_path"], + "succeeded", + output_path=offer["output_path"], + output_probe=node_probe, + ), + ) + row = await job(env, jid) + assert row.status == JobStatus.FAILED and row.error is not None + assert not output.exists() + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -6437,33 +6488,37 @@ async def fail_rescan(*args, **kwargs): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - o = c.messages("job")[0]["job"] - output = materialize_offer(h, o) - probe = ProbeData.model_validate( - {"file_path": o["output_path"], "container": "mkv", "duration_s": 60, "size_bytes": 6, "streams": []} - ) - await h._handle_finished( - h.entries["node-a"], - finished(jid, o["attempt_id"], o["log_path"], "succeeded", output_path=o["output_path"], output_probe=probe), - ) - row = await job(env, jid) - assert row.status == JobStatus.SUCCEEDED - assert row.warning is not None - async with env.factory() as session: - assert ( - await session.scalar( - select(JobEvent.type).where( - JobEvent.attempt_id == o["attempt_id"], - JobEvent.type.in_(["artifact_check_pending", "artifact_check_finished"]), + try: + await h.dispatch_once() + o = c.messages("job")[0]["job"] + output = materialize_offer(h, o) + probe = ProbeData.model_validate( + {"file_path": o["output_path"], "container": "mkv", "duration_s": 60, "size_bytes": 6, "streams": []} + ) + await h._handle_finished( + h.entries["node-a"], + finished( + jid, o["attempt_id"], o["log_path"], "succeeded", output_path=o["output_path"], output_probe=probe + ), + ) + row = await job(env, jid) + assert row.status == JobStatus.SUCCEEDED + assert row.warning is not None + async with env.factory() as session: + assert ( + await session.scalar( + select(JobEvent.type).where( + JobEvent.attempt_id == o["attempt_id"], + JobEvent.type.in_(["artifact_check_pending", "artifact_check_finished"]), + ) ) + == "artifact_check_finished" ) - == "artifact_check_finished" - ) - assert (env.library / "movie.mkv").exists() - assert not (env.library / "movie.avi").exists() - c.disconnect() - await asyncio.wait_for(task, 2) + assert (env.library / "movie.mkv").exists() + assert not (env.library / "movie.avi").exists() + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.asyncio @@ -7807,8 +7862,11 @@ async def send_json(self, data): dispatch.cancel() else: release.set() - await asyncio.wait_for(asyncio.gather(dispatch, return_exceptions=True), 2) - assert dispatch.cancelled() is (failure == "cancel") + if failure == "cancel": + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dispatch, 2) + else: + assert await asyncio.wait_for(dispatch, 2) is None assert not (env.workspace / f"jobs/{case.jid}").exists() assert not case.hub._action_reports assert (env.library / "movie.avi").read_bytes() == b"source" @@ -8057,7 +8115,11 @@ async def send_json(self, data): teardown_report() elif phase == "cancel": dispatch.cancel() - await asyncio.wait_for(asyncio.gather(dispatch, return_exceptions=True), 2) + if phase == "cancel" or failure == "cancelled": + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dispatch, 2) + else: + assert await asyncio.wait_for(dispatch, 2) is None assert not output.exists() assert "actions_finished" in await events(env, case.jid) assert not case.hub._action_reports @@ -8368,16 +8430,18 @@ async def test_log_parent_symlink(env): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - o = c.messages("job")[0]["job"] - p = h.workspace_dir / o["log_path"] - external = make_test_temp_dir("outside") - p.parent.rename(external / "moved") - p.parent.symlink_to(external / "moved") - await h._handle_finished(h.entries["node-a"], finished(jid, o["attempt_id"], o["log_path"])) - assert (await job(env, jid)).status == JobStatus.FAILED - c.disconnect() - await asyncio.wait_for(task, 2) + try: + await h.dispatch_once() + o = c.messages("job")[0]["job"] + p = h.workspace_dir / o["log_path"] + external = make_test_temp_dir("outside") + p.parent.rename(external / "moved") + p.parent.symlink_to(external / "moved") + await h._handle_finished(h.entries["node-a"], finished(jid, o["attempt_id"], o["log_path"])) + assert (await job(env, jid)).status == JobStatus.FAILED + finally: + c.disconnect() + await asyncio.wait_for(task, 2) @pytest.mark.parametrize("preserve", [False, True]) @@ -8398,48 +8462,52 @@ async def test_finished_transcode(env, monkeypatch, preserve, missing_log): h = hub(env) c = FakeConn([hello()]) task = await serve(h, c) - await h.dispatch_once() - o = c.messages("job")[0]["job"] - output = materialize_offer(h, o) - output.write_bytes(b"remux") - if missing_log: - (h.workspace_dir / o["log_path"]).unlink() - from common.media.probe import ProbeData + try: + await h.dispatch_once() + o = c.messages("job")[0]["job"] + output = materialize_offer(h, o) + output.write_bytes(b"remux") + if missing_log: + (h.workspace_dir / o["log_path"]).unlink() + from common.media.probe import ProbeData - probe = ProbeData.model_validate( - {"file_path": o["output_path"], "container": "mkv", "duration_s": 60, "size_bytes": 5, "streams": []} - ) - await h._handle_finished( - h.entries["node-a"], - finished(jid, o["attempt_id"], o["log_path"], "succeeded", output_path=o["output_path"], output_probe=probe), - ) - assert (await job(env, jid)).status == JobStatus.SUCCEEDED - assert list(env.trash.rglob("*")) - async with env.factory() as s: - event = await s.scalar( - select(JobEvent).where(JobEvent.attempt_id == o["attempt_id"], JobEvent.type == "publication") + probe = ProbeData.model_validate( + {"file_path": o["output_path"], "container": "mkv", "duration_s": 60, "size_bytes": 5, "streams": []} ) - assert event.payload["preserve_timestamps"] is preserve - media = ( - await s.execute(select(MediaFile).where(MediaFile.path == str(env.library / "movie.mkv"))) - ).scalar_one() - assert media.size == 5 - assert media.container == "mkv" - assert media.video_codec == "h264" - assert media.health_status == HealthStatus.PENDING.value - assert media.transcode_status == "transcoded" - attempt = await s.get(JobAttempt, o["attempt_id"]) - assert attempt is not None and attempt.output_probe is not None - assert attempt.output_probe["file_path"] == str(env.library / "movie.mkv") - assert attempt.output_probe["container"] == "mkv" - assert attempt.output_probe["streams"][0]["codec"] == "h264" - probe = await s.get(ProbeRow, media.probe_id) - assert json.loads(probe.raw_json)["format"]["duration"] == "600.0" - healthcheck = ( - await s.execute(select(Job).where(Job.kind == "healthcheck", Job.media_file_id == media.id)) - ).scalar_one() - c.disconnect() - await asyncio.wait_for(task, 2) + await h._handle_finished( + h.entries["node-a"], + finished( + jid, o["attempt_id"], o["log_path"], "succeeded", output_path=o["output_path"], output_probe=probe + ), + ) + assert (await job(env, jid)).status == JobStatus.SUCCEEDED + assert list(env.trash.rglob("*")) + async with env.factory() as s: + event = await s.scalar( + select(JobEvent).where(JobEvent.attempt_id == o["attempt_id"], JobEvent.type == "publication") + ) + assert event.payload["preserve_timestamps"] is preserve + media = ( + await s.execute(select(MediaFile).where(MediaFile.path == str(env.library / "movie.mkv"))) + ).scalar_one() + assert media.size == 5 + assert media.container == "mkv" + assert media.video_codec == "h264" + assert media.health_status == HealthStatus.PENDING.value + assert media.transcode_status == "transcoded" + attempt = await s.get(JobAttempt, o["attempt_id"]) + assert attempt is not None and attempt.output_probe is not None + assert attempt.output_probe["file_path"] == str(env.library / "movie.mkv") + assert attempt.output_probe["container"] == "mkv" + assert attempt.output_probe["streams"][0]["codec"] == "h264" + probe = await s.get(ProbeRow, media.probe_id) + assert json.loads(probe.raw_json)["format"]["duration"] == "600.0" + healthcheck = ( + await s.execute(select(Job).where(Job.kind == "healthcheck", Job.media_file_id == media.id)) + ).scalar_one() + finally: + c.disconnect() + await asyncio.wait_for(task, 2) assert media.first_seen.replace(tzinfo=UTC) == first_seen assert healthcheck.status == JobStatus.QUEUED and healthcheck.hold_until is None diff --git a/tests/test_job_service.py b/tests/test_job_service.py index 777b48c..f7e2f4a 100644 --- a/tests/test_job_service.py +++ b/tests/test_job_service.py @@ -315,41 +315,42 @@ async def blocked(*args, **kwargs): raise monkeypatch.setattr(service.staging, "prune_job", blocked) - assert (await api_env.client.post(f"/api/jobs/{job_id}/retry")).status_code == 202 - await asyncio.wait_for(entered.wait(), 2) - if operation == "shutdown": - ending = asyncio.create_task(service.close()) - elif operation == "reset": - api_env.app.state.services_running = True - api_env.app.state.auth.config.api_key = "test-job-service" - api_env.client.headers["Authorization"] = "Bearer test-job-service" - ending = asyncio.create_task(api_env.client.post("/api/backups/reset", json={"keep": []})) - # Keep reset's service restarts deterministic and avoid starting unrelated loops. - for owner, name in ( - (api_env.app.state.scanner, "start"), - (api_env.app.state.healthchecks, "start"), - (api_env.app.state.staging, "start"), - (api_env.app.state.scheduler, "start"), - (api_env.app.state.notifications, "start"), - ): - monkeypatch.setattr(owner, name, lambda: None) - - async def start(): - pass - - monkeypatch.setattr(api_env.app.state.probes, "start", start) - monkeypatch.setattr(api_env.app.state.flow_enqueue, "start", start) - else: - api_env.app.state.services_running = True - async with api_env.factory() as session: - job = await session.get(Job, job_id) - assert job is not None - library_id = job.library_id - await api_env.app.state.library_deletion.start() - response = await api_env.client.delete(f"/api/libraries/{library_id}") - assert response.status_code == 202 - ending = api_env.app.state.library_deletion._tasks[library_id] + ending = None try: + assert (await api_env.client.post(f"/api/jobs/{job_id}/retry")).status_code == 202 + await asyncio.wait_for(entered.wait(), 2) + if operation == "shutdown": + ending = asyncio.create_task(service.close()) + elif operation == "reset": + api_env.app.state.services_running = True + api_env.app.state.auth.config.api_key = "test-job-service" + api_env.client.headers["Authorization"] = "Bearer test-job-service" + ending = asyncio.create_task(api_env.client.post("/api/backups/reset", json={"keep": []})) + # Keep reset's service restarts deterministic and avoid starting unrelated loops. + for owner, name in ( + (api_env.app.state.scanner, "start"), + (api_env.app.state.healthchecks, "start"), + (api_env.app.state.staging, "start"), + (api_env.app.state.scheduler, "start"), + (api_env.app.state.notifications, "start"), + ): + monkeypatch.setattr(owner, name, lambda: None) + + async def start(): + pass + + monkeypatch.setattr(api_env.app.state.probes, "start", start) + monkeypatch.setattr(api_env.app.state.flow_enqueue, "start", start) + else: + api_env.app.state.services_running = True + async with api_env.factory() as session: + job = await session.get(Job, job_id) + assert job is not None + library_id = job.library_id + await api_env.app.state.library_deletion.start() + response = await api_env.client.delete(f"/api/libraries/{library_id}") + assert response.status_code == 202 + ending = api_env.app.state.library_deletion._tasks[library_id] await asyncio.wait_for(draining.wait(), 2) assert not ending.done() release.set() @@ -366,8 +367,9 @@ async def start(): assert await session.get(Library, library_id) is None finally: release.set() - await asyncio.wait_for(asyncio.gather(ending, return_exceptions=True), 5) - await api_env.app.state.library_deletion.close() + if ending is not None: + await asyncio.wait_for(asyncio.gather(ending, return_exceptions=True), 5) + await asyncio.wait_for(api_env.app.state.library_deletion.close(), 5) @pytest.mark.asyncio diff --git a/tests/test_job_store.py b/tests/test_job_store.py index dacff73..504b0d6 100644 --- a/tests/test_job_store.py +++ b/tests/test_job_store.py @@ -665,7 +665,11 @@ async def test_claims_interrupted_healthcheck(db: async_sessionmaker[AsyncSessio assert claimed is not None and claimed.status == JobStatus.CLAIMED async with db() as session: - attempts = list((await session.scalars(select(JobAttempt).where(JobAttempt.job_id == job.id))).all()) + attempts = list( + await session.scalars( + select(JobAttempt).where(JobAttempt.job_id == job.id).order_by(JobAttempt.attempt_number) + ) + ) assert [(attempt.attempt_number, attempt.status, attempt.result) for attempt in attempts] == [ (1, JobStatus.FAILED.value, "interrupted"), (2, JobStatus.CLAIMED.value, None), diff --git a/tests/test_library_deletion.py b/tests/test_library_deletion.py index 6813918..f150184 100644 --- a/tests/test_library_deletion.py +++ b/tests/test_library_deletion.py @@ -341,10 +341,11 @@ async def closing(tasks): monkeypatch.setattr(runtime.service.scanner, "drain_library", held) monkeypatch.setattr(runtime.service, "_close", closing) - await runtime.service.request(library_id) - await asyncio.wait_for(started.wait(), 1) - stopping = asyncio.create_task(runtime.service.close()) + stopping = None try: + await runtime.service.request(library_id) + await asyncio.wait_for(started.wait(), 1) + stopping = asyncio.create_task(runtime.service.close()) await asyncio.wait_for(closing_started.wait(), 1) assert not stopping.done() release.set() @@ -359,7 +360,8 @@ async def closing(tasks): assert await session.get(Library, library_id) is None finally: release.set() - await asyncio.gather(stopping, return_exceptions=True) + if stopping is not None: + await asyncio.wait_for(asyncio.gather(stopping, return_exceptions=True), 5) @pytest.mark.asyncio diff --git a/tests/test_lifecycle_safety.py b/tests/test_lifecycle_safety.py index 5e30e37..50d6566 100644 --- a/tests/test_lifecycle_safety.py +++ b/tests/test_lifecycle_safety.py @@ -188,11 +188,11 @@ async def blocked_pass() -> int: monkeypatch.setattr(service, "_pass", blocked_pass) monkeypatch.setattr(service, "start", type(service).start.__get__(service)) monkeypatch.setattr(service, "close", type(service).close.__get__(service)) - await service.start() - await asyncio.wait_for(started.wait(), 2) - first = asyncio.create_task(service.close()) - second = None + first = second = None try: + await service.start() + await asyncio.wait_for(started.wait(), 2) + first = asyncio.create_task(service.close()) await asyncio.wait_for(stopping.wait(), 2) second = asyncio.create_task(service.close()) checkpoint = asyncio.Event() @@ -205,6 +205,7 @@ async def blocked_pass() -> int: finally: release.set() await asyncio.wait_for(asyncio.gather(*(task for task in (first, second) if task is not None)), 2) + await asyncio.wait_for(service.close(), 2) @pytest.mark.asyncio diff --git a/tests/test_notification_delivery.py b/tests/test_notification_delivery.py index a34781d..fcd14c1 100644 --- a/tests/test_notification_delivery.py +++ b/tests/test_notification_delivery.py @@ -59,7 +59,9 @@ async def test_terminal_enqueue_is_transactional(db: async_sessionmaker[AsyncSes await session.flush() assert await finish(session, job_id, JobStatus.FAILED, now=NOW) deliveries = list((await session.execute(select(NotificationDelivery))).scalars()) - events = list((await session.execute(select(JobEvent).where(JobEvent.job_id == job_id))).scalars()) + events = list( + (await session.execute(select(JobEvent).where(JobEvent.job_id == job_id).order_by(JobEvent.id))).scalars() + ) assert [(item.target_name, item.job_event_id) for item in deliveries] == [("primary", events[-1].id)] await session.rollback() diff --git a/tests/test_queue_staging_limits.py b/tests/test_queue_staging_limits.py index 511a55e..ee6223c 100644 --- a/tests/test_queue_staging_limits.py +++ b/tests/test_queue_staging_limits.py @@ -283,15 +283,16 @@ async def test_bulk_metadata_order(api_env: ApiEnv, monkeypatch) -> None: for index in range(150) ], ) - identifiers = list( + identifiers = sorted( await session.scalars( insert(Job).returning(Job.id), [ {"library_id": library.id, "media_file_id": media_id, "kind": "healthcheck", "status": "failed"} for media_id in media_ids ], - ) - )[::-1] + ), + reverse=True, + ) for identifier in (identifiers[0], identifiers[-1]): job = await session.get(Job, identifier) assert job is not None diff --git a/tests/test_scanner.py b/tests/test_scanner.py index 673b15b..dbe7061 100644 --- a/tests/test_scanner.py +++ b/tests/test_scanner.py @@ -1330,7 +1330,7 @@ def capture(type_, payload, key=None, *, bulk=False): else: payload = {"action": "pruned", "media_file_id": media.id, "library_id": scan_env.library_id} assert removals == [(payload, f"file.status:file:{media.id}")] - assert job_removals == job_ids + assert sorted(job_removals) == sorted(job_ids) events = [] while subscription.backlog: events.append(await asyncio.wait_for(subscription.next(), 1)) diff --git a/tests/test_staging.py b/tests/test_staging.py index a49249c..f8d2a28 100644 --- a/tests/test_staging.py +++ b/tests/test_staging.py @@ -43,6 +43,11 @@ from tests.helpers import request_retry, database_context, make_test_disk_path, make_test_temp_dir +@pytest.fixture(autouse=True) +def workspace_capacity(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("server.staging.shutil.disk_usage", lambda _: SimpleNamespace(free=20_000_000_000)) + + @pytest_asyncio.fixture async def db() -> AsyncIterator[async_sessionmaker[AsyncSession]]: # Parallel staging jobs need independent database transactions. diff --git a/tests/test_staging_safety.py b/tests/test_staging_safety.py index 63f6794..5e8c4a0 100644 --- a/tests/test_staging_safety.py +++ b/tests/test_staging_safety.py @@ -382,8 +382,8 @@ async def test_continuation_refuses_changes(staged_env, change: str): @pytest.mark.asyncio -@pytest.mark.parametrize("state", ["drained", "exhausted", "unconfirmed", "unread", "transcode", "unavailable"]) -async def test_prune_healthcheck_logs(staged_env, state: str): +@pytest.mark.parametrize("workload", ["healthcheck", "transcode"]) +async def test_prune_healthcheck_logs(staged_env, workload: str): factory, service, job_id, source = staged_env relative = workspace.create_attempt_layout(service.workspace_dir, job_id, 1, ".mkv")["log"] log = service.workspace_dir / relative @@ -396,18 +396,14 @@ async def test_prune_healthcheck_logs(staged_env, state: str): attempt_number=1, status=JobStatus.FAILED, result="interrupted", - workload="transcode" if state == "transcode" else "healthcheck", + workload=workload, log_path=relative, - log_data=None if state == "unread" else log.read_bytes(), + log_data=log.read_bytes(), finished_at=utcnow(), - error="max attempts exceeded" - if state == "exhausted" - else ("healthcheck process exit is unconfirmed" if state == "unconfirmed" else "requeued"), + error="requeued", ) session.add(attempt) await session.flush() - if state == "unavailable": - session.add(JobEvent(job_id=job_id, attempt_id=attempt.id, type="healthcheck_log_unavailable")) await session.commit() eligible = False assert await service.prune_attempt(attempt.id) is eligible @@ -415,7 +411,7 @@ async def test_prune_healthcheck_logs(staged_env, state: str): async with factory() as session: saved = await session.get(JobAttempt, attempt.id) assert saved is not None and saved.result == "interrupted" - assert saved.log_data == (None if state == "unread" else b"healthcheck log") + assert saved.log_data == b"healthcheck log" assert source.read_bytes() == b"source" diff --git a/tests/test_workspace_recheck.py b/tests/test_workspace_recheck.py index b850d60..6a25ee9 100644 --- a/tests/test_workspace_recheck.py +++ b/tests/test_workspace_recheck.py @@ -287,7 +287,11 @@ async def continue_subtitle(*args, **kwargs) -> str: job = await session.get(Job, job_id) assert job is not None and job.status == JobStatus.RUNNING stages = list( - (await session.scalars(select(JobEvent).where(JobEvent.job_id == job_id, JobEvent.type == "stage"))) + await session.scalars( + select(JobEvent) + .where(JobEvent.job_id == job_id, JobEvent.type == "stage") + .order_by(JobEvent.id) + ) ) assert len(stages) == 2 and stages[-1].payload["plan"] == plan.model_dump(mode="json") finally: diff --git a/ui/tests/batchDryRun.test.ts b/ui/tests/batchDryRun.test.ts index d5b6125..7c822c6 100644 --- a/ui/tests/batchDryRun.test.ts +++ b/ui/tests/batchDryRun.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'bun:test'; -import { computed, ref } from 'vue'; +import { computed, ref, watch } from 'vue'; import ts from 'typescript'; import type { BatchDryRunReport, @@ -160,8 +160,20 @@ const setup = () => { tick: async () => { const [id, timer] = [...timers.entries()][0]!; timers.delete(id); - timer.callback(); - for (let turn = 0; turn < 20 && store.loading.value; turn++) await Promise.resolve(); + const completed = Promise.withResolvers(); + const stop = watch( + store.loading, + (loading) => { + if (!loading) completed.resolve(undefined); + }, + { flush: 'sync' }, + ); + try { + timer.callback(); + if (store.loading.value) await completed.promise; + } finally { + stop(); + } expect(store.loading.value).toBe(false); }, }; diff --git a/ui/tests/filesActions.test.ts b/ui/tests/filesActions.test.ts index dc51012..dd6eef2 100644 --- a/ui/tests/filesActions.test.ts +++ b/ui/tests/filesActions.test.ts @@ -651,9 +651,7 @@ test('transcode stale retry', async () => { : Response.json({ job_id: 10, kind: 'transcode', status: 'pending' }, { status: 202 }), }); expect(await state.queueTranscode(file(1))).toBe(false); - expect(state.messages).toEqual([ - { kind: 'error', text: 'Flow changed. Reload and try again.' }, - ]); + expect(state.messages).toEqual([{ kind: 'error', text: 'Flow changed. Reload and try again.' }]); expect(state.calls).toHaveLength(1); expect(state.actionBusy.value).toBe(false); state.flows.value = [savedFlow(2, 5)]; @@ -681,6 +679,9 @@ test('single override warning', async () => { const state = await handler({ libraries: [library(1, false)] }); expect(await state.queueHealthCheck(file(1))).toBe(true); expect(state.dialogs).toHaveLength(1); + expect(state.dialogs[0]?.message).toBe( + 'Automatic health checks are disabled for Library 1 (1 file).', + ); }); test('override cancellation', async () => { @@ -700,6 +701,9 @@ test('mixed batch warning', async () => { }); await state.queueBulk('healthcheck', [file(1), file(2), file(3, 2), file(4, 3)]); expect(state.dialogs).toHaveLength(1); + expect(state.dialogs[0]?.message).toBe( + 'Automatic health checks are disabled for Library 1 (2 files), Library 3 (1 file).', + ); expect(state.calls).toEqual([ { url: '/api/files/bulk', @@ -755,7 +759,9 @@ test('batch summarizes acceptance', async () => { }; const state = await handler({ result }); expect(await state.queueBulk('transcode', [file(1), file(2), file(3)])).toEqual(result); - expect(state.messages.map(({ kind }) => kind)).toEqual(['error']); + expect(state.messages).toEqual([ + { kind: 'error', text: '2 transcode requests accepted, 1 rejected. #3: active job' }, + ]); }); test('batch acceptance summary', async () => { @@ -766,7 +772,9 @@ test('batch acceptance summary', async () => { }, }); await state.queueBulk('transcode', [file(1)]); - expect(state.messages.map(({ kind }) => kind)).toEqual(['success']); + expect(state.messages).toEqual([ + { kind: 'success', text: '1 transcode request accepted, 0 rejected.' }, + ]); }); test('batch rejection summary', async () => { @@ -776,7 +784,9 @@ test('batch rejection summary', async () => { }; const state = await handler({ result }); expect(await state.queueBulk('transcode', [file(1)])).toEqual(result); - expect(state.messages.map(({ kind }) => kind)).toEqual(['error']); + expect(state.messages).toEqual([ + { kind: 'error', text: '0 transcode requests accepted, 1 rejected. #1: Source missing.' }, + ]); }); test('settings unavailable blocks', async () => { @@ -855,9 +865,7 @@ test('transcode selection snapshot', async () => { state.flows.value[1]!.version = 8; state.flows.value[1]!.definition.entry = 'changed'; await state.picker().onDryRun(); - expect(state.previewCalls).toEqual([ - { fileId: 1, definition: savedFlow(3, 7).definition }, - ]); + expect(state.previewCalls).toEqual([{ fileId: 1, definition: savedFlow(3, 7).definition }]); expect(state.preview().props.flows.map(({ id }) => id)).toEqual([3]); expect(state.preview().props.filePath).toBe('/media/1.mkv'); expect(state.preview().props.facts).toContain('1920×1080'); diff --git a/ui/tests/jobRequeue.test.ts b/ui/tests/jobRequeue.test.ts index f0ff377..e236480 100644 --- a/ui/tests/jobRequeue.test.ts +++ b/ui/tests/jobRequeue.test.ts @@ -116,19 +116,31 @@ test('bulk partial summary', async () => { ); expect(await state.requeueBulk([1, 2, 3, 4])).toEqual([1, 2, 3]); expect(state.calls).toEqual([{ kind: 'bulk', id: [1, 2, 3, 4], action: 'retry' }]); - expect(state.messages.map(({ kind }) => kind)).toEqual(['error']); + expect(state.messages).toEqual([ + { + kind: 'error', + text: 'Requeue requested for 3 jobs. 1 rejected. First rejection: #4: active job', + }, + ]); }); test('bulk accepted summary', async () => { const state = await handler([], [{ accepted: [2], failed: [] }]); expect(await state.requeueBulk([2])).toEqual([2]); - expect(state.messages.map(({ kind }) => kind)).toEqual(['success']); + expect(state.messages).toEqual([ + { kind: 'success', text: 'Requeue requested for 1 job. 0 rejected.' }, + ]); }); test('bulk rejection summary', async () => { const state = await handler([], [{ accepted: [], failed: [{ id: 2, detail: 'Missing file.' }] }]); expect(await state.requeueBulk([2])).toEqual([]); - expect(state.messages.map(({ kind }) => kind)).toEqual(['error']); + expect(state.messages).toEqual([ + { + kind: 'error', + text: 'Requeue requested for 0 jobs. 1 rejected. First rejection: #2: Missing file.', + }, + ]); }); test('bulk request failure', async () => { diff --git a/ui/tests/liveUpdates.test.ts b/ui/tests/liveUpdates.test.ts index d3ae02e..5a47ff8 100644 --- a/ui/tests/liveUpdates.test.ts +++ b/ui/tests/liveUpdates.test.ts @@ -1,5 +1,10 @@ -import { afterEach, beforeAll, describe, expect, mock, test } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import * as Vue from 'vue'; +import ts from 'typescript'; import * as realUtils from '../app/utils'; +import * as stateHelpers from '../app/composables/stateHelpers'; +import * as jobList from '../app/utils/jobList'; +import * as statusUtils from '../app/utils/status'; const callbacks: Array<(event: ServerEvent) => void> = []; const reconnects: Array<() => void> = []; @@ -19,7 +24,7 @@ let responseBulk: let responseTotal: number | undefined; let actionStatus = 200; -mock.module('~/utils', () => ({ +const transport = { ...realUtils, request: (url: string) => { calls.push(url); @@ -34,19 +39,13 @@ mock.module('~/utils', () => ({ return Promise.resolve(response(url)); }, parse_api_error: async (body: { detail?: string } | null) => body?.detail ?? null, -})); -mock.module('~/composables/useApiErrorMessage', () => ({ - useApiErrorMessage: () => ({ messageFor: () => '' }), -})); -mock.module('../app/composables/useEvents', () => ({ +}; +const events = { useEvents: () => ({ on: (callback: (event: ServerEvent) => void) => callbacks.push(callback), onReconnect: (callback: () => void) => reconnects.push(callback), }), -})); -mock.module('../app/composables/useTranscodeFlow', () => ({ - useTranscodeFlow: () => ({ pick: async () => null }), -})); +}; type ServerEvent = import('../app/composables/useEvents').ServerEvent; type JobRow = import('../app/types/api').JobRow; @@ -170,22 +169,86 @@ const releaseAll = () => { for (const finish of releases.keys()) finish(); }; -beforeAll(() => { - Bun.plugin({ - name: 'browser-composables', - setup(build) { - build.onLoad({ filter: /use(Jobs|Files)\.ts$/ }, async (args) => ({ - contents: (await Bun.file(args.path).text()).replaceAll('import.meta.client', 'true'), - loader: 'ts', - })); +const compiled = new Map(); +for (const name of ['useJobs', 'useFiles']) { + const source = await Bun.file(new URL(`../app/composables/${name}.ts`, import.meta.url)).text(); + compiled.set( + name, + ts.transpileModule(source.replaceAll('import.meta.client', 'true'), { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.CommonJS }, + }).outputText, + ); +} +// Failed assertions can leave held loads running until teardown. +const pendingLoads = new Set>(); +const track = (pending: Promise): Promise => { + pendingLoads.add(pending); + void pending.then( + () => pendingLoads.delete(pending), + () => pendingLoads.delete(pending), + ); + return pending; +}; +let composables: { + useJobs: typeof import('../app/composables/useJobs'); + useFiles: typeof import('../app/composables/useFiles'); +}; +beforeEach(() => { + const modules: Record = { + vue: Vue, + '~/utils': transport, + '~/utils/jobList': jobList, + '~/utils/status': statusUtils, + './stateHelpers': stateHelpers, + './useEvents': events, + './useTranscodeFlow': { useTranscodeFlow: () => ({ pick: async () => null }) }, + '~/composables/useApiErrorMessage': { + useApiErrorMessage: () => ({ messageFor: () => '' }), }, - }); + }; + const load = (name: string): T => { + const exports = {}; + new Function('require', 'exports', compiled.get(name)!)( + (name: string) => modules[name] ?? {}, + exports, + ); + return exports as T; + }; + const jobs = load('useJobs'); + const files = load('useFiles'); + composables = { + useJobs: { + ...jobs, + useJobs: () => { + const store = jobs.useJobs(); + return { + ...store, + load: (filters) => track(store.load(filters)), + loadProcessing: () => track(store.loadProcessing()), + }; + }, + }, + useFiles: { + ...files, + useFiles: () => { + const store = files.useFiles(); + return { + ...store, + load: (filters) => track(store.load(filters)), + loadRecent: () => track(store.loadRecent()), + }; + }, + }, + }; }); -afterEach(() => { +afterEach(async () => { + holdRequest = false; releaseAll(); + await Promise.allSettled([...pendingLoads]); + callbacks.length = 0; + reconnects.length = 0; calls.length = 0; - holdRequest = false; responseJob = undefined; responseJobs = undefined; responseFiles = undefined; @@ -199,7 +262,7 @@ afterEach(() => { describe('live updates', () => { test('retry acknowledges promptly', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = { ...job(1, 100), status: 'failed' }; await jobs.load(filters); @@ -216,7 +279,7 @@ describe('live updates', () => { }); test.each(['retry', 'delete'] as const)('bulk %s acknowledges', async (action) => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJobs = [job(1, 0), job(2, 0)]; responseTotal = 2; @@ -242,7 +305,7 @@ describe('live updates', () => { }); test('delete acknowledges promptly', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); await jobs.load(filters); calls.length = 0; @@ -266,7 +329,7 @@ describe('live updates', () => { }); test.each(['single', 'bulk'] as const)('refresh accepts %s', async (mode) => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); await jobs.load(filters); holdRequest = true; @@ -292,7 +355,7 @@ describe('live updates', () => { }); test('delete retains rejection', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); await jobs.load(filters); calls.length = 0; @@ -306,7 +369,7 @@ describe('live updates', () => { }); test('pending stays listed', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); const subscriptions = callbacks.length; useJobs(); @@ -355,7 +418,7 @@ describe('live updates', () => { ['pending', 'transcode'], ['queued', 'healthcheck'], ] as const)('file status %s', async (queuedStatus, kind) => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = [ { @@ -389,7 +452,7 @@ describe('live updates', () => { }); test('queued status updates', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = { ...job(1, 0), @@ -432,7 +495,7 @@ describe('live updates', () => { }); test('pending snapshots normalize', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = { ...job(1, 100), @@ -495,7 +558,7 @@ describe('live updates', () => { ['pending', 901], ['queued', 902], ] as const)('terminal rejects %s', async (queuedStatus, id) => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); const catalogue = { ...file(id), @@ -575,7 +638,7 @@ describe('live updates', () => { }); test.each(['cancel', 'bump'] as const)('bulk refreshes %s', async (action) => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); const selectedFilters = { ...filters, @@ -604,7 +667,7 @@ describe('live updates', () => { }); test('bulk partial refresh', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJobs = [job(1, 0), job(2, 0)]; responseTotal = 2; @@ -624,7 +687,7 @@ describe('live updates', () => { }); test.each(['cancel', 'bump'] as const)('action refreshes %s', async (action) => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); await jobs.load(filters); const listUrl = calls.at(-1)!; @@ -637,7 +700,7 @@ describe('live updates', () => { }); test.each(['cancel', 'delete'] as const)('bulk %s rejection', async (action) => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); await jobs.load(filters); calls.length = 0; @@ -650,7 +713,7 @@ describe('live updates', () => { }); test('bulk refresh failure', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); await jobs.load(filters); failedList = 'jobs'; @@ -664,7 +727,7 @@ describe('live updates', () => { }); test('files retain full page', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = Array.from({ length: 999 }, (_, index) => file(index + 1)); responseTotal = 1000; @@ -677,7 +740,7 @@ describe('live updates', () => { }); test('recent isolates filters', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); const retained = { ...file(9, 'retained.mkv'), library_id: 2 }; responseFiles = [retained]; @@ -709,7 +772,7 @@ describe('live updates', () => { }); test('recent stays bounded', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = Array.from({ length: 6 }, (_, index) => ({ ...file(index + 1), @@ -732,7 +795,7 @@ describe('live updates', () => { }); test('recent deletion merge', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = Array.from({ length: 6 }, (_, index) => file(index + 1)); for (const overlap of [false, true]) { @@ -759,7 +822,7 @@ describe('live updates', () => { }); test.each(['deleted', 'pruned'] as const)('removed file lists %s', async (action) => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = [file(1), file(2)]; responseTotal = 2; @@ -774,7 +837,7 @@ describe('live updates', () => { }); test('file deletion merge', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = [file(1), file(2)]; responseTotal = 2; @@ -801,7 +864,7 @@ describe('live updates', () => { }); test('recent failure retention', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = [file(2)]; await files.loadRecent(); @@ -819,7 +882,7 @@ describe('live updates', () => { }); test('recent progress overlaps', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); const active = { ...file(1), @@ -858,7 +921,7 @@ describe('live updates', () => { }); test('separates file states', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); await files.load({ ...fileFilters, transcode_status: 'pending' }); calls.length = 0; @@ -897,7 +960,7 @@ describe('live updates', () => { }); test('file progress updates', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); const active = { ...file(1), @@ -942,7 +1005,7 @@ describe('live updates', () => { }); test('file progress overlaps', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); const active = { ...file(1), @@ -974,7 +1037,7 @@ describe('live updates', () => { }); test('file terminal overlaps', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = [ { @@ -1007,7 +1070,7 @@ describe('live updates', () => { }); test('file job replacement', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = [ { @@ -1038,7 +1101,7 @@ describe('live updates', () => { }); test('jobs isolate filters', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); const view = { ...filters, query: 'failed-jobs' }; failedList = 'jobs'; @@ -1077,7 +1140,7 @@ describe('live updates', () => { }); test('files isolate filters', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); const view = { ...fileFilters, query: 'failed-files' }; failedList = 'files'; @@ -1113,8 +1176,8 @@ describe('live updates', () => { }); test('late failures obsolete', async () => { - const { useJobs } = await import('../app/composables/useJobs'); - const { useFiles } = await import('../app/composables/useFiles'); + const { useJobs } = composables.useJobs; + const { useFiles } = composables.useFiles; const jobs = useJobs(); const files = useFiles(); holdRequest = true; @@ -1143,7 +1206,7 @@ describe('live updates', () => { }); test('initial overlaps show rows', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = running(1, 0, stamps[0]!); holdRequest = true; @@ -1162,8 +1225,8 @@ describe('live updates', () => { }); test('events do not reload loaded lists', async () => { - const { useJobs } = await import('../app/composables/useJobs'); - const { useFiles } = await import('../app/composables/useFiles'); + const { useJobs } = composables.useJobs; + const { useFiles } = composables.useFiles; const jobs = useJobs(); const files = useFiles(); await jobs.load(filters); @@ -1240,8 +1303,8 @@ describe('live updates', () => { }); test('manual loads request once', async () => { - const { useJobs } = await import('../app/composables/useJobs'); - const { useFiles } = await import('../app/composables/useFiles'); + const { useJobs } = composables.useJobs; + const { useFiles } = composables.useFiles; await useJobs().load(filters); await useFiles().load(fileFilters); await useJobs().loadProcessing(); @@ -1254,7 +1317,7 @@ describe('live updates', () => { }); test('refresh removes inactive', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); for (const reconnect of [false, true]) { for (const missed of ['deleted', 'terminal']) { @@ -1293,7 +1356,7 @@ describe('live updates', () => { }); test('processing failure visible', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = running(1, 0, stamps[0]!); await jobs.loadProcessing(); @@ -1309,7 +1372,7 @@ describe('live updates', () => { }); test('processing order stays', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJobs = [3, 1, 2].map((id) => ({ ...job(id, 0), status: 'running' })); responseTotal = 3; @@ -1347,7 +1410,7 @@ describe('live updates', () => { }); test('cancellation retains jobs', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); for (const terminal of ['cancelled', 'succeeded'] as const) { holdRequest = false; @@ -1384,7 +1447,7 @@ describe('live updates', () => { }); test('dashboard splits jobs', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const { dashboardJobStatusOptions, processingJobStatuses } = await import('../app/utils/jobList'); const jobs = useJobs(); @@ -1435,7 +1498,7 @@ describe('live updates', () => { }); test('split preserves attempts', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const { dashboardJobStatusOptions } = await import('../app/utils/jobList'); const jobs = useJobs(); responseJob = running(2, 0, stamps[1]!); @@ -1463,7 +1526,7 @@ describe('live updates', () => { }); test('fetches job details', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; responseDetail = { id: 1, attempts: [ @@ -1481,8 +1544,8 @@ describe('live updates', () => { }); test('list totals update', async () => { - const { useJobs } = await import('../app/composables/useJobs'); - const { useFiles } = await import('../app/composables/useFiles'); + const { useJobs } = composables.useJobs; + const { useFiles } = composables.useFiles; const jobs = useJobs(); const files = useFiles(); await jobs.load(filters); @@ -1502,8 +1565,8 @@ describe('live updates', () => { }); test('search changes become snapshots', async () => { - const { useJobs } = await import('../app/composables/useJobs'); - const { useFiles } = await import('../app/composables/useFiles'); + const { useJobs } = composables.useJobs; + const { useFiles } = composables.useFiles; const jobs = useJobs(); const files = useFiles(); await jobs.load({ ...filters, query: 'file' }); @@ -1526,8 +1589,8 @@ describe('live updates', () => { }); test('partial pages keep snapshots', async () => { - const { useJobs } = await import('../app/composables/useJobs'); - const { useFiles } = await import('../app/composables/useFiles'); + const { useJobs } = composables.useJobs; + const { useFiles } = composables.useFiles; const jobs = useJobs(); const files = useFiles(); await jobs.load({ ...filters, limit: 1 }); @@ -1551,7 +1614,7 @@ describe('live updates', () => { }); test('file statuses combine', async () => { - const { useFiles } = await import('../app/composables/useFiles'); + const { useFiles } = composables.useFiles; const files = useFiles(); responseFiles = [ file(1), @@ -1623,7 +1686,7 @@ describe('live updates', () => { }); test('multiple statuses filter', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJobs = [ job(1, 0), @@ -1649,7 +1712,7 @@ describe('live updates', () => { }); test('updates preserve view choices', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); const choices = { ...filters, @@ -1672,7 +1735,7 @@ describe('live updates', () => { }); test('overlaps preserve live views', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = running(1, 0, stamps[0]!); const view = { ...filters, status: 'running' }; @@ -1696,7 +1759,7 @@ describe('live updates', () => { }); test('changed filters accept snapshots', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = running(1, 0, stamps[0]!); await jobs.load(filters); @@ -1714,7 +1777,7 @@ describe('live updates', () => { }); test('missed updates invalidate loads', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); for (const reconnect of [false, true]) { holdRequest = false; @@ -1740,7 +1803,7 @@ describe('live updates', () => { }); test('updates respect attempt boundaries', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = running(1, 0, stamps[0]!); await jobs.load(filters); @@ -1785,7 +1848,7 @@ describe('live updates', () => { }); test('late responses exclude deletions', async () => { - const { useJobs } = await import('../app/composables/useJobs'); + const { useJobs } = composables.useJobs; const jobs = useJobs(); responseJob = running(1, 0, stamps[0]!); await jobs.load(filters); diff --git a/ui/tests/logsLifecycle.test.ts b/ui/tests/logsLifecycle.test.ts index 5ac4af2..fec0019 100644 --- a/ui/tests/logsLifecycle.test.ts +++ b/ui/tests/logsLifecycle.test.ts @@ -33,26 +33,43 @@ const page = (ids: number[], next: number | null = null): LogsResponse => ({ oldest_id: 1, truncated: false, }); -const flush = async () => { - for (let i = 0; i < 6; i++) { - await Promise.resolve(); - await Vue.nextTick(); - } -}; const mount = async (pages: (LogsResponse | Promise)[], visible = false) => { const urls: string[] = []; let stream: ReadableStreamDefaultController | null = null; + const connected = Promise.withResolvers(); + const disconnected = Promise.withResolvers(); + let consumed: ReturnType> | null = null; const request = async (url: string, options: { signal?: AbortSignal }) => { urls.push(url); if (url.startsWith('/api/logs/stream')) { - return new Response( - new ReadableStream({ - start(controller) { - stream = controller; - options.signal?.addEventListener('abort', () => controller.close(), { once: true }); + const body = new ReadableStream({ + start(controller) { + stream = controller; + options.signal?.addEventListener('abort', () => controller.close(), { once: true }); + }, + }); + return { + ok: true, + body: { + getReader: () => { + const reader = body.getReader(); + return { + read: () => { + connected.resolve(undefined); + // The next read starts after the previous chunk was processed. + consumed?.resolve(undefined); + consumed = null; + return reader.read(); + }, + cancel: () => reader.cancel(), + releaseLock: () => { + reader.releaseLock(); + disconnected.resolve(undefined); + }, + }; }, - }), - ); + }, + }; } return Response.json((await pages.shift()) ?? page([])); }; @@ -89,16 +106,24 @@ const mount = async (pages: (LogsResponse | Promise)[], visible = return { state, urls, - emit(entries: LogEntry[]) { - stream?.enqueue( + ready: connected.promise, + async emit(entries: LogEntry[]) { + await connected.promise; + const processed = Promise.withResolvers(); + consumed = processed; + stream!.enqueue( new TextEncoder().encode( entries .map((entry) => `id: ${entry.id}\nevent: log_lines\ndata: ${JSON.stringify(entry)}\n\n`) .join(''), ), ); + await processed.promise; + }, + close: async () => { + renderer.render(null, root); + if (stream) await disconnected.promise; }, - close: () => renderer.render(null, root), }; }; @@ -113,8 +138,7 @@ test('applies initial persisted levels and pages empty results', async () => { expect(view.state.entries.value.map((entry) => entry.id)).toEqual([20, 30]); expect(view.state.live.value).toBe(false); } finally { - view.close(); - await flush(); + await view.close(); } }); @@ -134,9 +158,7 @@ test('keeps older history accessible after live eviction', async () => { ); try { await view.state.start(['warning']); - await flush(); - view.emit(Array.from({ length: 700 }, (_, index) => record(index + 101))); - await flush(); + await view.emit(Array.from({ length: 700 }, (_, index) => record(index + 101))); expect(view.state.entries.value.length).toBe(logs.LOG_WINDOW); expect(view.state.entries.value[0]?.id).toBe(501); expect(view.state.loggerNames.value).toEqual(['server.initial', 'test']); @@ -146,10 +168,8 @@ test('keeps older history accessible after live eviction', async () => { expect(view.state.entries.value[0]?.id).toBe(400); expect(view.state.entries.value.length).toBe(logs.LOG_WINDOW); expect(view.state.loggerNames.value).toEqual(['server.initial', 'server.older', 'test']); - await flush(); } finally { - view.close(); - await flush(); + await view.close(); } }); @@ -160,8 +180,20 @@ test('pages exact logger history', async () => { await view.state.searchHistory(' failure ', ['warning'], ' server.worker '); await view.state.loadOlder(); await view.state.refreshHistory(); - view.state.setLive(true); - await flush(); + const refreshed = Promise.withResolvers(); + const stop = Vue.watch( + view.state.loading, + (loading) => { + if (!loading) refreshed.resolve(undefined); + }, + { flush: 'sync' }, + ); + try { + view.state.setLive(true); + await refreshed.promise; + } finally { + stop(); + } for (const url of view.urls.slice(1)) { const params = new URL(url, 'http://localhost').searchParams; expect(params.get('logger')).toBe('server.worker'); @@ -172,8 +204,7 @@ test('pages exact logger history', async () => { await view.state.searchHistory('', ['warning'], ''); expect(new URL(view.urls.at(-1)!, 'http://localhost').searchParams.has('logger')).toBe(false); } finally { - view.close(); - await flush(); + await view.close(); } }); @@ -181,9 +212,9 @@ test('filters exact live loggers', async () => { const view = await mount([page([]), page([]), page([])], true); try { await view.state.start(); - await flush(); + await view.ready; await view.state.searchHistory('failure', ['warning'], 'server.worker'); - view.emit([ + await view.emit([ { ...record(101), logger: 'server.worker', message: 'Failure' }, { ...record(102), logger: 'server.worker.child', message: 'failure' }, { ...record(103), logger: 'Server.worker', message: 'failure' }, @@ -191,35 +222,30 @@ test('filters exact live loggers', async () => { { ...record(105), logger: 'server.worker', message: 'failure', level: 'info' }, { ...record(106), logger: 'server.worker', message: 'success' }, ]); - await flush(); expect(view.state.entries.value.map((entry) => entry.id)).toEqual([101]); await view.state.searchHistory('failure', ['warning'], ''); - view.emit([{ ...record(107), logger: 'other', message: 'failure' }]); - await flush(); + await view.emit([{ ...record(107), logger: 'other', message: 'failure' }]); expect(view.state.entries.value.map((entry) => entry.id)).toEqual([101, 107]); } finally { - view.close(); - await flush(); + await view.close(); } }); test('filters retained live entries', async () => { const history = Promise.withResolvers(); const view = await mount([page([]), history.promise, page([])], true); + let search: Promise | undefined; try { await view.state.start(); - await flush(); - view.emit([ + await view.emit([ { ...record(101), logger: 'server.worker' }, { ...record(102), logger: 'other' }, ]); - await flush(); - const search = view.state.searchHistory('', [], 'server.worker'); - view.emit([ + search = view.state.searchHistory('', [], 'server.worker'); + await view.emit([ { ...record(103), logger: 'server.worker' }, { ...record(104), logger: 'server.worker.child' }, ]); - await flush(); history.resolve(page([])); await search; expect(view.state.entries.value.map((entry) => entry.id)).toEqual([101, 103]); @@ -227,8 +253,8 @@ test('filters retained live entries', async () => { expect(view.state.entries.value.map((entry) => entry.id)).toEqual([101, 103]); } finally { history.resolve(page([])); - view.close(); - await flush(); + await search; + await view.close(); } }); @@ -243,19 +269,18 @@ test('retains observed logger suggestions', async () => { const view = await mount([initial, page([]), page([])], true); try { await view.state.start(['warning']); - await flush(); + await view.ready; expect(view.state.loggerNames.value).toEqual(['server.alpha', 'server.beta']); await view.state.searchHistory('', ['warning'], 'server.alpha'); expect(view.state.entries.value).toEqual([]); expect(view.state.loggerNames.value).toEqual(['server.alpha', 'server.beta']); const historyRequests = view.urls.filter((url) => !url.startsWith('/api/logs/stream')).length; - view.emit([ + await view.emit([ { ...record(101), logger: 'server.alpha' }, { ...record(102), logger: 'server.gamma', level: 'info' }, { ...record(103), logger: 'server.beta' }, { ...record(104), logger: 'Server.alpha' }, ]); - await flush(); expect(view.state.entries.value.map((entry) => entry.id)).toEqual([101]); expect(view.state.loggerNames.value).toEqual([ 'Server.alpha', @@ -271,7 +296,6 @@ test('retains observed logger suggestions', async () => { expect(view.state.loggerNames.value).toContain('server.alpha'); expect(view.state.loggerNames.value).toContain('server.beta'); } finally { - view.close(); - await flush(); + await view.close(); } }); diff --git a/ui/tests/sharedFeedback.test.ts b/ui/tests/sharedFeedback.test.ts index e6c7093..11db07f 100644 --- a/ui/tests/sharedFeedback.test.ts +++ b/ui/tests/sharedFeedback.test.ts @@ -26,12 +26,6 @@ import type { } from '../app/types/api'; import type { formatBytes, ensure_api_success } from '../app/utils'; -const flush = async () => { - for (let i = 0; i < 8; i++) { - await Promise.resolve(); - await Vue.nextTick(); - } -}; const deferred = () => { let resolve!: (value: T) => void; const promise = new Promise((done) => { @@ -565,6 +559,15 @@ test('search reloads and cancels', async () => { clearTimeout: (id: number) => timers.delete(id), }, ); + const lookupJob = + exported.lookupJob as typeof import('../app/composables/useGlobalSearch').lookupJob; + const lookups: ReturnType[] = []; + // The debounced watcher doesn't return its lookup promises. + exported.lookupJob = (...args: Parameters) => { + const pending = lookupJob(...args); + lookups.push(pending); + return pending; + }; const scope = Vue.effectScope(); const search = scope.run(() => (exported.useGlobalSearch as typeof useGlobalSearch)())!; try { @@ -574,40 +577,41 @@ test('search reloads and cancels', async () => { await search.openSearch(); expect(calls).toBe(6); search.searchTerm.value = '12'; - await flush(); + await Vue.nextTick(); timers.get(timerId)!(); - await flush(); search.searchTerm.value = '13'; - await flush(); + await Vue.nextTick(); expect(signals.at(-1)!.aborted).toBe(true); jobs[0]!.resolve(Response.json({ id: 12 })); + await lookups[0]; timers.get(timerId)!(); - await flush(); jobs[1]!.resolve(new Response(null, { status: 503 })); - await flush(); + await expect(lookups[1]!).rejects.toThrow('HTTP 503'); expect(search.matchedJob.value).toBeNull(); - expect(search.jobError.value).not.toBeNull(); + expect(search.jobError.value).toBe('Could not look up this job. Change the query to retry.'); search.searchTerm.value = '14'; - await flush(); + await Vue.nextTick(); timers.get(timerId)!(); - await flush(); scope.stop(); expect(signals.at(-1)!.aborted).toBe(true); jobs[2]!.resolve(Response.json({ id: 14 })); - await flush(); + await lookups[2]; expect(search.matchedJob.value).toBeNull(); failList = true; const nextScope = Vue.effectScope(); try { const next = nextScope.run(() => (exported.useGlobalSearch as typeof useGlobalSearch)())!; await next.openSearch(); - expect(next.listError.value).not.toBeNull(); + expect(next.listError.value).toBe( + 'Could not load search data. Close and reopen search to retry.', + ); } finally { nextScope.stop(); } } finally { jobs.forEach((job) => job.resolve(Response.json({ id: 0 }))); scope.stop(); + await Promise.allSettled(lookups); } }); @@ -1199,10 +1203,15 @@ test('batch scans summarize outcomes', async () => { options: { method: 'POST', body: JSON.stringify({ library_ids: [1, 2, 3] }) }, }, ]); - expect(feedback.map(({ level }) => level)).toEqual(['success']); + expect(feedback).toEqual([ + { level: 'success', message: '1 queued, 2 already queued or running, 0 suspended, 0 failed.' }, + ]); response = Response.json({ outcomes: [{ library_id: 1, status: 'suspended' }] }); await libraries.scanMany([1]); - expect(feedback.at(-1)?.level).toBe('info'); + expect(feedback.at(-1)).toEqual({ + level: 'info', + message: '0 queued, 0 already queued or running, 1 suspended, 0 failed.', + }); response = Response.json({ outcomes: [ { library_id: 1, status: 'stopped' }, @@ -1210,7 +1219,10 @@ test('batch scans summarize outcomes', async () => { ], }); await libraries.scanMany([1, 2]); - expect(feedback.at(-1)?.level).toBe('error'); + expect(feedback.at(-1)).toEqual({ + level: 'error', + message: '0 queued, 0 already queued or running, 0 suspended, 2 failed.', + }); response = Response.json({ detail: 'Scanner is paused.' }, { status: 409 }); await libraries.scanMany([1]); expect(feedback.at(-1)).toEqual({ level: 'error', message: 'Scanner is paused.' }); diff --git a/ui/tests/stateHelpers.test.ts b/ui/tests/stateHelpers.test.ts index 9bb0e45..d820c53 100644 --- a/ui/tests/stateHelpers.test.ts +++ b/ui/tests/stateHelpers.test.ts @@ -32,6 +32,6 @@ describe('shared state ordering', () => { { id: 2, priority: 1, progress: 0 }, { id: 1, priority: 0, progress: 0 }, ]); - expect(snapshot[0]?.progress).toBe(0); + expect(snapshot.map((item) => item.id)).toEqual([1, 2]); }); }); From 8b20ba2fe4103d9167c26876b4de301370864148 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Thu, 22 Oct 2026 07:40:17 +0300 Subject: [PATCH 3/8] Feat: add plex, jellyfin and emby as target for notify external --- common/src/common/actions.py | 40 ++- common/src/common/plan/models.py | 31 ++- common/src/common/plugin.py | 5 +- docs/plugins.md | 14 +- docs/recipes.md | 76 +++--- docs/usage.md | 3 +- plugins/action.embed_metadata/plugin.py | 8 +- plugins/action.extract_subs/plugin.py | 4 +- plugins/action.extract_subs/plugin.toml | 1 - plugins/action.notify/plugin.py | 6 +- plugins/action.notify_external/plugin.toml | 2 +- plugins/inspector.audio_languages/plugin.py | 4 +- plugins/inspector.bitrate_estimate/plugin.py | 12 +- .../inspector.bitrate_estimate/plugin.toml | 4 +- plugins/inspector.output_ratio/plugin.py | 13 +- plugins/inspector.output_ratio/plugin.toml | 4 +- plugins/inspector.stream_guard/plugin.py | 20 +- plugins/inspector.stream_guard/plugin.toml | 5 +- plugins/transcode.burn_subtitles/plugin.py | 36 +-- plugins/transcode.burn_subtitles/plugin.toml | 7 +- plugins/transcode.remux/plugin.py | 8 +- plugins/transcode.remux/plugin.toml | 4 +- plugins/transcode.video/plugin.py | 41 +-- plugins/transcode.video/plugin.toml | 10 +- .../server/external_notifications/__init__.py | 23 +- .../src/server/external_notifications/arr.py | 6 +- .../external_notifications/media_server.py | 22 ++ .../src/server/external_notifications/plex.py | 57 ++++ tests/test_external_notifications.py | 255 +++++++++++++++++- tests/test_plugin_migrations.py | 8 +- ui/app/components/StepInspector.vue | 102 ++++++- ui/app/types/api.ts | 6 +- ui/app/utils/stepDraft.ts | 24 +- ui/tests/flowInputs.test.ts | 63 +++++ 34 files changed, 744 insertions(+), 180 deletions(-) create mode 100644 server/src/server/external_notifications/media_server.py create mode 100644 server/src/server/external_notifications/plex.py diff --git a/common/src/common/actions.py b/common/src/common/actions.py index 17fb9d7..5ce9eab 100644 --- a/common/src/common/actions.py +++ b/common/src/common/actions.py @@ -1,8 +1,9 @@ from __future__ import annotations +from pathlib import PurePath, PurePosixPath, PureWindowsPath from typing import TYPE_CHECKING, Annotated, Literal -from pydantic import BaseModel, Field, HttpUrl, field_validator +from pydantic import BaseModel, Field, HttpUrl, field_validator, model_validator from common.media.language import normalize_language from common.metadata import MetadataValues @@ -55,16 +56,40 @@ class Notify(PluginAction): on: list[Literal["job.succeeded", "job.failed", "job.cancelled"]] = Field(min_length=1) +class PathMapping(BaseModel): + source: str = Field(min_length=1, description="Path used by Millwright.") + destination: str = Field(min_length=1, description="Path used by the application.") + + @model_validator(mode="after") + def check_paths(self) -> PathMapping: + source = PurePath(self.source) + destination = ( + PureWindowsPath(self.destination) + if PureWindowsPath(self.destination).is_absolute() + else PurePosixPath(self.destination) + ) + if not source.is_absolute() or not destination.is_absolute(): + raise ValueError("Use absolute paths in mappings.") + if ".." in source.parts or ".." in destination.parts: + raise ValueError("Mapping paths can't contain '..'.") + return self + + class ExternalNotificationConfig(BaseModel): - application: Literal["radarr", "sonarr"] + application: Literal["radarr", "sonarr", "plex", "jellyfin", "emby"] url: HttpUrl = Field(description="Application URL, including any URL base.") api_key: str = Field( - min_length=1, repr=False, description="Application API key.", json_schema_extra={"ui": "password"} + min_length=1, repr=False, description="Application API key or token.", json_schema_extra={"ui": "password"} ) unmonitor_after_refresh: bool = Field( default=False, description="Disable future automatic upgrades for the movie or associated episodes after refresh succeeds.", ) + path_mappings: list[PathMapping] = Field( + default=[], + description="Add mappings if an application uses different paths for your files.", + json_schema_extra={"ui": "path_mappings"}, + ) @field_validator("url") @classmethod @@ -73,6 +98,15 @@ def check_url(cls, value: HttpUrl) -> HttpUrl: raise ValueError("Use an application URL without credentials, a query, or a fragment.") return value + @model_validator(mode="after") + def check_options(self) -> ExternalNotificationConfig: + if self.application not in {"radarr", "sonarr"} and self.unmonitor_after_refresh: + raise ValueError("Unmonitor after refresh is only supported for Radarr and Sonarr.") + sources = [PurePath(mapping.source) for mapping in self.path_mappings] + if len(sources) != len(set(sources)): + raise ValueError("Each Millwright path can appear only once.") + return self + class NotifyExternal(ExternalNotificationConfig, PluginAction): kind: Literal["notify_external"] = "notify_external" diff --git a/common/src/common/plan/models.py b/common/src/common/plan/models.py index b84bb17..4e1a7bd 100644 --- a/common/src/common/plan/models.py +++ b/common/src/common/plan/models.py @@ -57,24 +57,28 @@ class StreamPolicy(BaseModel): class StreamPolicyInputs(BaseModel): audio_keep_languages: list[str] = Field( - default=[], description="Comma-separated ISO-639-2 audio languages (eng, jpn). Empty keeps all" + default=[], description="Keep audio in these languages, such as eng or jpn. Leave empty to keep all languages." + ) + audio_drop_descriptive: bool = Field(default=False, description="Remove commentary and descriptive audio tracks.") + audio_keep_forced: bool = Field( + default=True, description="Keep forced audio tracks even if their language isn't selected." ) - audio_drop_descriptive: bool = Field(default=False, description="Drop commentary/description audio tracks") - audio_keep_forced: bool = Field(default=True, description="Keep forced audio tracks regardless of language") audio_keep_all_if_empty: bool = Field( - default=False, description="Keep all audio if the language policy would remove every track" + default=False, description="Keep all audio tracks if the other audio filters would remove them all." ) audio_max_per_language: int | None = Field( - default=None, ge=1, description="Keep at most this many audio tracks per language. Blank keeps all" + default=None, ge=1, description="Limit audio tracks per language. Leave empty for no limit." ) subtitle_keep_languages: list[str] = Field( default=[], - description="Comma-separated ISO-639-2 subtitle languages. Empty leaves subtitles untouched", + description="Keep subtitles in these languages, such as eng or jpn. Leave empty to keep all languages.", + ) + subtitle_strip_all: bool = Field(default=False, description="Remove subtitles, keeping forced tracks if enabled.") + subtitle_keep_forced: bool = Field( + default=True, description="Keep forced subtitles even if their language isn't selected." ) - subtitle_strip_all: bool = Field(default=False, description="Drop all subtitles except retained forced tracks") - subtitle_keep_forced: bool = Field(default=True, description="Keep forced subtitles regardless of language") - drop_data: bool = Field(default=False, description="Drop all data streams") - drop_cover_art: bool = Field(default=False, description="Drop embedded picture video streams") + drop_data: bool = Field(default=False, description="Remove data streams.") + drop_cover_art: bool = Field(default=False, description="Remove embedded cover art.") def stream_policy(self) -> StreamPolicy | None: audio = None @@ -101,10 +105,11 @@ def stream_policy(self) -> StreamPolicy | None: class AudioPlanInputs(StreamPolicyInputs): - audio_codec: str | None = Field(default=None, description="Codec for re-encoding matching audio tracks") - audio_bitrate: str | None = Field(default=None, description="Target bitrate for re-encoded audio tracks") + audio_codec: str | None = Field(default=None, description="Choose the output audio codec.") + audio_bitrate: str | None = Field(default=None, description="Set the output audio bitrate, such as 192k.") audio_when_codecs: list[str] = Field( - default=[], description="Re-encode audio with these source codecs. Empty matches all" + default=[], + description="Convert audio tracks with these source codecs. Leave empty to convert all selected tracks.", ) @field_validator("audio_codec", "audio_bitrate", mode="before") diff --git a/common/src/common/plugin.py b/common/src/common/plugin.py index 8609a59..068c0b7 100644 --- a/common/src/common/plugin.py +++ b/common/src/common/plugin.py @@ -12,7 +12,7 @@ TextField = Annotated[str, Field(json_schema_extra={"ui": "text"})] -FieldType = Literal["int", "float", "bool", "str", "text", "password", "enum", "strings"] +FieldType = Literal["int", "float", "bool", "str", "text", "password", "enum", "strings", "path_mappings"] def input_fields(model: type[BaseModel]) -> list[dict[str, Any]]: @@ -56,6 +56,9 @@ def _field_type(info: FieldInfo) -> FieldType: if get_origin(annotation) is Literal: return "enum" if get_origin(annotation) is list: + extra = info.json_schema_extra + if isinstance(extra, dict) and extra.get("ui") == "path_mappings": + return "path_mappings" return "strings" if annotation is bool: return "bool" diff --git a/docs/plugins.md b/docs/plugins.md index 385e702..2a06fdb 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -274,13 +274,13 @@ return { } ``` -| Action | Result | -| --------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| `extract_subtitles` | Extract selected subtitles and publish sidecars beside the final media | -| `notify` | Send selected job outcomes to named notification targets | -| `notify_external` | Notify a configured external application after replacement. Supports Radarr and Sonarr refreshes, with optional unmonitoring | -| `preserve_timestamps` | Preserve the file times captured for processing | -| `embed_metadata` | Apply sidecar metadata during planning | +| Action | Result | +| --------------------- | ---------------------------------------------------------------------- | +| `extract_subtitles` | Extract selected subtitles and publish sidecars beside the final media | +| `notify` | Send selected job outcomes to named notification targets | +| `notify_external` | Notify a configured external application after replacement | +| `preserve_timestamps` | Preserve the file times captured for processing | +| `embed_metadata` | Apply sidecar metadata during planning | Required subtitle extraction happens before encoding. If removed tracks would otherwise be lost when discarding the original, their sidecars must be saved first. A collision leaves the original protected. diff --git a/docs/recipes.md b/docs/recipes.md index 5f639dd..e8c223c 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -15,9 +15,8 @@ To convert video to HEVC in an MKV container: 3. Connect its `plan` output to a **Done** result and `skip` to **Skip**. 4. Save the flow and preview it against a few files. -With no encoder selected, HEVC uses `libx265`. AVC and AV1 use `libx264` and `libsvtav1`. Quality mode -lets the encoder choose the bitrate. Try a sample to judge both quality and file size rather than -expecting a fixed percentage saving. +Leave the encoder unset to use `libx265` for HEVC, `libx264` for AVC, or `libsvtav1` for AV1. Use quality +mode to choose quality without a fixed bitrate. Try a sample to judge the quality and file size.
Import this example as JSON @@ -48,9 +47,8 @@ Save this as a JSON file and choose **Flows -> Import flow**: ### Target a source bitrate ratio -If you prefer a bitrate target, select bitrate mode and set a ratio. With the video bitrate source, -`0.5` targets half the reported video bitrate. If that bitrate is unavailable, it uses the whole-file -average, including audio and container data. The ratio doesn't halve the output file size. +Select bitrate mode and set a ratio. A ratio of `0.5` targets half the source bitrate, not half the file +size. The step inputs are: @@ -58,8 +56,7 @@ The step inputs are: {"target": "hevc", "rate_control": "bitrate", "ratio": 0.5, "bitrate_source": "video"} ``` -Use quality mode when the required source bitrate is unavailable. Quality and bitrate modes are -alternative ways of choosing the encoding target. +Use quality mode if the source bitrate is unavailable. ### Prefer a hardware encoder @@ -70,9 +67,9 @@ use this priority list: {"target": "hevc", "encoder_priority": ["hevc_qsv", "libx265"]} ``` -The choice is made before processing begins. It doesn't switch encoders halfway through a failed job. -A fixed encoder setting overrides the priority list. See [GPU access](installation.md#hardware-acceleration) -if your preferred encoder isn't available. +The first available encoder is used. A fixed encoder overrides this list. Failed encodes aren't retried +with another encoder. See [GPU access](installation.md#hardware-acceleration) if your preferred encoder +isn't available. ## Route by size, codec, or language @@ -106,8 +103,7 @@ Connect `plan` to **Done** and `skip` to **Skip**, as in the video example. Leav empty to convert all selected audio. Use the language and commentary options if you also want to narrow the track selection. -Audio re-encoding counts as transcoding for the library's original-retention setting, even though the -video is copied. +The [`transcode` trash mode](recovery.md#retain-originals) keeps the original when audio is re-encoded. ## Reject an output that shrinks too far @@ -123,25 +119,22 @@ This checks size, not visual quality. Rejected output doesn't replace your origi ## Embed sidecar metadata -Use **Embed sidecar metadata** to copy titles, descriptions, dates, and genres from Kodi NFO or yt-dlp -JSON files into MKV or MP4 media. Episode NFOs can also provide the show name, season, and episode number. -yt-dlp JSON can provide the artist from its channel or uploader. +Use **Embed sidecar metadata** to copy metadata from Kodi NFO or yt-dlp JSON files into MKV or MP4 files. For `Example.mkv`, put `Example.nfo` or `Example.info.json` beside it. Add **Embed sidecar metadata** -before the remux or video step, and connect its `next` output to that step. Millwright reads the -sidecars without changing them. +before the remux or video step, and connect its `next` output to that step. The step offers these settings: | Setting | Default | Choice | | ----------------------- | ---------------------- | --------------------------------------------------------------------- | -| `existing_tags` | `fill_missing` | Fill absent fields, or use `overwrite` to replace supplied fields | -| `source_priority` | `["nfo", "info_json"]` | The first source with a value wins for each field | -| `when_no_media_changes` | `skip` | Use `remux` to allow tag changes without an audio or video conversion | +| `existing_tags` | `fill_missing` | Fill missing tags, or use `overwrite` to replace supplied fields | +| `source_priority` | `["nfo", "info_json"]` | Prefer NFO or JSON when both supply a tag | +| `when_no_media_changes` | `skip` | Use `remux` to update tags without re-encoding | NFO files must describe a Kodi movie or episode. JSON must describe one yt-dlp video, not a playlist or -channel. Missing sidecars are allowed. If tags aren't added, check the job log for sidecar errors and -confirm that the step runs before conversion. Matching tags don't trigger another conversion. +channel. Missing sidecars are allowed. If tags aren't added, check the job log and confirm that the step +runs before conversion. MP4 tag support varies by player. Try a file in your player before applying this to a library. @@ -157,29 +150,34 @@ Choose the target and the outcomes you want to receive. For success, failure, an {"targets": ["ops"], "on": ["job.succeeded", "job.failed", "job.cancelled"]} ``` -Putting Notify before the conversion means it's already selected if that conversion fails. Disable the -step to stop notifications without deleting it. A notification can occasionally be delivered more than once. +Turn off **Enabled** to stop notifications without removing the step. Notifications can occasionally +arrive more than once. ## Notify external applications Use **Notify external application** to tell another application when Millwright replaces a file. -The supported applications are Radarr and Sonarr, which rescan the published media. This is useful when -a conversion changes the extension or media information the application last saw. +Supported applications are Radarr, Sonarr, Plex, Jellyfin, and Emby. -Add the step to the flow's successful path. Select the application, enter its base URL and API key, and -save the flow. Include any URL base, such as `https://example.org/sonarr`, but don't append `/api/v3`. +1. Add the step to the flow's successful path. +2. Select the application and enter its URL and API key or token. +3. Add **Path mappings** if the application uses different paths for your files. +4. Save the flow. -Millwright sends the refresh only after the converted file has replaced the original. It doesn't send -one for dry runs, skips, or failed conversions. Delivery status and errors appear under the job's -**Notification deliveries**. A notification failure doesn't mark the conversion itself as failed. +Use the application's base URL, such as `https://example.org/sonarr`, without an API endpoint. Add a +separate step for each application. -Enable **Unmonitor after refresh** only if you want to stop future automatic upgrades. Radarr unmonitors -the movie. Sonarr unmonitors the episodes in that file, not the whole season or series. +Each mapping pairs a **Millwright path** with an **Application path**. For example: -If lookup fails, check whether the media manager can recognize the published path and filename. Radarr -can use `{imdb-...}` or `{tmdb-...}` identifiers in the path or filename before trying its filename parser. +| Millwright path | Application path | +| ------------------- | ---------------- | +| `/mnt/movies` | `/media/Movies` | +| `/mnt/series` | `/media/TV` | -Flow and job exports can contain this API key. Keep them private. +For Radarr and Sonarr, enable **Unmonitor after refresh** to stop automatic upgrades for the converted +movie or episodes. -For the other direction, where a media manager tells Millwright about new imports, use -[Incoming webhooks](webhooks.md). +Check the job's **Notification deliveries** for status and errors. + +Flow and job exports can contain the API key or token. Keep them private. + +To receive imports from applications, use [Incoming webhooks](webhooks.md). diff --git a/docs/usage.md b/docs/usage.md index 696333d..a058665 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -190,8 +190,7 @@ describe files still in the library, and deleting history can change earlier siz - Create an Apprise target under **Notifications** and add a [notification step](recipes.md#job-notifications) to the flows you want to hear about. - Use [incoming webhooks](webhooks.md) to receive imports and renames from other applications. -- Use [Notify external applications](recipes.md#notify-external-applications) to tell Radarr or Sonarr - when a converted file is ready. +- [Notify applications after conversion](recipes.md#notify-external-applications). ## Command-line and API access diff --git a/plugins/action.embed_metadata/plugin.py b/plugins/action.embed_metadata/plugin.py index 9d30bbf..6e56c0b 100644 --- a/plugins/action.embed_metadata/plugin.py +++ b/plugins/action.embed_metadata/plugin.py @@ -12,20 +12,20 @@ class Inputs(BaseModel): existing_tags: Literal["fill_missing", "overwrite"] = Field( - default="fill_missing", description="Fill absent tags or replace only supplied fields" + default="fill_missing", description="Fill missing tags or overwrite tags supplied by the sidecar." ) source_priority: list[MetadataSource] = Field( - default=["nfo", "info_json"], description="Source order. The first nonempty value wins for each field" + default=["nfo", "info_json"], description="Choose which sidecars to prefer when they provide the same tag." ) when_no_media_changes: Literal["skip", "remux"] = Field( - default="skip", description="Skip metadata-only work or permit changed tags alone to trigger processing" + default="skip", description="Skip files needing only tag changes, or remux them to update the tags." ) @field_validator("source_priority") @classmethod def check_priority(cls, value: list[MetadataSource]) -> list[MetadataSource]: if not value or len(value) != len(set(value)): - raise ValueError("source_priority must be nonempty and unique") + raise ValueError("Choose at least one metadata source without duplicates.") return value diff --git a/plugins/action.extract_subs/plugin.py b/plugins/action.extract_subs/plugin.py index 5376a07..b3d1ee9 100644 --- a/plugins/action.extract_subs/plugin.py +++ b/plugins/action.extract_subs/plugin.py @@ -10,8 +10,8 @@ class Inputs(BaseModel): - formats: list[str] = Field(default=["srt"], description="Subtitle output formats to extract: srt or vtt.") - languages: list[str] = Field(default=["eng"], description="ISO 639-2 language tags to extract.") + formats: list[str] = Field(default=["srt"], description="Extract subtitles as srt or vtt.") + languages: list[str] = Field(default=["eng"], description="Choose subtitle language codes, such as eng or jpn.") INPUTS = Inputs diff --git a/plugins/action.extract_subs/plugin.toml b/plugins/action.extract_subs/plugin.toml index db21b98..825f8f1 100644 --- a/plugins/action.extract_subs/plugin.toml +++ b/plugins/action.extract_subs/plugin.toml @@ -3,7 +3,6 @@ kind = "action" description = "Extract selected subtitle tracks beside converted media." help = """ Select formats and languages. The default is English SRT. -Extracted files are written beside the converted media. """ name = "Extract subtitles" author = "Millwright" diff --git a/plugins/action.notify/plugin.py b/plugins/action.notify/plugin.py index 42d26cb..5bd6127 100644 --- a/plugins/action.notify/plugin.py +++ b/plugins/action.notify/plugin.py @@ -12,11 +12,11 @@ class Inputs(BaseModel): enabled: bool = Field( default=True, - description="Enable notifications from this flow step. When off, this step adds no notification actions.", + description="Enable notifications from this step.", ) - targets: list[str] = Field(default=[], description="Notification target identifiers. Required when enabled.") + targets: list[str] = Field(default=[], description="Choose at least one notification target when enabled.") on: list[Literal["job.succeeded", "job.failed", "job.cancelled"]] = Field( - default=[], description="Job events that trigger notifications. Required when enabled." + default=[], description="Choose at least one job event to notify on when enabled." ) @model_validator(mode="after") diff --git a/plugins/action.notify_external/plugin.toml b/plugins/action.notify_external/plugin.toml index 2d0b8df..98624fa 100644 --- a/plugins/action.notify_external/plugin.toml +++ b/plugins/action.notify_external/plugin.toml @@ -1,6 +1,6 @@ id = "action.notify_external" kind = "action" -description = "Refresh Radarr or Sonarr after verified publication. Optionally unmonitor the movie or associated episodes after refresh succeeds." +description = "Notify applications after conversion." name = "Notify external application" author = "Millwright" url = "https://github.com/arabcoders/millwright" diff --git a/plugins/inspector.audio_languages/plugin.py b/plugins/inspector.audio_languages/plugin.py index 0dbdb7e..355a980 100644 --- a/plugins/inspector.audio_languages/plugin.py +++ b/plugins/inspector.audio_languages/plugin.py @@ -10,7 +10,9 @@ class Inputs(BaseModel): - expected: list[str] = Field(default=["eng", "und"], description="ISO 639-2 language tags allowed on audio streams.") + expected: list[str] = Field( + default=["eng", "und"], description="List allowed audio language codes, such as eng or und." + ) INPUTS = Inputs diff --git a/plugins/inspector.bitrate_estimate/plugin.py b/plugins/inspector.bitrate_estimate/plugin.py index 98f8b76..90e8545 100644 --- a/plugins/inspector.bitrate_estimate/plugin.py +++ b/plugins/inspector.bitrate_estimate/plugin.py @@ -10,16 +10,16 @@ class Inputs(BaseModel): source: Literal["video", "container", "average"] = Field( - default="video", description="Bitrate source: video, container, or their average when both are available" + default="video", description="Choose video bitrate, whole-file bitrate, or their average." ) avc_ratio: float = Field( - default=0.75, ge=0.05, le=1.0, description="AVC bitrate estimate as a source bitrate ratio." + default=0.75, ge=0.05, le=1.0, description="Estimate AVC bitrate as this fraction of the source bitrate." ) hevc_ratio: float = Field( - default=0.5, ge=0.05, le=1.0, description="HEVC bitrate estimate as a source bitrate ratio." + default=0.5, ge=0.05, le=1.0, description="Estimate HEVC bitrate as this fraction of the source bitrate." ) av1_ratio: float = Field( - default=0.45, ge=0.05, le=1.0, description="AV1 bitrate estimate as a source bitrate ratio." + default=0.45, ge=0.05, le=1.0, description="Estimate AV1 bitrate as this fraction of the source bitrate." ) @@ -29,9 +29,9 @@ class Inputs(BaseModel): def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: video = next(iter(ctx.probe.real_video), None) if video is None: - return {"type": "skip", "reason": "no video stream"} + return {"type": "skip", "reason": "No video stream found."} if ctx.probe.target_bitrate_kbps(inputs.source, 1) is None: - return {"type": "skip", "reason": "no derivable video bitrate"} + return {"type": "skip", "reason": "Cannot determine the source bitrate."} ratios = {"avc": inputs.avc_ratio, "hevc": inputs.hevc_ratio, "av1": inputs.av1_ratio} targets = ["avc", "hevc", "av1"] facets: dict[str, int] = {} diff --git a/plugins/inspector.bitrate_estimate/plugin.toml b/plugins/inspector.bitrate_estimate/plugin.toml index db5dfaf..cce695a 100644 --- a/plugins/inspector.bitrate_estimate/plugin.toml +++ b/plugins/inspector.bitrate_estimate/plugin.toml @@ -2,9 +2,7 @@ id = "inspector.bitrate_estimate" kind = "inspector" description = "Estimate AVC, HEVC, and AV1 bitrate and size." help = """ -Apply codec ratios to the selected bitrate source to estimate bitrate and size. -Video bitrate falls back to file size and duration. Skip files without video or -a derivable bitrate. +Set a bitrate ratio for each codec. Files without video or a usable bitrate are skipped. """ name = "Bitrate estimate" author = "Millwright" diff --git a/plugins/inspector.output_ratio/plugin.py b/plugins/inspector.output_ratio/plugin.py index 64f6812..1af2373 100644 --- a/plugins/inspector.output_ratio/plugin.py +++ b/plugins/inspector.output_ratio/plugin.py @@ -12,16 +12,13 @@ class Inputs(BaseModel): metric: Literal["size", "bitrate"] = Field( default="size", - description=( - "Compare output and source by file size or bitrate. Bitrate uses the first video stream's reported value, " - "or whole-file average bitrate when unavailable." - ), + description="Compare the output and input by file size or bitrate.", ) min_ratio: float = Field( default=0.5, ge=0.01, le=100.0, - description="Minimum output-to-source ratio. 0.5 permits output values down to half the source.", + description="Require output to be at least this fraction of the input. For example, 0.5 means half.", ) @@ -41,15 +38,15 @@ def metric_value(metric: str, probe: ProbeData) -> float | None: def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: if ctx.source_probe is None: - return {"type": "skip", "reason": "no source probe"} + return {"type": "skip", "reason": "Input file information is unavailable for this check."} output = metric_value(inputs.metric, ctx.probe) source = metric_value(inputs.metric, ctx.source_probe) if output is None or source is None or source == 0: - return {"type": "veto", "reason": f"cannot determine {inputs.metric} for guard comparison"} + return {"type": "veto", "reason": f"Cannot compare input and output {inputs.metric}."} ratio = output / source if ratio < inputs.min_ratio: return { "type": "veto", - "reason": f"output {inputs.metric} is {ratio:.2f}x of source, minimum is {inputs.min_ratio:.2f}x", + "reason": f"Output {inputs.metric} is {ratio:.2f} times the input, below the minimum of {inputs.min_ratio:.2f}.", } return {"type": "match"} diff --git a/plugins/inspector.output_ratio/plugin.toml b/plugins/inspector.output_ratio/plugin.toml index 341fdc7..05e7b03 100644 --- a/plugins/inspector.output_ratio/plugin.toml +++ b/plugins/inspector.output_ratio/plugin.toml @@ -2,9 +2,7 @@ id = "inspector.output_ratio" kind = "inspector" description = "Reject output below a minimum source size or bitrate ratio." help = """ -Reject output below the minimum output-to-source ratio (default: 0.5). -Compare file size or video bitrate. Bitrate falls back to size and duration. -Skip without a source probe. Reject if either metric is unavailable. +Place after a conversion step. Choose size or bitrate and set the minimum ratio. """ name = "Minimum output ratio" author = "Millwright" diff --git a/plugins/inspector.stream_guard/plugin.py b/plugins/inspector.stream_guard/plugin.py index e08d67a..b549372 100644 --- a/plugins/inspector.stream_guard/plugin.py +++ b/plugins/inspector.stream_guard/plugin.py @@ -11,17 +11,19 @@ class Inputs(BaseModel): - min_audio: int = Field(default=1, ge=0, description="Minimum number of audio streams. Zero disables the check.") + min_audio: int = Field( + default=1, ge=0, description="Require at least this many audio streams. Set to zero to disable." + ) min_subtitles: int = Field( - default=0, ge=0, description="Minimum number of subtitle streams. Zero disables the check." + default=0, ge=0, description="Require at least this many subtitle streams. Set to zero to disable." ) audio_languages: list[str] = Field( default=[], - description="Comma-separated ISO-639-2 audio languages. Blank counts all, and und includes missing tags.", + description=("Count these audio language codes. Leave empty to count all. Use und for untagged tracks."), ) subtitle_languages: list[str] = Field( default=[], - description="Comma-separated ISO-639-2 subtitle languages. Blank counts all, and und includes missing tags.", + description=("Count these subtitle language codes. Leave empty to count all. Use und for untagged tracks."), ) @@ -35,18 +37,18 @@ def _count(streams: list[ProbeStream], languages: list[str]) -> int: def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: if ctx.source_probe is None: - return {"type": "skip", "reason": "no source probe"} + return {"type": "skip", "reason": "Input file information is unavailable for this check."} audio = _count(ctx.probe.audio, inputs.audio_languages) subtitles = _count(ctx.probe.subtitle, inputs.subtitle_languages) if audio < inputs.min_audio: - reason = f"audio streams {audio} below minimum {inputs.min_audio}" + reason = f"Audio stream count is {audio}, below the minimum of {inputs.min_audio}" if inputs.audio_languages: reason += f" (languages: {', '.join(inputs.audio_languages)})" - return {"type": "veto", "reason": reason} + return {"type": "veto", "reason": reason + "."} if inputs.min_subtitles and subtitles < inputs.min_subtitles: - reason = f"subtitle streams {subtitles} below minimum {inputs.min_subtitles}" + reason = f"Subtitle stream count is {subtitles}, below the minimum of {inputs.min_subtitles}" if inputs.subtitle_languages: reason += f" (languages: {', '.join(inputs.subtitle_languages)})" - return {"type": "veto", "reason": reason} + return {"type": "veto", "reason": reason + "."} return {"type": "match"} diff --git a/plugins/inspector.stream_guard/plugin.toml b/plugins/inspector.stream_guard/plugin.toml index ce37796..90cec05 100644 --- a/plugins/inspector.stream_guard/plugin.toml +++ b/plugins/inspector.stream_guard/plugin.toml @@ -2,9 +2,8 @@ id = "inspector.stream_guard" kind = "inspector" description = "Reject output with too few audio or subtitle streams." help = """ -Run after transcoding. Check audio and subtitle counts separately. -Set a minimum to zero to disable its check. Optional comma-separated ISO-639-2 -filters count any listed language. Missing tags count as und. Blank filters count all streams. +Place after a conversion step. Set minimum audio and subtitle counts. +Use language filters to count only selected languages. """ name = "Stream guard" author = "Millwright" diff --git a/plugins/transcode.burn_subtitles/plugin.py b/plugins/transcode.burn_subtitles/plugin.py index 9101bb7..846efcb 100644 --- a/plugins/transcode.burn_subtitles/plugin.py +++ b/plugins/transcode.burn_subtitles/plugin.py @@ -12,32 +12,32 @@ class Inputs(BaseModel): - language: str = Field(default="eng", description="Lowercase ISO-639-2 language code, such as eng") + language: str = Field(default="eng", description="Use a lowercase ISO 639-2 language code, such as eng.") name_priority: list[str] = Field( default=["video_language", "video", "language"], - description="Name patterns in priority order: video_language=movie.eng.srt, video=movie.srt, language=eng.srt", + description="Rank filename patterns: video_language=movie.eng.srt, video=movie.srt, language=eng.srt.", ) format_priority: list[str] = Field( - default=["srt", "ass", "ssa", "vtt"], description="Subtitle extensions in priority order" + default=["srt", "ass", "ssa", "vtt"], description="Rank subtitle formats from srt, ass, ssa, and vtt." ) priority_order: Literal["name_first", "format_first"] = Field( - default="name_first", description="Rank matches by name first or format first" + default="name_first", description="Rank matches by name first or format first." ) on_multiple: Literal["first_match", "error"] = Field( - default="first_match", description="For multiple matches, choose by priority or return an error" + default="first_match", description="For multiple matches, choose by priority or report an error." ) on_missing: Literal["skip", "error"] = Field( - default="skip", description="When no sidecar matches, skip or return an error" + default="skip", description="When no sidecar matches, skip or report an error." ) - encoder: str = Field(default="libx264", description="FFmpeg video encoder used for burning subtitles") - container: Literal["mkv", "mp4"] = Field(default="mkv", description="Output container format") - quality: int = Field(default=23, ge=0, le=63, description="Video quality value passed to the encoder") + encoder: str = Field(default="libx264", description="Choose the FFmpeg video encoder for burning subtitles.") + container: Literal["mkv", "mp4"] = Field(default="mkv", description="Choose the output container.") + quality: int = Field(default=23, ge=0, le=63, description="Set the encoder's video quality value.") @field_validator("language") @classmethod def check_language(cls, value: str) -> str: if re.fullmatch(r"[a-z]{3}", value) is None: - raise ValueError("language must be lowercase ISO-639-2") + raise ValueError("Use a three-letter lowercase language code, such as eng.") return value @field_validator("name_priority") @@ -45,7 +45,9 @@ def check_language(cls, value: str) -> str: def check_names(cls, value: list[str]) -> list[str]: allowed = {"video_language", "video", "language"} if not value or len(value) != len(set(value)) or any(item not in allowed for item in value): - raise ValueError("name_priority must be nonempty, unique, and use allowed names") + raise ValueError( + "Choose at least one filename pattern without duplicates: video_language, video, or language." + ) return value @field_validator("format_priority") @@ -53,7 +55,7 @@ def check_names(cls, value: list[str]) -> list[str]: def check_formats(cls, value: list[str]) -> list[str]: allowed = {"srt", "ass", "ssa", "vtt"} if not value or len(value) != len(set(value)) or any(item not in allowed for item in value): - raise ValueError("format_priority must be nonempty, unique, and use allowed formats") + raise ValueError("Choose at least one subtitle format without duplicates: srt, ass, ssa, or vtt.") return value @@ -62,7 +64,7 @@ def check_formats(cls, value: list[str]) -> list[str]: def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: if not ctx.probe.real_video: - return {"type": "skip", "reason": "no video stream"} + return {"type": "skip", "reason": "No video stream found."} stem = PurePath(ctx.original_filename).stem candidates: list[tuple[int, int, str]] = [] seen: set[str] = set() @@ -85,11 +87,11 @@ def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: ) if len(candidates) > 1 and inputs.on_multiple == "error": matches = ", ".join(sorted(candidate[2] for candidate in candidates)) - return {"type": "error", "error": f"multiple matching subtitle sidecars: {matches}"} + return {"type": "error", "error": f"Multiple matching subtitle files: {matches}."} if not candidates: if inputs.on_missing == "error": - return {"type": "error", "error": "no matching subtitle sidecar"} - return {"type": "skip", "reason": "no matching subtitle sidecar"} + return {"type": "error", "error": "No matching subtitle file found."} + return {"type": "skip", "reason": "No matching subtitle file found."} selected = candidates[0][2] plan = FFmpegPlan( container=inputs.container, @@ -117,4 +119,4 @@ def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: ) name = inputs.name_priority[candidates[0][0]] extension = inputs.format_priority[candidates[0][1]] - return {"type": "plan", "plan": plan, "reason": f"selected {selected} ({name}, {extension} priority)"} + return {"type": "plan", "plan": plan, "reason": f"Selected {selected} ({name}, {extension} priority)."} diff --git a/plugins/transcode.burn_subtitles/plugin.toml b/plugins/transcode.burn_subtitles/plugin.toml index f06b72d..c13ac6e 100644 --- a/plugins/transcode.burn_subtitles/plugin.toml +++ b/plugins/transcode.burn_subtitles/plugin.toml @@ -2,11 +2,8 @@ id = "transcode.burn_subtitles" kind = "transcode" description = "Burn a matching sidecar subtitle into the video." help = """ -Place the sidecar beside the input and set its lowercase ISO-639-2 language code. -Rank matching names and formats by the configured priorities (default formats: SRT, ASS, SSA, VTT). -Skip files without video or with only cover art. Skip missing sidecars unless on_missing is error. -Re-encode video with the selected encoder and quality (defaults: libx264 and 23). -MP4 output drops embedded subtitle streams. +Place subtitle files beside the video. Choose a language and set filename and format priorities. +Burning subtitles re-encodes the video. MP4 output removes embedded subtitles. """ name = "Burn subtitles" author = "Millwright" diff --git a/plugins/transcode.remux/plugin.py b/plugins/transcode.remux/plugin.py index e88237f..0d58f33 100644 --- a/plugins/transcode.remux/plugin.py +++ b/plugins/transcode.remux/plugin.py @@ -12,8 +12,10 @@ class Inputs(AudioPlanInputs): - container: Literal["mkv", "mp4", "original"] = Field(default="mkv", description="Output container format") - size_ceiling_ratio: float = Field(default=1.05, gt=0, description="Maximum output-to-source file size ratio") + container: Literal["mkv", "mp4", "original"] = Field(default="mkv", description="Choose the output container.") + size_ceiling_ratio: float = Field( + default=1.05, gt=0, description="Reject output larger than this multiple of the input file size." + ) INPUTS = Inputs @@ -30,7 +32,7 @@ def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: media_changes = changes_media(plan, ctx.probe) plan = apply_metadata(plan, ctx) if not changes_media(plan, ctx.probe): - return {"type": "skip", "reason": "container and streams already satisfy the policy"} + return {"type": "skip", "reason": "The file already matches the requested settings."} reasons = change_reasons(plan, ctx.probe) # Generate missing packet timestamps during remuxing, not as a reason to remux again. plan.generate_timestamps = media_changes diff --git a/plugins/transcode.remux/plugin.toml b/plugins/transcode.remux/plugin.toml index 7d12832..272fc8e 100644 --- a/plugins/transcode.remux/plugin.toml +++ b/plugins/transcode.remux/plugin.toml @@ -2,8 +2,8 @@ id = "transcode.remux" kind = "transcode" description = "Change container or stream settings without re-encoding video." help = """ -Choose MKV (default), MP4, or the original container. Apply stream selection and optional -audio encoding. Skip files already matching the container, stream, and audio settings. +Choose MKV (default), MP4, or the original container. Select streams to keep and optionally +re-encode audio. Files already matching the requested settings are skipped. """ name = "Remux" author = "Millwright" diff --git a/plugins/transcode.video/plugin.py b/plugins/transcode.video/plugin.py index a1027a5..2aa2c39 100644 --- a/plugins/transcode.video/plugin.py +++ b/plugins/transcode.video/plugin.py @@ -13,37 +13,42 @@ class Inputs(AudioPlanInputs): - target: Literal["avc", "hevc", "av1"] = Field(default="hevc", description="Target video codec family") + target: Literal["avc", "hevc", "av1"] = Field(default="hevc", description="Choose the output video codec family.") rate_control: RateControlMode = Field( default=RateControlMode.QUALITY, - description="Quality sets encoder quality with variable size. Bitrate targets a source bitrate ratio", + description="Use encoder quality with variable output size, or target a source bitrate ratio.", ) quality: int | None = Field( - default=None, ge=0, le=63, description="Codec-specific video quality value. Used only in quality mode" + default=None, ge=0, le=63, description="Set the codec-specific quality value. Applies only in quality mode." ) ratio: float | None = Field( default=None, gt=0, le=1, - description="Target video bitrate as a fraction of the source. Used only in bitrate mode", + description="Set video bitrate as a fraction of the source. Applies only in bitrate mode.", ) - encoder: str | None = Field(default=None, description="FFmpeg video encoder override") + encoder: str | None = Field(default=None, description="Override the FFmpeg video encoder.") encoder_priority: list[str] = Field( default=[], - description="Encoder priority order. Override wins. Empty uses the software default", + description=( + "List encoders in priority order. The encoder override wins. Leave blank for the software default." + ), ) - pixel_format: str | None = Field(default=None, description="Pixel format for the encoded video") - ten_bit: bool = Field(default=False, description="Encode video with 10-bit precision") - preset: str | None = Field(default=None, description="Encoder speed and compression preset") - max_width: int | None = Field(default=None, ge=1, description="Maximum output video width in pixels") - max_height: int | None = Field(default=None, ge=1, description="Maximum output video height in pixels") - container: Literal["mkv", "mp4"] = Field(default="mkv", description="Output container format") + pixel_format: str | None = Field(default=None, description="Set the encoded video's pixel format.") + ten_bit: bool = Field(default=False, description="Encode video with 10-bit precision.") + preset: str | None = Field(default=None, description="Choose the encoder's speed and compression preset.") + max_width: int | None = Field(default=None, ge=1, description="Limit output video width in pixels.") + max_height: int | None = Field(default=None, ge=1, description="Limit output video height in pixels.") + container: Literal["mkv", "mp4"] = Field(default="mkv", description="Choose the output container.") bitrate_source: Literal["video", "container", "average"] = Field( - default="video", description="Bitrate mode source: video, container, or their average when both are available" + default="video", + description="In bitrate mode, choose video bitrate, whole-file bitrate, or their average.", + ) + size_ceiling_ratio: float = Field( + default=1.05, gt=0, description="Reject output larger than this multiple of the input file size." ) - size_ceiling_ratio: float = Field(default=1.05, gt=0, description="Maximum output-to-source file size ratio") - fix_timestamps: bool = Field(default=False, description="Generate missing timestamps for AVI and TS inputs") - extras: list[str] = Field(default=[], description="Additional FFmpeg arguments, one argument per entry") + fix_timestamps: bool = Field(default=False, description="Generate missing timestamps for AVI and TS inputs.") + extras: list[str] = Field(default=[], description="Add FFmpeg arguments, one argument per entry.") INPUTS = Inputs @@ -52,7 +57,7 @@ class Inputs(AudioPlanInputs): def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: video = ctx.probe.real_video if not video: - return {"type": "skip", "reason": "no video stream"} + return {"type": "skip", "reason": "No video stream found."} families = {"avc": {"avc", "h264", "avc1"}, "hevc": {"hevc", "h265"}, "av1": {"av1", "av01"}} defaults = {"avc": ("libx264", 0.75, 23), "hevc": ("libx265", 0.5, 21), "av1": ("libsvtav1", 0.45, 30)} encoder, ratio, quality = defaults[inputs.target] @@ -116,7 +121,7 @@ def decide(ctx: PluginContext, inputs: Inputs) -> dict[str, Any]: plan.video = None plan = apply_metadata(plan, ctx) if not changes_media(plan, ctx.probe): - return {"type": "skip", "reason": f"already {inputs.target}"} + return {"type": "skip", "reason": f"The file already matches the {inputs.target.upper()} settings."} reasons = change_reasons(plan, ctx.probe) return { "type": "plan", diff --git a/plugins/transcode.video/plugin.toml b/plugins/transcode.video/plugin.toml index 87e243d..7db8cba 100644 --- a/plugins/transcode.video/plugin.toml +++ b/plugins/transcode.video/plugin.toml @@ -1,13 +1,9 @@ id = "transcode.video" kind = "transcode" -description = "Encode AVC, HEVC, or AV1 with quality or bitrate rate control." +description = "Convert video to AVC, HEVC, or AV1." help = """ -Quality mode (default) uses encoder quality without a bitrate target. Bitrate mode uses -a source bitrate ratio without quality flags. The output size limit applies in both modes. -Optionally set encoder, pixel format, bit depth, preset, or maximum dimensions. -Skip matching files unless another video, audio, stream, or container change is requested. -Skip files without video or with only cover art. HEVC defaults to libx265, quality 21, -and bitrate ratio 0.5. AVC uses libx264. AV1 uses libsvtav1. Both have codec-specific defaults. +Choose quality or bitrate mode. Set an encoder or an encoder priority list if needed. +Files already matching your settings are skipped. """ name = "Video transcode" author = "Millwright" diff --git a/server/src/server/external_notifications/__init__.py b/server/src/server/external_notifications/__init__.py index 90103f4..13ce734 100644 --- a/server/src/server/external_notifications/__init__.py +++ b/server/src/server/external_notifications/__init__.py @@ -1,10 +1,13 @@ from __future__ import annotations +from pathlib import PurePath, PurePosixPath, PureWindowsPath from typing import TYPE_CHECKING from common.actions import NotifyExternal from server.external_notifications.arr import notify_arr +from server.external_notifications.media_server import notify_media_server +from server.external_notifications.plex import notify_plex if TYPE_CHECKING: from collections.abc import Awaitable, Callable @@ -12,6 +15,24 @@ async def notify_external(payload: dict, save_progress: Callable[[dict], Awaitable[None]] | None = None) -> bool: action = NotifyExternal.model_validate(payload["action"]) + path = PurePath(payload["path"]) + if action.path_mappings: + matches = [mapping for mapping in action.path_mappings if path.parent.is_relative_to(mapping.source)] + if not matches: + message = f"No path mapping matches {str(path.parent)!r}." + raise ValueError(message) + mapping = max(matches, key=lambda item: len(PurePath(item.source).parts)) + relative = path.relative_to(mapping.source) + destination = ( + PureWindowsPath(mapping.destination) + if PureWindowsPath(mapping.destination).is_absolute() + else PurePosixPath(mapping.destination) + ) + path = destination.joinpath(*relative.parts) match action.application: case "radarr" | "sonarr": - return await notify_arr(action, payload, save_progress) + return await notify_arr(action, payload, str(path), save_progress) + case "plex" | "jellyfin" | "emby": + if action.application == "plex": + return await notify_plex(action, str(path.parent)) + return await notify_media_server(action, str(path.parent)) diff --git a/server/src/server/external_notifications/arr.py b/server/src/server/external_notifications/arr.py index c62be39..80aff65 100644 --- a/server/src/server/external_notifications/arr.py +++ b/server/src/server/external_notifications/arr.py @@ -49,9 +49,9 @@ class _Progress(BaseModel): async def notify_arr( - action: NotifyExternal, payload: dict, save_progress: Callable[[dict], Awaitable[None]] | None = None + action: NotifyExternal, payload: dict, path: str, save_progress: Callable[[dict], Awaitable[None]] | None = None ) -> bool: - path, filename = payload["path"], payload["original_filename"] + filename = payload["original_filename"] async with httpx.AsyncClient( base_url=str(action.url).rstrip("/") + "/api/v3/", headers={"X-Api-Key": action.api_key}, @@ -64,7 +64,7 @@ async def notify_arr( response.raise_for_status() progress = _Command.model_validate(response.json()) elif action.application == "radarr": - identifiers = {kind.lower(): value for kind, value in _IDENTIFIER.findall(path + "/" + filename)} + identifiers = {kind.lower(): value for kind, value in _IDENTIFIER.findall(payload["path"] + "/" + filename)} if identifiers: params = {"tmdbId": identifiers["tmdb"]} if "tmdb" in identifiers else {} response = await client.get("movie", params=params) diff --git a/server/src/server/external_notifications/media_server.py b/server/src/server/external_notifications/media_server.py new file mode 100644 index 0000000..63eb416 --- /dev/null +++ b/server/src/server/external_notifications/media_server.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import httpx + +if TYPE_CHECKING: + from common.actions import NotifyExternal + + +async def notify_media_server(action: NotifyExternal, folder: str) -> bool: + headers = ( + {"Authorization": f'MediaBrowser Token="{action.api_key}"'} + if action.application == "jellyfin" + else {"X-Emby-Token": action.api_key} + ) + async with httpx.AsyncClient(base_url=str(action.url).rstrip("/") + "/", headers=headers, timeout=15) as client: + response = await client.post( + "Library/Media/Updated", json={"Updates": [{"Path": folder, "UpdateType": "Modified"}]} + ) + response.raise_for_status() + return True diff --git a/server/src/server/external_notifications/plex.py b/server/src/server/external_notifications/plex.py new file mode 100644 index 0000000..5082762 --- /dev/null +++ b/server/src/server/external_notifications/plex.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from pathlib import PurePosixPath, PureWindowsPath +from typing import TYPE_CHECKING + +import httpx +from pydantic import BaseModel, Field + +if TYPE_CHECKING: + from common.actions import NotifyExternal + + +class _Location(BaseModel): + path: str + + +class _Section(BaseModel): + key: int = Field(gt=0) + locations: list[_Location] = Field(default=[], alias="Location") + + +class _Sections(BaseModel): + sections: list[_Section] = Field(default=[], alias="Directory") + + +class _Library(BaseModel): + container: _Sections = Field(alias="MediaContainer") + + +async def notify_plex(action: NotifyExternal, folder: str) -> bool: + async with httpx.AsyncClient( + base_url=str(action.url).rstrip("/") + "/", + headers={ + "X-Plex-Token": action.api_key, + "X-Plex-Client-Identifier": "millwright", + "X-Plex-Product": "Millwright", + "Accept": "application/json", + }, + timeout=15, + ) as client: + response = await client.get("library/sections") + response.raise_for_status() + library = _Library.model_validate(response.json()) + path_type = PureWindowsPath if PureWindowsPath(folder).is_absolute() else PurePosixPath + path = path_type(folder) + sections = [ + section + for section in library.container.sections + if any(path.is_relative_to(path_type(location.path)) for location in section.locations) + ] + if not sections: + message = f"No Plex library contains {folder!r}. Check the path mappings and Plex library folders." + raise ValueError(message) + for section in sections: + response = await client.get(f"library/sections/{section.key}/refresh", params={"path": folder}) + response.raise_for_status() + return True diff --git a/tests/test_external_notifications.py b/tests/test_external_notifications.py index d0cc644..b66e30e 100644 --- a/tests/test_external_notifications.py +++ b/tests/test_external_notifications.py @@ -2,9 +2,12 @@ import asyncio import json +from pathlib import PurePath import httpx import pytest +from common.actions import ExternalNotificationConfig +from common.plugin import build_inputs from sqlalchemy import select from server.external_notifications import notify_external @@ -28,12 +31,253 @@ def notification(application: str = "radarr", *, unmonitor: bool = True) -> dict } +@pytest.mark.parametrize( + "options, error", + [ + ({"application": "plex", "unmonitor_after_refresh": True}, "Unmonitor after refresh"), + ({"application": "plex", "path_mappings": [{"destination": "/media"}]}, "source"), + ({"application": "plex", "path_mappings": [{"source": "/library"}]}, "destination"), + ( + {"application": "plex", "path_mappings": [{"source": "library", "destination": "/media"}]}, + "Use absolute paths", + ), + ( + {"application": "plex", "path_mappings": [{"source": "/library", "destination": "media"}]}, + "Use absolute paths", + ), + ( + {"application": "plex", "path_mappings": [{"source": "/library", "destination": "/media/../other"}]}, + "can't contain", + ), + ( + { + "application": "plex", + "path_mappings": [ + {"source": "/library", "destination": "/media"}, + {"source": "/library/", "destination": "/other"}, + ], + }, + "appear only once", + ), + ], +) +def test_external_options_rejected(options, error) -> None: + with pytest.raises(ValueError, match=error): + build_inputs(ExternalNotificationConfig, {"url": "https://server.test", "api_key": "key", **options}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mappings, root, folder, prefix", + [ + ([], "/library", "/library/Series", "/library/Ser"), + ( + [ + {"source": "/library/Movies", "destination": "/movies"}, + {"source": "/library/", "destination": "/media/"}, + ], + "/media", + "/media/Series", + "/media/Ser", + ), + ( + [ + {"source": "/library", "destination": "/wrong"}, + {"source": "/library/Series", "destination": "/media/Series"}, + ], + "/media", + "/media/Series", + "/media/Ser", + ), + ( + [ + {"source": "/library/Series", "destination": "/media/Series"}, + {"source": "/library", "destination": "/wrong"}, + ], + "/media", + "/media/Series", + "/media/Ser", + ), + ( + [{"source": "/library", "destination": "D:\\Media"}], + "d:\\media", + "D:\\Media\\Series", + "D:\\Media\\Ser", + ), + ], +) +async def test_plex_folder_refresh(monkeypatch, mappings, root, folder, prefix) -> None: + payload = notification("plex", unmonitor=False) + payload["action"]["path_mappings"] = mappings + requests = [] + + def respond(request): + requests.append(request) + assert request.headers["X-Plex-Token"] == "private-api-key" + assert request.headers["Accept"] == "application/json" + if request.url.path == "/base/library/sections": + return httpx.Response( + 200, + json={ + "MediaContainer": { + "Directory": [ + {"key": "7", "Location": [{"path": root}, {"path": folder}]}, + {"key": "8", "Location": [{"path": root}]}, + {"key": "9", "Location": [{"path": prefix}]}, + ] + } + }, + ) + assert request.method == "GET" + assert dict(request.url.params) == {"path": folder} + return httpx.Response(200) + + client = httpx.AsyncClient + monkeypatch.setattr( + "server.external_notifications.plex.httpx.AsyncClient", + lambda **kwargs: client(**kwargs, transport=httpx.MockTransport(respond)), + ) + assert await notify_external(payload) + assert [(request.method, request.url.path) for request in requests] == [ + ("GET", "/base/library/sections"), + ("GET", "/base/library/sections/7/refresh"), + ("GET", "/base/library/sections/8/refresh"), + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mismatch", ["mapping", "library"]) +async def test_plex_path_mismatch(monkeypatch, mismatch) -> None: + payload = notification("plex", unmonitor=False) + payload["path"] = "/library/SeriesOther/Episode.mp4" + if mismatch == "mapping": + payload["action"]["path_mappings"] = [ + {"source": "/library/Series", "destination": "/media/Series"}, + {"source": "/library/Movies", "destination": "/media/Movies"}, + ] + requests = [] + + def respond(request): + requests.append(request) + assert request.url.path == "/base/library/sections" + return httpx.Response( + 200, + json={"MediaContainer": {"Directory": [{"key": "7", "Location": [{"path": "/library/Series"}]}]}}, + ) + + client = httpx.AsyncClient + monkeypatch.setattr( + "server.external_notifications.plex.httpx.AsyncClient", + lambda **kwargs: client(**kwargs, transport=httpx.MockTransport(respond)), + ) + with pytest.raises(ValueError, match="No path mapping matches" if mismatch == "mapping" else "No Plex library"): + await notify_external(payload) + assert len(requests) == (0 if mismatch == "mapping" else 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "application, path, folder", + [ + ("jellyfin", "/library/Series & Friends/Season 1/Episode.mp4", "/media/Series & Friends/Season 1"), + ("emby", "/library/Movies/A Movie/Movie.mp4", "/movies/A Movie"), + ], +) +async def test_media_folder_update(monkeypatch, application, path, folder) -> None: + payload = notification(application, unmonitor=False) + payload["path"] = path + payload["action"]["path_mappings"] = [ + {"source": "/library/Movies", "destination": "/movies"}, + {"source": "/library", "destination": "/media"}, + ] + requests = [] + + def respond(request): + requests.append(request) + assert request.method == "POST" + assert request.url.path == "/base/Library/Media/Updated" + assert not request.url.params + if application == "jellyfin": + assert request.headers["Authorization"] == 'MediaBrowser Token="private-api-key"' + else: + assert request.headers["X-Emby-Token"] == "private-api-key" + assert json.loads(request.content) == {"Updates": [{"Path": folder, "UpdateType": "Modified"}]} + return httpx.Response(204 if application == "jellyfin" else 200) + + client = httpx.AsyncClient + monkeypatch.setattr( + "server.external_notifications.media_server.httpx.AsyncClient", + lambda **kwargs: client(**kwargs, transport=httpx.MockTransport(respond)), + ) + assert await notify_external(payload) + assert len(requests) == 1 + + +@pytest.mark.asyncio +async def test_plex_retry_persisted(monkeypatch, caplog) -> None: + refreshes = [] + + def respond(request): + if request.url.path == "/base/library/sections": + return httpx.Response( + 200, + json={ + "MediaContainer": { + "Directory": [{"key": key, "Location": [{"path": "/library"}]} for key in ["7", "8"]] + } + }, + ) + refreshes.append(request.url.path) + if len(refreshes) == 2: + raise httpx.ConnectError("connection refused private-api-key", request=request) + return httpx.Response(200) + + client = httpx.AsyncClient + monkeypatch.setattr( + "server.external_notifications.plex.httpx.AsyncClient", + lambda **kwargs: client(**kwargs, transport=httpx.MockTransport(respond)), + ) + async with database_context("plex-retry") as factory: + async with factory() as session: + job_id = await make_job(session, notify=False) + job = await session.get(Job, job_id) + assert job is not None + job.status = "succeeded" + event = JobEvent(job_id=job_id, type="external_notification", payload=notification("plex", unmonitor=False)) + session.add(event) + await session.flush() + session.add(NotificationDelivery(job_id=job_id, job_event_id=event.id, target_name="external:plex")) + await session.commit() + await NotificationService(factory).deliver_pending() + async with factory() as session: + delivery = (await session.scalars(select(NotificationDelivery))).one() + assert (delivery.status, delivery.attempts) == ("pending", 1) + assert delivery.error and "connection refused" in delivery.error + assert "private-api-key" not in delivery.error + await NotificationService(factory).deliver_pending() + async with factory() as session: + delivery = (await session.scalars(select(NotificationDelivery))).one() + assert (delivery.status, delivery.attempts, delivery.error) == ("sent", 2, None) + job = await session.get(Job, job_id) + assert job is not None and job.status == "succeeded" + assert refreshes == [ + "/base/library/sections/7/refresh", + "/base/library/sections/8/refresh", + "/base/library/sections/7/refresh", + "/base/library/sections/8/refresh", + ] + assert all("private-api-key" not in str(record.__dict__) for record in caplog.records) + + @pytest.mark.asyncio @pytest.mark.parametrize("identifier", [None, "tmdb-42", "imdb-tt123"]) async def test_radarr_refresh_order(monkeypatch, identifier) -> None: payload = notification() if identifier: payload["path"] = f"/library/Movie {{{identifier}}}/Movie.mp4" + payload["action"]["path_mappings"] = [ + {"source": str(PurePath(payload["path"]).parent), "destination": "/radarr/Movie"} + ] requests = [] movie = {"id": 7, "tmdbId": 42, "imdbId": "tt123"} @@ -79,12 +323,19 @@ async def tick(_seconds): @pytest.mark.parametrize("failure", [None, "lookup", "refresh", "unmonitor"]) async def test_sonarr_delivery_persisted(monkeypatch, caplog, failure) -> None: payload = notification("sonarr") + payload["action"]["path_mappings"] = [ + {"source": "/other", "destination": "/unused"}, + {"source": "/library", "destination": "/sonarr"}, + ] requests = [] def respond(request): requests.append(request) if request.url.path.endswith("/parse"): - assert dict(request.url.params) == {"title": payload["original_filename"], "path": payload["path"]} + assert dict(request.url.params) == { + "title": payload["original_filename"], + "path": "/sonarr/Series/Series.S01E01E02.mp4", + } return httpx.Response( 200, json={ @@ -128,6 +379,8 @@ def respond(request): assert delivery.attempts == (3 if failure else 1) assert (delivery.error is not None) is bool(failure) assert "private-api-key" not in (delivery.error or "") + event = await session.get(JobEvent, delivery.job_event_id) + assert event is not None and event.payload["path"] == "/library/Series/Series.S01E01E02.mp4" job = await session.get(Job, job_id) assert job is not None and job.status == "succeeded" if failure in {"lookup", "refresh"}: diff --git a/tests/test_plugin_migrations.py b/tests/test_plugin_migrations.py index e9b1628..6a99097 100644 --- a/tests/test_plugin_migrations.py +++ b/tests/test_plugin_migrations.py @@ -200,7 +200,13 @@ async def test_legacy_flow_http() -> None: assert canonical["nodes"][0]["config_version"] == 2 assert canonical["nodes"][0]["inputs"] == inputs assert canonical["actions"] == [ - {**action, "kind": "notify_external", "plugin_id": "action.notify_external", "config_version": 2} + { + **action, + "kind": "notify_external", + "plugin_id": "action.notify_external", + "config_version": 2, + "path_mappings": [], + } ] assert canonical["_ui"] == raw["_ui"] async with factory() as session: diff --git a/ui/app/components/StepInspector.vue b/ui/app/components/StepInspector.vue index 127eafd..b93c128 100644 --- a/ui/app/components/StepInspector.vue +++ b/ui/app/components/StepInspector.vue @@ -155,7 +155,7 @@ class="w-full" @update:model-value="pick(String($event))" /> +
+
+ + + + + + + +
+ Add mapping +