diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfile index c09234479ff..5a1244bd8f3 100644 --- a/apps/media-server/Dockerfile +++ b/apps/media-server/Dockerfile @@ -15,13 +15,15 @@ COPY apps/media-server/capcodec.md ./capcodec.md RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/audio-quality.integration.test.ts \ && bun test src/__tests__/lib/audio-quality-formats.integration.test.ts src/__tests__/lib/audio-quality-benchmark.test.ts \ && bun test src/__tests__/lib/audio-levels.integration.test.ts \ - && bun test src/__tests__/lib/drive-resumable-upload.test.ts src/__tests__/lib/storage-upload.test.ts src/__tests__/lib/container-memory.test.ts \ + && bun test src/__tests__/lib/drive-resumable-upload.test.ts src/__tests__/lib/storage-upload.test.ts src/__tests__/lib/container-memory.test.ts src/__tests__/lib/parallel-download.test.ts \ && bun test src/__tests__/lib/media-size.test.ts src/__tests__/lib/media-probe.integration.test.ts \ && bun test src/__tests__/lib/media-audio.integration.test.ts \ && bun test src/__tests__/lib/recording-verification.integration.test.ts src/__tests__/lib/job-manager.test.ts \ - && bun test src/__tests__/lib/media-transfer.test.ts src/__tests__/lib/capcodec-encoder.test.ts \ + && bun test src/__tests__/lib/media-transfer.test.ts src/__tests__/lib/capcodec-encoder.test.ts src/__tests__/lib/nvenc-encoder.test.ts \ && bun test src/__tests__/routes/recording-verification.test.ts \ && bun test src/__tests__/routes/video.test.ts \ + && bun test src/__tests__/lib/lifecycle.test.ts src/__tests__/lib/temp-files.test.ts \ + && bun test src/__tests__/routes/lifecycle.test.ts \ && bun test src/__tests__/lib/media-routes-real-world.integration.test.ts \ && bun test src/__tests__/lib/media-video.integration.test.ts @@ -37,6 +39,6 @@ ENV PORT=3456 EXPOSE 3456 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"] + CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then(async (response) => process.exit(response.ok || (await response.json().catch(() => ({}))).status === 'draining' ? 0 : 1)).catch(() => process.exit(1))"] CMD ["bun", "run", "src/index.ts"] diff --git a/apps/media-server/src/__tests__/lib/job-manager.test.ts b/apps/media-server/src/__tests__/lib/job-manager.test.ts index ff2b43e690b..5bada4e694c 100644 --- a/apps/media-server/src/__tests__/lib/job-manager.test.ts +++ b/apps/media-server/src/__tests__/lib/job-manager.test.ts @@ -6,7 +6,9 @@ import { cleanupExpiredJobs, createJob, deleteJob, + extendJobLifetime, getJob, + getJobTotals, type JobProgress, type RecordingWorkerAcknowledgement, sendWebhook, @@ -522,3 +524,83 @@ describe("job cleanup", () => { expect(currentJob?.error).toContain("maximum lifetime of 60 minutes"); }); }); + +describe("job lifetime", () => { + test("extends the lifetime cap for long work without shortening it", () => { + const job = createTrackedJob("job-extended-lifetime"); + const now = Date.now(); + job.phase = "processing"; + job.createdAt = now - 10 * 60 * 1000; + + expect(extendJobLifetime(job.jobId, 60_000)).toBe( + job.createdAt + 60 * 60 * 1000, + ); + const deadline = extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000); + expect(deadline).toBeGreaterThanOrEqual(now + 2 * 60 * 60 * 1000); + expect(extendJobLifetime(job.jobId, 60_000)).toBe(deadline); + + job.createdAt = now - 90 * 60 * 1000; + job.updatedAt = Date.now(); + expect(cleanupExpiredJobs()).toBe(0); + expect(getJob(job.jobId)?.phase).toBe("processing"); + }); + + test("keeps a hard upper bound on extended lifetimes", () => { + const job = createTrackedJob("job-capped-lifetime"); + job.phase = "uploading"; + job.createdAt = Date.now() - 10 * 60 * 1000; + + expect(extendJobLifetime(job.jobId, 24 * 60 * 60 * 1000)).toBe( + job.createdAt + 4 * 60 * 60 * 1000, + ); + expect(extendJobLifetime(job.jobId, Number.NaN)).toBeUndefined(); + expect(extendJobLifetime(job.jobId, -1)).toBeUndefined(); + }); + + test("expires a job past its extended deadline", () => { + const job = createTrackedJob("job-past-extended-lifetime"); + const abortController = new AbortController(); + updateJob(job.jobId, { phase: "processing", abortController }); + job.createdAt = Date.now() - 10 * 60 * 1000; + extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000); + job.createdAt -= 3 * 60 * 60 * 1000; + job.lifetimeDeadlineAt = Date.now() - 1; + job.updatedAt = Date.now(); + + expect(cleanupExpiredJobs()).toBe(1); + expect(job.phase).toBe("error"); + expect(job.error).toContain("maximum lifetime"); + expect(abortController.signal.aborted).toBe(true); + }); + + test("does not extend terminal jobs", () => { + const job = createTrackedJob("job-terminal-lifetime"); + updateJob(job.jobId, { phase: "complete" }); + expect(extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000)).toBeUndefined(); + expect(job.lifetimeDeadlineAt).toBeUndefined(); + }); +}); + +describe("job totals", () => { + test("counts created, terminal, and expired jobs", () => { + const before = getJobTotals(); + const completed = createTrackedJob("job-total-complete"); + const errored = createTrackedJob("job-total-error"); + const cancelled = createTrackedJob("job-total-cancelled"); + const stale = createTrackedJob("job-total-stale"); + updateJob(completed.jobId, { phase: "complete" }); + updateJob(completed.jobId, { phase: "error" }); + updateJob(errored.jobId, { phase: "error" }); + updateJob(cancelled.jobId, { phase: "cancelled" }); + updateJob(stale.jobId, { phase: "processing" }); + stale.updatedAt = Date.now() - 16 * 60 * 1000; + cleanupExpiredJobs(); + + const after = getJobTotals(); + expect(after.created - before.created).toBe(4); + expect(after.completed - before.completed).toBe(1); + expect(after.errored - before.errored).toBe(2); + expect(after.cancelled - before.cancelled).toBe(1); + expect(after.expired - before.expired).toBe(1); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/lifecycle.test.ts b/apps/media-server/src/__tests__/lib/lifecycle.test.ts new file mode 100644 index 00000000000..bf1aa4f7eba --- /dev/null +++ b/apps/media-server/src/__tests__/lib/lifecycle.test.ts @@ -0,0 +1,207 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + beginRequest, + createShutdownHandler, + getDrainBlockers, + getIdleTimeoutSeconds, + getShutdownGraceMs, + isDraining, + isIdle, + restoreIdleTimeoutOnFirstChunk, + setDraining, + setRequestIdleTimeout, + trackBackgroundTask, + waitForDrain, +} from "../../lib/lifecycle"; + +const originalGrace = process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS; +const originalIdle = process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS; + +function deferred() { + let resolve = () => {}; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +afterEach(() => { + setDraining(false); + if (originalGrace === undefined) + delete process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS; + else process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = originalGrace; + if (originalIdle === undefined) + delete process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS; + else process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = originalIdle; +}); + +describe("shutdown configuration", () => { + test("defaults to the immediate shutdown path", () => { + delete process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS; + expect(getShutdownGraceMs()).toBe(0); + process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = "not-a-number"; + expect(getShutdownGraceMs()).toBe(0); + process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = "120000"; + expect(getShutdownGraceMs()).toBe(120_000); + }); + + test("bounds the idle timeout to Bun's maximum", () => { + delete process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS; + expect(getIdleTimeoutSeconds()).toBe(255); + process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = "90"; + expect(getIdleTimeoutSeconds()).toBe(90); + process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = "10000"; + expect(getIdleTimeoutSeconds()).toBe(255); + }); + + test("ignores servers without per-request timeouts", () => { + const calls: Array<[Request, number]> = []; + const request = new Request("http://localhost/video/convert"); + setRequestIdleTimeout(undefined, request, 0); + setRequestIdleTimeout( + { + timeout: (req: Request, seconds: number) => calls.push([req, seconds]), + }, + request, + 0, + ); + expect(calls).toEqual([[request, 0]]); + }); +}); + +describe("drain", () => { + test("waits for background work and in-flight requests", async () => { + const work = deferred(); + const task = trackBackgroundTask(work.promise); + const endRequest = beginRequest(); + expect(isIdle()).toBe(false); + expect(getDrainBlockers().backgroundTasks).toBeGreaterThanOrEqual(1); + + const drained = waitForDrain(5_000); + await Bun.sleep(50); + work.resolve(); + await task; + await Bun.sleep(50); + endRequest(); + + expect(await drained).toBe(true); + expect(isIdle()).toBe(true); + }); + + test("gives up when the grace period elapses", async () => { + const work = deferred(); + const task = trackBackgroundTask(work.promise); + const startedAt = Date.now(); + + expect(await waitForDrain(300)).toBe(false); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(250); + + work.resolve(); + await task; + }); + + test("stops tracking rejected background work", async () => { + const failure = trackBackgroundTask(Promise.reject(new Error("failed"))); + await failure.catch(() => {}); + await Bun.sleep(0); + expect(await waitForDrain(1_000)).toBe(true); + }); +}); + +describe("shutdown handler", () => { + test("keeps today's immediate shutdown when no grace period is set", async () => { + const work = deferred(); + const task = trackBackgroundTask(work.promise); + let finalized = 0; + const shutdown = createShutdownHandler( + async () => { + finalized++; + }, + () => 0, + ); + + await shutdown("SIGTERM"); + await shutdown("SIGTERM"); + + expect(finalized).toBe(1); + expect(isDraining()).toBe(false); + work.resolve(); + await task; + }); + + test("drains before finalizing when a grace period is set", async () => { + const work = deferred(); + const task = trackBackgroundTask(work.promise); + let finalizedWhileBusy: boolean | undefined; + const shutdown = createShutdownHandler( + async () => { + finalizedWhileBusy = !isIdle(); + }, + () => 10_000, + ); + + const shuttingDown = shutdown("SIGTERM"); + await Bun.sleep(50); + expect(isDraining()).toBe(true); + expect(finalizedWhileBusy).toBeUndefined(); + + work.resolve(); + await task; + await shuttingDown; + expect(finalizedWhileBusy).toBe(false); + }); + + test("forces the immediate path on a second signal", async () => { + const work = deferred(); + const task = trackBackgroundTask(work.promise); + let finalized = 0; + const shutdown = createShutdownHandler( + async () => { + finalized++; + }, + () => 60_000, + ); + + const shuttingDown = shutdown("SIGTERM"); + await Bun.sleep(50); + expect(finalized).toBe(0); + await shutdown("SIGINT"); + await shuttingDown; + expect(finalized).toBe(1); + + work.resolve(); + await task; + }); +}); + +describe("streamed response idle timeout", () => { + test("comes back once, on the first chunk, and cancel still reaches the source", async () => { + let restores = 0; + let cancelled = false; + let push: ReadableStreamDefaultController | undefined; + const source = new ReadableStream({ + start(controller) { + push = controller; + }, + cancel() { + cancelled = true; + }, + }); + const reader = restoreIdleTimeoutOnFirstChunk(source, () => { + restores++; + }).getReader(); + + const first = reader.read(); + await Bun.sleep(10); + expect(restores).toBe(0); + + push?.enqueue(new Uint8Array([1])); + expect((await first).value).toEqual(new Uint8Array([1])); + push?.enqueue(new Uint8Array([2])); + await reader.read(); + expect(restores).toBe(1); + + await reader.cancel(); + expect(cancelled).toBe(true); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts new file mode 100644 index 00000000000..597855b4463 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts @@ -0,0 +1,711 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import type { VideoMetadata } from "../../lib/job-manager"; +import { decodeThreadsFromEnv } from "../../lib/media-common"; +import { + buildTranscodeEditArgs, + renderEditedVideo, +} from "../../lib/media-edit"; +import { probeVideo } from "../../lib/media-probe"; +import { + buildProcessVideoArgs, + type ProcessVideoArgsInput, + pickMobileSafeH264Level, + processVideo, +} from "../../lib/media-video"; +import { + buildNvencEncodeArgs, + canDecodeOnGpu, + encodeAttemptsBudgetMs, + fitWithinEven, + getVideoEncoderStatus, + type NvencOptions, + nvencCq, + nvencFallbackReason, + nvencOptionsFromEnv, + recordNvencAttempt, + selectedVideoEncoder, +} from "../../lib/nvenc-encoder"; + +const enabled = { CAP_MEDIA_VIDEO_ENCODER: "nvenc" }; +const defaults: NvencOptions = { + preset: "p1", + cqOffset: 8, + lookahead: 0, + spatialAq: true, + temporalAq: false, + gpuDecodeCodecs: ["hevc", "av1"], +}; +const opts: ProcessVideoArgsInput["opts"] = { + maxWidth: 1920, + maxHeight: 1080, + preset: "medium", + crf: 23, + audioBitrate: "128k", +}; +const nvencVideoArgs = [ + "-c:v", + "h264_nvenc", + "-preset", + "p1", + "-tune", + "hq", + "-rc", + "vbr", + "-cq", + "31", + "-b:v", + "0", + "-spatial-aq", + "1", + "-profile:v", + "high", + "-g", + "250", +]; +const gpuScale = "scale_cuda=format=nv12"; +const cpuScale = + "scale='min(1920,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2"; + +function metadata( + width: number, + height: number, + videoCodec: string, +): VideoMetadata { + return { + duration: 10, + width, + height, + fps: 30, + videoCodec, + audioCodec: "aac", + audioChannels: 2, + sampleRate: 48_000, + bitrate: 0, + fileSize: 0, + }; +} + +function processArgs( + source: VideoMetadata, + overrides: Partial = {}, +): ProcessVideoArgsInput { + return { + inputPath: "in.mp4", + outputPath: "out.mp4", + videoCodec: source.videoCodec, + hasAudio: true, + opts, + targetH264Level: pickMobileSafeH264Level(source, opts).ffmpegValue, + normalizeH264Level: false, + videoTranscode: true, + audioTranscode: false, + extraInputArgs: [], + extraOutputArgs: [], + decodeThreads: "2", + sourceWidth: source.width, + sourceHeight: source.height, + ...overrides, + }; +} + +const outputTail = [ + "-c:a", + "copy", + "-movflags", + "+faststart", + "-progress", + "pipe:2", + "-y", + "out.mp4", +]; + +describe("nvenc encoder selection", () => { + test("is off unless CAP_MEDIA_VIDEO_ENCODER is nvenc", () => { + expect(selectedVideoEncoder({})).toBe("libx264"); + expect(selectedVideoEncoder({ CAP_MEDIA_VIDEO_ENCODER: "x264" })).toBe( + "libx264", + ); + expect(selectedVideoEncoder({ CAP_MEDIA_VIDEO_ENCODER: "capcodec" })).toBe( + "capcodec", + ); + expect(selectedVideoEncoder(enabled)).toBe("nvenc"); + expect(nvencOptionsFromEnv({})).toBeNull(); + expect( + nvencOptionsFromEnv({ CAP_MEDIA_VIDEO_ENCODER: "capcodec" }), + ).toBeNull(); + expect(getVideoEncoderStatus({})).toEqual({ + active: "libx264", + nvenc: null, + }); + expect(getVideoEncoderStatus(enabled, false)).toEqual({ + active: "nvenc", + nvenc: null, + }); + }); + + test("uses p1, cq = crf + 8, no lookahead and spatial AQ by default", () => { + expect(nvencOptionsFromEnv(enabled)).toEqual(defaults); + }); + + test("reads tuning overrides from the environment", () => { + expect( + nvencOptionsFromEnv({ + ...enabled, + CAP_NVENC_PRESET: " P7 ", + CAP_NVENC_CQ_OFFSET: "-3", + CAP_NVENC_LOOKAHEAD: "0", + CAP_NVENC_SPATIAL_AQ: "off", + CAP_NVENC_TEMPORAL_AQ: "1", + }), + ).toEqual({ + preset: "p7", + cqOffset: -3, + lookahead: 0, + spatialAq: false, + temporalAq: true, + gpuDecodeCodecs: ["hevc", "av1"], + }); + expect( + nvencOptionsFromEnv({ + ...enabled, + CAP_NVENC_GPU_DECODE_CODECS: "h264, HEVC,vp8,nope", + })?.gpuDecodeCodecs, + ).toEqual(["h264", "hevc"]); + }); + + test("ignores invalid tuning overrides", () => { + expect( + nvencOptionsFromEnv({ + ...enabled, + CAP_NVENC_PRESET: "slow", + CAP_NVENC_CQ_OFFSET: "2.5", + CAP_NVENC_LOOKAHEAD: "64", + CAP_NVENC_SPATIAL_AQ: "maybe", + CAP_NVENC_TEMPORAL_AQ: "", + }), + ).toEqual(defaults); + }); + + test("maps crf to a bounded cq", () => { + expect(nvencCq(23, 2)).toBe("25"); + expect(nvencCq(18, 2)).toBe("20"); + expect(nvencCq(20.5, 0)).toBe("20.5"); + expect(nvencCq(50, 5)).toBe("51"); + expect(nvencCq(0, -4)).toBe("1"); + expect(nvencCq(Number.NaN, 2)).toBe("25"); + }); + + test("builds constant-quality VBR encoder args", () => { + expect(buildNvencEncodeArgs(defaults, 23)).toEqual(nvencVideoArgs); + expect( + buildNvencEncodeArgs( + { ...defaults, lookahead: 0, spatialAq: false, temporalAq: true }, + 18, + ), + ).toEqual([ + "-c:v", + "h264_nvenc", + "-preset", + "p1", + "-tune", + "hq", + "-rc", + "vbr", + "-cq", + "26", + "-b:v", + "0", + "-temporal-aq", + "1", + "-profile:v", + "high", + "-g", + "250", + ]); + }); + + test("fits the GPU resize to the CPU path's box and even dimensions", () => { + expect(fitWithinEven(3840, 2160, 1920, 1080)).toEqual({ + width: 1920, + height: 1080, + }); + expect(fitWithinEven(3024, 1964, 1920, 1080)).toEqual({ + width: 1662, + height: 1080, + }); + expect(fitWithinEven(1079, 1919, 1920, 1080)).toEqual({ + width: 606, + height: 1080, + }); + expect(fitWithinEven(1281, 721, 1920, 1080)).toEqual({ + width: 1280, + height: 720, + }); + }); + + test("decodes on the GPU only for configured, unrotated codecs", () => { + for (const codec of ["hevc", "av1"]) { + expect(canDecodeOnGpu(codec, 0)).toBe(true); + } + expect(canDecodeOnGpu("h264", 0)).toBe(false); + expect(canDecodeOnGpu("h264", 0, ["h264"])).toBe(true); + expect(canDecodeOnGpu("vp8", 0)).toBe(false); + expect(canDecodeOnGpu("prores", 0)).toBe(false); + expect(canDecodeOnGpu("hevc", 90)).toBe(false); + expect(canDecodeOnGpu("h264", null)).toBe(false); + }); +}); + +describe("nvenc fallback", () => { + test("reserves job time for the libx264 retry only when nvenc is on", () => { + expect(encodeAttemptsBudgetMs(60_000, {})).toBe(60_000); + expect(encodeAttemptsBudgetMs(60_000, enabled)).toBe(120_000); + }); + + test("falls back to libx264 on any failure that was not a cancellation", () => { + expect( + nvencFallbackReason( + new Error("FFmpeg exited with code 1. Last stderr: No device"), + false, + ), + ).toBe("FFmpeg exited with code 1. Last stderr: No device"); + expect(nvencFallbackReason("boom", false)).toBe("boom"); + expect( + nvencFallbackReason( + new Error( + "https://bucket.s3.amazonaws.com/u/v/raw.mp4?X-Amz-Signature=abc: Server returned 403", + ), + false, + ), + ).toBe(" Server returned 403"); + expect(nvencFallbackReason(new Error(""), false)).toBe("unknown error"); + expect( + nvencFallbackReason(new Error("x".repeat(2000)), false), + ).toHaveLength(500); + expect(nvencFallbackReason(new Error("killed"), true)).toBeNull(); + }); + + test("reports the last attempt through the encoder status", () => { + const before = getVideoEncoderStatus(enabled).nvenc; + recordNvencAttempt("process", "No NVENC capable devices found"); + const fellBack = getVideoEncoderStatus(enabled).nvenc; + expect(fellBack?.attempts).toBe((before?.attempts ?? 0) + 1); + expect(fellBack?.fallbacks).toBe((before?.fallbacks ?? 0) + 1); + expect(fellBack?.lastAttempt).toMatchObject({ + job: "process", + fellBack: true, + reason: "No NVENC capable devices found", + }); + recordNvencAttempt("edit"); + const succeeded = getVideoEncoderStatus(enabled).nvenc; + expect(succeeded?.fallbacks).toBe(fellBack?.fallbacks); + expect(succeeded?.lastAttempt).toMatchObject({ + job: "edit", + fellBack: false, + reason: null, + }); + }); +}); + +describe("process video args", () => { + test("keeps the default libx264 args unchanged", () => { + const source = metadata(3024, 1964, "h264"); + expect(buildProcessVideoArgs(processArgs(source))).toEqual([ + "ffmpeg", + "-threads", + "2", + "-i", + "in.mp4", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "23", + "-vf", + cpuScale, + "-pix_fmt", + "yuv420p", + "-level:v", + "4.2", + ...outputTail, + ]); + }); + + test("runs a 3024x1964 H.264 source fully on the GPU", () => { + const source = metadata(3024, 1964, "h264"); + expect( + buildProcessVideoArgs( + processArgs(source, { + nvenc: { options: defaults, gpuDecode: true }, + }), + ), + ).toEqual([ + "ffmpeg", + "-threads", + "2", + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", + "-c:v", + "h264_cuvid", + "-resize", + "1662x1080", + "-i", + "in.mp4", + ...nvencVideoArgs, + "-vf", + gpuScale, + "-bsf:v", + "h264_metadata=level=4.2", + ...outputTail, + ]); + }); + + test("runs a 4K HEVC source fully on the GPU", () => { + const source = metadata(3840, 2160, "hevc"); + expect( + buildProcessVideoArgs( + processArgs(source, { + inputPath: "in.mov", + audioTranscode: true, + nvenc: { options: defaults, gpuDecode: true }, + }), + ), + ).toEqual([ + "ffmpeg", + "-threads", + "2", + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", + "-c:v", + "hevc_cuvid", + "-resize", + "1920x1080", + "-i", + "in.mov", + ...nvencVideoArgs, + "-vf", + gpuScale, + "-bsf:v", + "h264_metadata=level=4.2", + "-c:a", + "aac", + "-b:a", + "128k", + "-movflags", + "+faststart", + "-progress", + "pipe:2", + "-y", + "out.mp4", + ]); + }); + + test("keeps -bf 0 for a 1080p VP9 webm", () => { + const source = metadata(1920, 1080, "vp9"); + expect( + buildProcessVideoArgs( + processArgs(source, { + inputPath: "in.webm", + audioTranscode: true, + nvenc: { options: defaults, gpuDecode: true }, + }), + ), + ).toEqual([ + "ffmpeg", + "-threads", + "2", + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", + "-c:v", + "vp9_cuvid", + "-i", + "in.webm", + ...nvencVideoArgs, + "-vf", + gpuScale, + "-bsf:v", + "h264_metadata=level=4.2", + "-bf", + "0", + "-c:a", + "aac", + "-b:a", + "128k", + "-movflags", + "+faststart", + "-progress", + "pipe:2", + "-y", + "out.mp4", + ]); + }); + + test("scales on the CPU before NVENC when the GPU cannot decode", () => { + const source = metadata(1280, 720, "vp8"); + expect( + buildProcessVideoArgs( + processArgs(source, { + inputPath: "in.webm", + decodeThreads: "4", + extraInputArgs: ["-fflags", "+genpts"], + extraOutputArgs: ["-max_muxing_queue_size", "4096"], + nvenc: { options: defaults, gpuDecode: false }, + }), + ), + ).toEqual([ + "ffmpeg", + "-threads", + "4", + "-fflags", + "+genpts", + "-i", + "in.webm", + ...nvencVideoArgs, + "-vf", + cpuScale, + "-pix_fmt", + "yuv420p", + "-bsf:v", + "h264_metadata=level=4.2", + "-bf", + "0", + "-c:a", + "copy", + "-movflags", + "+faststart", + "-max_muxing_queue_size", + "4096", + "-progress", + "pipe:2", + "-y", + "out.mp4", + ]); + }); + + test("never touches level normalization or stream copies", () => { + const source = metadata(1920, 1080, "h264"); + const nvenc = { options: defaults, gpuDecode: false }; + expect( + buildProcessVideoArgs( + processArgs(source, { normalizeH264Level: true, nvenc }), + ), + ).toEqual([ + "ffmpeg", + "-threads", + "2", + "-i", + "in.mp4", + "-c:v", + "copy", + "-bsf:v", + "h264_metadata=level=4.2", + ...outputTail, + ]); + expect( + buildProcessVideoArgs( + processArgs(source, { videoTranscode: false, hasAudio: false }), + ), + ).toEqual([ + "ffmpeg", + "-threads", + "2", + "-i", + "in.mp4", + "-c:v", + "copy", + "-an", + ...outputTail.slice(2), + ]); + }); +}); + +describe("edit re-encode args", () => { + const range = [{ start: 1, end: 3 }]; + + test("swaps libx264 for NVENC and keeps the rest of the edit graph", () => { + const libx264 = buildTranscodeEditArgs("in.mp4", range, "out.mp4", true); + const nvenc = buildTranscodeEditArgs( + "in.mp4", + range, + "out.mp4", + true, + 30, + defaults, + ); + expect(libx264.slice(libx264.indexOf("[v]") + 1)).toEqual([ + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "18", + "-pix_fmt", + "yuv420p", + "-enc_time_base:v", + "1/30", + "-map", + "[a]", + "-c:a", + "aac", + "-b:a", + "160k", + "-movflags", + "+faststart", + "out.mp4", + ]); + expect(nvenc).toEqual([ + "ffmpeg", + "-hide_banner", + "-y", + "-ss", + "1.000", + "-t", + "2.000", + "-i", + "in.mp4", + "-filter_complex", + "[0:v:0]fps=30,setpts=PTS-STARTPTS[v0];[0:a:0]asetpts=PTS-STARTPTS[a0];[v0][a0]concat=n=1:v=1:a=1[v][a]", + "-map", + "[v]", + "-c:v", + "h264_nvenc", + "-preset", + "p1", + "-tune", + "hq", + "-rc", + "vbr", + "-cq", + "26", + "-b:v", + "0", + "-spatial-aq", + "1", + "-profile:v", + "high", + "-g", + "250", + "-pix_fmt", + "yuv420p", + "-enc_time_base:v", + "1/30", + "-map", + "[a]", + "-c:a", + "aac", + "-b:a", + "160k", + "-movflags", + "+faststart", + "out.mp4", + ]); + }); +}); + +describe("decode thread overrides", () => { + test("keep today's defaults unless set to a valid count", () => { + const name = "MEDIA_SERVER_VERIFY_DECODE_THREADS"; + expect(decodeThreadsFromEnv(name, 1, {})).toBe("1"); + expect(decodeThreadsFromEnv(name, 1, { [name]: " " })).toBe("1"); + expect(decodeThreadsFromEnv(name, 1, { [name]: "4" })).toBe("4"); + expect(decodeThreadsFromEnv(name, 2, { [name]: "0" })).toBe("0"); + expect(decodeThreadsFromEnv(name, 2, { [name]: "-1" })).toBe("2"); + expect(decodeThreadsFromEnv(name, 2, { [name]: "1.5" })).toBe("2"); + expect(decodeThreadsFromEnv(name, 2, { [name]: "65" })).toBe("2"); + expect(decodeThreadsFromEnv(name, 2, { [name]: "auto" })).toBe("2"); + }); +}); + +describe("nvenc jobs on hosts without an NVENC device", () => { + const fixture = join( + import.meta.dir, + "..", + "fixtures", + "test-with-audio.mp4", + ); + + async function withNvencEnv(run: () => Promise): Promise { + const previous = process.env.CAP_MEDIA_VIDEO_ENCODER; + process.env.CAP_MEDIA_VIDEO_ENCODER = "nvenc"; + try { + return await run(); + } finally { + if (previous === undefined) delete process.env.CAP_MEDIA_VIDEO_ENCODER; + else process.env.CAP_MEDIA_VIDEO_ENCODER = previous; + } + } + + function expectRecordedAttempt(job: "process" | "edit", attempts: number) { + const status = getVideoEncoderStatus(enabled).nvenc; + expect(status?.attempts).toBe(attempts + 1); + expect(status?.lastAttempt?.job).toBe(job); + if (status?.lastAttempt?.fellBack) { + expect(status.lastAttempt.reason).toBeString(); + } + } + + test("does not start the encoder once the job is cancelled", async () => { + const metadata = await probeVideo(`file://${fixture}`); + const attempts = getVideoEncoderStatus(enabled).nvenc?.attempts ?? 0; + const controller = new AbortController(); + controller.abort(); + await expect( + withNvencEnv(() => + processVideo( + fixture, + metadata, + { maxWidth: 160, maxHeight: 120 }, + undefined, + controller.signal, + ), + ), + ).rejects.toThrow("Video processing was cancelled"); + expect(getVideoEncoderStatus(enabled).nvenc?.attempts ?? 0).toBe(attempts); + }); + + test("still produces the transcode", async () => { + const metadata = await probeVideo(`file://${fixture}`); + const attempts = getVideoEncoderStatus(enabled).nvenc?.attempts ?? 0; + const output = await withNvencEnv(() => + processVideo(fixture, metadata, { maxWidth: 160, maxHeight: 120 }), + ); + try { + const result = await probeVideo(`file://${output.path}`); + expect(result.videoCodec).toBe("h264"); + expect(result.width).toBe(160); + expect(result.height).toBe(120); + expect(result.audioCodec).toBe("aac"); + expectRecordedAttempt("process", attempts); + } finally { + await output.cleanup(); + } + }); + + test("still renders the edit", async () => { + const metadata = await probeVideo(`file://${fixture}`); + const attempts = getVideoEncoderStatus(enabled).nvenc?.attempts ?? 0; + const output = await withNvencEnv(() => + renderEditedVideo({ + inputPath: fixture, + keepRanges: [{ start: 0, end: 0.5 }], + metadata, + }), + ); + try { + const result = await probeVideo(`file://${output.path}`); + expect(result.videoCodec).toBe("h264"); + expect(result.audioCodec).toBe("aac"); + expectRecordedAttempt("edit", attempts); + } finally { + await output.cleanup(); + } + }); +}); diff --git a/apps/media-server/src/__tests__/lib/parallel-download.test.ts b/apps/media-server/src/__tests__/lib/parallel-download.test.ts new file mode 100644 index 00000000000..ce8290f87fd --- /dev/null +++ b/apps/media-server/src/__tests__/lib/parallel-download.test.ts @@ -0,0 +1,193 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { randomBytes } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + downloadUrlToPathParallel, + parallelDownloadOptionsFromEnv, + planRanges, +} from "../../lib/parallel-download"; + +const MB = 1024 * 1024; +const payload = randomBytes(5 * MB + 12345); +let dir: string; +let server: ReturnType; +let rangeRequests = 0; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "parallel-download-")); + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + const range = req.headers.get("range")?.match(/^bytes=(\d+)-(\d+)$/); + if (url.pathname === "/plain" || !range) { + return new Response(payload, { headers: { ETag: '"v1"' } }); + } + rangeRequests++; + if (url.pathname === "/no-total" || url.pathname === "/capped") { + const end = + url.pathname === "/capped" ? 64 * 1024 - 1 : Number(range[2]); + return new Response(payload.subarray(0, end + 1), { + status: 206, + headers: { + "Content-Range": + url.pathname === "/capped" + ? `bytes 0-${end}/${payload.length}` + : `bytes 0-${end}/*`, + }, + }); + } + if (url.pathname === "/huge") { + return new Response(payload.subarray(0, MB), { + status: 206, + headers: { + "Content-Range": `bytes 0-${MB - 1}/${Number.MAX_SAFE_INTEGER}`, + }, + }); + } + if (url.pathname === "/changed" && Number(range[1]) > 0) { + return new Response("precondition", { status: 412 }); + } + const start = Number(range[1]); + const end = Math.min(Number(range[2]), payload.length - 1); + const body = + url.pathname === "/short" && start > 0 + ? payload.subarray(start, end) + : payload.subarray(start, end + 1); + return new Response(body, { + status: 206, + headers: { + ETag: '"v1"', + "Content-Range": `bytes ${start}-${end}/${payload.length}`, + }, + }); + }, + }); +}); + +afterAll(async () => { + server.stop(true); + await rm(dir, { recursive: true, force: true }); +}); + +describe("parallel download", () => { + test("env defaults keep the single-stream path", () => { + expect(parallelDownloadOptionsFromEnv({})).toEqual({ + concurrency: 1, + partBytes: 16 * MB, + }); + expect( + parallelDownloadOptionsFromEnv({ + MEDIA_SERVER_DOWNLOAD_CONCURRENCY: "8", + MEDIA_SERVER_DOWNLOAD_PART_MB: "32", + }), + ).toEqual({ concurrency: 8, partBytes: 32 * MB }); + expect( + parallelDownloadOptionsFromEnv({ + MEDIA_SERVER_DOWNLOAD_CONCURRENCY: "500", + }).concurrency, + ).toBe(32); + }); + + test("plans contiguous ranges after the first part", () => { + const ranges = planRanges(10, 4, 3); + expect(ranges.count).toBe(2); + expect([ranges.at(0), ranges.at(1)]).toEqual([ + { start: 4, end: 6 }, + { start: 7, end: 9 }, + ]); + expect(planRanges(4, 4, 3).count).toBe(0); + expect(planRanges(Number.MAX_SAFE_INTEGER, MB, MB).count).toBeGreaterThan( + 1e9, + ); + }); + + test("reassembles ranged parts byte for byte", async () => { + const path = join(dir, "ranged.bin"); + const result = await downloadUrlToPathParallel( + `http://localhost:${server.port}/file`, + path, + { concurrency: 4, partBytes: MB }, + ); + expect(result.ranged).toBe(true); + expect(result.parts).toBe(6); + expect(Buffer.compare(await readFile(path), payload)).toBe(0); + }); + + test("falls back to one stream when ranges are unsupported", async () => { + const path = join(dir, "plain.bin"); + const result = await downloadUrlToPathParallel( + `http://localhost:${server.port}/plain`, + path, + { concurrency: 4, partBytes: MB }, + ); + expect(result).toEqual({ bytes: payload.length, parts: 1, ranged: false }); + expect(Buffer.compare(await readFile(path), payload)).toBe(0); + }); + + test("fails when the object changes mid-download", async () => { + await expect( + downloadUrlToPathParallel( + `http://localhost:${server.port}/changed`, + join(dir, "changed.bin"), + { concurrency: 4, partBytes: MB }, + ), + ).rejects.toThrow("Ranged download part failed: 412"); + }); + + test.each(["no-total", "capped"])( + "restarts as one stream when the first range can't be used (%s)", + async (name) => { + const path = join(dir, `${name}.bin`); + const result = await downloadUrlToPathParallel( + `http://localhost:${server.port}/${name}`, + path, + { concurrency: 4, partBytes: MB }, + ); + expect(result).toEqual({ + bytes: payload.length, + parts: 1, + ranged: false, + }); + expect(Buffer.compare(await readFile(path), payload)).toBe(0); + }, + ); + + test("rejects an absurd Content-Range total before planning parts", async () => { + const before = rangeRequests; + await expect( + downloadUrlToPathParallel( + `http://localhost:${server.port}/huge`, + join(dir, "huge.bin"), + { concurrency: 4, partBytes: MB }, + ), + ).rejects.toThrow("invalid size"); + expect(rangeRequests).toBe(before + 1); + }); + + test("fails when a part comes back short", async () => { + await expect( + downloadUrlToPathParallel( + `http://localhost:${server.port}/short`, + join(dir, "short.bin"), + { concurrency: 4, partBytes: MB }, + ), + ).rejects.toThrow("expected"); + }); + + test("stops on abort", async () => { + const controller = new AbortController(); + controller.abort(); + const before = rangeRequests; + await expect( + downloadUrlToPathParallel( + `http://localhost:${server.port}/file`, + join(dir, "aborted.bin"), + { concurrency: 4, partBytes: MB, signal: controller.signal }, + ), + ).rejects.toThrow(); + expect(rangeRequests).toBe(before); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts index 0d11a2fb7f9..129c807bdad 100644 --- a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts @@ -960,6 +960,26 @@ describe("complete recording decode", () => { ); }); + test("skips decoded content hashes only when nothing compares them", async () => { + const options = { expectedDuration: 5, requireAudio: true }; + const hashed = await verifyRecording(silent, options); + const unhashed = await verifyRecording(silent, { + ...options, + hashDecodedContent: false, + }); + expect(unhashed.integrity).toBeUndefined(); + expect(unhashed.video).toEqual(hashed.video); + expect(unhashed.audio).toEqual(hashed.audio); + const evidence = await inspectRecordingSources(silent, silent); + const preserved = await verifyRecording(silent, { + requireAudio: false, + hashDecodedContent: false, + sourceEvidence: evidence, + }); + expect(preserved.sourcePreserved).toBe(true); + expect(preserved.integrity).toEqual(evidence.integrity); + }); + test("accepts legacy video without audio only when audio was not requested", async () => { const input = join(FIXTURES, "test-no-audio.mp4"); const evidence = await verifyRecording(input, { diff --git a/apps/media-server/src/__tests__/lib/storage-upload.test.ts b/apps/media-server/src/__tests__/lib/storage-upload.test.ts index f260ea8ae61..4b57d3ed527 100644 --- a/apps/media-server/src/__tests__/lib/storage-upload.test.ts +++ b/apps/media-server/src/__tests__/lib/storage-upload.test.ts @@ -3,7 +3,10 @@ import { createHash } from "node:crypto"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { uploadFileToStorage } from "../../lib/media-video"; +import { + multipartUploadConcurrencyFromEnv, + uploadFileToStorage, +} from "../../lib/media-video"; import { verifyRemoteRecordingBytes } from "../../lib/recording-verification"; const originalFetch = globalThis.fetch; @@ -314,6 +317,74 @@ describe("uploadFileToStorage", () => { }); }); + test("uploads multipart parts concurrently and completes them in order", async () => { + const uploadFile = await createTempUploadFile(partSize * 4 + 1); + let inFlight = 0; + let maxInFlight = 0; + let completed: unknown; + globalThis.fetch = (async (input, init) => { + const url = String(input); + if (init?.method === "POST") { + const json = JSON.parse(String(init.body)); + if (url.endsWith("/sign")) + return Response.json({ + url: `https://storage.example.com/part-${json.partNumber}`, + }); + completed = json; + return Response.json({ objectIdentity: '"completed"' }); + } + const partNumber = Number(url.split("part-")[1]); + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await Bun.sleep((6 - partNumber) * 5); + inFlight--; + return new Response(null, { + status: 200, + headers: { etag: `"etag-${partNumber}"` }, + }); + }) as typeof fetch; + process.env.MEDIA_SERVER_UPLOAD_CONCURRENCY = "3"; + try { + await uploadFileToStorage( + uploadFile.path, + { + type: "multipart", + videoId: "video-id", + key: "user-id/video-id/result.mp4", + uploadId: "upload-id", + partSize, + signPartUrl: "https://cap.example.com/sign", + completeUrl: "https://cap.example.com/complete", + abortUrl: "https://cap.example.com/abort", + }, + "video/mp4", + ); + } finally { + delete process.env.MEDIA_SERVER_UPLOAD_CONCURRENCY; + await uploadFile.cleanup(); + } + expect(maxInFlight).toBe(3); + expect( + (completed as { parts: Array<{ partNumber: number }> }).parts.map( + (part) => part.partNumber, + ), + ).toEqual([1, 2, 3, 4, 5]); + }); + + test("reads a bounded multipart upload concurrency", () => { + expect(multipartUploadConcurrencyFromEnv({})).toBe(1); + expect( + multipartUploadConcurrencyFromEnv({ + MEDIA_SERVER_UPLOAD_CONCURRENCY: "8", + }), + ).toBe(8); + expect( + multipartUploadConcurrencyFromEnv({ + MEDIA_SERVER_UPLOAD_CONCURRENCY: "99", + }), + ).toBe(16); + }); + test("aborts multipart uploads when a part fails", async () => { const uploadFile = await createTempUploadFile(partSize + 1); const requests: Array<{ url: string; method: string }> = []; diff --git a/apps/media-server/src/__tests__/lib/temp-files.test.ts b/apps/media-server/src/__tests__/lib/temp-files.test.ts new file mode 100644 index 00000000000..04c6b136cb6 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/temp-files.test.ts @@ -0,0 +1,180 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, rm, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "bun"; +import { + cleanupMediaTransferCache, + sweepMediaTransferCache, +} from "../../lib/media-transfer"; +import { + cleanupStaleTempFiles, + createTempFile, + ensureTempDir, + getInstanceDir, + getTempDir, + getTempDirUsage, + getTempRoot, + removeOwnTempDir, + retainTempPath, + sweepOrphanedTempDirs, +} from "../../lib/temp-files"; + +const created: string[] = []; + +afterEach(async () => { + for (const path of created.splice(0)) { + await rm(path, { recursive: true, force: true }); + } +}); + +async function makeTree(path: string, ageMs: number, lease = false) { + created.push(path); + await mkdir(join(path, "nested", "deeper"), { recursive: true }); + await writeFile(join(path, "nested", "deeper", "segment.m4s"), "segment"); + const past = new Date(Date.now() - ageMs); + if (lease) { + await writeFile(join(path, ".lease"), ""); + await utimes(join(path, ".lease"), past, past); + } + await utimes(path, past, past); +} + +async function setAge(path: string, ageMs: number) { + const past = new Date(Date.now() - ageMs); + await utimes(path, past, past); +} + +describe("startup temp sweep", () => { + test("removes orphaned instance dirs and old legacy entries only", async () => { + await ensureTempDir(); + const root = getTempRoot(); + const staleInstance = join(root, `instance-${randomUUID()}`); + const liveInstance = join(root, `instance-${randomUUID()}`); + const oldLegacyDir = join(root, `mux-job_${randomUUID()}`); + const youngLegacyFile = join(root, `${randomUUID()}.mp4`); + await makeTree(staleInstance, 10 * 60 * 1000, true); + await makeTree(liveInstance, 10 * 60 * 1000); + await writeFile(join(liveInstance, ".lease"), ""); + await makeTree(oldLegacyDir, 2 * 60 * 60 * 1000); + created.push(youngLegacyFile); + await writeFile(youngLegacyFile, "in use by an older server version"); + + const removed = await sweepOrphanedTempDirs(); + + expect(removed).toBeGreaterThanOrEqual(2); + expect(existsSync(staleInstance)).toBe(false); + expect(existsSync(oldLegacyDir)).toBe(false); + expect(existsSync(liveInstance)).toBe(true); + expect(existsSync(youngLegacyFile)).toBe(true); + expect(existsSync(getTempDir())).toBe(true); + expect(existsSync(join(getTempDir(), ".lease"))).toBe(true); + }); + + test("never treats the server's own instance dir as orphaned", async () => { + await ensureTempDir(); + await setAge(join(getTempDir(), ".lease"), 24 * 60 * 60 * 1000); + await setAge(getTempDir(), 24 * 60 * 60 * 1000); + + await sweepOrphanedTempDirs(); + + expect(existsSync(getTempDir())).toBe(true); + }); +}); + +describe("stale temp cleanup", () => { + test("removes stale directories recursively and keeps in-use entries", async () => { + await ensureTempDir(); + const staleDir = join(getTempDir(), `stream-${randomUUID()}`); + await makeTree(staleDir, 2 * 60 * 60 * 1000); + const retainedDir = join(getTempDir(), `mux-job_${randomUUID()}`); + await makeTree(retainedDir, 2 * 60 * 60 * 1000); + const releaseRetainedDir = retainTempPath(retainedDir); + const inUse = await createTempFile(".mp4"); + created.push(inUse.path); + await writeFile(inUse.path, "input still read by ffmpeg"); + await setAge(inUse.path, 2 * 60 * 60 * 1000); + const fresh = await createTempFile(".mp4"); + created.push(fresh.path); + await writeFile(fresh.path, "fresh"); + + try { + expect(await cleanupStaleTempFiles()).toBeGreaterThanOrEqual(1); + expect(existsSync(staleDir)).toBe(false); + expect(existsSync(retainedDir)).toBe(true); + expect(existsSync(inUse.path)).toBe(true); + expect(existsSync(fresh.path)).toBe(true); + + releaseRetainedDir(); + await cleanupStaleTempFiles(); + expect(existsSync(retainedDir)).toBe(false); + } finally { + releaseRetainedDir(); + await inUse.cleanup(); + await fresh.cleanup(); + } + }); + + test("expires entries that were retained far beyond any job lifetime", async () => { + await ensureTempDir(); + const leaked = await createTempFile(".mp4"); + created.push(leaked.path); + await writeFile(leaked.path, "leaked handle"); + await setAge(leaked.path, 7 * 60 * 60 * 1000); + + await cleanupStaleTempFiles(); + + expect(existsSync(leaked.path)).toBe(false); + }); + + test("reports temp usage including nested files", async () => { + await ensureTempDir(); + const dir = join(getTempDir(), `usage-${randomUUID()}`); + await makeTree(dir, 0); + + const usage = await getTempDirUsage(); + + expect(usage.entries).toBeGreaterThanOrEqual(4); + expect(usage.bytes).toBeGreaterThanOrEqual("segment".length); + expect(usage.truncated).toBe(false); + }); + + test("removes the server's own dir on shutdown and recreates it on demand", async () => { + const handle = await createTempFile(".mp4"); + await writeFile(handle.path, "shutdown"); + + await removeOwnTempDir(); + expect(existsSync(getTempDir())).toBe(false); + + const next = await createTempFile(".mp4"); + expect(existsSync(join(getTempDir(), ".lease"))).toBe(true); + await next.cleanup(); + }); +}); + +describe("media transfer cache sweep", () => { + test("removes orphaned instance and dead-pid cache dirs but keeps a live pid's cache however old", async () => { + const base = join(tmpdir(), "cap-media-download-cache"); + const staleInstance = join(base, `instance-${randomUUID()}`); + const liveInstance = join(base, `instance-${randomUUID()}`); + const exited = spawn({ cmd: ["true"] }); + await exited.exited; + const deadPidDir = join(base, String(exited.pid)); + const oldLivePidDir = join(base, String(process.ppid)); + await makeTree(staleInstance, 10 * 60 * 1000, true); + await makeTree(liveInstance, 0, true); + await makeTree(deadPidDir, 0); + await makeTree(oldLivePidDir, 24 * 60 * 60 * 1000); + + await sweepMediaTransferCache(); + + expect(existsSync(staleInstance)).toBe(false); + expect(existsSync(deadPidDir)).toBe(false); + expect(existsSync(liveInstance)).toBe(true); + expect(existsSync(oldLivePidDir)).toBe(true); + expect(existsSync(getInstanceDir(base))).toBe(false); + await cleanupMediaTransferCache(); + }); +}); diff --git a/apps/media-server/src/__tests__/routes/lifecycle.test.ts b/apps/media-server/src/__tests__/routes/lifecycle.test.ts new file mode 100644 index 00000000000..ae93d131ace --- /dev/null +++ b/apps/media-server/src/__tests__/routes/lifecycle.test.ts @@ -0,0 +1,468 @@ +import { + afterEach, + beforeEach, + describe, + expect, + mock, + spyOn, + test, +} from "bun:test"; +import { existsSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import app from "../../app"; +import * as jobManager from "../../lib/job-manager"; +import { + cleanupExpiredJobs, + createJob, + deleteJob, + getActiveVideoProcessCount, + getJob, + type VideoMetadata, +} from "../../lib/job-manager"; +import { + DRAINING_DETAILS, + setDraining, + waitForDrain, +} from "../../lib/lifecycle"; +import * as mediaEdit from "../../lib/media-edit"; +import * as mediaProbe from "../../lib/media-probe"; +import * as mediaVideo from "../../lib/media-video"; +import { createTempFile, type TempFileHandle } from "../../lib/temp-files"; +import { getActiveDirectVideoProcessCount } from "../../lib/video-capacity"; + +const MEDIA_SERVER_SECRET = "test-secret"; +const AUTH_HEADERS = { + "Content-Type": "application/json", + "x-media-server-secret": MEDIA_SERVER_SECRET, +}; +const originalPublicStatus = process.env.MEDIA_SERVER_PUBLIC_STATUS; + +process.env.MEDIA_SERVER_WEBHOOK_SECRET = MEDIA_SERVER_SECRET; + +const metadata: VideoMetadata = { + duration: 3 * 60 * 60, + width: 1920, + height: 1080, + fps: 30, + videoCodec: "h264", + audioCodec: "aac", + audioChannels: 2, + sampleRate: 48_000, + bitrate: 5_000_000, + fileSize: 1024 ** 3, +}; + +const createdJobs: string[] = []; + +function post(path: string, body: unknown): Request { + return new Request(`http://localhost${path}`, { + method: "POST", + headers: AUTH_HEADERS, + body: JSON.stringify(body), + }); +} + +function deferred() { + let resolve: (value: T) => void = () => {}; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function tempFileWith(extension: string, contents: string) { + const handle = await createTempFile(extension); + await writeFile(handle.path, contents); + return handle; +} + +async function waitFor(condition: () => boolean, timeoutMs = 2_000) { + const deadline = Date.now() + timeoutMs; + while (!condition()) { + if (Date.now() > deadline) throw new Error("Timed out waiting"); + await Bun.sleep(10); + } +} + +beforeEach(() => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true); +}); + +afterEach(async () => { + mock.restore(); + setDraining(false); + if (originalPublicStatus === undefined) + delete process.env.MEDIA_SERVER_PUBLIC_STATUS; + else process.env.MEDIA_SERVER_PUBLIC_STATUS = originalPublicStatus; + await waitForDrain(2_000); + for (const jobId of createdJobs.splice(0)) deleteJob(jobId); +}); + +describe("draining", () => { + const url = "https://example.com/video.mp4"; + const admissions: Array<[string, unknown]> = [ + [ + "/video/process", + { videoId: "v", userId: "u", videoUrl: url, outputPresignedUrl: url }, + ], + [ + "/video/import", + { + videoId: "v", + userId: "u", + videoUrl: url, + outputPresignedUrl: url, + sourcePresignedUrl: url, + }, + ], + [ + "/video/edit", + { + videoId: "v", + userId: "u", + sourceUrl: url, + outputPresignedUrl: url, + keepRanges: [{ start: 0, end: 1 }], + }, + ], + [ + "/video/mux-segments", + { + videoId: "v", + userId: "u", + outputPresignedUrl: url, + videoInitUrl: url, + videoSegmentUrls: ["https://example.com/segment.m4s"], + }, + ], + [ + "/video/verify-recording", + { + videoId: "v", + userId: "u", + videoUrl: url, + fileSize: 1, + duration: 1, + requiredAudio: false, + objectIdentity: '"etag"', + webhookUrl: "https://example.com/webhook", + }, + ], + ["/video/convert", { videoUrl: url }], + ["/video/probe", { videoUrl: url }], + ["/video/thumbnail", { videoUrl: url }], + ["/audio/check", { videoUrl: url }], + ["/audio/extract", { videoUrl: url }], + ["/audio/convert", { audioUrl: url }], + ]; + + test.each(admissions)("rejects new work on %s", async (path, body) => { + setDraining(true); + const response = await app.fetch(post(path, body)); + + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("15"); + const data = await response.json(); + expect(data.code).toBe("SERVER_BUSY"); + expect(data.details).toBe(DRAINING_DETAILS); + }); + + test("fails health checks but keeps status endpoints working", async () => { + const job = createJob("job-draining-status", "video-id", "user-id"); + createdJobs.push(job.jobId); + setDraining(true); + + const health = await app.fetch(new Request("http://localhost/health")); + expect(health.status).toBe(503); + expect((await health.json()).status).toBe("draining"); + + const status = await app.fetch( + new Request(`http://localhost/video/process/${job.jobId}/status`), + ); + expect(status.status).toBe(200); + expect((await status.json()).jobId).toBe(job.jobId); + + const serverStatus = await app.fetch( + new Request("http://localhost/video/status"), + ); + expect(serverStatus.status).toBe(200); + }); +}); + +describe("status access", () => { + test("requires the secret when public status is disabled", async () => { + process.env.MEDIA_SERVER_PUBLIC_STATUS = "false"; + for (const path of [ + "/video/status", + "/audio/status", + "/video/process/job-missing/status", + ]) { + const anonymous = await app.fetch(new Request(`http://localhost${path}`)); + expect(anonymous.status).toBe(401); + const authenticated = await app.fetch( + new Request(`http://localhost${path}`, { headers: AUTH_HEADERS }), + ); + expect(authenticated.status).not.toBe(401); + } + }); + + test("reports runtime counters only to authenticated callers", async () => { + const job = createJob("job_runtime_report", "video-runtime", "user-id"); + createdJobs.push(job.jobId); + const anonymous = await app.fetch(new Request("http://localhost/health")); + expect((await anonymous.json()).runtime).toBeUndefined(); + + for (const path of ["/health", "/video/status"]) { + const response = await app.fetch( + new Request(`http://localhost${path}`, { headers: AUTH_HEADERS }), + ); + const { runtime } = await response.json(); + expect(runtime.activeJobs).toBeGreaterThanOrEqual(1); + expect(runtime.jobTotals.created).toBeGreaterThanOrEqual(1); + expect(typeof runtime.activeSubprocesses).toBe("number"); + expect(typeof runtime.tempDir.bytes).toBe("number"); + expect(typeof runtime.tempDir.entries).toBe("number"); + expect(typeof runtime.mediaTransferCache.bytes).toBe("number"); + expect(typeof runtime.slots.video.max).toBe("number"); + expect(typeof runtime.slots.directVideo.active).toBe("number"); + expect(typeof runtime.slots.audio.max).toBe("number"); + expect(typeof runtime.slots.probe.max).toBe("number"); + expect(JSON.stringify(runtime)).not.toContain(job.jobId); + expect(JSON.stringify(runtime)).not.toContain(job.videoId); + } + }); +}); + +describe("process jobs", () => { + test("heartbeats a slow output upload and derives the lifetime from the input", async () => { + const intervals: Array<{ + callback: () => void; + delay?: number; + handle: ReturnType; + }> = []; + const cleared = new Set(); + const originalSetInterval = globalThis.setInterval; + const originalClearInterval = globalThis.clearInterval; + spyOn(globalThis, "setInterval").mockImplementation((( + callback: () => void, + delay?: number, + ) => { + const handle = originalSetInterval(callback, delay); + intervals.push({ callback, delay, handle }); + return handle; + }) as unknown as typeof setInterval); + spyOn(globalThis, "clearInterval").mockImplementation((( + handle: ReturnType, + ) => { + cleared.add(handle); + originalClearInterval(handle); + }) as unknown as typeof clearInterval); + const files: TempFileHandle[] = []; + spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => { + const input = await tempFileWith(".mp4", "input"); + files.push(input); + return input; + }); + spyOn(mediaProbe, "probeVideoFile").mockResolvedValue(metadata); + spyOn(mediaVideo, "processVideo").mockImplementation(async () => { + const output = await tempFileWith(".mp4", "output"); + files.push(output); + return output; + }); + const uploadStarted = deferred(); + const upload = deferred(); + spyOn(mediaVideo, "uploadFileToS3").mockImplementation(async () => { + uploadStarted.resolve(); + return await upload.promise; + }); + + const response = await app.fetch( + post("/video/process", { + videoId: "video-slow-upload", + userId: "user-id", + videoUrl: "https://example.com/input.mp4", + outputPresignedUrl: "https://example.com/output.mp4", + }), + ); + const { jobId } = await response.json(); + createdJobs.push(jobId); + await uploadStarted.promise; + + const job = getJob(jobId); + expect(job?.phase).toBe("uploading"); + if (!job) throw new Error("Job missing"); + expect(job.lifetimeDeadlineAt).toBeGreaterThan( + job.createdAt + 2 * 60 * 60 * 1000, + ); + expect(job.lifetimeDeadlineAt).toBeLessThanOrEqual( + job.createdAt + 4 * 60 * 60 * 1000, + ); + + job.updatedAt = Date.now() - 16 * 60 * 1000; + const heartbeats = intervals.filter( + ({ delay, handle }) => delay === 60_000 && !cleared.has(handle), + ); + expect(heartbeats).toHaveLength(1); + heartbeats[0]?.callback(); + expect(cleanupExpiredJobs()).toBe(0); + expect(job.phase).toBe("uploading"); + + upload.resolve({}); + await waitFor(() => job.phase === "complete"); + await waitForDrain(2_000); + expect(files.every((handle) => !existsSync(handle.path))).toBe(true); + }); + + test("releases the job slot and temp files when processing fails", async () => { + const before = getActiveVideoProcessCount(); + let input: TempFileHandle | undefined; + spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => { + input = await tempFileWith(".mp4", "input"); + return input; + }); + spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({ + ...metadata, + duration: 10, + }); + spyOn(mediaVideo, "processVideo").mockRejectedValue( + new Error("FFmpeg exited with code 1"), + ); + + const response = await app.fetch( + post("/video/process", { + videoId: "video-failed-process", + userId: "user-id", + videoUrl: "https://example.com/input.mp4", + outputPresignedUrl: "https://example.com/output.mp4", + }), + ); + const { jobId } = await response.json(); + createdJobs.push(jobId); + + expect(await waitForDrain(2_000)).toBe(true); + expect(getJob(jobId)?.phase).toBe("error"); + expect(getActiveVideoProcessCount()).toBe(before); + expect(input && existsSync(input.path)).toBe(false); + }); +}); + +describe("edit jobs", () => { + test("cancellation aborts the upload and skips preview work", async () => { + const files: TempFileHandle[] = []; + spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => { + const input = await tempFileWith(".mp4", "input"); + files.push(input); + return input; + }); + spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({ + ...metadata, + duration: 10, + }); + spyOn(mediaEdit, "renderEditedVideo").mockImplementation(async () => { + const output = await tempFileWith(".mp4", "edited"); + files.push(output); + return output; + }); + const thumbnail = spyOn(mediaVideo, "generateThumbnail"); + const uploadStarted = deferred(); + let uploadSignal: AbortSignal | undefined; + spyOn(mediaVideo, "uploadFileToS3").mockImplementation( + (_path, _url, _contentType, signal) => { + uploadSignal = signal; + uploadStarted.resolve(); + return new Promise((_resolve, reject) => { + signal?.addEventListener("abort", () => reject(signal.reason), { + once: true, + }); + }); + }, + ); + + const response = await app.fetch( + post("/video/edit", { + videoId: "video-cancelled-edit", + userId: "user-id", + sourceUrl: "https://example.com/source.mp4", + outputPresignedUrl: "https://example.com/output.mp4", + thumbnailPresignedUrl: "https://example.com/thumbnail.jpg", + keepRanges: [{ start: 0, end: 5 }], + }), + ); + const { jobId } = await response.json(); + createdJobs.push(jobId); + await uploadStarted.promise; + + const cancel = await app.fetch(post(`/video/process/${jobId}/cancel`, {})); + expect(cancel.status).toBe(200); + expect(uploadSignal?.aborted).toBe(true); + + expect(await waitForDrain(2_000)).toBe(true); + expect(getJob(jobId)?.phase).toBe("cancelled"); + expect(thumbnail).not.toHaveBeenCalled(); + expect(files).toHaveLength(2); + expect(files.every((handle) => !existsSync(handle.path))).toBe(true); + }); +}); + +describe("direct conversions", () => { + test("release the slot when the download fails", async () => { + spyOn(mediaVideo, "downloadVideoToTemp").mockRejectedValue( + new Error("Failed to download video: 404"), + ); + + const response = await app.fetch( + post("/video/convert", { videoUrl: "https://example.com/missing.mp4" }), + ); + + expect(response.status).toBe(500); + expect(getActiveDirectVideoProcessCount()).toBe(0); + }); + + test("release the slot and temp files when the client cancels the body", async () => { + const files: TempFileHandle[] = []; + spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => { + const input = await tempFileWith(".webm", "input"); + files.push(input); + return input; + }); + spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({ + ...metadata, + duration: 10, + }); + spyOn(mediaVideo, "processVideo").mockImplementation(async () => { + const output = await tempFileWith(".mp4", "x".repeat(1024 * 1024)); + files.push(output); + return output; + }); + + const response = await app.fetch( + post("/video/convert", { videoUrl: "https://example.com/input.webm" }), + ); + expect(response.status).toBe(200); + expect(getActiveDirectVideoProcessCount()).toBe(1); + + await response.body?.cancel(); + await waitFor(() => getActiveDirectVideoProcessCount() === 0); + await waitFor(() => files.every((handle) => !existsSync(handle.path))); + }); + + test("release the slot after the body is fully read", async () => { + spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(() => + tempFileWith(".webm", "input"), + ); + spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({ + ...metadata, + duration: 10, + }); + spyOn(mediaVideo, "processVideo").mockImplementation(() => + tempFileWith(".mp4", "converted"), + ); + + const response = await app.fetch( + post("/video/convert", { videoUrl: "https://example.com/input.webm" }), + ); + + expect(await response.text()).toBe("converted"); + await waitFor(() => getActiveDirectVideoProcessCount() === 0); + }); +}); diff --git a/apps/media-server/src/app.ts b/apps/media-server/src/app.ts index e50979955ad..162a7652dd0 100644 --- a/apps/media-server/src/app.ts +++ b/apps/media-server/src/app.ts @@ -1,5 +1,6 @@ import { Hono } from "hono"; import { logger } from "hono/logger"; +import { beginRequest } from "./lib/lifecycle"; import audio from "./routes/audio"; import health from "./routes/health"; import video from "./routes/video"; @@ -7,6 +8,15 @@ import video from "./routes/video"; const app = new Hono(); app.use("*", logger()); +app.use("*", async (c, next) => { + if (c.req.method === "GET" || c.req.method === "HEAD") return await next(); + const end = beginRequest(); + try { + await next(); + } finally { + end(); + } +}); app.route("/health", health); app.route("/audio", audio); diff --git a/apps/media-server/src/index.ts b/apps/media-server/src/index.ts index 7a5b2151dfd..05dbdac4fd3 100644 --- a/apps/media-server/src/index.ts +++ b/apps/media-server/src/index.ts @@ -1,38 +1,52 @@ import app from "./app"; import { abortAllJobs } from "./lib/job-manager"; +import { createShutdownHandler, getIdleTimeoutSeconds } from "./lib/lifecycle"; import { cancelAllMediaOperations } from "./lib/media-operations"; -import { cleanupMediaTransferCache } from "./lib/media-transfer"; +import { + cleanupMediaTransferCache, + sweepMediaTransferCache, +} from "./lib/media-transfer"; +import { removeOwnTempDir, startTempDirMaintenance } from "./lib/temp-files"; const port = Number(process.env.PORT) || 3456; console.log(`[media-server] Starting on port ${port}`); -let shuttingDown = false; +void Promise.allSettled([ + startTempDirMaintenance(), + sweepMediaTransferCache(), +]).then(([temp, cache]) => { + const removed = + (temp.status === "fulfilled" ? temp.value : 0) + + (cache.status === "fulfilled" ? cache.value : 0); + if (removed > 0) { + console.log(`[media-server] Removed ${removed} orphaned temp entries`); + } +}); -const shutdown = async () => { - if (shuttingDown) return; - shuttingDown = true; - console.log("[media-server] Shutting down..."); +const shutdown = createShutdownHandler(async () => { const abortedJobs = await abortAllJobs(); if (abortedJobs > 0) { console.log(`[media-server] Aborted ${abortedJobs} active jobs`); } await cancelAllMediaOperations(); await cleanupMediaTransferCache(); + await removeOwnTempDir(); process.exit(0); -}; +}); process.on("SIGINT", () => { - void shutdown(); + void shutdown("SIGINT"); }); process.on("SIGTERM", () => { - void shutdown(); + void shutdown("SIGTERM"); }); process.on("SIGHUP", () => { - void shutdown(); + void shutdown("SIGHUP"); }); export default { port, + idleTimeout: getIdleTimeoutSeconds(), fetch: app.fetch, }; diff --git a/apps/media-server/src/lib/audio-quality.ts b/apps/media-server/src/lib/audio-quality.ts index d51fe5b7691..455824b258c 100644 --- a/apps/media-server/src/lib/audio-quality.ts +++ b/apps/media-server/src/lib/audio-quality.ts @@ -1,7 +1,6 @@ import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { lstat, mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; import { isAbsolute, join } from "node:path"; import { spawn } from "bun"; import { z } from "zod"; @@ -13,6 +12,7 @@ import { } from "./audio-quality-policy"; import { proveRecordingPackets } from "./recording-packet-proof"; import { registerSubprocess, unregisterSubprocess } from "./subprocess"; +import { ensureTempDir, getTempDir } from "./temp-files"; const streamSchema = z.object({ codec_type: z.string(), @@ -307,7 +307,8 @@ export async function createAudioQualityCandidate( Math.abs(videoDuration - input.duration) > 0.1 ) return { status: "unchanged", reason: "source-duration-mismatch" }; - directory = await mkdtemp(join(tmpdir(), "cap-audio-quality-")); + await ensureTempDir(); + directory = await mkdtemp(join(getTempDir(), "audio-quality-")); let path = join(directory, "candidate.mp4"); const renderArgs = [ "ffmpeg", diff --git a/apps/media-server/src/lib/auth.ts b/apps/media-server/src/lib/auth.ts index a8e322d2861..9641d67c496 100644 --- a/apps/media-server/src/lib/auth.ts +++ b/apps/media-server/src/lib/auth.ts @@ -12,3 +12,17 @@ export function validateMediaServerSecret(c: MediaServerContext): boolean { } return c.req.header("x-media-server-secret") === secret; } + +export function hasMediaServerSecret(c: MediaServerContext): boolean { + return ( + c.req.header("x-media-server-secret") !== undefined && + validateMediaServerSecret(c) + ); +} + +export function canReadStatus(c: MediaServerContext): boolean { + const publicStatus = process.env.MEDIA_SERVER_PUBLIC_STATUS?.toLowerCase(); + return publicStatus === "false" || publicStatus === "0" + ? validateMediaServerSecret(c) + : true; +} diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts index 74caa0deaa7..5c05238f20c 100644 --- a/apps/media-server/src/lib/job-manager.ts +++ b/apps/media-server/src/lib/job-manager.ts @@ -124,6 +124,7 @@ export interface Job { webhookLastAttemptAt?: number; recordingVerificationDeadlineAt?: number; recordingProcessingDeadlineAt?: number; + lifetimeDeadlineAt?: number; recordingVerification?: RecordingVerificationProof; manifestSha256?: string; jobId: string; @@ -147,9 +148,17 @@ export interface Job { } const jobs = new Map(); +const jobTotals = { + created: 0, + completed: 0, + errored: 0, + cancelled: 0, + expired: 0, +}; const JOB_TTL_MS = 60 * 60 * 1000; const STALE_JOB_MS = 15 * 60 * 1000; const MAX_JOB_LIFETIME_MS = 60 * 60 * 1000; +const MAX_EXTENDED_JOB_LIFETIME_MS = 4 * 60 * 60 * 1000; const MAX_RECORDING_PROCESSING_BUDGET_MS = 3 * 60 * 60 * 1000; const WEBHOOK_MAX_ATTEMPTS = 3; const WEBHOOK_RETRY_BASE_MS = 500; @@ -366,9 +375,14 @@ export function createJob( webhookSecret, }; jobs.set(jobId, job); + jobTotals.created++; return job; } +export function getJobTotals(): typeof jobTotals { + return { ...jobTotals }; +} + export function getJob(jobId: string): Job | undefined { return jobs.get(jobId); } @@ -410,12 +424,37 @@ export function updateJob( job.terminalAt ??= job.updatedAt; if (job.recordingWorkerLeaseTimer) clearTimeout(job.recordingWorkerLeaseTimer); + if (job.phase === "complete") jobTotals.completed++; + else if (job.phase === "error") jobTotals.errored++; + else jobTotals.cancelled++; } if (!isActivePhase(job.phase) && job.webhookInFlight) job.webhookPending = true; return job; } +export function extendJobLifetime( + jobId: string, + remainingBudgetMs: number, +): number | undefined { + const job = jobs.get(jobId); + if ( + !job || + !isActivePhase(job.phase) || + !Number.isFinite(remainingBudgetMs) || + remainingBudgetMs <= 0 + ) + return undefined; + job.lifetimeDeadlineAt = Math.min( + job.createdAt + MAX_EXTENDED_JOB_LIFETIME_MS, + Math.max( + job.lifetimeDeadlineAt ?? job.createdAt + MAX_JOB_LIFETIME_MS, + Date.now() + remainingBudgetMs, + ), + ); + return job.lifetimeDeadlineAt; +} + export function touchJob(jobId: string): Job | undefined { const job = jobs.get(jobId); if (!job) return undefined; @@ -556,11 +595,14 @@ export function cleanupExpiredJobs(): number { `[job-manager] Cleaning up expired job ${jobId} (phase=${job.phase}, age=${Math.round(age / 60000)}m)`, ); job.abortController?.abort(); - updateJob(jobId, { - phase: "error", - error: `Job expired: no progress update for ${Math.round(staleness / 60000)} minutes`, - message: "Processing failed (expired)", - }); + if ( + updateJob(jobId, { + phase: "error", + error: `Job expired: no progress update for ${Math.round(staleness / 60000)} minutes`, + message: "Processing failed (expired)", + }) + ) + jobTotals.expired++; void sendWebhook(job); } cleaned++; @@ -572,34 +614,40 @@ export function cleanupExpiredJobs(): number { `[job-manager] Marking stale job ${jobId} as error (phase=${job.phase}, no update for ${Math.round(staleness / 60000)}m)`, ); job.abortController?.abort(); - updateJob(jobId, { - phase: "error", - error: `Job stale: no progress update for ${Math.round(staleness / 60000)} minutes`, - message: "Processing failed (stale)", - }); + if ( + updateJob(jobId, { + phase: "error", + error: `Job stale: no progress update for ${Math.round(staleness / 60000)} minutes`, + message: "Processing failed (stale)", + }) + ) + jobTotals.expired++; void sendWebhook(job); cleaned++; continue; } + const lifetimeDeadline = + job.lifetimeDeadlineAt ?? job.createdAt + MAX_JOB_LIFETIME_MS; const deadline = job.recordingVerificationDeadlineAt ?? job.recordingProcessingDeadlineAt ?? - job.createdAt + MAX_JOB_LIFETIME_MS; + lifetimeDeadline; if (isActivePhase(job.phase) && now > deadline) { console.warn( `[job-manager] Marking long-running job ${jobId} as error (phase=${job.phase}, age=${Math.round(age / 60000)}m)`, ); job.abortController?.abort(); - updateJob(jobId, { + const expired = updateJob(jobId, { phase: "error", error: job.recordingVerificationDeadlineAt === undefined && job.recordingProcessingDeadlineAt === undefined - ? `Job exceeded maximum lifetime of ${Math.round(MAX_JOB_LIFETIME_MS / 60000)} minutes` + ? `Job exceeded maximum lifetime of ${Math.round((lifetimeDeadline - job.createdAt) / 60000)} minutes` : "Recording verification timed out", message: "Processing failed (timeout)", }); + if (expired) jobTotals.expired++; void sendWebhook(job); cleaned++; continue; diff --git a/apps/media-server/src/lib/lifecycle.ts b/apps/media-server/src/lib/lifecycle.ts new file mode 100644 index 00000000000..fd60e537a30 --- /dev/null +++ b/apps/media-server/src/lib/lifecycle.ts @@ -0,0 +1,192 @@ +import { getActiveVideoProcessCount } from "./job-manager"; +import { + getActiveAudioOperationCount, + getActiveProbeOperationCount, +} from "./media-operations"; +import { getActiveSubprocessCount } from "./subprocess"; + +const DRAIN_POLL_INTERVAL_MS = 250; +const DRAIN_LOG_INTERVAL_MS = 30_000; +export const DRAIN_RETRY_AFTER_SECONDS = 15; +export const DRAINING_DETAILS = + "Server is draining for shutdown, please retry later"; + +const MAX_IDLE_TIMEOUT_SECONDS = 255; + +let draining = false; +let inFlightRequests = 0; +const backgroundTasks = new Set>(); + +// Bun closes a connection after idleTimeout seconds without I/O, including +// while a handler is still working, and its 10 s default is shorter than the +// AWS ALB keep-alive idle timeout (60 s), which turns reused sockets into 502s. +export function getIdleTimeoutSeconds(): number { + const value = Number.parseInt( + process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS ?? "", + 10, + ); + return Number.isFinite(value) && value >= 0 + ? Math.min(MAX_IDLE_TIMEOUT_SECONDS, value) + : MAX_IDLE_TIMEOUT_SECONDS; +} + +// A streamed response can stay silent while ffmpeg opens its input, so the +// idle timeout only comes back once the first bytes are on their way. +export function restoreIdleTimeoutOnFirstChunk( + stream: ReadableStream, + restore: () => void, +): ReadableStream { + let restored = false; + return stream.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + if (!restored) { + restored = true; + restore(); + } + controller.enqueue(chunk); + }, + }), + ); +} + +type IdleTimeoutServer = { + timeout?: (request: Request, seconds: number) => void; +}; + +export function setRequestIdleTimeout( + server: unknown, + request: Request, + seconds: number, +): void { + const timeout = (server as IdleTimeoutServer | undefined)?.timeout; + if (typeof timeout !== "function") return; + try { + timeout.call(server, request, seconds); + } catch {} +} + +export function getShutdownGraceMs(): number { + const value = Number.parseInt( + process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS ?? "0", + 10, + ); + return Number.isFinite(value) && value > 0 ? value : 0; +} + +export function isDraining(): boolean { + return draining; +} + +export function setDraining(value: boolean): void { + draining = value; +} + +export function trackBackgroundTask(task: Promise): Promise { + backgroundTasks.add(task); + const settle = () => { + backgroundTasks.delete(task); + }; + task.then(settle, settle); + return task; +} + +export function beginRequest(): () => void { + inFlightRequests++; + let ended = false; + return () => { + if (ended) return; + ended = true; + inFlightRequests--; + }; +} + +export function getDrainBlockers() { + return { + activeVideoProcesses: getActiveVideoProcessCount(), + backgroundTasks: backgroundTasks.size, + inFlightRequests, + audioOperations: getActiveAudioOperationCount(), + probeOperations: getActiveProbeOperationCount(), + subprocesses: getActiveSubprocessCount(), + }; +} + +export function isIdle(): boolean { + return Object.values(getDrainBlockers()).every((count) => count === 0); +} + +function wait(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + const done = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", done); + resolve(); + }; + const timer = setTimeout(done, ms); + signal?.addEventListener("abort", done, { once: true }); + }); +} + +export async function waitForDrain( + graceMs: number, + signal?: AbortSignal, +): Promise { + const startedAt = Date.now(); + const deadline = startedAt + graceMs; + let lastLogAt = startedAt; + while (!isIdle()) { + const now = Date.now(); + if (signal?.aborted || now >= deadline) return false; + if (now - lastLogAt >= DRAIN_LOG_INTERVAL_MS) { + lastLogAt = now; + console.log("[media-server] Draining", { + remainingMs: deadline - now, + ...getDrainBlockers(), + }); + } + await wait(Math.min(DRAIN_POLL_INTERVAL_MS, deadline - now), signal); + } + return true; +} + +export function createShutdownHandler( + finalize: () => Promise, + graceMs = getShutdownGraceMs, +): (signal: string) => Promise { + let shuttingDown = false; + let forceDrain: AbortController | undefined; + + return async (signal) => { + if (forceDrain) { + console.log( + `[media-server] Received ${signal} while draining, shutting down now`, + ); + forceDrain.abort(); + return; + } + if (shuttingDown) return; + shuttingDown = true; + + const drainMs = graceMs(); + if (drainMs > 0) { + forceDrain = new AbortController(); + setDraining(true); + console.log( + `[media-server] Received ${signal}, draining for up to ${drainMs}ms`, + getDrainBlockers(), + ); + const drained = await waitForDrain(drainMs, forceDrain.signal); + console.log( + drained + ? "[media-server] Drain complete" + : "[media-server] Drain ended with work still active", + getDrainBlockers(), + ); + } else { + console.log("[media-server] Shutting down..."); + } + + await finalize(); + }; +} diff --git a/apps/media-server/src/lib/media-common.ts b/apps/media-server/src/lib/media-common.ts index ad3c692242e..875a8faba37 100644 --- a/apps/media-server/src/lib/media-common.ts +++ b/apps/media-server/src/lib/media-common.ts @@ -32,6 +32,20 @@ export const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1000; export const UPLOAD_TIMEOUT_MS = 10 * 60 * 1000; const mediaFetch: typeof fetch = globalThis.fetch.bind(globalThis); +export function decodeThreadsFromEnv( + name: string, + fallback: number, + env: Record = process.env, +): string { + const raw = env[name]?.trim(); + if (!raw) return String(fallback); + const value = Number(raw); + if (!Number.isInteger(value) || value < 0 || value > 64) { + return String(fallback); + } + return String(value); +} + export async function withTimeout( promise: Promise, timeoutMs: number, diff --git a/apps/media-server/src/lib/media-edit.ts b/apps/media-server/src/lib/media-edit.ts index aa8b69c5326..91a9f048371 100644 --- a/apps/media-server/src/lib/media-edit.ts +++ b/apps/media-server/src/lib/media-edit.ts @@ -1,6 +1,14 @@ import { writeFile } from "node:fs/promises"; import { file, spawn } from "bun"; import type { VideoMetadata } from "./job-manager"; +import { + buildNvencEncodeArgs, + logNvencFallback, + type NvencOptions, + nvencFallbackReason, + nvencOptionsFromEnv, + recordNvencAttempt, +} from "./nvenc-encoder"; import { registerSubprocess, terminateProcess } from "./subprocess"; import { createTempFile, type TempFileHandle } from "./temp-files"; @@ -24,6 +32,7 @@ const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000; const MAX_TIMEOUT_MS = 60 * 60 * 1000; const DEFAULT_OUTPUT_FPS = 30; const MAX_TRANSCODE_RANGES_PER_BATCH = 4; +const EDIT_CRF = 18; function roundTime(value: number) { return Math.round(value * 1000) / 1000; @@ -37,7 +46,7 @@ function getTotalRangeDuration(ranges: EditRange[]) { return ranges.reduce((total, range) => total + getRangeDuration(range), 0); } -function getTimeoutMs(ranges: EditRange[]) { +export function getTimeoutMs(ranges: EditRange[]) { const durationMs = getTotalRangeDuration(ranges) * 20_000; return Math.min(MAX_TIMEOUT_MS, Math.max(DEFAULT_TIMEOUT_MS, durationMs)); } @@ -154,6 +163,7 @@ export function buildTranscodeEditArgs( outputPath: string, hasAudio: boolean, fps = DEFAULT_OUTPUT_FPS, + nvenc: NvencOptions | null = null, ) { if (ranges.length === 0 || ranges.length > MAX_TRANSCODE_RANGES_PER_BATCH) { throw new Error( @@ -187,12 +197,9 @@ export function buildTranscodeEditArgs( [...filters, concat].join(";"), "-map", "[v]", - "-c:v", - "libx264", - "-preset", - "fast", - "-crf", - "18", + ...(nvenc + ? buildNvencEncodeArgs(nvenc, EDIT_CRF) + : ["-c:v", "libx264", "-preset", "fast", "-crf", String(EDIT_CRF)]), "-pix_fmt", "yuv420p", "-enc_time_base:v", @@ -398,6 +405,7 @@ async function renderTranscodedEdit( hasAudio: boolean, fps: number | undefined, timeoutMs: number, + nvenc: NvencOptions | null, onProgress?: ProgressCallback, abortSignal?: AbortSignal, ) { @@ -411,7 +419,14 @@ async function renderTranscodedEdit( const batchFile = await createTempFile(".mp4"); batchFiles.push(batchFile); await runFfmpegCommand( - buildTranscodeEditArgs(inputPath, batch, batchFile.path, hasAudio, fps), + buildTranscodeEditArgs( + inputPath, + batch, + batchFile.path, + hasAudio, + fps, + nvenc, + ), getRemainingTimeoutMs(startedAt, timeoutMs), abortSignal, ); @@ -456,14 +471,36 @@ export async function renderEditedVideo({ } const timeoutMs = getTimeoutMs(normalizedRanges); + const render = (nvenc: NvencOptions | null) => + renderTranscodedEdit( + inputPath, + normalizedRanges, + Boolean(metadata.audioCodec), + metadata.fps, + timeoutMs, + nvenc, + onProgress, + abortSignal, + ); - return await renderTranscodedEdit( - inputPath, - normalizedRanges, - Boolean(metadata.audioCodec), - metadata.fps, - timeoutMs, - onProgress, - abortSignal, - ); + const nvenc = nvencOptionsFromEnv(); + if (nvenc) { + try { + const outputFile = await render(nvenc); + recordNvencAttempt("edit"); + return outputFile; + } catch (error) { + const reason = nvencFallbackReason(error, abortSignal?.aborted ?? false); + if (reason === null) throw error; + recordNvencAttempt("edit", reason); + logNvencFallback({ + job: "edit", + sourceCodec: metadata.videoCodec, + gpuDecode: false, + reason, + }); + } + } + + return await render(null); } diff --git a/apps/media-server/src/lib/media-operations.ts b/apps/media-server/src/lib/media-operations.ts index 1c9901dd89e..c1af412092f 100644 --- a/apps/media-server/src/lib/media-operations.ts +++ b/apps/media-server/src/lib/media-operations.ts @@ -30,6 +30,14 @@ export function getActiveAudioOperationCount(): number { return count; } +export function getMaxConcurrentAudioOperations(): number { + return MAX_CONCURRENT_AUDIO_OPERATIONS; +} + +export function getMaxConcurrentProbeOperations(): number { + return MAX_CONCURRENT_PROBE_OPERATIONS; +} + export function canAcceptNewAudioOperation(): boolean { return ( !hasCriticalMemoryPressure() && diff --git a/apps/media-server/src/lib/media-transfer.ts b/apps/media-server/src/lib/media-transfer.ts index 62720779397..a74cbc40932 100644 --- a/apps/media-server/src/lib/media-transfer.ts +++ b/apps/media-server/src/lib/media-transfer.ts @@ -1,9 +1,15 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { createHash } from "node:crypto"; -import { mkdir, open, readdir, rm, stat } from "node:fs/promises"; +import { open, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { z } from "zod"; +import { + ensureInstanceDir, + getInstanceDir, + releaseInstanceDir, + sweepOrphanedInstanceDirs, +} from "./temp-files"; const targetSchema = z.object({ version: z.literal(1), @@ -36,46 +42,59 @@ const transfers = new AsyncLocalStorage(); const cache = new Map(); const pendingTransfers = new Map>(); const cacheBase = join(tmpdir(), "cap-media-download-cache"); -const cacheRoot = join(cacheBase, String(process.pid)); +const cacheRoot = getInstanceDir(cacheBase); const CACHE_BYTES = 2 * 1024 ** 3; const CACHE_TTL_MS = 30 * 60_000; let cacheBytes = 0; let lastPruneAt = 0; -async function initializeCache() { - await mkdir(cacheRoot, { recursive: true, mode: 0o700 }); - for (const entry of await readdir(cacheBase, { withFileTypes: true })) { - if ( - !entry.isDirectory() || - !/^\d+$/.test(entry.name) || - entry.name === String(process.pid) - ) - continue; - try { - process.kill(Number(entry.name), 0); - } catch (error) { - if ( - typeof error === "object" && - error !== null && - "code" in error && - error.code === "ESRCH" - ) - await rm(join(cacheBase, entry.name), { recursive: true, force: true }); - } +// Pre-instance-dir caches are named after their owner's pid. Only remove one +// once that process is confirmed gone; an old server sharing this volume during +// a rolling deploy may still be reading from its cache. +function isLegacyCacheOrphan(name: string) { + if (!/^\d+$/.test(name)) return false; + if (name === String(process.pid)) return false; + try { + process.kill(Number(name), 0); + return false; + } catch (error) { + return ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ESRCH" + ); } } + +export function sweepMediaTransferCache(): Promise { + return sweepOrphanedInstanceDirs(cacheBase, isLegacyCacheOrphan); +} + +async function initializeCache() { + await ensureInstanceDir(cacheBase, 0o700); + await sweepMediaTransferCache(); +} let cacheReady: Promise | undefined; const pruneTimer = setInterval(() => { void pruneCache().catch(() => {}); }, 60_000); pruneTimer.unref?.(); +const sweepTimer = setInterval(() => { + void sweepMediaTransferCache().catch(() => {}); +}, 5 * 60_000); +sweepTimer.unref?.(); export async function cleanupMediaTransferCache() { - await rm(cacheRoot, { recursive: true, force: true }).catch(() => {}); + await releaseInstanceDir(cacheBase); cache.clear(); cacheBytes = 0; cacheReady = undefined; } +export function getMediaTransferCacheUsage() { + return { bytes: cacheBytes, entries: cache.size }; +} + export function isInternalMediaUrl(input: string): boolean { try { const url = new URL(input); diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 48158979327..bdff52896c7 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -13,6 +13,7 @@ import { uploadDriveResumable } from "./drive-resumable-upload"; import type { VideoMetadata } from "./job-manager"; import { DOWNLOAD_TIMEOUT_MS, + decodeThreadsFromEnv, normalizeLocalPath, PROCESS_TIMEOUT_MS, type ProgressCallback, @@ -21,6 +22,23 @@ import { } from "./media-common"; import { probeVideoFile } from "./media-probe"; import { fetchMedia, materializeMedia } from "./media-transfer"; +import { + buildGpuDecodeInputArgs, + buildNvencEncodeArgs, + canDecodeOnGpu, + fitWithinEven, + GPU_DECODE_FORMAT_FILTER, + logNvencFallback, + type NvencOptions, + nvencFallbackReason, + nvencOptionsFromEnv, + recordNvencAttempt, + selectedVideoEncoder, +} from "./nvenc-encoder"; +import { + downloadUrlToPathParallel, + parallelDownloadOptionsFromEnv, +} from "./parallel-download"; import { readRecordingAudioTail } from "./recording-packet-proof"; import { RecordingTimingError, @@ -1252,6 +1270,18 @@ export async function downloadVideoToTemp( ? AbortSignal.any([abortSignal, timeoutSignal]) : timeoutSignal; + const parallel = parallelDownloadOptionsFromEnv(); + if (parallel.concurrency > 1) { + await downloadUrlToPathParallel(videoUrl, tempFile.path, { + ...parallel, + signal: combinedSignal, + }); + if (file(tempFile.path).size === 0) { + throw new Error("Downloaded video is empty"); + } + return tempFile; + } + const response = await fetchMedia(videoUrl, { signal: combinedSignal, }); @@ -1269,15 +1299,18 @@ export async function downloadVideoToTemp( const reader = response.body.getReader(); const writer = file(tempFile.path).writer(); + let writerEnded = false; try { while (true) { const { done, value } = await reader.read(); if (done) break; writer.write(value); } + writerEnded = true; await writer.end(); } finally { reader.releaseLock(); + if (!writerEnded) await Promise.resolve(writer.end()).catch(() => {}); } if (file(tempFile.path).size === 0) { @@ -1311,7 +1344,7 @@ function needsAudioTranscode(metadata: VideoMetadata): boolean { return Boolean(metadata.audioCodec && metadata.audioCodec !== "aac"); } -function getProcessTimeoutMs( +export function getProcessTimeoutMs( durationSeconds: number, baseTimeoutMs: number, ): number { @@ -1425,10 +1458,11 @@ export function pickMobileSafeH264Level( return { value: 52, ffmpegValue: "5.2" }; } -async function probeH264Level( +async function probeVideoStreamEntries( inputPath: string, + showEntries: string, abortSignal?: AbortSignal, -): Promise { +): Promise { const proc = registerSubprocess( spawn({ cmd: [ @@ -1438,7 +1472,7 @@ async function probeH264Level( "-select_streams", "v:0", "-show_entries", - "stream=level", + showEntries, "-of", "default=noprint_wrappers=1:nokey=1", inputPath, @@ -1472,10 +1506,7 @@ async function probeH264Level( ); const exitCode = await proc.exited; if (exitCode !== 0) return null; - - const rawLevel = stdout.trim().split(/\s+/)[0] ?? ""; - const level = Number.parseInt(rawLevel, 10); - return Number.isFinite(level) && level > 0 ? level : null; + return stdout.trim(); } catch { return null; } finally { @@ -1486,6 +1517,37 @@ async function probeH264Level( } } +async function probeH264Level( + inputPath: string, + abortSignal?: AbortSignal, +): Promise { + const stdout = await probeVideoStreamEntries( + inputPath, + "stream=level", + abortSignal, + ); + if (stdout === null) return null; + const rawLevel = stdout.split(/\s+/)[0] ?? ""; + const level = Number.parseInt(rawLevel, 10); + return Number.isFinite(level) && level > 0 ? level : null; +} + +async function probeVideoRotation( + inputPath: string, + abortSignal?: AbortSignal, +): Promise { + const stdout = await probeVideoStreamEntries( + inputPath, + "stream_side_data=rotation", + abortSignal, + ); + if (stdout === null) return null; + const rawRotation = stdout.split(/\s+/)[0] ?? ""; + if (rawRotation === "") return 0; + const rotation = Number.parseFloat(rawRotation); + return Number.isFinite(rotation) ? rotation : null; +} + export async function processVideo( inputPath: string, metadata: VideoMetadata, @@ -1550,38 +1612,170 @@ export async function processVideo( throw err; } } + const argsInput: ProcessVideoArgsInput = { + inputPath, + outputPath: outputTempFile.path, + videoCodec: metadata.videoCodec, + hasAudio: Boolean(metadata.audioCodec), + opts, + targetH264Level: targetH264Level.ffmpegValue, + normalizeH264Level, + videoTranscode, + audioTranscode, + extraInputArgs, + extraOutputArgs, + sourceWidth: metadata.width, + sourceHeight: metadata.height, + // NVENC leaves the CPU free, so decode (and scale) can use every core. + decodeThreads: decodeThreadsFromEnv( + "MEDIA_SERVER_PROCESS_DECODE_THREADS", + selectedVideoEncoder() === "nvenc" ? 0 : 2, + ), + }; + const totalDurationUs = metadata.duration * 1_000_000; + const nvenc = + videoTranscode && !normalizeH264Level ? nvencOptionsFromEnv() : null; + + try { + if (nvenc) { + const gpuDecode = canDecodeOnGpu( + metadata.videoCodec, + await probeVideoRotation(inputPath, abortSignal), + nvenc.gpuDecodeCodecs, + ); + try { + await runTranscodeCommand( + buildProcessVideoArgs({ + ...argsInput, + nvenc: { options: nvenc, gpuDecode }, + }), + outputTempFile.path, + totalDurationUs, + processTimeoutMs, + onProgress, + abortSignal, + ); + recordNvencAttempt("process"); + return outputTempFile; + } catch (error) { + const reason = nvencFallbackReason( + error, + abortSignal?.aborted ?? false, + ); + if (reason === null) throw error; + recordNvencAttempt("process", reason); + logNvencFallback({ + job: "process", + sourceCodec: metadata.videoCodec, + gpuDecode, + reason, + }); + } + } + + await runTranscodeCommand( + buildProcessVideoArgs(argsInput), + outputTempFile.path, + totalDurationUs, + processTimeoutMs, + onProgress, + abortSignal, + ); + return outputTempFile; + } catch (err) { + await outputTempFile.cleanup(); + throw err; + } +} + +export interface ProcessVideoArgsInput { + inputPath: string; + outputPath: string; + videoCodec: string; + hasAudio: boolean; + opts: Pick< + Required, + "maxWidth" | "maxHeight" | "preset" | "crf" | "audioBitrate" + >; + targetH264Level: string; + normalizeH264Level: boolean; + videoTranscode: boolean; + audioTranscode: boolean; + extraInputArgs: string[]; + extraOutputArgs: string[]; + decodeThreads: string; + sourceWidth?: number; + sourceHeight?: number; + nvenc?: { options: NvencOptions; gpuDecode: boolean } | null; +} + +export function buildProcessVideoArgs(input: ProcessVideoArgsInput): string[] { + const { opts, nvenc } = input; + const gpuDecode = Boolean( + nvenc?.gpuDecode && input.sourceWidth && input.sourceHeight, + ); + const source = { + width: input.sourceWidth ?? 0, + height: input.sourceHeight ?? 0, + }; + const cpuScaleFilter = `scale='min(${opts.maxWidth},iw)':'min(${opts.maxHeight},ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2`; const ffmpegArgs: string[] = [ "ffmpeg", "-threads", - "2", - ...extraInputArgs, + input.decodeThreads, + ...(gpuDecode + ? buildGpuDecodeInputArgs( + input.videoCodec, + source, + fitWithinEven( + source.width, + source.height, + opts.maxWidth, + opts.maxHeight, + ), + ) + : []), + ...input.extraInputArgs, "-i", - inputPath, + input.inputPath, ]; - if (normalizeH264Level) { + if (input.normalizeH264Level) { ffmpegArgs.push( "-c:v", "copy", "-bsf:v", - `h264_metadata=level=${targetH264Level.ffmpegValue}`, - ); - } else if (videoTranscode) { - ffmpegArgs.push( - "-c:v", - "libx264", - "-preset", - opts.preset, - "-crf", - opts.crf.toString(), - "-vf", - `scale='min(${opts.maxWidth},iw)':'min(${opts.maxHeight},ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2`, - "-pix_fmt", - "yuv420p", - "-level:v", - targetH264Level.ffmpegValue, + `h264_metadata=level=${input.targetH264Level}`, ); - if (metadata.videoCodec === "vp8" || metadata.videoCodec === "vp9") { + } else if (input.videoTranscode) { + if (nvenc) { + ffmpegArgs.push( + ...buildNvencEncodeArgs(nvenc.options, opts.crf), + ...(gpuDecode + ? ["-vf", GPU_DECODE_FORMAT_FILTER] + : ["-vf", cpuScaleFilter, "-pix_fmt", "yuv420p"]), + ); + } else { + ffmpegArgs.push( + "-c:v", + "libx264", + "-preset", + opts.preset, + "-crf", + opts.crf.toString(), + "-vf", + cpuScaleFilter, + "-pix_fmt", + "yuv420p", + ); + } + if (nvenc) { + // h264_nvenc refuses to open when the nominal frame rate exceeds the level (browser WebM often reports 1000 fps); libx264 only writes it into the SPS, so tag it the same way. + ffmpegArgs.push("-bsf:v", `h264_metadata=level=${input.targetH264Level}`); + } else { + ffmpegArgs.push("-level:v", input.targetH264Level); + } + if (input.videoCodec === "vp8" || input.videoCodec === "vp9") { // Reordered B-frames can make MP4 duration exclude the tail of sparse browser captures. ffmpegArgs.push("-bf", "0"); } @@ -1589,8 +1783,8 @@ export async function processVideo( ffmpegArgs.push("-c:v", "copy"); } - if (metadata.audioCodec) { - if (audioTranscode) { + if (input.hasAudio) { + if (input.audioTranscode) { ffmpegArgs.push("-c:a", "aac", "-b:a", opts.audioBitrate); } else { ffmpegArgs.push("-c:a", "copy"); @@ -1602,13 +1796,24 @@ export async function processVideo( ffmpegArgs.push( "-movflags", "+faststart", - ...extraOutputArgs, + ...input.extraOutputArgs, "-progress", "pipe:2", "-y", - outputTempFile.path, + input.outputPath, ); + return ffmpegArgs; +} +async function runTranscodeCommand( + ffmpegArgs: string[], + outputPath: string, + totalDurationUs: number, + processTimeoutMs: number, + onProgress?: ProgressCallback, + abortSignal?: AbortSignal, +): Promise { + if (abortSignal?.aborted) throw new Error("Video processing was cancelled"); const proc = registerSubprocess( spawn({ cmd: ffmpegArgs, @@ -1616,7 +1821,6 @@ export async function processVideo( stderr: "pipe", }), ); - const totalDurationUs = metadata.duration * 1_000_000; let abortCleanup: (() => void) | undefined; if (abortSignal) { abortCleanup = () => { @@ -1670,7 +1874,7 @@ export async function processVideo( ); } - const outputSize = await file(outputTempFile.path).size; + const outputSize = await file(outputPath).size; if (outputSize === 0) { throw new Error("FFmpeg produced empty output file"); } @@ -1678,11 +1882,6 @@ export async function processVideo( processTimeoutMs, () => terminateProcess(proc), ); - - return outputTempFile; - } catch (err) { - await outputTempFile.cleanup(); - throw err; } finally { if (abortCleanup) { abortSignal?.removeEventListener("abort", abortCleanup); @@ -2537,6 +2736,13 @@ async function abortMultipartUpload( await response.body?.cancel().catch(() => {}); } +export function multipartUploadConcurrencyFromEnv( + env: Record = process.env, +): number { + const value = Number.parseInt(env.MEDIA_SERVER_UPLOAD_CONCURRENCY ?? "", 10); + return Number.isFinite(value) && value > 0 ? Math.min(value, 16) : 1; +} + async function uploadFileMultipart( filePath: string, target: Extract, @@ -2564,25 +2770,45 @@ async function uploadFileMultipart( ); } - for (let partNumber = 1; partNumber <= partCount; partNumber++) { - const start = (partNumber - 1) * partSize; - const end = Math.min(start + partSize, contentLength); - const partLength = end - start; - const url = await getMultipartPartUrl( - target, - partNumber, - partLength, - abortSignal, - ); - const etag = await uploadMultipartPart( - url, - fileHandle.slice(start, end), - partLength, - partNumber, - abortSignal, - ); - parts.push({ partNumber, etag, size: partLength }); + const failed = new AbortController(); + const signal = abortSignal + ? AbortSignal.any([abortSignal, failed.signal]) + : failed.signal; + let nextPart = 1; + const worker = async () => { + while (nextPart <= partCount) { + const partNumber = nextPart++; + const start = (partNumber - 1) * partSize; + const end = Math.min(start + partSize, contentLength); + const partLength = end - start; + const url = await getMultipartPartUrl( + target, + partNumber, + partLength, + signal, + ); + const etag = await uploadMultipartPart( + url, + fileHandle.slice(start, end), + partLength, + partNumber, + signal, + ); + parts.push({ partNumber, etag, size: partLength }); + } + }; + const workers = Array.from( + { length: Math.min(multipartUploadConcurrencyFromEnv(), partCount) }, + () => worker(), + ); + try { + await Promise.all(workers); + } catch (error) { + failed.abort(); + await Promise.allSettled(workers); + throw error; } + parts.sort((a, b) => a.partNumber - b.partNumber); return await completeMultipartUpload(target, parts, abortSignal); } catch (error) { diff --git a/apps/media-server/src/lib/nvenc-encoder.ts b/apps/media-server/src/lib/nvenc-encoder.ts new file mode 100644 index 00000000000..eb2848ba9c2 --- /dev/null +++ b/apps/media-server/src/lib/nvenc-encoder.ts @@ -0,0 +1,246 @@ +const DEFAULT_PRESET = "p1"; +const DEFAULT_CQ_OFFSET = 8; +const DEFAULT_LOOKAHEAD = 0; +const MAX_LOOKAHEAD = 32; +const GOP_LENGTH = "250"; +const NVENC_PRESETS = new Set(["p1", "p2", "p3", "p4", "p5", "p6", "p7"]); +const NVDEC_CODECS = new Set(["h264", "hevc", "vp9", "av1"]); +// H.264 and VP9 decode faster on the host CPU than on L4 NVDEC; HEVC and AV1 do not. +const DEFAULT_GPU_DECODE_CODECS = ["hevc", "av1"]; +const MAX_FALLBACK_REASON_LENGTH = 500; + +export const NVENC_HWACCEL_INPUT_ARGS = [ + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", +]; + +export type VideoEncoderName = "libx264" | "capcodec" | "nvenc"; +export type NvencJobKind = "process" | "edit"; + +export interface NvencOptions { + preset: string; + cqOffset: number; + lookahead: number; + spatialAq: boolean; + temporalAq: boolean; + gpuDecodeCodecs: string[]; +} + +interface NvencAttempt { + job: NvencJobKind; + at: string; + fellBack: boolean; + reason: string | null; +} + +const nvencStatus: { + attempts: number; + fallbacks: number; + lastAttempt: NvencAttempt | null; +} = { attempts: 0, fallbacks: 0, lastAttempt: null }; + +type Env = Record; + +export function selectedVideoEncoder(env: Env = process.env): VideoEncoderName { + const value = env.CAP_MEDIA_VIDEO_ENCODER; + if (value === "capcodec" || value === "nvenc") return value; + return "libx264"; +} + +function parseBoundedInt( + raw: string | undefined, + min: number, + max: number, +): number | undefined { + if (raw === undefined || raw.trim() === "") return undefined; + const value = Number(raw); + if (!Number.isInteger(value) || value < min || value > max) return undefined; + return value; +} + +function parseToggle(raw: string | undefined): boolean | undefined { + const normalized = raw?.trim().toLowerCase(); + if (normalized === "1" || normalized === "true" || normalized === "on") { + return true; + } + if (normalized === "0" || normalized === "false" || normalized === "off") { + return false; + } + return undefined; +} + +export function nvencOptionsFromEnv( + env: Env = process.env, +): NvencOptions | null { + if (selectedVideoEncoder(env) !== "nvenc") return null; + const preset = env.CAP_NVENC_PRESET?.trim().toLowerCase() ?? ""; + return { + preset: NVENC_PRESETS.has(preset) ? preset : DEFAULT_PRESET, + cqOffset: + parseBoundedInt(env.CAP_NVENC_CQ_OFFSET, -51, 51) ?? DEFAULT_CQ_OFFSET, + lookahead: + parseBoundedInt(env.CAP_NVENC_LOOKAHEAD, 0, MAX_LOOKAHEAD) ?? + DEFAULT_LOOKAHEAD, + spatialAq: parseToggle(env.CAP_NVENC_SPATIAL_AQ) ?? true, + temporalAq: parseToggle(env.CAP_NVENC_TEMPORAL_AQ) ?? false, + gpuDecodeCodecs: parseGpuDecodeCodecs(env.CAP_NVENC_GPU_DECODE_CODECS), + }; +} + +function parseGpuDecodeCodecs(raw: string | undefined): string[] { + if (raw === undefined || raw.trim() === "") return DEFAULT_GPU_DECODE_CODECS; + return raw + .split(",") + .map((codec) => codec.trim().toLowerCase()) + .filter((codec) => NVDEC_CODECS.has(codec)); +} + +export function nvencCq(crf: number, cqOffset: number): string { + const base = Number.isFinite(crf) ? crf : 23; + const cq = Math.min(51, Math.max(1, base + cqOffset)); + return String(Math.round(cq * 1000) / 1000); +} + +export function buildNvencEncodeArgs( + options: NvencOptions, + crf: number, +): string[] { + return [ + "-c:v", + "h264_nvenc", + "-preset", + options.preset, + "-tune", + "hq", + "-rc", + "vbr", + "-cq", + nvencCq(crf, options.cqOffset), + "-b:v", + "0", + ...(options.spatialAq ? ["-spatial-aq", "1"] : []), + ...(options.temporalAq ? ["-temporal-aq", "1"] : []), + ...(options.lookahead > 0 + ? ["-rc-lookahead", String(options.lookahead)] + : []), + "-profile:v", + "high", + // h264_nvenc otherwise inherits the preset's GOP; pin libx264's default keyint. + "-g", + GOP_LENGTH, + ]; +} + +// Mirrors the CPU filter chain: fit within the box keeping aspect (rounded like libavfilter), then floor to even. +export function fitWithinEven( + width: number, + height: number, + maxWidth: number, + maxHeight: number, +): { width: number; height: number } { + let w = Math.min(maxWidth, width); + let h = Math.min(maxHeight, height); + const tmpW = Math.round((h * width) / height); + const tmpH = Math.round((w * height) / width); + w = Math.min(tmpW, w); + h = Math.min(tmpH, h); + return { width: Math.trunc(w / 2) * 2, height: Math.trunc(h / 2) * 2 }; +} + +// cuvid resizes inside NVDEC for free; scale_cuda then only converts 10-bit surfaces and passes 8-bit through. +export function buildGpuDecodeInputArgs( + videoCodec: string, + source: { width: number; height: number }, + target: { width: number; height: number }, +): string[] { + return [ + ...NVENC_HWACCEL_INPUT_ARGS, + "-c:v", + `${videoCodec}_cuvid`, + ...(target.width !== source.width || target.height !== source.height + ? ["-resize", `${target.width}x${target.height}`] + : []), + ]; +} + +export const GPU_DECODE_FORMAT_FILTER = "scale_cuda=format=nv12"; + +// FFmpeg skips autorotation for hardware frames, so rotated sources decode on the CPU to keep the libx264 geometry. +export function canDecodeOnGpu( + videoCodec: string, + rotation: number | null, + gpuDecodeCodecs: string[] = DEFAULT_GPU_DECODE_CODECS, +): boolean { + return gpuDecodeCodecs.includes(videoCodec) && rotation === 0; +} + +// A failed NVENC encode is retried on libx264 with a fresh timeout, so the job +// deadline has to leave room for both attempts. +export function encodeAttemptsBudgetMs( + encodeTimeoutMs: number, + env: Env = process.env, +): number { + return selectedVideoEncoder(env) === "nvenc" + ? encodeTimeoutMs * 2 + : encodeTimeoutMs; +} + +export function nvencFallbackReason( + error: unknown, + aborted: boolean, +): string | null { + if (aborted) return null; + const message = ( + error instanceof Error ? error.message : String(error) + ).replace(/[a-z][a-z0-9+.-]*:\/\/\S+/gi, ""); + return message.slice(-MAX_FALLBACK_REASON_LENGTH) || "unknown error"; +} + +export function recordNvencAttempt( + job: NvencJobKind, + fallbackReason: string | null = null, +): void { + nvencStatus.attempts += 1; + if (fallbackReason !== null) nvencStatus.fallbacks += 1; + nvencStatus.lastAttempt = { + job, + at: new Date().toISOString(), + fellBack: fallbackReason !== null, + reason: fallbackReason, + }; +} + +export function logNvencFallback(details: { + job: NvencJobKind; + sourceCodec: string; + gpuDecode: boolean; + reason: string; +}): void { + console.warn( + JSON.stringify({ + event: "nvenc_fallback_to_libx264", + timestamp: new Date().toISOString(), + ...details, + }), + ); +} + +export function getVideoEncoderStatus( + env: Env = process.env, + includeDetails = true, +) { + const active = selectedVideoEncoder(env); + return { + active, + nvenc: + active === "nvenc" && includeDetails + ? { + attempts: nvencStatus.attempts, + fallbacks: nvencStatus.fallbacks, + lastAttempt: nvencStatus.lastAttempt, + } + : null, + }; +} diff --git a/apps/media-server/src/lib/parallel-download.ts b/apps/media-server/src/lib/parallel-download.ts new file mode 100644 index 00000000000..f582f3537ba --- /dev/null +++ b/apps/media-server/src/lib/parallel-download.ts @@ -0,0 +1,195 @@ +import { open } from "node:fs/promises"; + +const MIN_PART_BYTES = 1024 * 1024; +// Far above any recording we store (the largest so far is ~47 GB). Anything +// bigger is a bad Content-Range header, not a real object. +export const MAX_RANGED_TOTAL_BYTES = 1024 ** 4; + +export interface ParallelDownloadOptions { + concurrency: number; + partBytes: number; + signal?: AbortSignal; + fetchImpl?: typeof fetch; +} + +export interface ParallelDownloadResult { + bytes: number; + parts: number; + ranged: boolean; +} + +export function parallelDownloadOptionsFromEnv( + env: Record = process.env, +): Pick { + const concurrency = Number.parseInt( + env.MEDIA_SERVER_DOWNLOAD_CONCURRENCY ?? "", + 10, + ); + const partMb = Number.parseInt(env.MEDIA_SERVER_DOWNLOAD_PART_MB ?? "", 10); + return { + concurrency: + Number.isFinite(concurrency) && concurrency > 0 + ? Math.min(concurrency, 32) + : 1, + partBytes: Math.max( + MIN_PART_BYTES, + (Number.isFinite(partMb) && partMb > 0 ? partMb : 16) * 1024 * 1024, + ), + }; +} + +export function planRanges( + totalBytes: number, + firstPartBytes: number, + partBytes: number, +): { count: number; at: (index: number) => { start: number; end: number } } { + return { + count: Math.max(0, Math.ceil((totalBytes - firstPartBytes) / partBytes)), + at: (index) => { + const start = firstPartBytes + index * partBytes; + return { start, end: Math.min(totalBytes, start + partBytes) - 1 }; + }, + }; +} + +function parseContentRange( + value: string | null, +): { start: number; end: number; total: number } | null { + const match = value?.match(/^bytes\s+(\d+)-(\d+)\/(\d+)$/i); + if (!match) return null; + const [start, end, total] = match.slice(1).map(Number) as [ + number, + number, + number, + ]; + return Number.isSafeInteger(total) && start <= end && end < total + ? { start, end, total } + : null; +} + +async function readInto( + response: Response, + write: (chunk: Uint8Array, offset: number) => Promise, + start: number, + expected: number | null, +): Promise { + if (!response.body) throw new Error("No response body"); + const reader = response.body.getReader(); + let offset = start; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + await write(value, offset); + offset += value.byteLength; + } + } finally { + reader.releaseLock(); + } + const read = offset - start; + if (expected !== null && read !== expected) { + throw new Error( + `Ranged download returned ${read} bytes, expected ${expected}`, + ); + } + return read; +} + +export async function downloadUrlToPathParallel( + url: string, + path: string, + options: ParallelDownloadOptions, +): Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const partBytes = Math.max(MIN_PART_BYTES, options.partBytes); + const handle = await open(path, "w"); + const write = async (chunk: Uint8Array, offset: number) => { + let written = 0; + while (written < chunk.byteLength) { + const { bytesWritten } = await handle.write( + chunk, + written, + chunk.byteLength - written, + offset + written, + ); + written += bytesWritten; + } + }; + const controller = new AbortController(); + const signal = options.signal + ? AbortSignal.any([options.signal, controller.signal]) + : controller.signal; + const fetchOk = async (headers?: Record) => { + const response = await fetchImpl(url, { headers, signal }); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + throw new Error( + `Failed to download video: ${response.status} ${response.statusText}`, + ); + } + return response; + }; + try { + const first = await fetchOk({ Range: `bytes=0-${partBytes - 1}` }); + if (first.status !== 206) { + const bytes = await readInto(first, write, 0, null); + return { bytes, parts: 1, ranged: false }; + } + const range = parseContentRange(first.headers.get("content-range")); + if ( + range === null || + range.start !== 0 || + range.end + 1 < Math.min(partBytes, range.total) + ) { + // A partial reply we can't place or that the server capped below our + // part size: start over as one stream rather than keep a fragment. + await first.body?.cancel().catch(() => {}); + const bytes = await readInto(await fetchOk(), write, 0, null); + return { bytes, parts: 1, ranged: false }; + } + const { total } = range; + if (total > MAX_RANGED_TOTAL_BYTES) { + await first.body?.cancel().catch(() => {}); + throw new Error(`Ranged download reported an invalid size: ${total}`); + } + const firstBytes = range.end + 1; + const etag = first.headers.get("etag"); + const ranges = planRanges(total, firstBytes, partBytes); + let next = 0; + const worker = async () => { + while (next < ranges.count) { + signal.throwIfAborted(); + const part = ranges.at(next++); + const response = await fetchImpl(url, { + headers: { + Range: `bytes=${part.start}-${part.end}`, + ...(etag ? { "If-Match": etag } : {}), + }, + signal, + }); + if (response.status !== 206) { + await response.body?.cancel().catch(() => {}); + throw new Error( + `Ranged download part failed: ${response.status} ${response.statusText}`, + ); + } + await readInto(response, write, part.start, part.end - part.start + 1); + } + }; + const workers = Array.from( + { length: Math.max(1, Math.min(options.concurrency - 1, ranges.count)) }, + () => worker(), + ); + const firstRead = readInto(first, write, 0, firstBytes); + try { + await Promise.all([firstRead, ...workers]); + } catch (error) { + controller.abort(); + await Promise.allSettled([firstRead, ...workers]); + throw error; + } + return { bytes: total, parts: ranges.count + 1, ranged: true }; + } finally { + await handle.close(); + } +} diff --git a/apps/media-server/src/lib/recording-verification.ts b/apps/media-server/src/lib/recording-verification.ts index 9a599fbf010..59b6b8547d2 100644 --- a/apps/media-server/src/lib/recording-verification.ts +++ b/apps/media-server/src/lib/recording-verification.ts @@ -5,7 +5,7 @@ import { lstat } from "node:fs/promises"; import { isAbsolute } from "node:path"; import { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; -import { PROCESS_TIMEOUT_MS } from "./media-common"; +import { decodeThreadsFromEnv, PROCESS_TIMEOUT_MS } from "./media-common"; import { fetchMedia, MediaTransferBudgetError, @@ -53,6 +53,9 @@ export interface RecordingVerificationOptions { hasAudio?: boolean; sourceEvidence?: RecordingSourceEvidence; allowObservedDuration?: boolean; + // Hashing every decoded frame costs as much as the decode; skip it when the + // caller has no source evidence to compare against. + hashDecodedContent?: boolean; abortSignal?: AbortSignal; timeoutMs?: number; onProgress?: (progress: { frames: number; totalFrames?: number }) => void; @@ -846,8 +849,11 @@ async function decodeRecording( ? Boolean(options.sourceEvidence.audio) : options.hasAudio : sourceAudioInput !== null; + const hashContent = + !packetProof && + (options.hashDecodedContent !== false || Boolean(options.sourceEvidence)); const streamDecodedContent = - !packetProof && process.platform !== "win32" && knownAudio !== undefined; + hashContent && process.platform !== "win32" && knownAudio !== undefined; if (streamDecodedContent && !Bun.semver.satisfies(Bun.version, ">=1.4.0")) { throw new Error( "Recording content verification requires Bun 1.4.0 or newer; upgrade the media-server runtime to avoid double-closing decoded-content pipes", @@ -890,8 +896,9 @@ async function decodeRecording( "1", "-filter_complex_threads", "1", + // FFmpeg's H.264/HEVC decoders are bit-exact across thread counts, so decoded hashes do not depend on this. "-threads", - "1", + decodeThreadsFromEnv("MEDIA_SERVER_VERIFY_DECODE_THREADS", 1), "-protocol_whitelist", remote ? "http,https,tcp,tls" : "file", "-f", @@ -958,7 +965,7 @@ async function decodeRecording( ] : []), ] - : packetProof + : !hashContent ? [] : [ ...outputOptions, @@ -1089,7 +1096,7 @@ async function decodeRecording( } stream.contentSha256 = digest.sha256; } - } else if (!packetProof) { + } else if (hashContent) { for (const digest of diagnostics.digests) decodeLine(digest, streams); } result = validateEvidence( @@ -1097,7 +1104,7 @@ async function decodeRecording( options, sourceAudioInput !== undefined || packetProof !== undefined, videoTiming, - !packetProof, + hashContent, ); } catch (error) { throw failure ?? error; diff --git a/apps/media-server/src/lib/runtime-report.ts b/apps/media-server/src/lib/runtime-report.ts new file mode 100644 index 00000000000..bfe0bace96d --- /dev/null +++ b/apps/media-server/src/lib/runtime-report.ts @@ -0,0 +1,60 @@ +import { + getActiveVideoProcessCount, + getAllJobs, + getJobTotals, + getMaxConcurrentVideoProcesses, + getSystemResources, +} from "./job-manager"; +import { getDrainBlockers, getShutdownGraceMs, isDraining } from "./lifecycle"; +import { + getMaxConcurrentAudioOperations, + getMaxConcurrentProbeOperations, +} from "./media-operations"; +import { getMediaTransferCacheUsage } from "./media-transfer"; +import { getTempDirUsage } from "./temp-files"; +import { + getActiveDirectVideoProcessCount, + getMaxConcurrentDirectVideoProcesses, +} from "./video-capacity"; + +export async function getRuntimeReport() { + const blockers = getDrainBlockers(); + const jobs = getAllJobs(); + return { + draining: isDraining(), + shutdownGraceMs: getShutdownGraceMs(), + processUptimeSeconds: Math.round(process.uptime()), + activeJobs: jobs.filter( + (job) => + job.phase !== "complete" && + job.phase !== "error" && + job.phase !== "cancelled", + ).length, + jobsInMemory: jobs.length, + jobTotals: getJobTotals(), + activeSubprocesses: blockers.subprocesses, + backgroundTasks: blockers.backgroundTasks, + inFlightRequests: blockers.inFlightRequests, + slots: { + video: { + active: getActiveVideoProcessCount(), + max: getMaxConcurrentVideoProcesses(), + effectiveMax: getSystemResources().effectiveMax, + }, + directVideo: { + active: getActiveDirectVideoProcessCount(), + max: getMaxConcurrentDirectVideoProcesses(), + }, + audio: { + active: blockers.audioOperations, + max: getMaxConcurrentAudioOperations(), + }, + probe: { + active: blockers.probeOperations, + max: getMaxConcurrentProbeOperations(), + }, + }, + tempDir: await getTempDirUsage(), + mediaTransferCache: getMediaTransferCacheUsage(), + }; +} diff --git a/apps/media-server/src/lib/subprocess.ts b/apps/media-server/src/lib/subprocess.ts index 31ac5ee4e7a..ffdf7c3f670 100644 --- a/apps/media-server/src/lib/subprocess.ts +++ b/apps/media-server/src/lib/subprocess.ts @@ -19,6 +19,10 @@ export function unregisterSubprocess(proc: ManagedSubprocess): void { activeSubprocesses.delete(proc.pid); } +export function getActiveSubprocessCount(): number { + return activeSubprocesses.size; +} + async function waitForProcessExit( proc: Pick, timeoutMs: number, diff --git a/apps/media-server/src/lib/temp-files.ts b/apps/media-server/src/lib/temp-files.ts index 5141ad6c427..0c8ec1fdf4f 100644 --- a/apps/media-server/src/lib/temp-files.ts +++ b/apps/media-server/src/lib/temp-files.ts @@ -1,18 +1,139 @@ import { randomUUID } from "node:crypto"; -import { mkdir, readdir, rm, stat, unlink } from "node:fs/promises"; +import type { Dirent, Stats } from "node:fs"; +import { + lstat, + mkdir, + readdir, + rm, + unlink, + utimes, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -const TEMP_DIR = join(tmpdir(), "cap-media-server"); +const TEMP_ROOT = join(tmpdir(), "cap-media-server"); +const INSTANCE_ID = randomUUID(); +const INSTANCE_DIR_PREFIX = "instance-"; +const INSTANCE_LEASE_FILE = ".lease"; +const INSTANCE_LEASE_REFRESH_MS = 30_000; +const INSTANCE_LEASE_STALE_MS = 5 * 60 * 1000; const STALE_FILE_AGE_MS = 60 * 60 * 1000; +const MAX_ACTIVE_TEMP_AGE_MS = 6 * 60 * 60 * 1000; +const TEMP_MAINTENANCE_INTERVAL_MS = 5 * 60 * 1000; +const TEMP_USAGE_CACHE_MS = 5_000; +const TEMP_USAGE_MAX_ENTRIES = 20_000; + +const leasedDirs = new Set(); +const activeTempPaths = new Set(); +let leaseTimer: ReturnType | undefined; +let maintenanceTimer: ReturnType | undefined; export interface TempFileHandle { path: string; cleanup: () => Promise; } +export interface TempDirUsage { + bytes: number; + entries: number; + truncated: boolean; +} + +export function getInstanceDir(base: string): string { + return join(base, `${INSTANCE_DIR_PREFIX}${INSTANCE_ID}`); +} + +const TEMP_DIR = getInstanceDir(TEMP_ROOT); + +async function refreshLeases(): Promise { + const now = new Date(); + await Promise.allSettled( + Array.from(leasedDirs, async (dir) => { + const lease = join(dir, INSTANCE_LEASE_FILE); + try { + await utimes(lease, now, now); + } catch { + await mkdir(dir, { recursive: true }); + await writeFile(lease, ""); + } + }), + ); +} + +export async function ensureInstanceDir( + base: string, + mode?: number, +): Promise { + const dir = getInstanceDir(base); + await mkdir(dir, { recursive: true, mode }); + if (!leasedDirs.has(dir)) { + await writeFile(join(dir, INSTANCE_LEASE_FILE), ""); + leasedDirs.add(dir); + if (!leaseTimer) { + leaseTimer = setInterval(() => { + void refreshLeases(); + }, INSTANCE_LEASE_REFRESH_MS); + leaseTimer.unref?.(); + } + } + return dir; +} + +export async function releaseInstanceDir(base: string): Promise { + const dir = getInstanceDir(base); + leasedDirs.delete(dir); + await rm(dir, { recursive: true, force: true }).catch(() => {}); +} + +// Instance directories carry a lease so a host that shares one temp volume +// between an old (draining) and a new container never sweeps live work. +export async function sweepOrphanedInstanceDirs( + base: string, + isLegacyOrphan: (name: string, stats: Stats) => boolean, +): Promise { + let entries: Dirent[]; + try { + entries = await readdir(base, { withFileTypes: true }); + } catch { + return 0; + } + const ownDir = getInstanceDir(base); + const now = Date.now(); + let removed = 0; + + for (const entry of entries) { + const path = join(base, entry.name); + if (path === ownDir) continue; + try { + const orphaned = + entry.isDirectory() && entry.name.startsWith(INSTANCE_DIR_PREFIX) + ? now - + ( + await lstat(join(path, INSTANCE_LEASE_FILE)).catch(() => + lstat(path), + ) + ).mtimeMs > + INSTANCE_LEASE_STALE_MS + : isLegacyOrphan(entry.name, await lstat(path)); + if (!orphaned) continue; + await rm(path, { recursive: true, force: true }); + removed++; + } catch {} + } + + return removed; +} + export async function ensureTempDir(): Promise { - await mkdir(TEMP_DIR, { recursive: true }); + await ensureInstanceDir(TEMP_ROOT); +} + +export function retainTempPath(path: string): () => void { + activeTempPaths.add(path); + return () => { + activeTempPaths.delete(path); + }; } export async function createTempFile( @@ -21,6 +142,7 @@ export async function createTempFile( await ensureTempDir(); const filename = `${randomUUID()}${extension.startsWith(".") ? extension : `.${extension}`}`; const path = join(TEMP_DIR, filename); + const release = retainTempPath(path); return { path, @@ -28,32 +150,116 @@ export async function createTempFile( try { await unlink(path); } catch {} + release(); }, }; } +export function sweepOrphanedTempDirs(): Promise { + return sweepOrphanedInstanceDirs( + TEMP_ROOT, + (_name, stats) => Date.now() - stats.mtimeMs > STALE_FILE_AGE_MS, + ); +} + export async function cleanupStaleTempFiles(): Promise { + let cleaned = await sweepOrphanedTempDirs(); + let entries: string[]; try { - await ensureTempDir(); - const files = await readdir(TEMP_DIR); - const now = Date.now(); - let cleaned = 0; + entries = await readdir(TEMP_DIR); + } catch { + return cleaned; + } + const now = Date.now(); - for (const file of files) { - const filePath = join(TEMP_DIR, file); - try { - const fileStat = await stat(filePath); - if (now - fileStat.mtimeMs > STALE_FILE_AGE_MS) { - await rm(filePath, { force: true }); - cleaned++; - } - } catch {} - } + for (const entry of entries) { + if (entry === INSTANCE_LEASE_FILE) continue; + const path = join(TEMP_DIR, entry); + try { + const age = now - (await lstat(path)).mtimeMs; + if ( + age <= STALE_FILE_AGE_MS || + (activeTempPaths.has(path) && age <= MAX_ACTIVE_TEMP_AGE_MS) + ) + continue; + await rm(path, { recursive: true, force: true }); + activeTempPaths.delete(path); + cleaned++; + } catch {} + } - return cleaned; + return cleaned; +} + +export async function startTempDirMaintenance(): Promise { + if (!maintenanceTimer) { + maintenanceTimer = setInterval(() => { + void cleanupStaleTempFiles() + .then((cleaned) => { + if (cleaned > 0) + console.log(`[temp-files] Removed ${cleaned} stale temp entries`); + }) + .catch(() => {}); + }, TEMP_MAINTENANCE_INTERVAL_MS); + maintenanceTimer.unref?.(); + } + return await sweepOrphanedTempDirs(); +} + +export async function removeOwnTempDir(): Promise { + activeTempPaths.clear(); + await releaseInstanceDir(TEMP_ROOT); +} + +async function measureDir( + dir: string, + usage: TempDirUsage, +): Promise { + let entries: Dirent[]; + try { + entries = await readdir(dir, { withFileTypes: true }); } catch { - return 0; + return usage; + } + for (const entry of entries) { + if (usage.entries >= TEMP_USAGE_MAX_ENTRIES) { + usage.truncated = true; + return usage; + } + usage.entries++; + const path = join(dir, entry.name); + if (entry.isDirectory()) { + await measureDir(path, usage); + } else { + usage.bytes += await lstat(path).then( + (stats) => stats.size, + () => 0, + ); + } } + return usage; +} + +let tempUsageCache: + | { measuredAt: number; usage: Promise } + | undefined; + +export function getTempDirUsage(): Promise { + const now = Date.now(); + if ( + !tempUsageCache || + now - tempUsageCache.measuredAt > TEMP_USAGE_CACHE_MS + ) { + tempUsageCache = { + measuredAt: now, + usage: measureDir(TEMP_ROOT, { bytes: 0, entries: 0, truncated: false }), + }; + } + return tempUsageCache.usage; +} + +export function getTempRoot(): string { + return TEMP_ROOT; } export function getTempDir(): string { diff --git a/apps/media-server/src/lib/video-input-validation.ts b/apps/media-server/src/lib/video-input-validation.ts index 41e26532384..2adb2dbbe4d 100644 --- a/apps/media-server/src/lib/video-input-validation.ts +++ b/apps/media-server/src/lib/video-input-validation.ts @@ -1,5 +1,9 @@ import { spawn } from "bun"; -import { PROCESS_TIMEOUT_MS, withTimeout } from "./media-common"; +import { + decodeThreadsFromEnv, + PROCESS_TIMEOUT_MS, + withTimeout, +} from "./media-common"; import { registerSubprocess, terminateProcess } from "./subprocess"; const MAX_DIAGNOSTIC_LENGTH = 8_192; @@ -22,7 +26,7 @@ export async function validateVideoInput( "-err_detect", "explode+crccheck", "-threads", - "2", + decodeThreadsFromEnv("MEDIA_SERVER_VALIDATE_DECODE_THREADS", 2), "-i", inputPath, "-map", diff --git a/apps/media-server/src/routes/audio.ts b/apps/media-server/src/routes/audio.ts index d2f8cc71e2b..bd4c3a12c5d 100644 --- a/apps/media-server/src/routes/audio.ts +++ b/apps/media-server/src/routes/audio.ts @@ -1,6 +1,14 @@ -import { Hono } from "hono"; +import { type Context, Hono } from "hono"; import { z } from "zod"; -import { validateMediaServerSecret } from "../lib/auth"; +import { canReadStatus, validateMediaServerSecret } from "../lib/auth"; +import { + DRAIN_RETRY_AFTER_SECONDS, + DRAINING_DETAILS, + getIdleTimeoutSeconds, + isDraining, + restoreIdleTimeoutOnFirstChunk, + setRequestIdleTimeout, +} from "../lib/lifecycle"; import { canAcceptNewAudioOperation as canAcceptNewProcess, checkHasAudioTrack, @@ -35,7 +43,22 @@ function isTimeoutError(err: unknown): boolean { return err instanceof Error && err.message.includes("timed out"); } +function drainingResponse(c: Context) { + c.header("Retry-After", DRAIN_RETRY_AFTER_SECONDS.toString()); + return c.json( + { + error: "Server is busy", + code: "SERVER_BUSY", + details: DRAINING_DETAILS, + }, + 503, + ); +} + audio.get("/status", (c) => { + if (!canReadStatus(c)) { + return c.json({ error: "Unauthorized" }, 401); + } return c.json({ activeProcesses: getActiveProcessCount(), canAcceptNewProcess: canAcceptNewProcess(), @@ -61,6 +84,8 @@ audio.post("/check", async (c) => { ); } + if (isDraining()) return drainingResponse(c); + try { const hasAudio = await checkHasAudioTrack(result.data.videoUrl); return c.json({ hasAudio }); @@ -120,6 +145,11 @@ audio.post("/extract", async (c) => { } const { videoUrl, stream: useStreaming } = result.data; + if (isDraining()) return drainingResponse(c); + setRequestIdleTimeout(c.env, c.req.raw, 0); + const restoreIdleTimeout = () => + setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds()); + let streaming = false; try { const hasAudio = await checkHasAudioTrack(videoUrl); @@ -137,12 +167,16 @@ audio.post("/extract", async (c) => { cleanup(); }); - return new Response(stream, { - headers: { - "Content-Type": "audio/mpeg", - "Transfer-Encoding": "chunked", + streaming = true; + return new Response( + restoreIdleTimeoutOnFirstChunk(stream, restoreIdleTimeout), + { + headers: { + "Content-Type": "audio/mpeg", + "Transfer-Encoding": "chunked", + }, }, - }); + ); } const audioData = await extractAudio(videoUrl); @@ -186,6 +220,8 @@ audio.post("/extract", async (c) => { }, 500, ); + } finally { + if (!streaming) restoreIdleTimeout(); } }); @@ -209,6 +245,11 @@ audio.post("/convert", async (c) => { } const { audioUrl, bitrate } = result.data; + if (isDraining()) return drainingResponse(c); + setRequestIdleTimeout(c.env, c.req.raw, 0); + const restoreIdleTimeout = () => + setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds()); + let streaming = false; try { const { stream, cleanup } = extractAudioStream(audioUrl, { @@ -220,12 +261,16 @@ audio.post("/convert", async (c) => { cleanup(); }); - return new Response(stream, { - headers: { - "Content-Type": "audio/mpeg", - "Transfer-Encoding": "chunked", + streaming = true; + return new Response( + restoreIdleTimeoutOnFirstChunk(stream, restoreIdleTimeout), + { + headers: { + "Content-Type": "audio/mpeg", + "Transfer-Encoding": "chunked", + }, }, - }); + ); } catch (err) { console.error("[audio/convert] Error:", err); @@ -259,6 +304,8 @@ audio.post("/convert", async (c) => { }, 500, ); + } finally { + if (!streaming) restoreIdleTimeout(); } }); diff --git a/apps/media-server/src/routes/health.ts b/apps/media-server/src/routes/health.ts index 07b2731ab43..c3fef26c6ad 100644 --- a/apps/media-server/src/routes/health.ts +++ b/apps/media-server/src/routes/health.ts @@ -1,11 +1,15 @@ import os from "node:os"; import { Hono } from "hono"; +import { hasMediaServerSecret } from "../lib/auth"; import { getSystemResources } from "../lib/job-manager"; +import { isDraining } from "../lib/lifecycle"; import { getMediaEngineStatus } from "../lib/media-engine"; +import { getVideoEncoderStatus } from "../lib/nvenc-encoder"; +import { getRuntimeReport } from "../lib/runtime-report"; const health = new Hono(); -health.get("/", (c) => { +health.get("/", async (c) => { const mediaEngine = getMediaEngineStatus(); const [, loadAvg5m, loadAvg15m] = os.loadavg(); const totalMemoryBytes = os.totalmem(); @@ -16,26 +20,36 @@ health.get("/", (c) => { (1 - freeMemoryBytes / totalMemoryBytes) * 100, ); const resources = getSystemResources(); + const draining = isDraining(); + const authenticated = hasMediaServerSecret(c); - return c.json({ - status: - mediaEngine.available && resources.effectiveMax > 0 ? "ok" : "degraded", - mediaEngine, - ["ff" + "mpeg"]: { - available: mediaEngine.available, - version: mediaEngine.version, + return c.json( + { + status: draining + ? "draining" + : mediaEngine.available && resources.effectiveMax > 0 + ? "ok" + : "degraded", + mediaEngine, + ["ff" + "mpeg"]: { + available: mediaEngine.available, + version: mediaEngine.version, + }, + videoEncoder: getVideoEncoderStatus(process.env, authenticated), + system: { + ...resources, + loadAvg5m, + loadAvg15m, + totalMemoryMB, + freeMemoryMB, + memoryUsagePercent, + uptimeSeconds: Math.round(os.uptime()), + processUptimeSeconds: Math.round(process.uptime()), + }, + ...(authenticated && { runtime: await getRuntimeReport() }), }, - system: { - ...resources, - loadAvg5m, - loadAvg15m, - totalMemoryMB, - freeMemoryMB, - memoryUsagePercent, - uptimeSeconds: Math.round(os.uptime()), - processUptimeSeconds: Math.round(process.uptime()), - }, - }); + draining ? 503 : 200, + ); }); export default health; diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index cd2b44e68b5..09495309fb6 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -4,7 +4,11 @@ import { file } from "bun"; import { Hono } from "hono"; import { z } from "zod"; import { enhanceLocalRecording } from "../lib/audio-levels"; -import { validateMediaServerSecret } from "../lib/auth"; +import { + canReadStatus, + hasMediaServerSecret, + validateMediaServerSecret, +} from "../lib/auth"; import type { Job, RecordingErrorCode, @@ -17,6 +21,7 @@ import { claimRecordingWorker, createJob, deleteJob, + extendJobLifetime, forceCleanupActiveJobs, generateJobId, getActiveVideoProcessCount, @@ -30,8 +35,18 @@ import { touchJob, updateJob, } from "../lib/job-manager"; -import { PROCESS_TIMEOUT_MS } from "../lib/media-common"; -import { renderEditedVideo } from "../lib/media-edit"; +import { + DRAINING_DETAILS, + getIdleTimeoutSeconds, + isDraining, + setRequestIdleTimeout, + trackBackgroundTask, +} from "../lib/lifecycle"; +import { PROCESS_TIMEOUT_MS, UPLOAD_TIMEOUT_MS } from "../lib/media-common"; +import { + getTimeoutMs as getEditRenderTimeoutMs, + renderEditedVideo, +} from "../lib/media-edit"; import { canAcceptNewProbeOperation as canAcceptNewProbeProcess, getActiveProbeOperationCount as getActiveProbeProcessCount, @@ -54,6 +69,7 @@ import { downloadVideoToTemp, generatePreviewGif, generateThumbnail, + getProcessTimeoutMs, muxMediaTracksToMp4, processVideo, repairContainer, @@ -61,6 +77,7 @@ import { uploadFileToStorage, uploadToS3, } from "../lib/media-video"; +import { encodeAttemptsBudgetMs } from "../lib/nvenc-encoder"; import { RecordingTimingError } from "../lib/recording-timing"; import { hashRecordingFile, @@ -69,6 +86,7 @@ import { verifyRemoteRecordingBytes, verifyRemuxedRecording, } from "../lib/recording-verification"; +import { getRuntimeReport } from "../lib/runtime-report"; import type { TempFileHandle } from "../lib/temp-files"; import { cleanupStaleTempFiles } from "../lib/temp-files"; import { @@ -93,6 +111,8 @@ const VIDEO_MEMORY_PRESSURE_ERROR = "SERVER_BUSY: container memory pressure is too high"; const SEGMENT_DOWNLOAD_MAX_ATTEMPTS = 3; const SEGMENT_DOWNLOAD_RETRY_BASE_MS = 250; +const JOB_TRANSFER_ALLOWANCE_MS = 2 * UPLOAD_TIMEOUT_MS; +const JOB_TRANSFER_BYTES_PER_SECOND = 2 * 1024 ** 2; const probeSchema = z.object({ videoUrl: z.string().url(), @@ -220,6 +240,12 @@ function getMuxBusyResponseBody( }; } +const DRAINING_RESPONSE_BODY = { + error: "Server is busy", + code: "SERVER_BUSY", + details: DRAINING_DETAILS, +}; + function summarizeVideoInput(videoUrl: string, inputExtension?: string) { const extension = inputExtension?.toLowerCase() ?? null; try { @@ -286,25 +312,26 @@ async function createVideoDownloadResponse( onCleanup?.(); }; + const reader = outputFile.stream().getReader(); const stream = new ReadableStream({ - async start(controller) { - const reader = outputFile.stream().getReader(); - + async pull(controller) { try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - if (value) controller.enqueue(value); + const { done, value } = await reader.read(); + if (done) { + controller.close(); + reader.releaseLock(); + await cleanup(); + } else if (value) { + controller.enqueue(value); } - controller.close(); } catch (error) { controller.error(error); - } finally { reader.releaseLock(); await cleanup(); } }, async cancel() { + await reader.cancel().catch(() => {}); await cleanup(); }, }); @@ -345,6 +372,23 @@ async function withJobHeartbeat( } } +function extendJobLifetimeForWork( + jobId: string, + workTimeoutMs: number, + transferBytes: number, +): void { + extendJobLifetime( + jobId, + encodeAttemptsBudgetMs(workTimeoutMs) + + JOB_TRANSFER_ALLOWANCE_MS + + POST_VERIFICATION_ASSET_BUDGET_MS + + Math.ceil( + (Math.max(0, transferBytes || 0) / JOB_TRANSFER_BYTES_PER_SECOND) * + 1000, + ), + ); +} + async function withMuxMemoryGuard( abortController: AbortController, operation: () => Promise, @@ -373,7 +417,10 @@ async function withMuxMemoryGuard( } } -video.get("/status", (c) => { +video.get("/status", async (c) => { + if (!canReadStatus(c)) { + return c.json({ error: "Unauthorized" }, 401); + } const jobs = getAllJobs(); const resources = getSystemResources(); const now = Date.now(); @@ -401,6 +448,7 @@ video.get("/status", (c) => { stalenessMinutes: Math.round((now - j.updatedAt) / 60000), error: j.error, })), + ...(hasMediaServerSecret(c) && { runtime: await getRuntimeReport() }), }); }); @@ -423,6 +471,11 @@ video.post("/probe", async (c) => { ); } + if (isDraining()) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json(DRAINING_RESPONSE_BODY, 503); + } + try { const metadata = await probeVideo(result.data.videoUrl); return c.json({ metadata }); @@ -481,6 +534,11 @@ video.post("/thumbnail", async (c) => { ); } + if (isDraining()) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json(DRAINING_RESPONSE_BODY, 503); + } + try { const metadata = await probeVideo(result.data.videoUrl); @@ -562,6 +620,17 @@ video.post("/convert", async (c) => { const requestId = crypto.randomUUID(); const startedAt = Date.now(); + if (isDraining()) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json( + { + ...getBusyResponseBody(getVideoCapacitySnapshot()), + details: DRAINING_DETAILS, + }, + 503, + ); + } + try { slot = tryAcquireDirectVideoProcessSlot(canAcceptNewVideoProcess); const capacity = getVideoCapacitySnapshot(); @@ -583,6 +652,7 @@ video.post("/convert", async (c) => { ), ...capacity, }); + setRequestIdleTimeout(c.env, c.req.raw, 0); inputTempFile = await downloadVideoToTemp( result.data.videoUrl, @@ -618,12 +688,14 @@ video.post("/convert", async (c) => { ...getVideoCapacitySnapshot(), }); + setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds()); return await createVideoDownloadResponse( outputTempFile, [inputTempFile, outputTempFile], slot.release, ); } catch (err) { + setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds()); slot?.release(); await cleanupTempFiles([outputTempFile, inputTempFile]); logVideoEvent("video_convert_failed", { @@ -692,6 +764,13 @@ video.on("POST", ["/process", "/import"], async (c) => { } const capacity = getVideoCapacitySnapshot(); + if (isDraining()) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json( + { ...getBusyResponseBody(capacity), details: DRAINING_DETAILS }, + 503, + ); + } const bulkCapacityReached = result.data.priority === "bulk" && capacity.effectiveMaxVideoProcesses > 1 && @@ -715,34 +794,36 @@ video.on("POST", ["/process", "/import"], async (c) => { const jobId = generateJobId(); const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret); - withMediaTransfers(32 * 1024 ** 3, () => - processVideoAsync( - job.jobId, - videoUrl, - outputPresignedUrl, - thumbnailPresignedUrl, - previewGifPresignedUrl, - result.data, - ), - ).catch((err) => { - console.error( - `[video/process] Async processing error for job ${jobId}:`, - err, - ); - const currentJob = getJob(jobId); - if ( - currentJob && - currentJob.phase !== "error" && - currentJob.phase !== "complete" && - currentJob.phase !== "cancelled" - ) { - updateJob(jobId, { - phase: "error", - error: err instanceof Error ? err.message : String(err), - message: "Processing failed (unhandled)", - }); - } - }); + void trackBackgroundTask( + withMediaTransfers(32 * 1024 ** 3, () => + processVideoAsync( + job.jobId, + videoUrl, + outputPresignedUrl, + thumbnailPresignedUrl, + previewGifPresignedUrl, + result.data, + ), + ).catch((err) => { + console.error( + `[video/process] Async processing error for job ${jobId}:`, + err, + ); + const currentJob = getJob(jobId); + if ( + currentJob && + currentJob.phase !== "error" && + currentJob.phase !== "complete" && + currentJob.phase !== "cancelled" + ) { + updateJob(jobId, { + phase: "error", + error: err instanceof Error ? err.message : String(err), + message: "Processing failed (unhandled)", + }); + } + }), + ); return c.json({ jobId, @@ -775,6 +856,17 @@ video.post("/edit", async (c) => { ); } + if (isDraining()) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json( + { + ...getBusyResponseBody(getVideoCapacitySnapshot()), + details: DRAINING_DETAILS, + }, + 503, + ); + } + if (!canAcceptNewVideoProcess()) { c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); return c.json(getBusyResponseBody(getVideoCapacitySnapshot()), 503); @@ -794,31 +886,33 @@ video.post("/edit", async (c) => { const jobId = generateJobId(); const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret); - withMediaTransfers(32 * 1024 ** 3, () => - editVideoAsync( - job.jobId, - sourceUrl, - outputPresignedUrl, - thumbnailPresignedUrl, - previewGifPresignedUrl, - result.data, - ), - ).catch((err) => { - console.error(`[video/edit] Async edit error for job ${jobId}:`, err); - const currentJob = getJob(jobId); - if ( - currentJob && - currentJob.phase !== "error" && - currentJob.phase !== "complete" && - currentJob.phase !== "cancelled" - ) { - updateJob(jobId, { - phase: "error", - error: err instanceof Error ? err.message : String(err), - message: "Edit failed (unhandled)", - }); - } - }); + void trackBackgroundTask( + withMediaTransfers(32 * 1024 ** 3, () => + editVideoAsync( + job.jobId, + sourceUrl, + outputPresignedUrl, + thumbnailPresignedUrl, + previewGifPresignedUrl, + result.data, + ), + ).catch((err) => { + console.error(`[video/edit] Async edit error for job ${jobId}:`, err); + const currentJob = getJob(jobId); + if ( + currentJob && + currentJob.phase !== "error" && + currentJob.phase !== "complete" && + currentJob.phase !== "cancelled" + ) { + updateJob(jobId, { + phase: "error", + error: err instanceof Error ? err.message : String(err), + message: "Edit failed (unhandled)", + }); + } + }), + ); return c.json({ jobId, @@ -963,6 +1057,11 @@ async function processWithResilientRetry( progress: 10, message: "Retrying with error recovery...", }); + extendJobLifetimeForWork( + jobId, + getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS), + metadata.fileSize, + ); try { const outputFile = await processVideo( @@ -988,6 +1087,11 @@ async function processWithResilientRetry( progress: 10, message: "Attempting full repair...", }); + extendJobLifetimeForWork( + jobId, + getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS), + metadata.fileSize, + ); let lastResortRepairFile: TempFileHandle | null = null; try { @@ -1075,21 +1179,25 @@ function isDurationClose(actual: number, expected: number) { ); } -async function waitForVerificationRetry(attempt: number) { - await new Promise((resolve) => - setTimeout(resolve, UPLOAD_VERIFICATION_RETRY_MS * (attempt + 1)), - ); +async function waitForVerificationRetry( + attempt: number, + abortSignal?: AbortSignal, +) { + await Bun.sleep(UPLOAD_VERIFICATION_RETRY_MS * (attempt + 1)); + abortSignal?.throwIfAborted(); } async function verifyUploadedVideo( outputVerificationUrl: string | undefined, expectedMetadata: VideoMetadata, + abortSignal?: AbortSignal, ) { if (!outputVerificationUrl) return; let lastError: Error | undefined; for (let attempt = 0; attempt < UPLOAD_VERIFICATION_ATTEMPTS; attempt++) { + abortSignal?.throwIfAborted(); try { const actualMetadata = await probeVideo(outputVerificationUrl); if (isDurationClose(actualMetadata.duration, expectedMetadata.duration)) { @@ -1104,7 +1212,7 @@ async function verifyUploadedVideo( } if (attempt < UPLOAD_VERIFICATION_ATTEMPTS - 1) { - await waitForVerificationRetry(attempt); + await waitForVerificationRetry(attempt, abortSignal); } } @@ -1124,7 +1232,10 @@ async function editVideoAsync( } const abortController = new AbortController(); + const signal = abortController.signal; updateJob(jobId, { abortController }); + let inputTempFile: TempFileHandle | null = null; + let outputTempFile: TempFileHandle | null = null; try { updateJob(jobId, { @@ -1137,10 +1248,11 @@ async function editVideoAsync( await sendWebhook(downloadingJob); } - const inputTempFile = await withJobHeartbeat(jobId, () => - downloadVideoToTemp(sourceUrl, ".mp4", abortController.signal), + const input = await withJobHeartbeat(jobId, () => + downloadVideoToTemp(sourceUrl, ".mp4", signal), ); - updateJob(jobId, { inputTempFile }); + inputTempFile = input; + updateJob(jobId, { inputTempFile: input }); updateJob(jobId, { phase: "probing", @@ -1152,7 +1264,15 @@ async function editVideoAsync( await sendWebhook(probingJob); } - const sourceMetadata = await probeVideoFile(inputTempFile.path); + const sourceMetadata = await withJobHeartbeat(jobId, () => + probeVideoFile(input.path), + ); + signal.throwIfAborted(); + extendJobLifetimeForWork( + jobId, + getEditRenderTimeoutMs(options.keepRanges), + sourceMetadata.fileSize, + ); updateJob(jobId, { phase: "processing", @@ -1164,12 +1284,12 @@ async function editVideoAsync( await sendWebhook(processingJob); } - const outputTempFile = await withJobHeartbeat(jobId, () => + const output = await withJobHeartbeat(jobId, () => renderEditedVideo({ - inputPath: inputTempFile.path, + inputPath: input.path, keepRanges: options.keepRanges, metadata: sourceMetadata, - abortSignal: abortController.signal, + abortSignal: signal, onProgress: (progress, message) => { updateJob(jobId, { progress: Math.min(80, 10 + progress * 0.9), @@ -1187,9 +1307,13 @@ async function editVideoAsync( }, }), ); - updateJob(jobId, { outputTempFile }); + outputTempFile = output; + updateJob(jobId, { outputTempFile: output }); - const outputMetadata = await probeVideoFile(outputTempFile.path); + const outputMetadata = await withJobHeartbeat(jobId, () => + probeVideoFile(output.path), + ); + signal.throwIfAborted(); updateJob(jobId, { metadata: outputMetadata }); updateJob(jobId, { @@ -1202,8 +1326,19 @@ async function editVideoAsync( await sendWebhook(uploadingJob); } - await uploadFileToS3(outputTempFile.path, outputPresignedUrl, "video/mp4"); - await verifyUploadedVideo(options.outputVerificationUrl, outputMetadata); + await withJobHeartbeat(jobId, async () => { + await uploadFileToS3( + output.path, + outputPresignedUrl, + "video/mp4", + signal, + ); + await verifyUploadedVideo( + options.outputVerificationUrl, + outputMetadata, + signal, + ); + }); if (thumbnailPresignedUrl || previewGifPresignedUrl) { updateJob(jobId, { @@ -1217,21 +1352,31 @@ async function editVideoAsync( } } - if (thumbnailPresignedUrl) { - const thumbnailData = await generateThumbnail( - outputTempFile.path, + await withJobHeartbeat(jobId, async () => { + if (thumbnailPresignedUrl) { + const thumbnailData = await generateThumbnail( + output.path, + outputMetadata.duration, + {}, + signal, + ); + await uploadToS3( + thumbnailData, + thumbnailPresignedUrl, + "image/jpeg", + signal, + ); + } + + await generateAndUploadPreviewGif( + output.path, outputMetadata.duration, + previewGifPresignedUrl, + signal, + "video/edit", ); - await uploadToS3(thumbnailData, thumbnailPresignedUrl, "image/jpeg"); - } - - await generateAndUploadPreviewGif( - outputTempFile.path, - outputMetadata.duration, - previewGifPresignedUrl, - abortController.signal, - "video/edit", - ); + }); + signal.throwIfAborted(); updateJob(jobId, { phase: "complete", @@ -1243,12 +1388,11 @@ async function editVideoAsync( await sendWebhook(completedJob); } - await inputTempFile.cleanup(); - await outputTempFile.cleanup(); - setTimeout(() => deleteJob(jobId), 5 * 60 * 1000); } catch (err) { - console.error(`[video/edit] Error editing job ${jobId}:`, err); + if (!signal.aborted) { + console.error(`[video/edit] Error editing job ${jobId}:`, err); + } const updatedJob = updateJob(jobId, { phase: "error", @@ -1261,16 +1405,10 @@ async function editVideoAsync( await sendWebhook(updatedJob); } } finally { - const currentJob = getJob(jobId); - if (currentJob) { - await Promise.allSettled([ - currentJob.inputTempFile?.cleanup(), - currentJob.outputTempFile?.cleanup(), - ]); - } - setTimeout(() => deleteJob(jobId), 5 * 60 * 1000); } + } finally { + await cleanupTempFiles([inputTempFile, outputTempFile]); } } @@ -1288,8 +1426,11 @@ async function processVideoAsync( } const abortController = new AbortController(); + const signal = abortController.signal; updateJob(jobId, { abortController }); + let inputTempFile: TempFileHandle | null = null; + let outputTempFile: TempFileHandle | null = null; let repairedTempFile: TempFileHandle | null = null; let lastResortRepairFile: TempFileHandle | null = null; @@ -1301,26 +1442,18 @@ async function processVideoAsync( }); await sendWebhook(job); - const inputTempFile = await withJobHeartbeat(jobId, () => - downloadVideoToTemp( - videoUrl, - options.inputExtension, - abortController.signal, - ), + const input = await withJobHeartbeat(jobId, () => + downloadVideoToTemp(videoUrl, options.inputExtension, signal), ); - updateJob(jobId, { inputTempFile }); + inputTempFile = input; + updateJob(jobId, { inputTempFile: input }); const sourcePresignedUrl = options.sourcePresignedUrl; if (sourcePresignedUrl) { updateJob(jobId, { message: "Saving original video..." }); await sendWebhook(job); await withJobHeartbeat(jobId, () => - uploadFileToS3( - inputTempFile.path, - sourcePresignedUrl, - "video/mp4", - abortController.signal, - ), + uploadFileToS3(input.path, sourcePresignedUrl, "video/mp4", signal), ); } @@ -1328,7 +1461,7 @@ async function processVideoAsync( if (isWebm) { updateJob(jobId, { message: "Checking the original recording..." }); await withJobHeartbeat(jobId, () => - validateVideoInput(inputTempFile.path, abortController.signal), + validateVideoInput(input.path, signal), ); } @@ -1339,17 +1472,19 @@ async function processVideoAsync( }); await sendWebhook(job); - const { metadata, repairedFile } = await probeWithRepairFallback( - inputTempFile.path, - isWebm, - abortController.signal, + const { metadata, repairedFile } = await withJobHeartbeat(jobId, () => + probeWithRepairFallback(input.path, isWebm, signal), ); repairedTempFile = repairedFile; + signal.throwIfAborted(); updateJob(jobId, { metadata }); + extendJobLifetimeForWork( + jobId, + getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS), + metadata.fileSize, + ); - const processingInputPath = repairedFile - ? repairedFile.path - : inputTempFile.path; + const processingInputPath = repairedFile ? repairedFile.path : input.path; updateJob(jobId, { phase: "processing", @@ -1360,18 +1495,19 @@ async function processVideoAsync( }); await sendWebhook(job); - const { outputFile: outputTempFile, lastResortRepairFile: lrrf } = - await processWithResilientRetry( - processingInputPath, - inputTempFile.path, - metadata, - options, - isWebm, - jobId, - abortController.signal, - ); - lastResortRepairFile = lrrf; - updateJob(jobId, { outputTempFile }); + const processed = await processWithResilientRetry( + processingInputPath, + input.path, + metadata, + options, + isWebm, + jobId, + signal, + ); + outputTempFile = processed.outputFile; + lastResortRepairFile = processed.lastResortRepairFile; + const outputPath = processed.outputFile.path; + updateJob(jobId, { outputTempFile: processed.outputFile }); updateJob(jobId, { phase: "uploading", @@ -1380,11 +1516,8 @@ async function processVideoAsync( }); await sendWebhook(job); - const uploadReceipt = await uploadFileToS3( - outputTempFile.path, - outputPresignedUrl, - "video/mp4", - abortController.signal, + const uploadReceipt = await withJobHeartbeat(jobId, () => + uploadFileToS3(outputPath, outputPresignedUrl, "video/mp4", signal), ); if (thumbnailPresignedUrl || previewGifPresignedUrl) { @@ -1396,38 +1529,40 @@ async function processVideoAsync( await sendWebhook(job); } - if (thumbnailPresignedUrl) { - // The uploaded MP4 has not been independently verified, so decode failures must still fail the job. - const thumbnailData = await generateThumbnail( - outputTempFile.path, - metadata.duration, - {}, - abortController.signal, - ); - try { - await uploadToS3( - thumbnailData, - thumbnailPresignedUrl, - "image/jpeg", - abortController.signal, - ); - } catch (error) { - abortController.signal.throwIfAborted(); - console.warn( - `[video/process] Thumbnail upload failed for ${jobId}:`, - error, + await withJobHeartbeat(jobId, async () => { + if (thumbnailPresignedUrl) { + // The uploaded MP4 has not been independently verified, so decode failures must still fail the job. + const thumbnailData = await generateThumbnail( + outputPath, + metadata.duration, + {}, + signal, ); + try { + await uploadToS3( + thumbnailData, + thumbnailPresignedUrl, + "image/jpeg", + signal, + ); + } catch (error) { + signal.throwIfAborted(); + console.warn( + `[video/process] Thumbnail upload failed for ${jobId}:`, + error, + ); + } } - } - await generateAndUploadPreviewGif( - outputTempFile.path, - metadata.duration, - previewGifPresignedUrl, - abortController.signal, - "video/process", - ); - abortController.signal.throwIfAborted(); + await generateAndUploadPreviewGif( + outputPath, + metadata.duration, + previewGifPresignedUrl, + signal, + "video/process", + ); + }); + signal.throwIfAborted(); updateJob(jobId, { phase: "complete", @@ -1441,7 +1576,7 @@ async function processVideoAsync( if (options.audioLevels) await enhanceLocalRecording({ - path: outputTempFile.path, + path: outputPath, videoId: options.videoId, userId: options.userId, jobId, @@ -1452,14 +1587,9 @@ async function processVideoAsync( webhookSecret: options.webhookSecret, }); - await inputTempFile.cleanup(); - await outputTempFile.cleanup(); - await repairedTempFile?.cleanup(); - await lastResortRepairFile?.cleanup(); - setTimeout(() => deleteJob(jobId), 5 * 60 * 1000); } catch (err) { - if (!abortController.signal.aborted) { + if (!signal.aborted) { console.error(`[video/process] Error processing job ${jobId}:`, err); } const failedJob = getJob(jobId); @@ -1469,7 +1599,7 @@ async function processVideoAsync( failedJob.phase !== "cancelled" && failedJob.phase !== "error" ) { - const cancelled = abortController.signal.aborted; + const cancelled = signal.aborted; const updatedJob = updateJob(jobId, { phase: cancelled ? "cancelled" : "error", error: cancelled @@ -1483,18 +1613,20 @@ async function processVideoAsync( await sendWebhook(updatedJob); } } - - const currentJob = getJob(jobId); - if (currentJob) { - await currentJob.inputTempFile?.cleanup(); - await currentJob.outputTempFile?.cleanup(); - } - await repairedTempFile?.cleanup(); - await lastResortRepairFile?.cleanup(); + } finally { + await cleanupTempFiles([ + inputTempFile, + outputTempFile, + repairedTempFile, + lastResortRepairFile, + ]); } } video.get("/process/:jobId/status", async (c) => { + if (!canReadStatus(c)) { + return c.json({ error: "Unauthorized" }, 401); + } const jobId = c.req.param("jobId"); const job = getJob(jobId); @@ -1845,6 +1977,16 @@ video.post("/verify-recording", async (c) => { ); return c.json(recordingWorkerResponse(existing, ownerJobId)); } + if (isDraining()) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json( + { + ...getMuxBusyResponseBody(getVideoCapacitySnapshot()), + details: DRAINING_DETAILS, + }, + 503, + ); + } if (!canAcceptNewVideoProcess(body.processingPriority)) { c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); return c.json(getMuxBusyResponseBody(getVideoCapacitySnapshot()), 503); @@ -1877,19 +2019,24 @@ video.post("/verify-recording", async (c) => { deleteJob(jobId); return c.json(recordingWorkerResponse(job, ownerJobId)); } - void withMediaTransfers(body.downloadBudgetBytes ?? body.fileSize * 3, () => - verifyUploadedRecordingAsync(jobId, body), - ).catch((error: unknown) => { - const current = getJob(jobId); - if (!current || ["complete", "error", "cancelled"].includes(current.phase)) - return; - updateJob(jobId, { - phase: "error", - error: "Recording transfer could not complete", - errorCode: classifySourceError(error), - }); - sendCurrentJobWebhook(jobId); - }); + void trackBackgroundTask( + withMediaTransfers(body.downloadBudgetBytes ?? body.fileSize * 3, () => + verifyUploadedRecordingAsync(jobId, body), + ).catch((error: unknown) => { + const current = getJob(jobId); + if ( + !current || + ["complete", "error", "cancelled"].includes(current.phase) + ) + return; + updateJob(jobId, { + phase: "error", + error: "Recording transfer could not complete", + errorCode: classifySourceError(error), + }); + sendCurrentJobWebhook(jobId); + }), + ); return c.json(recordingWorkerResponse(job, jobId)); }); @@ -1933,6 +2080,7 @@ async function verifyUploadedRecordingAsync( body.sourceObjectIdentity ?? body.objectIdentity, expectedFileSize: body.fileSize, hashContent: Boolean(body.generation), + hashDecodedContent: false, abortSignal: abortController.signal, }), ), @@ -2174,6 +2322,17 @@ video.post("/mux-segments", async (c) => { }); } + if (isDraining()) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json( + { + ...getMuxBusyResponseBody(getVideoCapacitySnapshot()), + details: DRAINING_DETAILS, + }, + 503, + ); + } + if (!canAcceptNewVideoProcess(body.data.processingPriority)) { c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); return c.json(getMuxBusyResponseBody(getVideoCapacitySnapshot()), 503); @@ -2215,36 +2374,38 @@ video.post("/mux-segments", async (c) => { }); } - withMediaTransfers(body.data.downloadBudgetBytes ?? 32 * 1024 ** 3, () => - muxSegmentsAsync( - jobId, - videoId, - outputUpload, - thumbnailPresignedUrl, - previewGifPresignedUrl, - videoInitUrl, - videoSegUrls, - audioInitUrl ?? null, - audioSegUrls ?? null, - body.data.outputVerificationUrl, - body.data, - ), - ).catch((err) => { - console.error(`[mux-segments] Async mux error for job ${jobId}:`, err); - const currentJob = getJob(jobId); - if ( - currentJob && - currentJob.phase !== "error" && - currentJob.phase !== "complete" && - currentJob.phase !== "cancelled" - ) { - updateJob(jobId, { - phase: "error", - error: err instanceof Error ? err.message : String(err), - }); - sendCurrentJobWebhook(jobId); - } - }); + void trackBackgroundTask( + withMediaTransfers(body.data.downloadBudgetBytes ?? 32 * 1024 ** 3, () => + muxSegmentsAsync( + jobId, + videoId, + outputUpload, + thumbnailPresignedUrl, + previewGifPresignedUrl, + videoInitUrl, + videoSegUrls, + audioInitUrl ?? null, + audioSegUrls ?? null, + body.data.outputVerificationUrl, + body.data, + ), + ).catch((err) => { + console.error(`[mux-segments] Async mux error for job ${jobId}:`, err); + const currentJob = getJob(jobId); + if ( + currentJob && + currentJob.phase !== "error" && + currentJob.phase !== "complete" && + currentJob.phase !== "cancelled" + ) { + updateJob(jobId, { + phase: "error", + error: err instanceof Error ? err.message : String(err), + }); + sendCurrentJobWebhook(jobId); + } + }), + ); return c.json({ ...recordingWorkerResponse(job, jobId), @@ -2568,15 +2729,14 @@ async function muxSegmentsAsync( outputVerificationUrl?: string, context: MuxContext = {}, ): Promise { - const { ensureTempDir } = await import("../lib/temp-files"); + const { ensureTempDir, getTempDir, retainTempPath } = await import( + "../lib/temp-files" + ); const { lstat, mkdir, rm } = await import("node:fs/promises"); const { join } = await import("node:path"); - const workDir = join( - (await import("node:os")).tmpdir(), - "cap-media-server", - `mux-${jobId}`, - ); + const workDir = join(getTempDir(), `mux-${jobId}`); + const releaseWorkDir = retainTempPath(workDir); const abortController = new AbortController(); updateJob(jobId, { abortController }); let outputUploadStarted = false; @@ -2666,10 +2826,12 @@ async function muxSegmentsAsync( const combinedVideoPath = join(workDir, "combined_video.mp4"); const videoInitPath = join(videoDir, "init.mp4"); - await streamConcatFiles( - [videoInitPath, ...videoSegmentFiles], - combinedVideoPath, - abortController.signal, + await withJobHeartbeat(jobId, () => + streamConcatFiles( + [videoInitPath, ...videoSegmentFiles], + combinedVideoPath, + abortController.signal, + ), ); await rm(videoDir, { recursive: true, force: true }); @@ -2679,10 +2841,13 @@ async function muxSegmentsAsync( combinedAudioPath = join(workDir, "combined_audio.mp4"); const audioInitPath = join(audioDir, "init.mp4"); - await streamConcatFiles( - [audioInitPath, ...audioSegmentFiles], - combinedAudioPath, - abortController.signal, + const audioOutputPath = combinedAudioPath; + await withJobHeartbeat(jobId, () => + streamConcatFiles( + [audioInitPath, ...audioSegmentFiles], + audioOutputPath, + abortController.signal, + ), ); await rm(audioDir, { recursive: true, force: true }); } @@ -2789,7 +2954,9 @@ async function muxSegmentsAsync( resources: getSystemResources(), }); - let metadata = await probeVideoFile(resultPath); + let metadata = await withJobHeartbeat(jobId, () => + probeVideoFile(resultPath), + ); if ( !Number.isFinite(metadata.duration) || metadata.duration <= 0 || @@ -2903,37 +3070,39 @@ async function muxSegmentsAsync( sendCurrentJobWebhook(jobId); } - if (thumbnailPresignedUrl) { - try { - const duration = metadata?.duration ?? 0; - const thumbnailData = await generateThumbnail( - resultPath, - duration, - {}, - abortController.signal, - ); - await uploadToS3( - thumbnailData, - thumbnailPresignedUrl, - "image/jpeg", - abortController.signal, - ); - } catch (thumbErr) { - abortController.signal.throwIfAborted(); - console.warn( - `[mux-segments] Thumbnail generation failed for ${videoId}:`, - thumbErr, - ); + const assetDuration = metadata?.duration ?? 0; + await withJobHeartbeat(jobId, async () => { + if (thumbnailPresignedUrl) { + try { + const thumbnailData = await generateThumbnail( + resultPath, + assetDuration, + {}, + abortController.signal, + ); + await uploadToS3( + thumbnailData, + thumbnailPresignedUrl, + "image/jpeg", + abortController.signal, + ); + } catch (thumbErr) { + abortController.signal.throwIfAborted(); + console.warn( + `[mux-segments] Thumbnail generation failed for ${videoId}:`, + thumbErr, + ); + } } - } - await generateAndUploadPreviewGif( - resultPath, - metadata?.duration ?? 0, - previewGifPresignedUrl, - abortController.signal, - "mux-segments", - ); + await generateAndUploadPreviewGif( + resultPath, + assetDuration, + previewGifPresignedUrl, + abortController.signal, + "mux-segments", + ); + }); abortController.signal.throwIfAborted(); updateJob(jobId, { @@ -3014,6 +3183,7 @@ async function muxSegmentsAsync( } finally { if (processingTimeout) clearTimeout(processingTimeout); await rm(workDir, { recursive: true, force: true }).catch(() => {}); + releaseWorkDir(); } }