diff --git a/electron/app-settings.test.ts b/electron/app-settings.test.ts index c6c9e74f3..f3781734f 100644 --- a/electron/app-settings.test.ts +++ b/electron/app-settings.test.ts @@ -129,4 +129,40 @@ describe("app settings store", () => { store.dismissStarPrompt(); expect(store.getSnapshot().recording).toMatchObject({ micEnabled: true, micDeviceId: "mic" }); }); + + it("stores a camera quality and rejects one it cannot capture at", () => { + const dir = temp(); + const store = new AppSettingsStore(dir); + + expect(store.setRecordingPreferences({ camQuality: "1080p" }).recording.camQuality).toBe( + "1080p", + ); + expect(() => store.setRecordingPreferences({ camQuality: "4320p" as never })).toThrow( + TypeError, + ); + expect(store.getSnapshot().recording.camQuality).toBe("1080p"); + }); + + it("leaves the camera quality alone when a patch does not mention it", () => { + // Every other window writes narrow patches; a camera toggle from the HUD + // must not quietly reset the resolution the user picked. + const dir = temp(); + const store = new AppSettingsStore(dir); + store.setRecordingPreferences({ camQuality: "1440p" }); + + store.setRecordingPreferences({ camEnabled: true }); + + expect(store.getSnapshot().recording.camQuality).toBe("1440p"); + }); + + it("reads a settings file written before the camera had a quality", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ micEnabled: true }), + "utf8", + ); + + expect(new AppSettingsStore(dir).getSnapshot().recording.camQuality).toBe("2160p"); + }); }); diff --git a/electron/app-settings.ts b/electron/app-settings.ts index 2cc1cee25..bef025bf3 100644 --- a/electron/app-settings.ts +++ b/electron/app-settings.ts @@ -1,5 +1,11 @@ import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { + DEFAULT_WEBCAM_QUALITY, + WEBCAM_QUALITY_IDS, + type WebcamQualityId, + webcamQualityFrom, +} from "../src/hooks/webcamCaptureTarget"; import type { CursorCaptureMode } from "../src/lib/recordingSession"; export interface RecordingPreferences { @@ -9,6 +15,8 @@ export interface RecordingPreferences { camEnabled: boolean; camDeviceId: string | null; camDeviceName: string | null; + /** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */ + camQuality: WebcamQualityId; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; /** Display captures on macOS and Windows. Opt-in: on Windows the icons also leave the real desktop while recording. */ @@ -30,6 +38,7 @@ export const DEFAULT_RECORDING_PREFERENCES: RecordingPreferences = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: DEFAULT_WEBCAM_QUALITY, systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, @@ -100,6 +109,9 @@ function parseRecording(raw: RawSettings): RecordingPreferences { camEnabled: bool(raw.camEnabled, DEFAULT_RECORDING_PREFERENCES.camEnabled), camDeviceId: nullableString(raw.camDeviceId, DEFAULT_RECORDING_PREFERENCES.camDeviceId), camDeviceName: nullableString(raw.camDeviceName, DEFAULT_RECORDING_PREFERENCES.camDeviceName), + // Unset in every settings file written before the camera had a quality + // setting, and `webcamQualityFrom` answers those with the default. + camQuality: webcamQualityFrom(raw.camQuality), systemAudioEnabled: bool( raw.systemAudioEnabled, DEFAULT_RECORDING_PREFERENCES.systemAudioEnabled, @@ -167,6 +179,12 @@ function validateRecordingPatch(patch: Partial): void { if (key === "cursorCaptureMode" && value !== "system" && value !== "editable-overlay") { throw new TypeError("cursorCaptureMode is invalid"); } + // Rejected here rather than coerced on read, so a bad write is a visible + // error at its source instead of a resolution that silently is not the + // one the caller asked for. + if (key === "camQuality" && !WEBCAM_QUALITY_IDS.includes(value as WebcamQualityId)) { + throw new TypeError("camQuality is invalid"); + } } } diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 6e6bac45f..c3bd11c01 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -17,6 +17,7 @@ import { shell, systemPreferences, } from "electron"; +import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget"; import { type AxcutDocument, isAxcutDocumentFile, @@ -649,6 +650,8 @@ export interface RecordingPrefs { camDeviceId: string | null; /** Camera label paired with the preferred id for restart-safe resolution. */ camDeviceName: string | null; + /** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */ + camQuality: WebcamQualityId; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; hideDesktopIcons: boolean; @@ -662,6 +665,7 @@ const defaultRecordingPrefs: RecordingPrefs = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: DEFAULT_WEBCAM_QUALITY, systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, diff --git a/electron/ipc/recordingPrefs.test.ts b/electron/ipc/recordingPrefs.test.ts index fe8cdce9d..2c3317af9 100644 --- a/electron/ipc/recordingPrefs.test.ts +++ b/electron/ipc/recordingPrefs.test.ts @@ -19,6 +19,7 @@ const defaults: RecordingPrefs = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: "2160p", systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index c0921486f..7a80f3214 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -41,6 +41,8 @@ add_executable(wgc-capture src/desktop_icon_cover.cpp src/desktop_icon_cover.h src/dpi_awareness.h + src/frame_visibility.cpp + src/frame_visibility.h src/dshow_webcam_capture.cpp src/dshow_webcam_capture.h src/main.cpp @@ -55,6 +57,8 @@ add_executable(wgc-capture src/wasapi_render_keepalive.cpp src/wasapi_render_keepalive.h src/webcam_capture.cpp + src/webcam_format.cpp + src/webcam_format.h src/webcam_capture.h src/wgc_session.cpp src/wgc_session.h @@ -117,3 +121,31 @@ target_link_libraries(audio_sample_utils_test PRIVATE mfuuid ole32 ) + +add_executable(webcam_format_test + src/webcam_format.cpp + src/webcam_format.h + src/webcam_format_test.cpp +) + +target_compile_definitions(webcam_format_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_format_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(frame_visibility_test + src/frame_visibility.cpp + src/frame_visibility.h + src/frame_visibility_test.cpp +) + +target_compile_definitions(frame_visibility_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(frame_visibility_test PRIVATE /EHsc /W4 /utf-8) diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp index f39e4b065..fcbb04d58 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp @@ -1,5 +1,7 @@ #include "dshow_webcam_capture.h" +#include "webcam_format.h" + #include #include #include @@ -29,6 +31,19 @@ ISampleGrabber : public IUnknown { virtual HRESULT STDMETHODCALLTYPE SetCallback(IUnknown* callback, long whichMethodToCallback) = 0; }; +/** + * Is this subtype something the graph must DECODE before this class can unpack it? + * + * Listed positively -- an unknown subtype counts as compressed -- because + * guessing wrong this way only costs resolution, while guessing wrong the other + * way costs the camera track entirely. + */ +bool isCompressedDshowSubtype(const GUID& subtype) { + return !(subtype == MEDIASUBTYPE_YUY2 || subtype == MEDIASUBTYPE_NV12 || + subtype == MEDIASUBTYPE_RGB32 || subtype == MEDIASUBTYPE_RGB24 || + subtype == MEDIASUBTYPE_UYVY || subtype == MEDIASUBTYPE_YV12); +} + bool succeeded(HRESULT hr, const char* label) { if (SUCCEEDED(hr)) { return true; @@ -112,7 +127,93 @@ DirectShowWebcamCapture::~DirectShowWebcamCapture() { delete impl_; } -bool DirectShowWebcamCapture::buildGraph(const CLSID& sourceClsid, const GUID* preferredSubtype) { +void DirectShowWebcamCapture::applyPreferredFormat(int requestedWidth, int requestedHeight, int requestedFps) { + Microsoft::WRL::ComPtr streamConfig; + if (FAILED(impl_->captureGraph->FindInterface( + &PIN_CATEGORY_CAPTURE, + &MEDIATYPE_Video, + impl_->captureFilter.Get(), + IID_PPV_ARGS(&streamConfig)))) { + // Virtual cameras commonly expose a single fixed format and no + // IAMStreamConfig at all. Nothing to choose from; let the graph connect. + return; + } + + int capCount = 0; + int capSize = 0; + if (FAILED(streamConfig->GetNumberOfCapabilities(&capCount, &capSize)) || + capSize != sizeof(VIDEO_STREAM_CONFIG_CAPS)) { + return; + } + + std::vector formats; + formats.reserve(static_cast(std::max(0, capCount))); + for (int index = 0; index < capCount; ++index) { + VIDEO_STREAM_CONFIG_CAPS caps{}; + AM_MEDIA_TYPE* mediaType = nullptr; + if (FAILED(streamConfig->GetStreamCaps(index, &mediaType, reinterpret_cast(&caps))) || !mediaType) { + continue; + } + if (mediaType->formattype == FORMAT_VideoInfo && mediaType->pbFormat) { + const auto* videoInfo = reinterpret_cast(mediaType->pbFormat); + // AvgTimePerFrame is in 100ns units; 333333 is 30 fps. + const int fps = videoInfo->AvgTimePerFrame > 0 + ? static_cast((10'000'000LL + videoInfo->AvgTimePerFrame / 2) / videoInfo->AvgTimePerFrame) + : 0; + formats.push_back(WebcamFormat{ + std::abs(videoInfo->bmiHeader.biWidth), + std::abs(videoInfo->bmiHeader.biHeight), + fps, + isCompressedDshowSubtype(mediaType->subtype)}); + } + freeMediaType(*mediaType); + CoTaskMemFree(mediaType); + } + + const WebcamFormat chosen = chooseWebcamFormat( + formats, + requestedWidth > 0 ? requestedWidth : 1920, + requestedHeight > 0 ? requestedHeight : 1080, + std::max(1, requestedFps)); + + // Second pass: set the first capability that matches the choice. GetStreamCaps + // hands back the media type we must give to SetFormat, so it has to be + // re-read rather than cached across the loop above. + for (int index = 0; index < capCount; ++index) { + VIDEO_STREAM_CONFIG_CAPS caps{}; + AM_MEDIA_TYPE* mediaType = nullptr; + if (FAILED(streamConfig->GetStreamCaps(index, &mediaType, reinterpret_cast(&caps))) || !mediaType) { + continue; + } + bool matched = false; + if (mediaType->formattype == FORMAT_VideoInfo && mediaType->pbFormat) { + auto* videoInfo = reinterpret_cast(mediaType->pbFormat); + if (std::abs(videoInfo->bmiHeader.biWidth) == chosen.width && + std::abs(videoInfo->bmiHeader.biHeight) == chosen.height && + isCompressedDshowSubtype(mediaType->subtype) == chosen.compressed) { + if (chosen.fps > 0) { + videoInfo->AvgTimePerFrame = 10'000'000LL / chosen.fps; + } + matched = SUCCEEDED(streamConfig->SetFormat(mediaType)); + if (matched) { + std::cerr << "INFO: DirectShow webcam format " << chosen.width << "x" << chosen.height + << "@" << chosen.fps << std::endl; + } + } + } + freeMediaType(*mediaType); + CoTaskMemFree(mediaType); + if (matched) { + return; + } + } +} + +bool DirectShowWebcamCapture::buildGraph( + const CLSID& sourceClsid, + const GUID* preferredSubtype, + int preferredWidth, + int preferredHeight) { // Every attempt starts from empty filters. A RenderStream that fails can // leave pins connected behind it, and retrying on top of that half-built // graph is how you get a second failure that says nothing about the format. @@ -175,6 +276,8 @@ bool DirectShowWebcamCapture::buildGraph(const CLSID& sourceClsid, const GUID* p return false; } + applyPreferredFormat(preferredWidth, preferredHeight, fps_); + return succeeded(impl_->captureGraph->RenderStream( &PIN_CATEGORY_CAPTURE, &MEDIATYPE_Video, @@ -231,14 +334,14 @@ bool DirectShowWebcamCapture::initialize( // it connects with a subtype this file cannot read // (getopenscreen/openscreen#387). Naming a concrete subtype on the retry is // what makes DirectShow insert a colour converter for it. - if (!buildGraph(selectedClsid, nullptr)) { + if (!buildGraph(selectedClsid, nullptr, requestedWidth, requestedHeight)) { return false; } if (!resolveConnectedFormat(requestedWidth, requestedHeight, false)) { std::cerr << "WARNING: DirectShow webcam speaks a format this build cannot unpack; " "asking for RGB32 so the graph converts it" << std::endl; - if (!buildGraph(selectedClsid, &MEDIASUBTYPE_RGB32)) { + if (!buildGraph(selectedClsid, &MEDIASUBTYPE_RGB32, requestedWidth, requestedHeight)) { return false; } if (!resolveConnectedFormat(requestedWidth, requestedHeight, true)) { @@ -291,9 +394,28 @@ bool DirectShowWebcamCapture::resolveConnectedFormat( sourceStride_ = ((width_ * bitsPerPixel + 31) / 32) * 4; } sourceTopDown_ = pixelFormat_ != PixelFormat::Bgra || videoInfo->bmiHeader.biHeight < 0; + // The rate the graph settled on, not the one that was asked for. + // + // `chooseWebcamFormat` deliberately settles for less than the target + // when a camera offers nothing faster, and a driver may pick its own + // nearest rate after SetFormat regardless. `fps()` feeds three things + // that all have to agree with the frames actually arriving: the + // `webcam-format` event, the webcam encoder's nominal rate, and the + // constant-rate write interval in main.cpp. Left at the requested + // value, a 24 fps camera asked for 30 is encoded and paced as 30. + if (videoInfo->AvgTimePerFrame > 0) { + const int negotiated = static_cast( + (10'000'000LL + videoInfo->AvgTimePerFrame / 2) / videoInfo->AvgTimePerFrame); + if (negotiated > 0 && negotiated != fps_) { + std::cerr << "INFO: DirectShow webcam negotiated " << negotiated << " fps (asked for " + << fps_ << ")" << std::endl; + fps_ = std::clamp(negotiated, 1, 60); + } + } } std::cerr << "INFO: DirectShow webcam connected subtype " << guidToString(connectedType.subtype) - << " " << width_ << "x" << height_ << " stride=" << sourceStride_ << std::endl; + << " " << width_ << "x" << height_ << "@" << fps_ << " stride=" << sourceStride_ + << std::endl; freeMediaType(connectedType); if (width_ <= 0 || height_ <= 0) { width_ = requestedWidth > 0 ? requestedWidth : 1280; diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.h b/electron/native/wgc-capture/src/dshow_webcam_capture.h index cf510e2df..c0ba3503d 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.h +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.h @@ -59,7 +59,21 @@ class DirectShowWebcamCapture { * without leaving the graph half-built, so the caller can retry with a * different constraint. */ - bool buildGraph(const CLSID& sourceClsid, const GUID* preferredSubtype); + bool buildGraph( + const CLSID& sourceClsid, + const GUID* preferredSubtype, + int preferredWidth, + int preferredHeight); + /** + * Pins the capture pin to the best format the device offers, before the + * graph is rendered. + * + * Skipped, RenderStream's intelligent connect takes the pin's default + * format, which on a UVC camera is the first one it enumerates -- 640x480 + * on hardware that can do far better. Best-effort: a device without + * IAMStreamConfig, or one that rejects the format, still gets a graph. + */ + void applyPreferredFormat(int requestedWidth, int requestedHeight, int requestedFps); /** * Reads back what the graph actually negotiated and records how to unpack it. * diff --git a/electron/native/wgc-capture/src/frame_visibility.cpp b/electron/native/wgc-capture/src/frame_visibility.cpp new file mode 100644 index 000000000..d296b5f0e --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility.cpp @@ -0,0 +1,78 @@ +#include "frame_visibility.h" + +#include + +namespace { + +// Sampling at most this many pixels keeps the probe cheap on a 4K frame while +// still covering it evenly. +constexpr size_t kMaxSamples = 4096; + +// Luma is judged on a full-range 0..255 scale. A frame counts as visible when +// any sample is clearly above black, or when the picture is dim but not empty. +constexpr int kMaxLumaThreshold = 24; +constexpr int kAverageLumaThreshold = 4; + +// BT.709/BT.601 studio range: black sits at 16, white at 235. +constexpr int kStudioBlack = 16; +constexpr int kStudioSpan = 235 - kStudioBlack; + +bool isVisible(uint64_t lumaTotal, int maxLuma, size_t sampled) { + const uint64_t averageLuma = sampled > 0 ? lumaTotal / sampled : 0; + return maxLuma > kMaxLumaThreshold || averageLuma > kAverageLumaThreshold; +} + +} // namespace + +bool hasVisibleBgraContent(const std::vector& frame) { + if (frame.size() < 4) { + return false; + } + + uint64_t lumaTotal = 0; + int maxLuma = 0; + const size_t pixelCount = frame.size() / 4; + const size_t step = std::max(1, pixelCount / kMaxSamples); + size_t sampled = 0; + for (size_t pixel = 0; pixel < pixelCount; pixel += step) { + const size_t offset = pixel * 4; + const int b = frame[offset + 0]; + const int g = frame[offset + 1]; + const int r = frame[offset + 2]; + const int luma = (r * 54 + g * 183 + b * 19) >> 8; + lumaTotal += static_cast(luma); + maxLuma = std::max(maxLuma, luma); + sampled += 1; + } + + return isVisible(lumaTotal, maxLuma, sampled); +} + +bool hasVisibleNv12Content(const std::vector& frame) { + if (frame.size() < 6) { + return false; + } + + const size_t lumaCount = frame.size() * 2 / 3; + const size_t step = std::max(1, lumaCount / kMaxSamples); + uint64_t lumaTotal = 0; + int maxLuma = 0; + size_t sampled = 0; + for (size_t offset = 0; offset < lumaCount; offset += step) { + // Studio range mapped onto 0..255, so the thresholds mean the same here + // as they do for BGRA. Skip this and an all-black frame averages 16 -- + // past the average threshold on its own, so every black frame would + // count as a picture. + const int studio = static_cast(frame[offset]); + const int luma = std::clamp(((studio - kStudioBlack) * 255) / kStudioSpan, 0, 255); + lumaTotal += static_cast(luma); + maxLuma = std::max(maxLuma, luma); + sampled += 1; + } + + return isVisible(lumaTotal, maxLuma, sampled); +} + +bool hasVisibleWebcamContent(const std::vector& frame, bool isNv12) { + return isNv12 ? hasVisibleNv12Content(frame) : hasVisibleBgraContent(frame); +} diff --git a/electron/native/wgc-capture/src/frame_visibility.h b/electron/native/wgc-capture/src/frame_visibility.h new file mode 100644 index 000000000..9f62c2298 --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +/** + * Does this frame contain any picture, or is it the blank one a camera emits + * while it is still warming up? + * + * Used before recording starts, to tell "the camera is running" from "the + * camera is open but has not produced anything yet". + */ +bool hasVisibleBgraContent(const std::vector& frame); + +/** + * The NV12 twin: luma is the Y plane, one byte per pixel, so no colour + * conversion is needed. The Y plane is the first two thirds of the buffer. + * + * A camera's NV12 is studio-range, where black is 16 rather than 0. Those + * values are normalised to full range before the thresholds are applied -- + * without that, the average of an all-black frame is 16, clears the average + * threshold on its own, and every black frame counts as visible. + */ +bool hasVisibleNv12Content(const std::vector& frame); + +/** Dispatches to the probe matching the frame's layout. */ +bool hasVisibleWebcamContent(const std::vector& frame, bool isNv12); diff --git a/electron/native/wgc-capture/src/frame_visibility_test.cpp b/electron/native/wgc-capture/src/frame_visibility_test.cpp new file mode 100644 index 000000000..51e9f7531 --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility_test.cpp @@ -0,0 +1,66 @@ +#include "frame_visibility.h" + +#include +#include +#include + +namespace { + +int failures = 0; + +void expectVisible(const std::string& label, bool actual, bool expected) { + if (actual == expected) { + return; + } + std::printf("FAIL %s: expected %s, got %s\n", label.c_str(), expected ? "visible" : "blank", + actual ? "visible" : "blank"); + ++failures; +} + +std::vector nv12(int width, int height, std::uint8_t luma) { + std::vector frame(static_cast(width) * height * 3 / 2, 128); + std::fill(frame.begin(), frame.begin() + static_cast(width) * height, luma); + return frame; +} + +std::vector bgra(int width, int height, std::uint8_t value) { + return std::vector(static_cast(width) * height * 4, value); +} + +} // namespace + +int main() { + // The regression this file exists for. A camera's NV12 is studio-range, so an + // all-black frame is a plane of 16, not 0. Averaged raw that is 16 -- past the + // average threshold on its own -- and every black frame counted as a picture, + // which is exactly what the warm-up probe exists to rule out. + expectVisible("studio-range black is blank", hasVisibleNv12Content(nv12(640, 480, 16)), false); + expectVisible("true zero is blank", hasVisibleNv12Content(nv12(640, 480, 0)), false); + + // Just above black stays blank; a clearly lit frame does not. + expectVisible("near-black is blank", hasVisibleNv12Content(nv12(640, 480, 20)), false); + expectVisible("mid grey is visible", hasVisibleNv12Content(nv12(640, 480, 128)), true); + expectVisible("studio white is visible", hasVisibleNv12Content(nv12(640, 480, 235)), true); + + // BGRA keeps its own behaviour: full-range, black is 0. + expectVisible("bgra black is blank", hasVisibleBgraContent(bgra(640, 480, 0)), false); + expectVisible("bgra grey is visible", hasVisibleBgraContent(bgra(640, 480, 128)), true); + + // Buffers too small to carry a frame must not be read past their end. + expectVisible("empty nv12", hasVisibleNv12Content({}), false); + expectVisible("empty bgra", hasVisibleBgraContent({}), false); + expectVisible("runt nv12", hasVisibleNv12Content({16, 16, 16}), false); + + // The dispatcher has to pick the layout it is told, not guess. The same bytes + // read as BGRA are bright; read as studio-range NV12 they are black. + const std::vector blackNv12 = nv12(320, 240, 16); + expectVisible("dispatch nv12", hasVisibleWebcamContent(blackNv12, true), false); + expectVisible("dispatch bgra", hasVisibleWebcamContent(bgra(320, 240, 200), false), true); + + if (failures == 0) { + std::printf("frame_visibility_test: all assertions passed\n"); + return 0; + } + std::printf("frame_visibility_test: %d assertion(s) failed\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index c86524d6d..996540de7 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -6,6 +6,7 @@ #include "wasapi_device_watcher.h" #include "wasapi_loopback_capture.h" #include "wasapi_render_keepalive.h" +#include "frame_visibility.h" #include "webcam_capture.h" #include "wgc_session.h" @@ -393,30 +394,6 @@ void reportCaptureAdapters(ID3D11Device* device, HMONITOR targetMonitor) { } } -bool hasVisibleBgraContent(const std::vector& frame) { - if (frame.size() < 4) { - return false; - } - - uint64_t lumaTotal = 0; - BYTE maxLuma = 0; - const size_t pixelCount = frame.size() / 4; - const size_t step = std::max(1, pixelCount / 4096); - size_t sampledPixels = 0; - for (size_t pixel = 0; pixel < pixelCount; pixel += step) { - const size_t offset = pixel * 4; - const BYTE b = frame[offset + 0]; - const BYTE g = frame[offset + 1]; - const BYTE r = frame[offset + 2]; - const BYTE luma = static_cast((static_cast(r) * 54 + static_cast(g) * 183 + static_cast(b) * 19) >> 8); - lumaTotal += luma; - maxLuma = std::max(maxLuma, luma); - sampledPixels += 1; - } - - const uint64_t averageLuma = sampledPixels > 0 ? lumaTotal / sampledPixels : 0; - return maxLuma > 24 || averageLuma > 4; -} bool findBool(const std::string& json, const std::string& key, bool fallback) { auto pos = json.find("\"" + key + "\""); @@ -777,7 +754,10 @@ int wmain(int argc, wchar_t* argv[]) { WebcamCapture webcamCapture; bool webcamActive = false; - bool writeSeparateWebcam = false; + // Decided before initialize(), not after: it selects the capture pixel + // format, and only a camera going to its own file can use NV12 -- an inline + // picture-in-picture composite needs the frame as BGRA. + bool writeSeparateWebcam = config.webcamEnabled && !config.webcamOutputPath.empty(); if (config.webcamEnabled) { if (!webcamCapture.initialize( utf8ToWide(config.webcamDeviceId), @@ -785,7 +765,8 @@ int wmain(int argc, wchar_t* argv[]) { utf8ToWide(config.webcamDirectShowClsid), config.webcamWidth, config.webcamHeight, - config.webcamFps > 0 ? config.webcamFps : config.fps)) { + config.webcamFps > 0 ? config.webcamFps : config.fps, + writeSeparateWebcam)) { // Non-fatal: a screen+audio recording the user can still use is far // better than losing the whole recording because one camera device // didn't match. Report it so the renderer can inform the user (and, @@ -797,13 +778,14 @@ int wmain(int argc, wchar_t* argv[]) { "\"Failed to initialize native webcam capture\"}" << std::endl; config.webcamEnabled = false; + writeSeparateWebcam = false; } else { std::cout << "{\"event\":\"webcam-format\",\"schemaVersion\":2,\"width\":" << webcamCapture.width() << ",\"height\":" << webcamCapture.height() << ",\"fps\":" << webcamCapture.fps() << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(webcamCapture.selectedDeviceName())) << "\"}" << std::endl; - writeSeparateWebcam = !config.webcamOutputPath.empty(); + // writeSeparateWebcam was decided above, before the pixel format. } } @@ -980,8 +962,18 @@ int wmain(int argc, wchar_t* argv[]) { MFEncoderOptions webcamEncoderOptions = encoderOptions; webcamEncoderOptions.injectDefaultSinkWriterFailureOnce = false; webcamEncoderOptions.useDxgiInput = false; + webcamEncoderOptions.cpuInputIsNv12 = webcamCapture.deliversNv12(); + // The two-step ladder this replaces topped out at 8 Mbit/s for anything + // 720p or larger. That was sized for a camera nobody had configured + // above 640x480; now that the capture runs at the camera's real + // resolution, 8 Mbit/s starves a 1440p or 2160p frame badly enough to + // undo the extra pixels. The tiers mirror the screen ladder above. const int webcamPixels = std::max(1, webcamCapture.width()) * std::max(1, webcamCapture.height()); - const int webcamBitrate = webcamPixels >= 1280 * 720 ? 8'000'000 : 4'000'000; + const int webcamBitrate = webcamPixels >= 3840 * 2160 ? 40'000'000 + : webcamPixels >= 2560 * 1440 ? 24'000'000 + : webcamPixels >= 1920 * 1080 ? 16'000'000 + : webcamPixels >= 1280 * 720 ? 8'000'000 + : 4'000'000; if (!webcamEncoder.initialize( utf8ToWide(config.webcamOutputPath), webcamCapture.width(), @@ -1175,7 +1167,7 @@ int wmain(int argc, wchar_t* argv[]) { WebcamFrameSnapshot candidateWebcamFrame; if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && candidateWebcamFrame.sequence != latestWebcamSequence && - hasVisibleBgraContent(candidateWebcamFrame.data)) { + hasVisibleWebcamContent(candidateWebcamFrame.data, webcamCapture.deliversNv12())) { latestWebcamFrame = std::move(candidateWebcamFrame.data); latestWebcamWidth = candidateWebcamFrame.width; latestWebcamHeight = candidateWebcamFrame.height; @@ -1230,7 +1222,15 @@ int wmain(int argc, wchar_t* argv[]) { // Capture the sample here, but submit it to the sink // writer OUTSIDE this block below (issue #115) so a // slow WriteSample can't hold up the next frame pull. - hasWebcamSample = webcamEncoder.captureBgraSample(webcamFrame, webcamTimestampHns, webcamSample); + hasWebcamSample = + webcamCapture.deliversNv12() + ? webcamEncoder.captureNv12Sample( + Nv12FrameView{ + webcamFrame.data, webcamFrame.width, webcamFrame.height}, + webcamTimestampHns, + webcamSample) + : webcamEncoder.captureBgraSample( + webcamFrame, webcamTimestampHns, webcamSample); if (!hasWebcamSample) { encodeFailed = true; control.requestStop(); @@ -1461,7 +1461,7 @@ int wmain(int argc, wchar_t* argv[]) { while (std::chrono::steady_clock::now() < webcamDeadline && !hasVisibleWebcamFrame) { WebcamFrameSnapshot candidateWebcamFrame; if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && - hasVisibleBgraContent(candidateWebcamFrame.data)) { + hasVisibleWebcamContent(candidateWebcamFrame.data, webcamCapture.deliversNv12())) { latestWebcamFrame = std::move(candidateWebcamFrame.data); latestWebcamWidth = candidateWebcamFrame.width; latestWebcamHeight = candidateWebcamFrame.height; diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index b4dabcc54..786881e00 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -690,6 +690,7 @@ bool MFEncoder::initialize( // attempt would eat the injection and the run would land on the plain CPU // encoder, never reaching the software encoder the knob is aimed at. useDxgiInput_ = options.useDxgiInput && !options.injectDefaultSinkWriterFailureOnce; + cpuInputIsNv12_ = options.cpuInputIsNv12; videoEncoderSelection_ = kVideoEncoderSelectionDefault; videoEncoderRuntime_ = kVideoEncoderRuntimeUnknown; @@ -725,9 +726,26 @@ bool MFEncoder::initialize( // type the RGB32 path would have produced from scratch. Every attribute // one mode sets is deleted by the other; nothing carries over. auto configureVideoInputType = [&](bool dxgi) { + // Three input shapes, not two: GPU NV12, system-memory NV12 (the + // webcam, whose camera hands us NV12 already) and system-memory RGB32. + const bool nv12 = dxgi || cpuInputIsNv12_; inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); - inputType->SetGUID(MF_MT_SUBTYPE, dxgi ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + inputType->SetGUID(MF_MT_SUBTYPE, nv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + if (!dxgi && cpuInputIsNv12_) { + // NV12's declared stride is the Y plane's, which is one byte per + // pixel -- not the four an RGB32 row needs. + inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, static_cast(width_)); + // A camera's NV12 is studio-range, and at these sizes BT.709. + // Untagged, the encoder and the player each fall back to their own + // default and the recording comes back with shifted colours. + inputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); + inputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); + setFrameSize(inputType.Get(), static_cast(width_), static_cast(height_)); + setFrameRate(inputType.Get(), static_cast(fps_)); + setPixelAspectRatio(inputType.Get()); + return; + } if (dxgi) { inputType->DeleteItem(MF_MT_DEFAULT_STRIDE); // The video processor below converts full-range BGRA into @@ -750,7 +768,7 @@ bool MFEncoder::initialize( // Carried on the H.264 type as well so the MP4 sink writes the matching // colour tags instead of leaving players to guess from the frame size. auto configureOutputColorTags = [&](bool dxgi) { - if (dxgi) { + if (dxgi || cpuInputIsNv12_) { outputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); outputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); } else { @@ -1110,12 +1128,16 @@ bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destinat } if (frame.width == width_ && frame.height == height_) { - for (DWORD i = 0; i < requiredBytes; i += 4) { - destination[i] = frame.data[i]; - destination[i + 1] = frame.data[i + 1]; - destination[i + 2] = frame.data[i + 2]; - destination[i + 3] = 255; - } + // One memcpy, not a per-pixel loop forcing alpha to 255. + // + // The loop this replaces ran once per BYTE: at 3840x2160 that is 8.3 + // million iterations per frame, which measured out at ~12 fps of real + // camera motion inside a file whose container claimed 30 -- the encoder + // padded the gap with duplicates. The alpha it was writing is dead + // weight anyway: this buffer feeds an H.264 encoder through + // MFVideoFormat_RGB32, and RGB-to-YUV conversion ignores the alpha + // channel entirely. + std::memcpy(destination, frame.data, requiredBytes); return true; } @@ -1649,6 +1671,56 @@ bool MFEncoder::captureVideoSample( return true; } +bool MFEncoder::captureNv12Sample( + const Nv12FrameView& frame, + int64_t timestampHns, + Microsoft::WRL::ComPtr& outSample) { + outSample.Reset(); + + if (!frame.data || frame.width != width_ || frame.height != height_) { + // No rescaler here on purpose: the webcam encoder is created with the + // capture's own dimensions, so a mismatch means a bug upstream rather + // than a frame worth stretching. + std::cerr << "ERROR: NV12 webcam frame does not match the encoder's size" << std::endl; + return false; + } + + const int64_t sampleDuration = 10'000'000LL / fps_; + const int64_t sampleTime = nextSampleTime(timestampHns, sampleDuration); + const DWORD frameBytes = static_cast(width_ * height_ * 3 / 2); + + Microsoft::WRL::ComPtr buffer; + if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer(webcam NV12)")) { + return false; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&data, &maxLength, ¤tLength), "IMFMediaBuffer::Lock(webcam NV12)")) { + return false; + } + if (maxLength < frameBytes) { + buffer->Unlock(); + std::cerr << "ERROR: Media Foundation webcam NV12 buffer is too small" << std::endl; + return false; + } + std::memcpy(data, frame.data, frameBytes); + buffer->Unlock(); + buffer->SetCurrentLength(frameBytes); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(webcam NV12)")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(sampleTime); + sample->SetSampleDuration(sampleDuration); + + outSample = sample; + return true; +} + bool MFEncoder::captureBgraSample( const BgraFrameView& frame, int64_t timestampHns, diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index 8d1d6ae6e..148693408 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -18,6 +18,21 @@ struct BgraFrameView { int height = 0; }; +/** + * A planar NV12 frame: a width*height Y plane followed by an interleaved + * width/2 * height/2 UV plane, so width*height*3/2 bytes in total. + * + * Used for the webcam, whose camera can hand Media Foundation NV12 directly. + * Asking that same camera for RGB32 made the source reader decode and convert + * every frame, which measured at 92ms per 4K frame -- a hard ceiling near 11 + * fps -- against 32ms for NV12. + */ +struct Nv12FrameView { + const BYTE* data = nullptr; + int width = 0; + int height = 0; +}; + struct AudioInputFormat { GUID subtype = MFAudioFormat_PCM; UINT32 sampleRate = 0; @@ -47,6 +62,14 @@ struct MFEncoderOptions { // driver that refuses shared keyed-mutex textures records exactly as it did // before the path existed. Ask usesDxgiInput() for what actually happened. bool useDxgiInput = false; + /** + * Feed this encoder NV12 from system memory instead of RGB32. + * + * Only meaningful when `useDxgiInput` is false. The webcam encoder sets it + * when the camera itself delivers NV12; the screen encoder's CPU path + * still produces BGRA and leaves it alone. + */ + bool cpuInputIsNv12 = false; }; constexpr const char* kVideoEncoderSelectionDefault = "default"; @@ -122,6 +145,10 @@ class MFEncoder { ID3D11Texture2D* texture, int64_t timestampHns, Microsoft::WRL::ComPtr& outSample); + bool captureNv12Sample( + const Nv12FrameView& frame, + int64_t timestampHns, + Microsoft::WRL::ComPtr& outSample); bool captureBgraSample( const BgraFrameView& frame, int64_t timestampHns, @@ -237,6 +264,7 @@ class MFEncoder { int64_t lastTimestampHns_ = -1; bool finalized_ = false; bool useDxgiInput_ = false; + bool cpuInputIsNv12_ = false; const char* videoEncoderSelection_ = kVideoEncoderSelectionDefault; const char* videoEncoderRuntime_ = kVideoEncoderRuntimeUnknown; const char* containerFormat_ = kContainerFormatMp4; diff --git a/electron/native/wgc-capture/src/webcam_capture.cpp b/electron/native/wgc-capture/src/webcam_capture.cpp index 6377a799b..27145e227 100644 --- a/electron/native/wgc-capture/src/webcam_capture.cpp +++ b/electron/native/wgc-capture/src/webcam_capture.cpp @@ -1,5 +1,7 @@ #include "webcam_capture.h" +#include "webcam_format.h" + #include #include #include @@ -153,7 +155,8 @@ bool WebcamCapture::initialize( const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps) { + int requestedFps, + bool preferNv12) { fps_ = std::clamp(requestedFps > 0 ? requestedFps : 30, 1, 60); usingDirectShow_ = false; selectedMatchScore_ = 0; @@ -196,7 +199,7 @@ bool WebcamCapture::initialize( return false; } - return configureReader(requestedWidth, requestedHeight, fps_); + return configureReader(requestedWidth, requestedHeight, fps_, preferNv12); } bool WebcamCapture::selectDevice(const std::wstring& deviceId, const std::wstring& deviceName) { @@ -252,7 +255,133 @@ bool WebcamCapture::selectDevice(const std::wstring& deviceId, const std::wstrin return succeeded(hr, "ActivateObject(webcam)"); } -bool WebcamCapture::configureReader(int requestedWidth, int requestedHeight, int requestedFps) { +namespace { + +/** + * Is this subtype something the source reader must DECODE before it can convert? + * + * Listed positively -- an unknown subtype counts as compressed -- because the + * cost of guessing wrong the other way is a camera that silently records + * nothing, while guessing wrong this way only costs resolution. + */ +bool isCompressedVideoSubtype(const GUID& subtype) { + static const GUID* const kUncompressed[] = { + &MFVideoFormat_NV12, + &MFVideoFormat_YUY2, + &MFVideoFormat_RGB32, + &MFVideoFormat_RGB24, + &MFVideoFormat_ARGB32, + &MFVideoFormat_UYVY, + &MFVideoFormat_YV12, + &MFVideoFormat_I420, + &MFVideoFormat_IYUV, + }; + for (const GUID* candidate : kUncompressed) { + if (subtype == *candidate) { + return false; + } + } + return true; +} + +/** + * Every video format the camera advertises, read off the source reader. + * + * MF_MT_FRAME_RATE is a ratio; a 30000/1001 entry is NTSC 29.97 and rounds to + * 30, which is what we want to compare against the requested rate. A type + * missing either attribute is skipped rather than guessed at -- it cannot be + * requested by size anyway. + */ +std::vector readNativeFormats(IMFSourceReader* reader) { + std::vector formats; + for (DWORD index = 0;; ++index) { + Microsoft::WRL::ComPtr nativeType; + const HRESULT hr = + reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &nativeType); + if (hr == MF_E_NO_MORE_TYPES || FAILED(hr)) { + break; + } + + UINT32 width = 0; + UINT32 height = 0; + if (FAILED(MFGetAttributeSize(nativeType.Get(), MF_MT_FRAME_SIZE, &width, &height))) { + continue; + } + + UINT32 numerator = 0; + UINT32 denominator = 0; + int fps = 0; + if (SUCCEEDED(MFGetAttributeRatio(nativeType.Get(), MF_MT_FRAME_RATE, &numerator, &denominator)) && + denominator > 0) { + fps = static_cast((numerator + denominator / 2) / denominator); + } + + GUID subtype{}; + const bool compressed = SUCCEEDED(nativeType->GetGUID(MF_MT_SUBTYPE, &subtype)) + ? isCompressedVideoSubtype(subtype) + : true; + + formats.push_back( + WebcamFormat{static_cast(width), static_cast(height), fps, compressed}); + } + return formats; +} + + +/** + * Puts the camera itself on `wanted` by selecting the matching native type. + * + * Best-effort: a driver that refuses leaves the reader on whatever it had, and + * the RGB32 request that follows still produces a usable -- if smaller -- frame. + */ +void selectNativeFormat(IMFSourceReader* reader, const WebcamFormat& wanted) { + for (DWORD index = 0;; ++index) { + Microsoft::WRL::ComPtr nativeType; + const HRESULT hr = + reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &nativeType); + if (hr == MF_E_NO_MORE_TYPES || FAILED(hr)) { + return; + } + + UINT32 width = 0; + UINT32 height = 0; + if (FAILED(MFGetAttributeSize(nativeType.Get(), MF_MT_FRAME_SIZE, &width, &height))) { + continue; + } + if (static_cast(width) != wanted.width || static_cast(height) != wanted.height) { + continue; + } + + GUID subtype{}; + if (FAILED(nativeType->GetGUID(MF_MT_SUBTYPE, &subtype)) || + isCompressedVideoSubtype(subtype) != wanted.compressed) { + continue; + } + + UINT32 numerator = 0; + UINT32 denominator = 0; + if (wanted.fps > 0 && + SUCCEEDED(MFGetAttributeRatio(nativeType.Get(), MF_MT_FRAME_RATE, &numerator, &denominator)) && + denominator > 0) { + const int fps = static_cast((numerator + denominator / 2) / denominator); + if (fps != wanted.fps) { + continue; + } + } + + if (SUCCEEDED(reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, nativeType.Get()))) { + return; + } + } +} + +} // namespace + +bool WebcamCapture::configureReader( + int requestedWidth, + int requestedHeight, + int requestedFps, + bool preferNv12) { Microsoft::WRL::ComPtr attributes; if (!succeeded(MFCreateAttributes(&attributes, 3), "MFCreateAttributes(webcam reader)")) { return false; @@ -274,15 +403,57 @@ bool WebcamCapture::configureReader(int requestedWidth, int requestedHeight, int return false; } mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); - if (requestedWidth > 0 && requestedHeight > 0) { - MFSetAttributeSize(mediaType.Get(), MF_MT_FRAME_SIZE, static_cast(requestedWidth), static_cast(requestedHeight)); - } + // NV12 when the frame only has to reach the webcam encoder, which wants + // NV12 anyway. Asking for RGB32 instead makes the source reader decode AND + // convert every frame: measured at 92ms per 3840x2160 frame against 32ms + // for NV12, which is the difference between 11 fps and the camera's full + // 30. BGRA is still used when the frame is composited into the screen + // recording inline, which needs it in that layout. + deliversNv12_ = preferNv12; + mediaType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + + // Pick the capture format, then SELECT it on the source before asking for + // RGB32 output. + // + // Both halves matter. Handing Media Foundation an output type with no + // MF_MT_FRAME_SIZE leaves the device on its default format -- 640x480 on a + // BRIO that offers 3840x2160 -- and that default is what the overlay was + // upscaling. But naming the size on the *output* type alone is not enough + // either: with MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING the reader answers + // S_OK by inserting a converter in front of whichever native type is + // already selected, so the camera keeps running at 640x480 and the size we + // asked for is quietly dropped. Selecting the native type first is what + // actually reconfigures the camera. + const WebcamFormat chosen = chooseWebcamFormat( + readNativeFormats(sourceReader_.Get()), + requestedWidth > 0 ? requestedWidth : 1920, + requestedHeight > 0 ? requestedHeight : 1080, + std::max(1, requestedFps)); + std::cerr << "INFO: Native webcam format " << chosen.width << "x" << chosen.height << "@" + << chosen.fps << (chosen.compressed ? " (compressed)" : " (uncompressed)") << std::endl; + selectNativeFormat(sourceReader_.Get(), chosen); + + MFSetAttributeSize( + mediaType.Get(), MF_MT_FRAME_SIZE, static_cast(chosen.width), static_cast(chosen.height)); MFSetAttributeRatio(mediaType.Get(), MF_MT_FRAME_RATE, static_cast(std::max(1, requestedFps)), 1); - if (!succeeded(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()), - "SetCurrentMediaType(webcam RGB32)")) { - return false; + if (FAILED(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()))) { + // Some drivers refuse a size they nonetheless enumerate. A slightly soft + // camera beats no camera, so drop the size and let the driver choose -- + // exactly what this code did before it asked for anything. + std::cerr << "WARNING: Webcam rejected " << chosen.width << "x" << chosen.height + << "; falling back to the device default format" << std::endl; + Microsoft::WRL::ComPtr fallbackType; + if (!succeeded(MFCreateMediaType(&fallbackType), "MFCreateMediaType(webcam fallback)")) { + return false; + } + fallbackType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + fallbackType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + if (!succeeded( + sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, fallbackType.Get()), + "SetCurrentMediaType(webcam RGB32)")) { + return false; + } } sourceReader_->SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS, FALSE); sourceReader_->SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM, TRUE); @@ -336,12 +507,14 @@ void WebcamCapture::stop() { void WebcamCapture::captureLoop() { CoInitializeEx(nullptr, COINIT_MULTITHREADED); + const auto loopStartedAt = std::chrono::steady_clock::now(); while (!stopRequested_) { DWORD streamIndex = 0; DWORD flags = 0; LONGLONG timestamp = 0; Microsoft::WRL::ComPtr sample; + const auto readStartedAt = std::chrono::steady_clock::now(); HRESULT hr = sourceReader_->ReadSample( MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, @@ -349,24 +522,45 @@ void WebcamCapture::captureLoop() { &flags, ×tamp, &sample); + readSampleUs_ += static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - readStartedAt) + .count()); (void)streamIndex; (void)timestamp; if (FAILED(hr)) { - std::cerr << "WARNING: Failed to read webcam sample (hr=0x" << std::hex << hr << std::dec << ")" - << std::endl; + // Counted, not printed: a reader that fails every call would other- + // wise write thousands of identical lines over a long take. + readFailures_ += 1; + lastReadFailure_ = hr; std::this_thread::sleep_for(std::chrono::milliseconds(20)); continue; } if ((flags & MF_SOURCE_READERF_ENDOFSTREAM) != 0) { + sawEndOfStream_ = true; break; } if (!sample) { + // Normal in small numbers (MF_SOURCE_READERF_STREAMTICK); a run of + // nothing else is a camera that never started producing. + emptySamples_ += 1; continue; } + // `Lock()` below is enough; row padding cannot reach `latestFrame_`. + // + // A 2-D media buffer may well pad each row out to a stride wider than + // the frame, but that padding is never what `Lock()` hands back: + // "The Lock method returns a buffer that is guaranteed to be + // contiguous. If the underlying buffer is not contiguous, the method + // copies the data into a new buffer" (IMF2DBuffer, Remarks). Reading + // the native pitch through IMF2DBuffer::Lock2D would only be worth it + // as a measured optimisation -- it saves that copy -- and mixing the + // two interfaces on one buffer is advised against in the same page. Microsoft::WRL::ComPtr buffer; if (FAILED(sample->ConvertToContiguousBuffer(&buffer)) || !buffer) { + bufferFailures_ += 1; continue; } @@ -374,19 +568,54 @@ void WebcamCapture::captureLoop() { DWORD maxLength = 0; DWORD currentLength = 0; if (FAILED(buffer->Lock(&data, &maxLength, ¤tLength)) || !data) { + bufferFailures_ += 1; continue; } - const DWORD expectedLength = static_cast(std::max(0, width_) * std::max(0, height_) * 4); + const int pixels = std::max(0, width_) * std::max(0, height_); + const DWORD expectedLength = + static_cast(deliversNv12_ ? pixels * 3 / 2 : pixels * 4); if (currentLength >= expectedLength && expectedLength > 0) { - std::scoped_lock lock(frameMutex_); - latestFrame_.assign(data, data + expectedLength); - latestFrameSequence_ += 1; + if (framesDelivered_ == 0) { + const auto waitedMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - loopStartedAt) + .count(); + std::cerr << "INFO: First webcam frame after " << waitedMs << "ms" << std::endl; + } + framesDelivered_ += 1; + const auto storeStartedAt = std::chrono::steady_clock::now(); + { + std::scoped_lock lock(frameMutex_); + latestFrame_.assign(data, data + expectedLength); + latestFrameSequence_ += 1; + } + storeUs_ += static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - storeStartedAt) + .count()); + } else if (shortBuffers_++, !reportedShortBuffer_) { + // Every frame arriving short means the reader is handing us something + // other than the RGB32 we asked for -- a compressed frame it never + // converted, most often. Silently dropped, that reads downstream as a + // camera that produced nothing at all, with no clue why. Said once, + // because it is true for every frame that follows. + reportedShortBuffer_ = true; + std::cerr << "WARNING: Webcam frame is " << currentLength << " bytes, expected " + << expectedLength << " for " << width_ << "x" << height_ << " " + << (deliversNv12_ ? "NV12" : "RGB32") << "; dropping frames" << std::endl; } buffer->Unlock(); } + std::cerr << "INFO: Webcam capture loop ended: delivered=" << framesDelivered_ + << " emptySamples=" << emptySamples_ << " shortBuffers=" << shortBuffers_ + << " bufferFailures=" << bufferFailures_ << " readFailures=" << readFailures_ + << " lastReadHr=0x" << std::hex << lastReadFailure_ << std::dec + << " endOfStream=" << (sawEndOfStream_ ? "yes" : "no") + << " readSampleMs=" << (readSampleUs_ / 1000) << " storeMs=" << (storeUs_ / 1000) + << std::endl; + CoUninitialize(); } @@ -406,6 +635,10 @@ bool WebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { return true; } +bool WebcamCapture::deliversNv12() const { + return !usingDirectShow_ && deliversNv12_; +} + int WebcamCapture::width() const { if (usingDirectShow_) { return directShowCapture_.width(); diff --git a/electron/native/wgc-capture/src/webcam_capture.h b/electron/native/wgc-capture/src/webcam_capture.h index 5b61aa6b9..612e36603 100644 --- a/electron/native/wgc-capture/src/webcam_capture.h +++ b/electron/native/wgc-capture/src/webcam_capture.h @@ -28,7 +28,8 @@ class WebcamCapture { const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps); + int requestedFps, + bool preferNv12); bool start(); void stop(); bool copyLatestFrame(WebcamFrameSnapshot& destination); @@ -36,11 +37,19 @@ class WebcamCapture { int width() const; int height() const; int fps() const; + /** + * Do the frames from `copyLatestFrame` carry NV12 rather than BGRA? + * + * Always false on the DirectShow fallback, which unpacks whatever the + * camera speaks into BGRA itself. Callers must ask rather than assume: + * the two layouts differ in size as well as in meaning. + */ + bool deliversNv12() const; const std::wstring& selectedDeviceName() const; private: bool selectDevice(const std::wstring& deviceId, const std::wstring& deviceName); - bool configureReader(int requestedWidth, int requestedHeight, int requestedFps); + bool configureReader(int requestedWidth, int requestedHeight, int requestedFps, bool preferNv12); void captureLoop(); Microsoft::WRL::ComPtr mediaSource_; @@ -56,6 +65,27 @@ class WebcamCapture { int fps_ = 30; bool mfStarted_ = false; bool usingDirectShow_ = false; + bool deliversNv12_ = false; + /** Latches the short-buffer warning so it is said once, not 30x a second. */ + bool reportedShortBuffer_ = false; + /** + * Why the capture loop did or did not produce frames. + * + * A camera that delivers nothing used to surface only as + * MF_E_SINK_NO_SAMPLES_PROCESSED from the encoder's Finalize -- the one + * place that cannot say which of the loop's five silent `continue` paths + * swallowed the frames. These are reported once when the loop ends. + */ + uint64_t framesDelivered_ = 0; + uint64_t emptySamples_ = 0; + uint64_t readFailures_ = 0; + uint64_t shortBuffers_ = 0; + uint64_t bufferFailures_ = 0; + bool sawEndOfStream_ = false; + HRESULT lastReadFailure_ = S_OK; + /** Where the loop's wall time goes, in microseconds. */ + uint64_t readSampleUs_ = 0; + uint64_t storeUs_ = 0; int selectedMatchScore_ = 0; std::wstring selectedDeviceName_; }; diff --git a/electron/native/wgc-capture/src/webcam_format.cpp b/electron/native/wgc-capture/src/webcam_format.cpp new file mode 100644 index 000000000..32e2020ed --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format.cpp @@ -0,0 +1,82 @@ +#include "webcam_format.h" + +#include + +namespace { + +bool isUsable(const WebcamFormat& format) { + return format.width > 0 && format.height > 0; +} + +bool fitsTarget(const WebcamFormat& format, int targetWidth, int targetHeight) { + return format.width <= targetWidth && format.height <= targetHeight; +} + +long long pixels(const WebcamFormat& format) { + return static_cast(format.width) * static_cast(format.height); +} + +// True when `candidate` is a better frame rate than `best` for a target of +// `targetFps`: the lowest rate that still reaches the target, or -- when no +// mode reaches it -- the highest one on offer. +bool betterFps(int candidate, int best, int targetFps) { + const bool candidateReaches = candidate >= targetFps; + const bool bestReaches = best >= targetFps; + if (candidateReaches != bestReaches) { + return candidateReaches; + } + return candidateReaches ? candidate < best : candidate > best; +} + +} // namespace + +WebcamFormat chooseWebcamFormat( + const std::vector& available, + int targetWidth, + int targetHeight, + int targetFps) { + const WebcamFormat target{targetWidth, targetHeight, targetFps, false}; + + // Compressed modes are a last resort, so they are only considered when the + // camera offers nothing else at all. + const bool anyUncompressed = std::any_of(available.begin(), available.end(), [](const WebcamFormat& format) { + return isUsable(format) && !format.compressed; + }); + + const auto eligible = [&](const WebcamFormat& format) { + return isUsable(format) && !(anyUncompressed && format.compressed); + }; + + const bool anyFits = std::any_of(available.begin(), available.end(), [&](const WebcamFormat& format) { + return eligible(format) && fitsTarget(format, targetWidth, targetHeight); + }); + + const WebcamFormat* best = nullptr; + for (const WebcamFormat& format : available) { + if (!eligible(format)) { + continue; + } + // Once anything fits, oversized modes are out of the running entirely. + // With nothing fitting we rank the oversized modes instead, smallest + // first, so the scaler has the least work to do. + if (anyFits != fitsTarget(format, targetWidth, targetHeight)) { + continue; + } + if (best == nullptr) { + best = &format; + continue; + } + if (pixels(format) != pixels(*best)) { + const bool wins = anyFits ? pixels(format) > pixels(*best) : pixels(format) < pixels(*best); + if (wins) { + best = &format; + } + continue; + } + if (betterFps(format.fps, best->fps, targetFps)) { + best = &format; + } + } + + return best ? *best : target; +} diff --git a/electron/native/wgc-capture/src/webcam_format.h b/electron/native/wgc-capture/src/webcam_format.h new file mode 100644 index 000000000..7cb329af7 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +// One capture format a camera advertises. +struct WebcamFormat { + int width = 0; + int height = 0; + int fps = 0; + /** + * True for MJPEG/H264 modes, which reach higher resolutions than a camera's + * uncompressed ones but need a decoder MFT in front of the RGB32 conversion. + */ + bool compressed = false; +}; + +// Picks the format to drive the camera at, given everything it advertises. +// +// Both capture backends used to leave this to the driver: Media Foundation was +// handed a media type with no MF_MT_FRAME_SIZE, and the DirectShow graph was +// rendered without touching IAMStreamConfig. In both cases the device answers +// with its *default* type, and for a UVC camera that is the first format it +// enumerates -- 640x480 even on a BRIO that offers 3840x2160. The recording was +// then upscaled into the overlay, which is the pixelation users reported. +// +// Rules, in order: +// * uncompressed modes beat compressed ones outright, whatever their size. +// Pinning a camera's MJPEG mode and asking the source reader for RGB32 +// leaves it delivering no samples at all on some hosts -- a recording with +// no camera in it, which is far worse than a smaller one that works; +// * only formats that fit inside the target in BOTH dimensions are eligible, +// so an ultrawide mode cannot sneak in on pixel count alone; +// * among those, the most pixels wins; +// * ties go to the lowest frame rate that still reaches the target, else the +// highest available -- a 1080p60 mode is no help when we encode at 30; +// * if nothing fits (a camera that only does 4K), the smallest mode wins, so +// we scale down rather than push 1 GB/s of RGB32 through the encoder; +// * an empty list returns the target unchanged, letting the caller fall back +// to whatever the driver would have picked on its own. +WebcamFormat chooseWebcamFormat( + const std::vector& available, + int targetWidth, + int targetHeight, + int targetFps); diff --git a/electron/native/wgc-capture/src/webcam_format_test.cpp b/electron/native/wgc-capture/src/webcam_format_test.cpp new file mode 100644 index 000000000..9e40f5cbf --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format_test.cpp @@ -0,0 +1,131 @@ +#include "webcam_format.h" + +#include +#include + +namespace { + +int failures = 0; + +void expectFormat(const std::string& label, WebcamFormat actual, int width, int height, int fps) { + if (actual.width == width && actual.height == height && actual.fps == fps) { + return; + } + std::printf( + "FAIL %s: expected %dx%d@%d, got %dx%d@%d\n", + label.c_str(), + width, + height, + fps, + actual.width, + actual.height, + actual.fps); + ++failures; +} + +// Every mode a Logitech BRIO advertises that this helper could drive, as read +// off the device with `ffmpeg -f dshow -list_options true`. +const std::vector kBrio = { + {640, 480, 30, false}, {640, 360, 30, false}, {1280, 720, 30, false}, + {1920, 1080, 30, false}, {1280, 720, 90, true}, {1920, 1080, 60, true}, + {2560, 1440, 30, true}, {3840, 2160, 30, true}, {160, 120, 30, false}, + {340, 340, 30, false}, {1600, 896, 30, false}, {1024, 576, 30, false}, +}; + +} // namespace + +int main() { + // The regression this file exists for: a BRIO asked for 1080p must not come + // back with the 640x480 default it enumerates first. + expectFormat("brio targets 1080p", chooseWebcamFormat(kBrio, 1920, 1080, 30), 1920, 1080, 30); + + // Asked for more than the camera can do uncompressed, it must stop at the + // best uncompressed mode rather than pin an MJPEG one. Pinning MJPEG and + // asking the source reader for RGB32 produced a reader that delivered no + // samples at all, so the take had no camera in it. + expectFormat( + "4K request settles for the best uncompressed mode", + chooseWebcamFormat(kBrio, 3840, 2160, 30), + 1920, + 1080, + 30); + if (chooseWebcamFormat(kBrio, 3840, 2160, 30).compressed) { + std::printf("FAIL 4K request must not pin a compressed mode\n"); + ++failures; + } + + // A camera with nothing but compressed modes still has to be usable. + expectFormat( + "compressed-only camera is still driven", + chooseWebcamFormat({{1920, 1080, 30, true}, {1280, 720, 30, true}}, 1920, 1080, 30), + 1920, + 1080, + 30); + + // Uncompressed wins even when it is much smaller than a compressed option. + expectFormat( + "uncompressed beats a larger compressed mode", + chooseWebcamFormat({{3840, 2160, 30, true}, {640, 480, 30, false}}, 3840, 2160, 30), + 640, + 480, + 30); + + // 1080p60 exists too; at a 30 fps encode the extra frames only cost bandwidth. + expectFormat( + "prefers the frame rate we encode at", + chooseWebcamFormat({{1920, 1080, 60}, {1920, 1080, 30}}, 1920, 1080, 30), + 1920, + 1080, + 30); + + // Nothing reaches 30, so take the best the camera can do rather than fail. + expectFormat( + "falls back to the highest rate offered", + chooseWebcamFormat({{1920, 1080, 15}, {1920, 1080, 24}}, 1920, 1080, 30), + 1920, + 1080, + 24); + + // A 720p-only camera keeps working, just smaller. + expectFormat( + "720p camera stays 720p", + chooseWebcamFormat({{1280, 720, 30}, {640, 480, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + // 2560x800 has fewer pixels than 1920x1080 but is far wider; picking it + // would letterbox the overlay and cost a needless downscale. + expectFormat( + "rejects an oversized dimension even when pixel count fits", + chooseWebcamFormat({{2560, 800, 30}, {1280, 720, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + // Only modes above the target: scale down from the cheapest of them. + expectFormat( + "oversized-only camera takes the smallest mode", + chooseWebcamFormat({{3840, 2160, 30}, {2560, 1440, 30}}, 1920, 1080, 30), + 2560, + 1440, + 30); + + // Nothing to choose from: hand the target back and let the driver decide. + expectFormat("empty list returns the target", chooseWebcamFormat({}, 1920, 1080, 30), 1920, 1080, 30); + + // Drivers do report junk; it must not win. + expectFormat( + "ignores degenerate entries", + chooseWebcamFormat({{0, 0, 30}, {-1920, 1080, 30}, {1280, 720, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + if (failures == 0) { + std::printf("webcam_format_test: all assertions passed\n"); + return 0; + } + std::printf("webcam_format_test: %d assertion(s) failed\n", failures); + return 1; +} diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs index 8d4cfc018..219d03b08 100644 --- a/scripts/build-windows-wgc-helper.mjs +++ b/scripts/build-windows-wgc-helper.mjs @@ -105,3 +105,23 @@ if (!fs.existsSync(audioUtilsTestPath)) { // Pack) instead of failing this packaging command. await run(audioUtilsTestPath, [], { cwd: BUILD_DIR }); console.log(`Passed ${audioUtilsTestPath}`); + +const webcamFormatTestPath = path.join(BUILD_DIR, "webcam_format_test.exe"); +if (!fs.existsSync(webcamFormatTestPath)) { + throw new Error(`WGC helper build completed but ${webcamFormatTestPath} was not found.`); +} +// Guards the capture resolution the camera is driven at. Left unpinned, both +// backends fall back to the device default -- 640x480 on hardware that offers +// far more -- and the overlay upscales it. +await run(webcamFormatTestPath, [], { cwd: BUILD_DIR }); +console.log(`Passed ${webcamFormatTestPath}`); + +const frameVisibilityTestPath = path.join(BUILD_DIR, "frame_visibility_test.exe"); +if (!fs.existsSync(frameVisibilityTestPath)) { + throw new Error(`WGC helper build completed but ${frameVisibilityTestPath} was not found.`); +} +// Guards the warm-up probe that decides whether the camera has produced a +// picture yet. Studio-range black is 16, not 0, so an unnormalised average +// reads every black frame as content. +await run(frameVisibilityTestPath, [], { cwd: BUILD_DIR }); +console.log(`Passed ${frameVisibilityTestPath}`); diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs index 785e48c1e..2bca55bd7 100644 --- a/scripts/test-windows-wgc-helper.mjs +++ b/scripts/test-windows-wgc-helper.mjs @@ -490,9 +490,13 @@ const config = { webcamDirectShowClsid: resolveDirectShowWebcamClsid( process.env.OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME ?? "", ), - webcamWidth: 640, - webcamHeight: 360, - webcamFps: 30, + // DEFAULT_WEBCAM_QUALITY from src/hooks/webcamCaptureTarget.ts -- the target the + // app actually sends, so this exercises the format negotiation rather than a + // size no shipped pipeline ever asks for. A camera that cannot reach it is + // driven at its own best format, which is the case worth covering anyway. + webcamWidth: Number(process.env.OPENSCREEN_WGC_TEST_WEBCAM_WIDTH ?? 3840), + webcamHeight: Number(process.env.OPENSCREEN_WGC_TEST_WEBCAM_HEIGHT ?? 2160), + webcamFps: Number(process.env.OPENSCREEN_WGC_TEST_WEBCAM_FPS ?? 30), outputs: { screenPath: outputPath, ...(webcamOutputPath ? { webcamPath: webcamOutputPath } : {}), @@ -631,9 +635,23 @@ const encoderSelectionLine = result.stdout .split(/\r?\n/) .find((line) => line.includes('"event":"encoder-selection"')); const encoderSelection = encoderSelectionLine ? JSON.parse(encoderSelectionLine) : null; -const nativeWebcamDiagnostics = result.stderr - .split(/\r?\n/) - .filter((line) => line.includes("Native webcam candidate")); +const nativeWebcamDiagnostics = result.stderr.split(/\r?\n/).filter( + (line) => + line.includes("Native webcam candidate") || + // Which capture format the camera was actually driven at. Without this + // the smoke test could pass on a 640x480 take from a 4K camera and say + // nothing about it. + line.includes("Native webcam format") || + line.includes("DirectShow webcam format") || + line.includes("DirectShow webcam connected") || + line.includes("falling back to the device default") || + // How long the camera took to produce its first frame, and the tally of what + // the capture loop did with everything it read. Without these, a camera that + // delivers nothing is only visible as a failed Finalize, which cannot say why. + line.includes("First webcam frame") || + line.includes("Webcam capture loop ended") || + line.includes("Webcam frame is"), +); const nativeMicrophoneDiagnostics = result.stderr .split(/\r?\n/) .filter( diff --git a/src/components/ai-edition/v4/RecStage.test.tsx b/src/components/ai-edition/v4/RecStage.test.tsx index f931783aa..0f7a5696c 100644 --- a/src/components/ai-edition/v4/RecStage.test.tsx +++ b/src/components/ai-edition/v4/RecStage.test.tsx @@ -224,6 +224,7 @@ describe("RecStage controls", () => { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: "2160p", systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, @@ -424,6 +425,7 @@ describe("RecStage controls", () => { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: "2160p", systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, diff --git a/src/components/launch/HudDeviceSettings.quality.test.tsx b/src/components/launch/HudDeviceSettings.quality.test.tsx new file mode 100644 index 000000000..67db4a267 --- /dev/null +++ b/src/components/launch/HudDeviceSettings.quality.test.tsx @@ -0,0 +1,97 @@ +// @vitest-environment jsdom +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import { WEBCAM_QUALITY_IDS } from "../../hooks/webcamCaptureTarget"; +import { HudDeviceSettings, type HudDeviceSettingsLabels } from "./HudDeviceSettings"; + +vi.mock("../../hooks/useAudioLevelMeter", () => ({ + useAudioLevelMeter: () => ({ level: 0 }), +})); +vi.mock("../../hooks/useCameraPreviewStream", () => ({ + useCameraPreviewStream: () => ({ stream: null, error: null }), +})); + +const labels: HudDeviceSettingsLabels = { + title: "Device settings", + done: "Done", + microphone: "Microphone", + camera: "Camera", + micLevel: "Input level", + micHint: "Speak to check", + noMicrophones: "No microphone found", + searching: "Searching...", + noCameras: "No camera found", + cameraUnavailable: "Camera unavailable", + preview: "Preview", + previewUnavailable: "Preview unavailable", + about: "About", + checkForUpdates: "Check for updates", + checkingForUpdates: "Checking…", + cameraQuality: "Camera quality", + cameraQualityOptions: { + "1080p": "1080p", + "1440p": "1440p", + "2160p": "4K", + }, +}; + +function renderPanel(overrides: Partial[0]> = {}) { + const onSelectCameraQuality = vi.fn(); + render( + undefined} + {...overrides} + />, + ); + return { onSelectCameraQuality }; +} + +describe("HudDeviceSettings camera quality", () => { + it("offers every preset the capture pipeline knows about", () => { + renderPanel(); + + for (const id of WEBCAM_QUALITY_IDS) { + expect(screen.getByTestId(`camera-quality-${id}`)).toBeTruthy(); + } + }); + + it("marks the stored choice as the checked one", () => { + renderPanel(); + + expect(screen.getByTestId("camera-quality-1440p").getAttribute("aria-checked")).toBe("true"); + expect(screen.getByTestId("camera-quality-2160p").getAttribute("aria-checked")).toBe("false"); + }); + + it("reports the picked preset so it can be persisted", () => { + const { onSelectCameraQuality } = renderPanel(); + + fireEvent.click(screen.getByTestId("camera-quality-2160p")); + + expect(onSelectCameraQuality).toHaveBeenCalledWith("2160p"); + }); + + it("hides the choice when there is no camera to apply it to", () => { + // A resolution picker above "No camera found" is a control that cannot do + // anything, and it pushes the real message out of view. + renderPanel({ cameraDevices: [], activeCameraId: undefined }); + + expect(screen.queryByTestId("camera-quality-2160p")).toBeNull(); + }); +}); diff --git a/src/components/launch/HudDeviceSettings.tsx b/src/components/launch/HudDeviceSettings.tsx index d807f752b..3bf43df74 100644 --- a/src/components/launch/HudDeviceSettings.tsx +++ b/src/components/launch/HudDeviceSettings.tsx @@ -4,6 +4,7 @@ import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter"; import type { CameraDevice } from "../../hooks/useCameraDevices"; import { useCameraPreviewStream } from "../../hooks/useCameraPreviewStream"; import type { MicrophoneDevice } from "../../hooks/useMicrophoneDevices"; +import { WEBCAM_QUALITY_IDS, type WebcamQualityId } from "../../hooks/webcamCaptureTarget"; import styles from "./LaunchWindow.module.css"; const LEVEL_SEGMENTS = 12; @@ -25,6 +26,8 @@ export interface HudDeviceSettingsLabels { about: string; checkForUpdates: string; checkingForUpdates: string; + cameraQuality: string; + cameraQualityOptions: Record; } /** Segmented input-level bar, driven by the live analyser. */ @@ -96,6 +99,8 @@ export const HudDeviceSettings = memo(function HudDeviceSettings({ versionLabel, canCheckForUpdates, checkingForUpdates, + cameraQuality, + onSelectCameraQuality, onSelectMic, onSelectCamera, onCheckForUpdates, @@ -116,6 +121,8 @@ export const HudDeviceSettings = memo(function HudDeviceSettings({ versionLabel: string | null; canCheckForUpdates: boolean; checkingForUpdates: boolean; + cameraQuality: WebcamQualityId; + onSelectCameraQuality: (quality: WebcamQualityId) => void; onSelectMic: (device: MicrophoneDevice) => void; onSelectCamera: (device: CameraDevice) => void; onCheckForUpdates: () => void; @@ -214,6 +221,29 @@ export const HudDeviceSettings = memo(function HudDeviceSettings({ )} {hasCamera ? ( <> + {/* Below the device list, because it qualifies the camera picked + above. Hidden with no camera present -- a resolution control + over "No camera found" cannot do anything. */} +
+ {labels.cameraQuality} +
+ {WEBCAM_QUALITY_IDS.map((quality) => { + const isActive = quality === cameraQuality; + return ( + + ); + })}
{labels.preview}
diff --git a/src/components/launch/LaunchWindow.test.tsx b/src/components/launch/LaunchWindow.test.tsx index ae4484864..6af724a8e 100644 --- a/src/components/launch/LaunchWindow.test.tsx +++ b/src/components/launch/LaunchWindow.test.tsx @@ -59,6 +59,8 @@ const recorderState = vi.hoisted(() => ({ webcamDeviceId: undefined, setWebcamDeviceId: vi.fn(), setWebcamDeviceName: vi.fn(), + webcamQuality: "2160p", + setWebcamQuality: vi.fn(), systemAudioEnabled: false, setSystemAudioEnabled: vi.fn(), cursorCaptureMode: "editable-overlay", @@ -96,11 +98,15 @@ vi.mock("../../hooks/useMicrophoneDevices", () => ({ const cameraDevicesState = vi.hoisted(() => ({ isReady: true, + // Empty by default, as before. The device-settings panel only renders its + // camera controls when a camera exists, so tests that reach for them fill + // this in. + devices: [] as Array<{ deviceId: string; label: string }>, })); vi.mock("../../hooks/useCameraDevices", () => ({ useCameraDevices: () => ({ - devices: [], + devices: cameraDevicesState.devices, selectedDeviceId: "", setSelectedDeviceId: vi.fn(), isLoading: false, @@ -320,6 +326,7 @@ function resetLaunchMocks() { recorderState.value.setWebcamEnabled.mockClear(); recorderState.value.recordingPrefsLoaded = true; cameraDevicesState.isReady = true; + cameraDevicesState.devices = []; micDevicesState.value = []; micDevicesState.enabled = undefined; systemVersionState.value = "15.5"; @@ -1437,6 +1444,29 @@ describe("LaunchWindow device settings", () => { expect(within(panel).getByText("Version 1.9.6")).toBeInTheDocument(); }); + // Changing the capture resolution re-runs the webcam acquisition effect, whose cleanup stops + // every track of the live stream — the same stream the browser, macOS and Linux paths hand to + // the webcam MediaRecorder. Mid-take that ends the camera partway through, without a word. + it("ignores a camera quality change made in a panel left open by a recording", async () => { + cameraDevicesState.devices = [{ deviceId: "cam-1", label: "Logitech BRIO" }]; + const { rerender } = renderLaunchWindow(); + + fireEvent.click(await screen.findByTestId("launch-device-settings-button")); + const panel = await screen.findByTestId("hud-device-settings"); + + recorderState.value.recording = true; + rerender( + + + , + ); + + fireEvent.click(within(panel).getByTestId("camera-quality-1080p")); + + expect(recorderState.value.setWebcamQuality).not.toHaveBeenCalled(); + expect(window.electronAPI.setRecordingPrefs).not.toHaveBeenCalled(); + }); + it("is unavailable while recording, when devices can't be changed anyway", async () => { recorderState.value.recording = true; diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index b7fcc9715..e72ea2b25 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -13,6 +13,7 @@ import { } from "../../hooks/useMicrophoneDevices"; import { usePortalOwnsSource } from "../../hooks/usePortalOwnsSource"; import { useScreenRecorder } from "../../hooks/useScreenRecorder"; +import type { WebcamQualityId } from "../../hooks/webcamCaptureTarget"; import { requestCameraAccess } from "../../lib/requestCameraAccess"; import { HudCameraButton, @@ -111,6 +112,8 @@ export function LaunchWindow() { setWebcamEnabled, webcamDeviceId, setWebcamDeviceId, + webcamQuality, + setWebcamQuality, webcamDeviceName, setWebcamDeviceName, cursorCaptureMode, @@ -862,6 +865,7 @@ export function LaunchWindow() { camEnabled?: boolean; camDeviceId?: string; camDeviceName?: string; + camQuality?: WebcamQualityId; micEnabled?: boolean; micDeviceId?: string; micDeviceName?: string; @@ -927,6 +931,22 @@ export function LaunchWindow() { [persistRecordingPrefs, setSelectedCameraId, setWebcamDeviceId, setWebcamDeviceName], ); + const handleSelectCameraQuality = useCallback( + (quality: WebcamQualityId) => { + // The same guard the other controls carry. The gear is disabled mid-take, + // but a settings panel already open stays mounted, so this stays + // clickable. `webcamQuality` is a dependency of the webcam acquisition + // effect: changing it re-runs that effect's cleanup, which stops every + // track of the live stream -- the very stream the browser, macOS and + // Linux paths are recording. The camera would end partway through the + // take, silently. + if (controlsLocked) return; + setWebcamQuality(quality); + persistRecordingPrefs({ camQuality: quality }); + }, + [controlsLocked, persistRecordingPrefs, setWebcamQuality], + ); + const toggleDeviceSettings = useCallback(() => { if (controlsLocked) return; setIsLanguageMenuOpen(false); @@ -1068,6 +1088,12 @@ export function LaunchWindow() { about: t("deviceSettings.about"), checkForUpdates: tCommon("actions.checkForUpdates"), checkingForUpdates: t("deviceSettings.checkingForUpdates"), + cameraQuality: t("webcam.quality"), + cameraQualityOptions: { + "1080p": t("webcam.quality1080p"), + "1440p": t("webcam.quality1440p"), + "2160p": t("webcam.quality2160p"), + }, }), [t, tCommon], ); @@ -1282,6 +1308,8 @@ export function LaunchWindow() { // main process refuses the check then — an offered button would be dead. canCheckForUpdates={(appInfo?.canCheckForUpdates ?? false) && !recording} checkingForUpdates={isCheckingForUpdates} + cameraQuality={webcamQuality} + onSelectCameraQuality={handleSelectCameraQuality} onSelectMic={handleSelectMicDevice} onSelectCamera={handleSelectCameraDevice} onCheckForUpdates={handleCheckForUpdates} diff --git a/src/hooks/useCameraPreviewStream.ts b/src/hooks/useCameraPreviewStream.ts index 9080e7c8d..9e04c0558 100644 --- a/src/hooks/useCameraPreviewStream.ts +++ b/src/hooks/useCameraPreviewStream.ts @@ -1,4 +1,5 @@ import { useEffect, useRef, useState } from "react"; +import { webcamVideoConstraints } from "./webcamCaptureTarget"; export interface CameraPreviewStreamOptions { enabled: boolean; @@ -28,7 +29,13 @@ export function useCameraPreviewStream({ enabled, deviceId }: CameraPreviewStrea let cancelled = false; navigator.mediaDevices .getUserMedia({ - video: deviceId ? { deviceId: { exact: deviceId } } : true, + // Pinned to the smallest preset, not to whatever the recording is set + // to. Left unconstrained this opened at 640x480 and made the camera + // look as soft in the HUD as it did in the take; driving it at the + // user's 4K choice would be the opposite mistake, since this renders + // into a thumbnail a couple of hundred pixels wide and the recorder + // opens its own stream anyway. + video: webcamVideoConstraints(deviceId, "1080p"), audio: false, }) .then((s) => { diff --git a/src/hooks/useScreenRecorder.prefsRace.test.tsx b/src/hooks/useScreenRecorder.prefsRace.test.tsx index 8c06cfc79..ae84c2cd7 100644 --- a/src/hooks/useScreenRecorder.prefsRace.test.tsx +++ b/src/hooks/useScreenRecorder.prefsRace.test.tsx @@ -25,6 +25,7 @@ function prefs(micEnabled: boolean): RecordingPrefs { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: "2160p", systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 3e036e46e..c827dd985 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -21,6 +21,15 @@ import { requestCameraAccess } from "@/lib/requestCameraAccess"; import { loadUserPreferences, saveUserPreferences } from "@/lib/userPreferences"; import { canRecordMicrophone } from "@/utils/platformUtils"; import { createRecorderHandle, type RecorderHandle } from "./recorderHandle"; +import { + DEFAULT_WEBCAM_QUALITY, + WEBCAM_TARGET_FRAME_RATE, + type WebcamQualityId, + webcamBitrateForStream, + webcamPresetFor, + webcamQualityFrom, + webcamVideoConstraints, +} from "./webcamCaptureTarget"; import { webcamDeviceIdentityFrom } from "./webcamDeviceIdentity"; const TARGET_FRAME_RATE = 60; @@ -77,8 +86,6 @@ function effectiveBrowserCursorMode( const AUDIO_BITRATE_VOICE = 128_000; const AUDIO_BITRATE_SYSTEM = 192_000; -const WEBCAM_TARGET_FRAME_RATE = 30; - type UseScreenRecorderReturn = { recording: boolean; paused: boolean; @@ -99,6 +106,8 @@ type UseScreenRecorderReturn = { setMicrophoneDeviceName: (deviceName: string | undefined) => void; webcamDeviceId: string | undefined; setWebcamDeviceId: (deviceId: string | undefined) => void; + webcamQuality: WebcamQualityId; + setWebcamQuality: (quality: WebcamQualityId) => void; webcamDeviceName: string | undefined; setWebcamDeviceName: (deviceName: string | undefined) => void; systemAudioEnabled: boolean; @@ -257,6 +266,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const [microphoneDeviceId, setMicrophoneDeviceId] = useState(undefined); const [microphoneDeviceName, setMicrophoneDeviceName] = useState(undefined); const [webcamDeviceId, setWebcamDeviceId] = useState(undefined); + const [webcamQuality, setWebcamQuality] = useState(DEFAULT_WEBCAM_QUALITY); const [webcamDeviceName, setWebcamDeviceName] = useState(undefined); const [systemAudioEnabled, setSystemAudioEnabled] = useState(false); const [webcamEnabled, setWebcamEnabledState] = useState(false); @@ -281,6 +291,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { camEnabled: boolean; camDeviceId?: string | null; camDeviceName?: string | null; + camQuality?: WebcamQualityId | null; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; }) => { @@ -298,6 +309,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setWebcamDeviceId(prefs.camDeviceId ?? undefined); setWebcamDeviceName(prefs.camDeviceName ?? undefined); } + setWebcamQuality(webcamQualityFrom(prefs.camQuality)); setSystemAudioEnabled(prefs.systemAudioEnabled); setCursorCaptureMode(prefs.cursorCaptureMode); setRecordingPrefsLoaded(true); @@ -366,6 +378,19 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return accumulatedDurationMs.current + segmentDuration; }, []); + /** + * Recorder options for a webcam sidecar. + * + * The bitrate comes from the camera's own frame, never from the screen + * recording. Repeated inline at each call site, that rule held in two of the + * three and left the browser pipeline encoding a 2160p camera at the + * monitor's rate, capped well below what the frame needs. + */ + const webcamRecorderOptions = (stream: MediaStream | null): MediaRecorderOptions => ({ + mimeType: selectMimeType(), + videoBitsPerSecond: webcamBitrateForStream(stream), + }); + const selectMimeType = () => { // H.264 first: hardware-accelerated, so sharp real-time output. AV1/VP9 are // better for distribution but too CPU-heavy for live 60 fps capture (software @@ -480,14 +505,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { try { const stream = await navigator.mediaDevices.getUserMedia({ audio: false, - video: webcamDeviceId - ? { - deviceId: { exact: webcamDeviceId }, - frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, - } - : { - frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, - }, + video: webcamVideoConstraints(webcamDeviceId, webcamQuality), }); if (cancelled || thisAcquireId !== webcamAcquireId.current) { @@ -545,7 +563,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamStream.current = null; } }; - }, [webcamEnabled, webcamDeviceId, webcamDeviceName, t]); + }, [webcamEnabled, webcamDeviceId, webcamDeviceName, webcamQuality, t]); const finalizeRecording = useCallback( ( @@ -1240,8 +1258,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { enabled: webcamEnabled, deviceId: webcamIdentity.deviceId, deviceName: webcamIdentity.deviceName, - width: 0, - height: 0, + width: webcamPresetFor(webcamQuality).width, + height: webcamPresetFor(webcamQuality).height, fps: WEBCAM_TARGET_FRAME_RATE, }, cursor: { @@ -1356,10 +1374,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // recordingId we send here, so this name is the one finalize rebuilds. nativeWebcamRecorder = createRecorderHandle( webcamStream.current, - { - mimeType: selectMimeType(), - videoBitsPerSecond: BITRATE_BASE, - }, + webcamRecorderOptions(webcamStream.current), `${RECORDING_FILE_PREFIX}${activeRecordingId}${WEBCAM_FILE_SUFFIX}${VIDEO_FILE_EXTENSION}`, ); } else { @@ -1403,8 +1418,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // Same pairing rule as the Windows path; here the stream is still // open, so the identity can be read at the point of use. ...readWebcamDeviceIdentity(), - width: 0, - height: 0, + width: webcamPresetFor(webcamQuality).width, + height: webcamPresetFor(webcamQuality).height, fps: WEBCAM_TARGET_FRAME_RATE, }, cursor: { @@ -1555,10 +1570,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // take is never flattened into one ArrayBuffer at finalize (#253). nativeWebcamRecorder = createRecorderHandle( webcamStream.current, - { - mimeType: selectMimeType(), - videoBitsPerSecond: BITRATE_BASE, - }, + webcamRecorderOptions(webcamStream.current), `${RECORDING_FILE_PREFIX}${activeRecordingId}${WEBCAM_FILE_SUFFIX}${VIDEO_FILE_EXTENSION}`, ); } else { @@ -2021,7 +2033,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (webcamStream.current) { webcamRecorder.current = createRecorderHandle( webcamStream.current, - { mimeType, videoBitsPerSecond: Math.min(videoBitsPerSecond, BITRATE_BASE) }, + webcamRecorderOptions(webcamStream.current), `${RECORDING_FILE_PREFIX}${activeRecordingId}${WEBCAM_FILE_SUFFIX}${VIDEO_FILE_EXTENSION}`, ); } @@ -2403,6 +2415,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setMicrophoneDeviceName, webcamDeviceId, setWebcamDeviceId, + webcamQuality, + setWebcamQuality, webcamDeviceName, setWebcamDeviceName, systemAudioEnabled, diff --git a/src/hooks/webcamCaptureTarget.test.ts b/src/hooks/webcamCaptureTarget.test.ts new file mode 100644 index 000000000..25f83efdd --- /dev/null +++ b/src/hooks/webcamCaptureTarget.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_WEBCAM_QUALITY, + WEBCAM_QUALITY_IDS, + WEBCAM_QUALITY_PRESETS, + WEBCAM_TARGET_FRAME_RATE, + webcamBitrateFor, + webcamBitrateForStream, + webcamPresetFor, + webcamQualityFrom, + webcamVideoConstraints, +} from "./webcamCaptureTarget"; + +describe("webcamVideoConstraints", () => { + it("asks for the chosen preset, not the browser's 640x480 default", () => { + expect(webcamVideoConstraints(undefined, "2160p")).toMatchObject({ + width: { ideal: 3840 }, + height: { ideal: 2160 }, + }); + expect(webcamVideoConstraints(undefined, "1080p")).toMatchObject({ + width: { ideal: 1920 }, + height: { ideal: 1080 }, + }); + }); + + it("keeps the size negotiable so a 720p-only camera still opens", () => { + // `exact` would make getUserMedia throw OverconstrainedError on every + // camera that cannot hit the target, losing the webcam entirely rather + // than recording it a little smaller. + const constraints = webcamVideoConstraints(undefined, "2160p"); + + expect(constraints.width).not.toHaveProperty("exact"); + expect(constraints.height).not.toHaveProperty("exact"); + expect(constraints.width).not.toHaveProperty("min"); + expect(constraints.height).not.toHaveProperty("min"); + }); + + it("still pins the chosen device and caps the frame rate", () => { + const constraints = webcamVideoConstraints("camera-7", "1440p"); + + expect(constraints.deviceId).toEqual({ exact: "camera-7" }); + expect(constraints.frameRate).toEqual({ + ideal: WEBCAM_TARGET_FRAME_RATE, + max: WEBCAM_TARGET_FRAME_RATE, + }); + }); + + it("omits deviceId entirely when no camera was chosen", () => { + expect(webcamVideoConstraints(undefined, "1080p")).not.toHaveProperty("deviceId"); + }); + + it("falls back to the default preset rather than an unconstrained request", () => { + // An unrecognised stored value must not reopen the 640x480 hole. + expect(webcamVideoConstraints(undefined, undefined)).toEqual( + webcamVideoConstraints(undefined, DEFAULT_WEBCAM_QUALITY), + ); + }); +}); + +describe("webcamQualityFrom", () => { + it("accepts every advertised preset", () => { + for (const id of WEBCAM_QUALITY_IDS) { + expect(webcamQualityFrom(id)).toBe(id); + } + }); + + it("rejects anything else, including settings files from a future build", () => { + expect(webcamQualityFrom("4320p")).toBe(DEFAULT_WEBCAM_QUALITY); + expect(webcamQualityFrom(undefined)).toBe(DEFAULT_WEBCAM_QUALITY); + expect(webcamQualityFrom(null)).toBe(DEFAULT_WEBCAM_QUALITY); + expect(webcamQualityFrom(1080)).toBe(DEFAULT_WEBCAM_QUALITY); + }); +}); + +describe("webcamPresetFor", () => { + it("never returns a size below 1080p", () => { + for (const id of WEBCAM_QUALITY_IDS) { + const preset = webcamPresetFor(id); + expect(preset.width).toBeGreaterThanOrEqual(1920); + expect(preset.height).toBeGreaterThanOrEqual(1080); + } + expect(Object.keys(WEBCAM_QUALITY_PRESETS)).toEqual([...WEBCAM_QUALITY_IDS]); + }); +}); + +describe("webcamBitrateFor", () => { + it("scales with the frame the camera actually delivers", () => { + // A flat rate sized for 640x480 is what made the extra pixels pointless: + // starve a 2160p frame and it blocks up worse than a clean 1080p one. + expect(webcamBitrateFor(3840, 2160)).toBeGreaterThan(webcamBitrateFor(2560, 1440)); + expect(webcamBitrateFor(2560, 1440)).toBeGreaterThan(webcamBitrateFor(1920, 1080)); + expect(webcamBitrateFor(1920, 1080)).toBeGreaterThan(webcamBitrateFor(1280, 720)); + expect(webcamBitrateFor(1280, 720)).toBeGreaterThan(webcamBitrateFor(640, 480)); + }); + + it("gives 4K enough headroom to be worth capturing", () => { + expect(webcamBitrateFor(3840, 2160)).toBe(40_000_000); + }); + + it("matches the tier boundaries the native helper uses", () => { + // One pixel under a tier must not claim that tier's rate; the ladder in + // wgc-capture/src/main.cpp draws the lines at the same places. + expect(webcamBitrateFor(2560, 1439)).toBe(webcamBitrateFor(1920, 1080)); + expect(webcamBitrateFor(1920, 1079)).toBe(webcamBitrateFor(1280, 720)); + }); + + it("rates a small camera by what it delivered, not by what we asked for", () => { + // The preset is a request. A 720p camera asked for 2160p must not be + // handed a 4K bitrate it cannot fill. + expect(webcamBitrateFor(1280, 720)).toBe(8_000_000); + }); + + it("survives a track that reports nothing", () => { + // getSettings() may omit width/height before the first frame arrives. + const preset = WEBCAM_QUALITY_PRESETS[DEFAULT_WEBCAM_QUALITY]; + expect(webcamBitrateFor(undefined, undefined)).toBe( + webcamBitrateFor(preset.width, preset.height), + ); + expect(webcamBitrateFor(0, 0)).toBeGreaterThan(0); + }); +}); + +describe("webcamBitrateForStream", () => { + const streamWith = (settings: MediaTrackSettings) => + ({ getVideoTracks: () => [{ getSettings: () => settings }] }) as unknown as MediaStream; + + it("reads the size off the live track", () => { + expect(webcamBitrateForStream(streamWith({ width: 2560, height: 1440 }))).toBe(24_000_000); + }); + + it("falls back when there is no stream or no video track", () => { + expect(webcamBitrateForStream(null)).toBe(webcamBitrateFor(undefined, undefined)); + expect(webcamBitrateForStream({ getVideoTracks: () => [] } as unknown as MediaStream)).toBe( + webcamBitrateFor(undefined, undefined), + ); + }); +}); diff --git a/src/hooks/webcamCaptureTarget.ts b/src/hooks/webcamCaptureTarget.ts new file mode 100644 index 000000000..906412b19 --- /dev/null +++ b/src/hooks/webcamCaptureTarget.ts @@ -0,0 +1,104 @@ +/** + * The resolution the webcam is captured at, and the rate it is encoded at. + * + * Nothing here used to name a size. `getUserMedia({ video: { deviceId } })` and + * a Media Foundation source reader left without MF_MT_FRAME_SIZE both fall back + * to the device's *default* media type, and for a UVC camera that is the first + * format it enumerates -- 640x480. A Logitech BRIO that offers MJPEG up to + * 3840x2160 was therefore recorded at 640x480 and then scaled up into the + * picture-in-picture, which is what "the camera looks pixelated" actually was. + */ + +export const WEBCAM_TARGET_FRAME_RATE = 30; + +export const WEBCAM_QUALITY_IDS = ["1080p", "1440p", "2160p"] as const; +export type WebcamQualityId = (typeof WEBCAM_QUALITY_IDS)[number]; + +export interface WebcamQualityPreset { + id: WebcamQualityId; + width: number; + height: number; +} + +/** + * Sizes are a *request*, never a floor. A camera that tops out below the chosen + * preset is driven at the best format it has; the native helper picks that with + * `chooseWebcamFormat`, and the browser does it through `ideal` constraints. + * + * Measured on a BRIO (Snapdragon X, alongside a screen capture), 8s takes, + * frames the camera actually delivered out of 240, over four runs each: + * 1080p 244 14.2 Mbit/s 4/4 + * 1440p 242-244 21.9 Mbit/s 4/4 + * 2160p 243-244 38.4 Mbit/s 4/4 + * + * All three run at the camera's full rate. That is recent: driving the capture + * as RGB32 made Media Foundation decode and convert every frame, costing 92ms + * per 2160p frame -- a ceiling near 11 fps, inside a file whose container still + * claimed 30 because the encoder padded the gap with duplicates. The capture + * now asks for NV12, which the camera produces directly and the encoder + * consumes directly; see webcam_capture.cpp. Higher presets cost file size, not + * frames, which is why this is a user choice. + */ +export const WEBCAM_QUALITY_PRESETS: Record = { + "1080p": { id: "1080p", width: 1920, height: 1080 }, + "1440p": { id: "1440p", width: 2560, height: 1440 }, + "2160p": { id: "2160p", width: 3840, height: 2160 }, +}; + +export const DEFAULT_WEBCAM_QUALITY: WebcamQualityId = "2160p"; + +/** Narrows a persisted or IPC-delivered value to a preset we can act on. */ +export function webcamQualityFrom(value: unknown): WebcamQualityId { + return WEBCAM_QUALITY_IDS.includes(value as WebcamQualityId) + ? (value as WebcamQualityId) + : DEFAULT_WEBCAM_QUALITY; +} + +export function webcamPresetFor(quality: WebcamQualityId | undefined): WebcamQualityPreset { + return WEBCAM_QUALITY_PRESETS[webcamQualityFrom(quality)]; +} + +/** + * Video constraints for a webcam stream, optionally pinned to one device. + * + * Sizes are `ideal`, never `exact` or `min`: a camera that tops out at 720p has + * to keep working. `exact` turns "record it a bit smaller" into + * OverconstrainedError, and the recording then has no camera at all. + */ +export function webcamVideoConstraints( + deviceId: string | undefined, + quality?: WebcamQualityId, +): MediaTrackConstraints { + const preset = webcamPresetFor(quality); + return { + ...(deviceId ? { deviceId: { exact: deviceId } } : {}), + width: { ideal: preset.width }, + height: { ideal: preset.height }, + frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, + }; +} + +/** + * Encoder bitrate for a frame the camera actually delivered. + * + * Keyed off the delivered size rather than the requested preset: a 720p camera + * asked for 2160p must not be encoded as if it were 4K. The tiers mirror the + * native ladder in electron/native/wgc-capture/src/main.cpp -- keep them in step. + */ +export function webcamBitrateFor(width: number | undefined, height: number | undefined): number { + const preset = WEBCAM_QUALITY_PRESETS[DEFAULT_WEBCAM_QUALITY]; + const pixels = + width && height && width > 0 && height > 0 ? width * height : preset.width * preset.height; + + if (pixels >= 3840 * 2160) return 40_000_000; + if (pixels >= 2560 * 1440) return 24_000_000; + if (pixels >= 1920 * 1080) return 16_000_000; + if (pixels >= 1280 * 720) return 8_000_000; + return 4_000_000; +} + +/** The bitrate for whatever the stream's first video track ended up delivering. */ +export function webcamBitrateForStream(stream: MediaStream | null): number { + const settings = stream?.getVideoTracks()[0]?.getSettings(); + return webcamBitrateFor(settings?.width, settings?.height); +} diff --git a/src/i18n/locales/ar/launch.json b/src/i18n/locales/ar/launch.json index 2b3db0f7e..b1dc21aff 100644 --- a/src/i18n/locales/ar/launch.json +++ b/src/i18n/locales/ar/launch.json @@ -50,7 +50,11 @@ "noneFound": "لم يتم العثور على كاميرا", "unavailable": "الكاميرا غير متوفرة", "camera": "الكاميرا", - "cameraDevice": "جهاز الكاميرا" + "cameraDevice": "جهاز الكاميرا", + "quality": "جودة الكاميرا", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "sourceSelector": { "loading": "جاري تحميل المصادر...", diff --git a/src/i18n/locales/cs/launch.json b/src/i18n/locales/cs/launch.json index b3ae9a56f..d4d3f3104 100644 --- a/src/i18n/locales/cs/launch.json +++ b/src/i18n/locales/cs/launch.json @@ -50,7 +50,11 @@ "noneFound": "Kamera nenalezena", "unavailable": "Kamera nedostupná", "camera": "Kamera", - "cameraDevice": "Zařízení kamery" + "cameraDevice": "Zařízení kamery", + "quality": "Kvalita kamery", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "Použít upravitelný kurzor: znovu zapne automatické přiblížení a efekty kurzoru", diff --git a/src/i18n/locales/de/launch.json b/src/i18n/locales/de/launch.json index 81c6eeec0..8d199afaa 100644 --- a/src/i18n/locales/de/launch.json +++ b/src/i18n/locales/de/launch.json @@ -50,7 +50,11 @@ "noneFound": "Keine Kamera gefunden", "unavailable": "Kamera nicht verfügbar", "camera": "Kamera", - "cameraDevice": "Kameragerät" + "cameraDevice": "Kameragerät", + "quality": "Kameraqualität", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "Bearbeitbaren Cursor verwenden: schaltet Auto-Zoom und Cursor-Effekte wieder ein", diff --git a/src/i18n/locales/en/launch.json b/src/i18n/locales/en/launch.json index 3963513c3..e02d96f8e 100644 --- a/src/i18n/locales/en/launch.json +++ b/src/i18n/locales/en/launch.json @@ -50,7 +50,11 @@ "noneFound": "No camera found", "unavailable": "Camera unavailable", "camera": "Camera", - "cameraDevice": "Camera device" + "cameraDevice": "Camera device", + "quality": "Camera quality", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "Use editable cursor: turns auto zoom and cursor effects back on", diff --git a/src/i18n/locales/es/launch.json b/src/i18n/locales/es/launch.json index 348f42b5e..34965561c 100644 --- a/src/i18n/locales/es/launch.json +++ b/src/i18n/locales/es/launch.json @@ -50,7 +50,11 @@ "noneFound": "No se encontró cámara", "unavailable": "Cámara no disponible", "camera": "Cámara", - "cameraDevice": "Dispositivo de cámara" + "cameraDevice": "Dispositivo de cámara", + "quality": "Calidad de la cámara", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "Usar cursor editable: vuelve a activar el zoom automático y los efectos del cursor", diff --git a/src/i18n/locales/fr/launch.json b/src/i18n/locales/fr/launch.json index 5782d07e6..3a8d35690 100644 --- a/src/i18n/locales/fr/launch.json +++ b/src/i18n/locales/fr/launch.json @@ -50,7 +50,11 @@ "noneFound": "Aucune caméra trouvée", "unavailable": "Caméra non disponible", "camera": "Caméra", - "cameraDevice": "Périphérique caméra" + "cameraDevice": "Périphérique caméra", + "quality": "Qualité de la caméra", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "Utiliser le curseur éditable : réactive le zoom automatique et les effets de curseur", diff --git a/src/i18n/locales/it/launch.json b/src/i18n/locales/it/launch.json index 2c863de0f..fae376c8d 100644 --- a/src/i18n/locales/it/launch.json +++ b/src/i18n/locales/it/launch.json @@ -50,7 +50,11 @@ "noneFound": "Nessuna fotocamera trovata", "unavailable": "Fotocamera non disponibile", "camera": "Fotocamera", - "cameraDevice": "Dispositivo fotocamera" + "cameraDevice": "Dispositivo fotocamera", + "quality": "Qualità della fotocamera", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "Usa cursore modificabile: riattiva lo zoom automatico e gli effetti del cursore", diff --git a/src/i18n/locales/ja-JP/launch.json b/src/i18n/locales/ja-JP/launch.json index 20c7e0530..0429770d8 100644 --- a/src/i18n/locales/ja-JP/launch.json +++ b/src/i18n/locales/ja-JP/launch.json @@ -50,7 +50,11 @@ "noneFound": "カメラが見つかりません", "unavailable": "カメラが利用できません", "camera": "カメラ", - "cameraDevice": "カメラデバイス" + "cameraDevice": "カメラデバイス", + "quality": "カメラの画質", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "編集可能なカーソルを使う：自動ズームとカーソル効果が再びオンになります", diff --git a/src/i18n/locales/ko-KR/launch.json b/src/i18n/locales/ko-KR/launch.json index 1aef23e07..70f0886dc 100644 --- a/src/i18n/locales/ko-KR/launch.json +++ b/src/i18n/locales/ko-KR/launch.json @@ -50,7 +50,11 @@ "noneFound": "카메라를 찾을 수 없음", "unavailable": "카메라를 사용할 수 없음", "camera": "카메라", - "cameraDevice": "카메라 장치" + "cameraDevice": "카메라 장치", + "quality": "카메라 화질", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "편집 가능한 커서 사용: 자동 줌과 커서 효과가 다시 켜집니다", diff --git a/src/i18n/locales/pt-BR/launch.json b/src/i18n/locales/pt-BR/launch.json index 35aedb23a..0ed727214 100644 --- a/src/i18n/locales/pt-BR/launch.json +++ b/src/i18n/locales/pt-BR/launch.json @@ -50,7 +50,11 @@ "noneFound": "Nenhuma câmera encontrada", "unavailable": "Câmera indisponível", "camera": "Câmera", - "cameraDevice": "Dispositivo de câmera" + "cameraDevice": "Dispositivo de câmera", + "quality": "Qualidade da câmera", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "Usar cursor editável: reativa o zoom automático e os efeitos do cursor", diff --git a/src/i18n/locales/ru/launch.json b/src/i18n/locales/ru/launch.json index 48ca06b7e..67ebf3da8 100644 --- a/src/i18n/locales/ru/launch.json +++ b/src/i18n/locales/ru/launch.json @@ -50,7 +50,11 @@ "noneFound": "Камера не найдена", "unavailable": "Камера недоступна", "camera": "Камера", - "cameraDevice": "Устройство камеры" + "cameraDevice": "Устройство камеры", + "quality": "Качество камеры", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "sourceSelector": { "loading": "Загрузка источников...", diff --git a/src/i18n/locales/tr/launch.json b/src/i18n/locales/tr/launch.json index e919ba448..7df10beb9 100644 --- a/src/i18n/locales/tr/launch.json +++ b/src/i18n/locales/tr/launch.json @@ -50,7 +50,11 @@ "noneFound": "Kamera bulunamadı", "unavailable": "Kamera kullanılamıyor", "camera": "Kamera", - "cameraDevice": "Kamera cihazı" + "cameraDevice": "Kamera cihazı", + "quality": "Kamera kalitesi", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "Düzenlenebilir imleci kullan: otomatik yakınlaştırmayı ve imleç efektlerini yeniden açar", diff --git a/src/i18n/locales/vi/launch.json b/src/i18n/locales/vi/launch.json index 4bc27b80a..a76f0db84 100644 --- a/src/i18n/locales/vi/launch.json +++ b/src/i18n/locales/vi/launch.json @@ -50,7 +50,11 @@ "noneFound": "Không tìm thấy máy ảnh", "unavailable": "Máy ảnh không khả dụng", "camera": "Máy ảnh", - "cameraDevice": "Thiết bị máy ảnh" + "cameraDevice": "Thiết bị máy ảnh", + "quality": "Chất lượng camera", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "sourceSelector": { "loading": "Đang tải nguồn...", diff --git a/src/i18n/locales/zh-CN/launch.json b/src/i18n/locales/zh-CN/launch.json index 4ee356acf..7a0b26cb9 100644 --- a/src/i18n/locales/zh-CN/launch.json +++ b/src/i18n/locales/zh-CN/launch.json @@ -50,7 +50,11 @@ "noneFound": "未找到摄像头", "unavailable": "摄像头不可用", "camera": "摄像头", - "cameraDevice": "摄像头设备" + "cameraDevice": "摄像头设备", + "quality": "摄像头画质", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "使用可编辑光标：将重新开启自动缩放和光标效果", diff --git a/src/i18n/locales/zh-TW/launch.json b/src/i18n/locales/zh-TW/launch.json index a899f50e0..423d585d6 100644 --- a/src/i18n/locales/zh-TW/launch.json +++ b/src/i18n/locales/zh-TW/launch.json @@ -50,7 +50,11 @@ "noneFound": "未找到攝影機", "unavailable": "攝影機不可用", "camera": "攝影機", - "cameraDevice": "攝影機裝置" + "cameraDevice": "攝影機裝置", + "quality": "攝影機畫質", + "quality1080p": "1080p", + "quality1440p": "1440p", + "quality2160p": "4K" }, "cursor": { "useEditableCursorHint": "使用可編輯游標：將重新開啟自動縮放與游標效果",