feat(recording): capture the webcam at its real resolution and frame rate

electron/app-settings.test.ts

@@ -129,4 +129,40 @@ describe("app settings store", () => {
 		store.dismissStarPrompt();
 		expect(store.getSnapshot().recording).toMatchObject({ micEnabled: true, micDeviceId: "mic" });
 	});
+
+	it("stores a camera quality and rejects one it cannot capture at", () => {
+		const dir = temp();
+		const store = new AppSettingsStore(dir);
+
+		expect(store.setRecordingPreferences({ camQuality: "1080p" }).recording.camQuality).toBe(
+			"1080p",
+		);
+		expect(() => store.setRecordingPreferences({ camQuality: "4320p" as never })).toThrow(
+			TypeError,
+		);
+		expect(store.getSnapshot().recording.camQuality).toBe("1080p");
+	});
+
+	it("leaves the camera quality alone when a patch does not mention it", () => {
+		// Every other window writes narrow patches; a camera toggle from the HUD
+		// must not quietly reset the resolution the user picked.
+		const dir = temp();
+		const store = new AppSettingsStore(dir);
+		store.setRecordingPreferences({ camQuality: "1440p" });
+
+		store.setRecordingPreferences({ camEnabled: true });
+
+		expect(store.getSnapshot().recording.camQuality).toBe("1440p");
+	});
+
+	it("reads a settings file written before the camera had a quality", () => {
+		const dir = temp();
+		writeFileSync(
+			path.join(dir, "recording-settings.json"),
+			JSON.stringify({ micEnabled: true }),
+			"utf8",
+		);
+
+		expect(new AppSettingsStore(dir).getSnapshot().recording.camQuality).toBe("2160p");
+	});
 });

electron/app-settings.ts

@@ -1,5 +1,11 @@
 import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import path from "node:path";
+import {
+	DEFAULT_WEBCAM_QUALITY,
+	WEBCAM_QUALITY_IDS,
+	type WebcamQualityId,
+	webcamQualityFrom,
+} from "../src/hooks/webcamCaptureTarget";
 import type { CursorCaptureMode } from "../src/lib/recordingSession";
 
 export interface RecordingPreferences {
@@ -9,6 +15,8 @@ export interface RecordingPreferences {
 	camEnabled: boolean;
 	camDeviceId: string | null;
 	camDeviceName: string | null;
+	/** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */
+	camQuality: WebcamQualityId;
 	systemAudioEnabled: boolean;
 	cursorCaptureMode: CursorCaptureMode;
 	/** Display captures on macOS and Windows. Opt-in: on Windows the icons also leave the real desktop while recording. */
@@ -30,6 +38,7 @@ export const DEFAULT_RECORDING_PREFERENCES: RecordingPreferences = {
 	camEnabled: false,
 	camDeviceId: null,
 	camDeviceName: null,
+	camQuality: DEFAULT_WEBCAM_QUALITY,
 	systemAudioEnabled: false,
 	cursorCaptureMode: "editable-overlay",
 	hideDesktopIcons: false,
@@ -100,6 +109,9 @@ function parseRecording(raw: RawSettings): RecordingPreferences {
 		camEnabled: bool(raw.camEnabled, DEFAULT_RECORDING_PREFERENCES.camEnabled),
 		camDeviceId: nullableString(raw.camDeviceId, DEFAULT_RECORDING_PREFERENCES.camDeviceId),
 		camDeviceName: nullableString(raw.camDeviceName, DEFAULT_RECORDING_PREFERENCES.camDeviceName),
+		// Unset in every settings file written before the camera had a quality
+		// setting, and `webcamQualityFrom` answers those with the default.
+		camQuality: webcamQualityFrom(raw.camQuality),
 		systemAudioEnabled: bool(
 			raw.systemAudioEnabled,
 			DEFAULT_RECORDING_PREFERENCES.systemAudioEnabled,
@@ -167,6 +179,12 @@ function validateRecordingPatch(patch: Partial<RecordingPreferences>): void {
 		if (key === "cursorCaptureMode" && value !== "system" && value !== "editable-overlay") {
 			throw new TypeError("cursorCaptureMode is invalid");
 		}
+		// Rejected here rather than coerced on read, so a bad write is a visible
+		// error at its source instead of a resolution that silently is not the
+		// one the caller asked for.
+		if (key === "camQuality" && !WEBCAM_QUALITY_IDS.includes(value as WebcamQualityId)) {
+			throw new TypeError("camQuality is invalid");
+		}
 	}
 }
 

electron/ipc/handlers.ts

@@ -17,6 +17,7 @@ import {
 	shell,
 	systemPreferences,
 } from "electron";
+import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget";
 import {
 	type AxcutDocument,
 	isAxcutDocumentFile,
@@ -649,6 +650,8 @@ export interface RecordingPrefs {
 	camDeviceId: string | null;
 	/** Camera label paired with the preferred id for restart-safe resolution. */
 	camDeviceName: string | null;
+	/** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */
+	camQuality: WebcamQualityId;
 	systemAudioEnabled: boolean;
 	cursorCaptureMode: CursorCaptureMode;
 	hideDesktopIcons: boolean;
@@ -662,6 +665,7 @@ const defaultRecordingPrefs: RecordingPrefs = {
 	camEnabled: false,
 	camDeviceId: null,
 	camDeviceName: null,
+	camQuality: DEFAULT_WEBCAM_QUALITY,
 	systemAudioEnabled: false,
 	cursorCaptureMode: "editable-overlay",
 	hideDesktopIcons: false,

electron/ipc/recordingPrefs.test.ts

@@ -19,6 +19,7 @@ const defaults: RecordingPrefs = {
 	camEnabled: false,
 	camDeviceId: null,
 	camDeviceName: null,
+	camQuality: "2160p",
 	systemAudioEnabled: false,
 	cursorCaptureMode: "editable-overlay",
 	hideDesktopIcons: false,

electron/native/wgc-capture/CMakeLists.txt

@@ -41,6 +41,8 @@ add_executable(wgc-capture
   src/desktop_icon_cover.cpp
   src/desktop_icon_cover.h
   src/dpi_awareness.h
+  src/frame_visibility.cpp
+  src/frame_visibility.h
   src/dshow_webcam_capture.cpp
   src/dshow_webcam_capture.h
   src/main.cpp
@@ -55,6 +57,8 @@ add_executable(wgc-capture
   src/wasapi_render_keepalive.cpp
   src/wasapi_render_keepalive.h
   src/webcam_capture.cpp
+  src/webcam_format.cpp
+  src/webcam_format.h
   src/webcam_capture.h
   src/wgc_session.cpp
   src/wgc_session.h
@@ -117,3 +121,31 @@ target_link_libraries(audio_sample_utils_test PRIVATE
   mfuuid
   ole32
 )
+
+add_executable(webcam_format_test
+  src/webcam_format.cpp
+  src/webcam_format.h
+  src/webcam_format_test.cpp
+)
+
+target_compile_definitions(webcam_format_test PRIVATE
+  NOMINMAX
+  WIN32_LEAN_AND_MEAN
+  _WIN32_WINNT=0x0A00
+)
+
+target_compile_options(webcam_format_test PRIVATE /EHsc /W4 /utf-8)
+
+add_executable(frame_visibility_test
+  src/frame_visibility.cpp
+  src/frame_visibility.h
+  src/frame_visibility_test.cpp
+)
+
+target_compile_definitions(frame_visibility_test PRIVATE
+  NOMINMAX
+  WIN32_LEAN_AND_MEAN
+  _WIN32_WINNT=0x0A00
+)
+
+target_compile_options(frame_visibility_test PRIVATE /EHsc /W4 /utf-8)

electron/native/wgc-capture/src/dshow_webcam_capture.cpp

@@ -1,5 +1,7 @@
 #include "dshow_webcam_capture.h"
 
+#include "webcam_format.h"
+
 #include <initguid.h>
 #include <dshow.h>
 #include <wrl/client.h>
@@ -29,6 +31,19 @@ ISampleGrabber : public IUnknown {
     virtual HRESULT STDMETHODCALLTYPE SetCallback(IUnknown* callback, long whichMethodToCallback) = 0;
 };
 
+/**
+ * Is this subtype something the graph must DECODE before this class can unpack it?
+ *
+ * Listed positively -- an unknown subtype counts as compressed -- because
+ * guessing wrong this way only costs resolution, while guessing wrong the other
+ * way costs the camera track entirely.
+ */
+bool isCompressedDshowSubtype(const GUID& subtype) {
+    return !(subtype == MEDIASUBTYPE_YUY2 || subtype == MEDIASUBTYPE_NV12 ||
+             subtype == MEDIASUBTYPE_RGB32 || subtype == MEDIASUBTYPE_RGB24 ||
+             subtype == MEDIASUBTYPE_UYVY || subtype == MEDIASUBTYPE_YV12);
+}
+
 bool succeeded(HRESULT hr, const char* label) {
     if (SUCCEEDED(hr)) {
         return true;
@@ -112,7 +127,93 @@ DirectShowWebcamCapture::~DirectShowWebcamCapture() {
     delete impl_;
 }
 
-bool DirectShowWebcamCapture::buildGraph(const CLSID& sourceClsid, const GUID* preferredSubtype) {
+void DirectShowWebcamCapture::applyPreferredFormat(int requestedWidth, int requestedHeight, int requestedFps) {
+    Microsoft::WRL::ComPtr<IAMStreamConfig> streamConfig;
+    if (FAILED(impl_->captureGraph->FindInterface(
+            &PIN_CATEGORY_CAPTURE,
+            &MEDIATYPE_Video,
+            impl_->captureFilter.Get(),
+            IID_PPV_ARGS(&streamConfig)))) {
+        // Virtual cameras commonly expose a single fixed format and no
+        // IAMStreamConfig at all. Nothing to choose from; let the graph connect.
+        return;
+    }
+
+    int capCount = 0;
+    int capSize = 0;
+    if (FAILED(streamConfig->GetNumberOfCapabilities(&capCount, &capSize)) ||
+        capSize != sizeof(VIDEO_STREAM_CONFIG_CAPS)) {
+        return;
+    }
+
+    std::vector<WebcamFormat> formats;
+    formats.reserve(static_cast<size_t>(std::max(0, capCount)));
+    for (int index = 0; index < capCount; ++index) {
+        VIDEO_STREAM_CONFIG_CAPS caps{};
+        AM_MEDIA_TYPE* mediaType = nullptr;
+        if (FAILED(streamConfig->GetStreamCaps(index, &mediaType, reinterpret_cast<BYTE*>(&caps))) || !mediaType) {
+            continue;
+        }
+        if (mediaType->formattype == FORMAT_VideoInfo && mediaType->pbFormat) {
+            const auto* videoInfo = reinterpret_cast<VIDEOINFOHEADER*>(mediaType->pbFormat);
+            // AvgTimePerFrame is in 100ns units; 333333 is 30 fps.
+            const int fps = videoInfo->AvgTimePerFrame > 0
+                                ? static_cast<int>((10'000'000LL + videoInfo->AvgTimePerFrame / 2) / videoInfo->AvgTimePerFrame)
+                                : 0;
+            formats.push_back(WebcamFormat{
+                std::abs(videoInfo->bmiHeader.biWidth),
+                std::abs(videoInfo->bmiHeader.biHeight),
+                fps,
+                isCompressedDshowSubtype(mediaType->subtype)});
+        }
+        freeMediaType(*mediaType);
+        CoTaskMemFree(mediaType);
+    }
+
+    const WebcamFormat chosen = chooseWebcamFormat(
+        formats,
+        requestedWidth > 0 ? requestedWidth : 1920,
+        requestedHeight > 0 ? requestedHeight : 1080,
+        std::max(1, requestedFps));
+
+    // Second pass: set the first capability that matches the choice. GetStreamCaps
+    // hands back the media type we must give to SetFormat, so it has to be
+    // re-read rather than cached across the loop above.
+    for (int index = 0; index < capCount; ++index) {
+        VIDEO_STREAM_CONFIG_CAPS caps{};
+        AM_MEDIA_TYPE* mediaType = nullptr;
+        if (FAILED(streamConfig->GetStreamCaps(index, &mediaType, reinterpret_cast<BYTE*>(&caps))) || !mediaType) {
+            continue;
+        }
+        bool matched = false;
+        if (mediaType->formattype == FORMAT_VideoInfo && mediaType->pbFormat) {
+            auto* videoInfo = reinterpret_cast<VIDEOINFOHEADER*>(mediaType->pbFormat);
+            if (std::abs(videoInfo->bmiHeader.biWidth) == chosen.width &&
+                std::abs(videoInfo->bmiHeader.biHeight) == chosen.height &&
+                isCompressedDshowSubtype(mediaType->subtype) == chosen.compressed) {
+                if (chosen.fps > 0) {
+                    videoInfo->AvgTimePerFrame = 10'000'000LL / chosen.fps;
+                }
+                matched = SUCCEEDED(streamConfig->SetFormat(mediaType));
+                if (matched) {
+                    std::cerr << "INFO: DirectShow webcam format " << chosen.width << "x" << chosen.height
+                              << "@" << chosen.fps << std::endl;
+                }
+            }
+        }
+        freeMediaType(*mediaType);
+        CoTaskMemFree(mediaType);
+        if (matched) {
+            return;
+        }
+    }
+}
+
+bool DirectShowWebcamCapture::buildGraph(
+    const CLSID& sourceClsid,
+    const GUID* preferredSubtype,
+    int preferredWidth,
+    int preferredHeight) {
     // Every attempt starts from empty filters. A RenderStream that fails can
     // leave pins connected behind it, and retrying on top of that half-built
     // graph is how you get a second failure that says nothing about the format.
@@ -175,6 +276,8 @@ bool DirectShowWebcamCapture::buildGraph(const CLSID& sourceClsid, const GUID* p
         return false;
     }
 
+    applyPreferredFormat(preferredWidth, preferredHeight, fps_);
+
     return succeeded(impl_->captureGraph->RenderStream(
                          &PIN_CATEGORY_CAPTURE,
                          &MEDIATYPE_Video,
@@ -231,14 +334,14 @@ bool DirectShowWebcamCapture::initialize(
     // it connects with a subtype this file cannot read
     // (getopenscreen/openscreen#387). Naming a concrete subtype on the retry is
     // what makes DirectShow insert a colour converter for it.
-    if (!buildGraph(selectedClsid, nullptr)) {
+    if (!buildGraph(selectedClsid, nullptr, requestedWidth, requestedHeight)) {
         return false;
     }
     if (!resolveConnectedFormat(requestedWidth, requestedHeight, false)) {
         std::cerr << "WARNING: DirectShow webcam speaks a format this build cannot unpack; "
                      "asking for RGB32 so the graph converts it"
                   << std::endl;
-        if (!buildGraph(selectedClsid, &MEDIASUBTYPE_RGB32)) {
+        if (!buildGraph(selectedClsid, &MEDIASUBTYPE_RGB32, requestedWidth, requestedHeight)) {
             return false;
         }
         if (!resolveConnectedFormat(requestedWidth, requestedHeight, true)) {
@@ -291,9 +394,28 @@ bool DirectShowWebcamCapture::resolveConnectedFormat(
             sourceStride_ = ((width_ * bitsPerPixel + 31) / 32) * 4;
         }
         sourceTopDown_ = pixelFormat_ != PixelFormat::Bgra || videoInfo->bmiHeader.biHeight < 0;
+        // The rate the graph settled on, not the one that was asked for.
+        //
+        // `chooseWebcamFormat` deliberately settles for less than the target
+        // when a camera offers nothing faster, and a driver may pick its own
+        // nearest rate after SetFormat regardless. `fps()` feeds three things
+        // that all have to agree with the frames actually arriving: the
+        // `webcam-format` event, the webcam encoder's nominal rate, and the
+        // constant-rate write interval in main.cpp. Left at the requested
+        // value, a 24 fps camera asked for 30 is encoded and paced as 30.
+        if (videoInfo->AvgTimePerFrame > 0) {
+            const int negotiated = static_cast<int>(
+                (10'000'000LL + videoInfo->AvgTimePerFrame / 2) / videoInfo->AvgTimePerFrame);
+            if (negotiated > 0 && negotiated != fps_) {
+                std::cerr << "INFO: DirectShow webcam negotiated " << negotiated << " fps (asked for "
+                          << fps_ << ")" << std::endl;
+                fps_ = std::clamp(negotiated, 1, 60);
+            }
+        }
     }
     std::cerr << "INFO: DirectShow webcam connected subtype " << guidToString(connectedType.subtype)
-              << " " << width_ << "x" << height_ << " stride=" << sourceStride_ << std::endl;
+              << " " << width_ << "x" << height_ << "@" << fps_ << " stride=" << sourceStride_
+              << std::endl;
     freeMediaType(connectedType);
     if (width_ <= 0 || height_ <= 0) {
         width_ = requestedWidth > 0 ? requestedWidth : 1280;

electron/native/wgc-capture/src/dshow_webcam_capture.h

@@ -59,7 +59,21 @@ class DirectShowWebcamCapture {
      * without leaving the graph half-built, so the caller can retry with a
      * different constraint.
      */
-    bool buildGraph(const CLSID& sourceClsid, const GUID* preferredSubtype);
+    bool buildGraph(
+        const CLSID& sourceClsid,
+        const GUID* preferredSubtype,
+        int preferredWidth,
+        int preferredHeight);
+    /**
+     * Pins the capture pin to the best format the device offers, before the
+     * graph is rendered.
+     *
+     * Skipped, RenderStream's intelligent connect takes the pin's default
+     * format, which on a UVC camera is the first one it enumerates -- 640x480
+     * on hardware that can do far better. Best-effort: a device without
+     * IAMStreamConfig, or one that rejects the format, still gets a graph.
+     */
+    void applyPreferredFormat(int requestedWidth, int requestedHeight, int requestedFps);
     /**
      * Reads back what the graph actually negotiated and records how to unpack it.
      *