feat(replays): play video-backed sessions behind a replay engine seam

apps/web/src/components/replays/engine/replay-engine.ts

@@ -0,0 +1,88 @@
+// The replay engine seam
+//
+// The player used to construct `new Replayer(...)` inline, which made rrweb the
+// only thing it could ever play. The iOS SDK records H.264 segments and wraps
+// each one in rrweb-*shaped* events so the chunk pipeline carries them
+// untouched — but there is no DOM to rebuild, so rrweb renders nothing.
+//
+// Everything above this interface (the provider's transport state, the trimmed
+// timeline, markers, idle bands, the chunk loader) is format-agnostic and stays
+// exactly as it was. Everything rrweb-specific lives in `rrweb-engine.ts`;
+// everything video-specific in `video-engine.ts`.
+//
+// The contract is deliberately shaped like the rrweb surface the provider
+// already depended on, so the refactor is behaviour-preserving:
+//
+//   new Replayer(events, {root})   -> ReplayEngineFactory.create({mount, events})
+//   getMetaData().totalTime        -> totalTimeMs
+//   getCurrentTime()               -> getCurrentTimeMs()
+//   play(o) / pause(o?)            -> play(o) / pause(o?)
+//   setConfig({speed})             -> setSpeed(speed)
+//   addEvent(e)                    -> addEvent(e)
+//   destroy()                      -> destroy()
+//   on(Finish) / on(Resize)        -> onFinish / onResize callbacks
+//   .iframe + .wrapper.style       -> fit(container)
+
+/** The recorded viewport, used to letterbox the recording inside the surface. */
+export interface Viewport {
+	readonly width: number
+	readonly height: number
+}
+
+export interface ReplayEngineCreateInput {
+	/** The surface's inner div. The engine owns its contents entirely. */
+	readonly mount: HTMLElement
+	/** The seed events. Later events arrive through `addEvent`, forward-only. */
+	readonly events: ReadonlyArray<unknown>
+	/**
+	 * Viewport to fall back to when the engine can't report its own — derived
+	 * from the stream's `meta` events by `deriveMeta`.
+	 */
+	readonly fallbackViewport: Viewport
+	/** Playback reached the end of the recording. */
+	onFinish(): void
+	/** The recorded viewport changed mid-session; the surface must re-fit. */
+	onResize(): void
+}
+
+export interface ReplayEngine {
+	/** Length of the loaded recording, in real ms. */
+	readonly totalTimeMs: number
+	/**
+	 * Playhead as a real-ms offset from session start, matching the clock the
+	 * trimmed `Timeline` and backend span alignment are built against.
+	 *
+	 * Implementations must never return a negative or non-finite value — the
+	 * provider feeds this straight back into `play()`.
+	 */
+	getCurrentTimeMs(): number
+	play(offsetMs: number): void
+	/** With no offset, hold at the current position. */
+	pause(offsetMs?: number): void
+	setSpeed(speed: number): void
+	/**
+	 * Append an event that arrived after construction.
+	 *
+	 * Forward-only by contract: every trailing event postdates the seed. A
+	 * backward seek rebuilds the engine from the nearest checkpoint instead
+	 * (see `requestSeek` in `use-replay-chunk-loader.ts`).
+	 */
+	addEvent(event: unknown): void
+	/** Fit the recording inside `container`, letterboxed and centred. */
+	fit(container: HTMLElement): void
+	destroy(): void
+}
+
+export interface ReplayEngineFactory {
+	create(input: ReplayEngineCreateInput): ReplayEngine
+}
+
+/**
+ * Which engine plays a session's chunks.
+ *
+ * Carried by the `maple.session.replay_format` resource attribute so the player
+ * can pick an engine from session metadata alone, without downloading a chunk
+ * to find out. An absent attribute means `rrweb` — every session recorded
+ * before the marker existed is a browser recording.
+ */
+export type ReplayFormat = "rrweb" | "video"

apps/web/src/components/replays/engine/rrweb-engine.ts

@@ -0,0 +1,107 @@
+import { Replayer } from "@rrweb/replay"
+import { ReplayerEvents } from "@rrweb/types"
+import type { ReplayEngine, ReplayEngineCreateInput, ReplayEngineFactory } from "./replay-engine"
+
+// The rrweb engine — browser recordings.
+//
+// This is the behaviour the player has always had, moved behind the engine
+// interface unchanged. Every quirk documented here was a shipped bug once.
+
+class RrwebEngine implements ReplayEngine {
+	private readonly replayer: Replayer
+	private readonly fallbackViewport: ReplayEngineCreateInput["fallbackViewport"]
+
+	constructor(input: ReplayEngineCreateInput) {
+		const accent =
+			getComputedStyle(document.documentElement).getPropertyValue("--primary").trim() || "#6366f1"
+
+		this.fallbackViewport = input.fallbackViewport
+		this.replayer = new Replayer(input.events as never, {
+			root: input.mount,
+			speed: 1,
+			// We skip idle ourselves by jumping (see the provider's rAF loop) —
+			// rrweb's own skipInactive only fast-forwards, which is slow. Keep it off.
+			skipInactive: false,
+			mouseTail: { duration: 600, lineCap: "round", lineWidth: 3, strokeStyle: accent },
+			showWarning: false,
+			showDebug: false,
+			liveMode: false,
+		})
+
+		// rrweb's own transport events are unreliable in @rrweb/replay (Start/Resume
+		// often don't fire); play/pause state is driven from the provider's handlers.
+		// We still honour Finish to flip back to the replay affordance at the end.
+		this.replayer.on(ReplayerEvents.Finish, () => input.onFinish())
+		// The recorded viewport can change mid-session (responsive / window resize);
+		// rrweb resizes its iframe and emits Resize. Also fires for the initial snapshot.
+		this.replayer.on(ReplayerEvents.Resize, () => input.onResize())
+	}
+
+	get totalTimeMs(): number {
+		return this.replayer.getMetaData().totalTime
+	}
+
+	/**
+	 * Read the playhead, treating "not started yet" as 0.
+	 *
+	 * rrweb builds its player context with `baselineTime: 0`, and
+	 * `getCurrentTime()` is `timer.timeOffset + (baselineTime - events[0].timestamp)`
+	 * — so until the engine has been driven by a `play()` / `pause(offset)` (the only
+	 * things that assign `baselineTime`), it reports `-events[0].timestamp`: a
+	 * negative epoch, ~55 years. Feeding that back into `play()` re-bases the whole
+	 * stream decades into the future and nothing ever casts, which is what left the
+	 * player frozen at 0:00 until the first scrub re-based it for us.
+	 */
+	getCurrentTimeMs(): number {
+		const ms = this.replayer.getCurrentTime()
+		return Number.isFinite(ms) && ms > 0 ? ms : 0
+	}
+
+	play(offsetMs: number): void {
+		this.replayer.play(offsetMs)
+	}
+
+	pause(offsetMs?: number): void {
+		this.replayer.pause(offsetMs)
+	}
+
+	setSpeed(speed: number): void {
+		this.replayer.setConfig({ speed })
+	}
+
+	addEvent(event: unknown): void {
+		this.replayer.addEvent(event as never)
+	}
+
+	/**
+	 * Fit the recorded page *inside* the surface (contain + letterbox), centered on
+	 * both axes. The surface keeps a constant box (CSS aspect-ratio / fullscreen
+	 * flex), so the player height never jumps between recordings.
+	 *
+	 * Scale against the iframe rrweb actually built, not the statically-derived
+	 * fallback — a session can carry several Meta events (viewport resizes), and
+	 * `deriveMeta` keeps the last one, which may not match the current frame. The
+	 * iframe's width/height *attributes* always reflect the current viewport, and
+	 * the `Resize` listener re-runs this when they change mid-playback.
+	 */
+	fit(container: HTMLElement): void {
+		const vw = Number(this.replayer.iframe?.getAttribute("width")) || this.fallbackViewport.width
+		const vh = Number(this.replayer.iframe?.getAttribute("height")) || this.fallbackViewport.height
+		const availW = container.clientWidth
+		const availH = container.clientHeight
+		if (!availW || !availH || !vw || !vh) return
+		const scale = Math.min(availW / vw, availH / vh)
+		const offsetX = Math.max(0, (availW - vw * scale) / 2)
+		const offsetY = Math.max(0, (availH - vh * scale) / 2)
+		this.replayer.wrapper.style.transformOrigin = "top left"
+		this.replayer.wrapper.style.transform = `translate(${offsetX}px, ${offsetY}px) scale(${scale})`
+	}
+
+	destroy(): void {
+		this.replayer.destroy()
+	}
+}
+
+export const rrwebEngineFactory: ReplayEngineFactory = {
+	create: (input) => new RrwebEngine(input),
+}

apps/web/src/components/replays/engine/video-engine.test.ts

@@ -0,0 +1,155 @@
+import { describe, expect, it } from "vitest"
+import {
+	extractVideoSegments,
+	resolveSegment,
+	segmentsTotalMs,
+	videoSegmentPayload,
+	type VideoSegment,
+} from "./video-engine"
+
+// A mobile recording is a sequence of independent MP4s, each opening on an IDR
+// keyframe. The segment math below is what turns a playhead offset into
+// (which file, how far into it) — the whole reason seeking is exact here.
+//
+// The DOM side (<video>, Blob URLs) is deliberately not tested: jsdom implements
+// neither URL.createObjectURL nor HTMLMediaElement.play/pause, so a test of it
+// would only exercise its own stubs.
+
+const T0 = 1_700_000_000_000
+
+/** An rrweb-shaped custom event carrying one H.264 segment, as the iOS SDK emits. */
+const videoEvent = (offsetMs: number, durationMs: number, overrides: Record<string, unknown> = {}) => ({
+	type: 5,
+	timestamp: T0 + offsetMs,
+	data: {
+		tag: "video",
+		payload: {
+			segmentId: `seg-${offsetMs}`,
+			size: 12_800,
+			duration: durationMs,
+			encoding: "h264",
+			container: "mp4",
+			width: 390,
+			height: 844,
+			frameCount: 60,
+			frameRateType: "constant",
+			frameRate: 2,
+			left: 0,
+			top: 0,
+			base64: "AAAAIGZ0eXA=",
+			...overrides,
+		},
+	},
+})
+
+const metaEvent = (offsetMs: number) => ({
+	type: 4,
+	timestamp: T0 + offsetMs,
+	data: { href: "app://main", width: 390, height: 844 },
+})
+
+const touchEvent = (offsetMs: number) => ({
+	type: 3,
+	timestamp: T0 + offsetMs,
+	data: { source: 2, type: 7, pointerType: 2 },
+})
+
+describe("videoSegmentPayload", () => {
+	it("recognises the SDK's video custom event", () => {
+		expect(videoSegmentPayload(videoEvent(0, 30_000))).toMatchObject({ encoding: "h264" })
+	})
+
+	it("rejects everything that is not one", () => {
+		// Type 5 but a different tag, right type/tag but wrong event type, and the
+		// ordinary rrweb events a browser session is made of.
+		expect(videoSegmentPayload({ type: 5, timestamp: T0, data: { tag: "breadcrumb" } })).toBeUndefined()
+		expect(videoSegmentPayload(metaEvent(0))).toBeUndefined()
+		expect(videoSegmentPayload(touchEvent(0))).toBeUndefined()
+		expect(videoSegmentPayload(null)).toBeUndefined()
+		expect(videoSegmentPayload(undefined)).toBeUndefined()
+	})
+})
+
+describe("extractVideoSegments", () => {
+	it("positions segments against the stream's first event, not the epoch", () => {
+		// The meta event opens the chunk, so time-zero is it — the first video
+		// segment lands 5ms later, not 1.7 trillion ms later.
+		const events = [metaEvent(0), videoEvent(5, 30_000), touchEvent(1_200), videoEvent(30_005, 30_000)]
+		const segments = extractVideoSegments(events, T0)
+		expect(segments).toHaveLength(2)
+		expect(segments[0]).toMatchObject({ startMs: 5, durationMs: 30_000, width: 390, height: 844 })
+		expect(segments[1]).toMatchObject({ startMs: 30_005, durationMs: 30_000 })
+	})
+
+	it("orders segments by start time regardless of arrival order", () => {
+		const segments = extractVideoSegments([videoEvent(60_000, 30_000), videoEvent(0, 30_000)], T0)
+		expect(segments.map((s) => s.startMs)).toEqual([0, 60_000])
+	})
+
+	it("skips unusable segments rather than failing the recording", () => {
+		// A chunk missing its payload must not take the surrounding footage down
+		// with it — same posture as `decodeRange` skipping a malformed chunk.
+		const events = [
+			videoEvent(0, 30_000),
+			videoEvent(30_000, 30_000, { base64: "" }),
+			{ type: 5, timestamp: T0 + 60_000, data: { tag: "video" } },
+			videoEvent(90_000, 30_000),
+		]
+		expect(extractVideoSegments(events, T0).map((s) => s.startMs)).toEqual([0, 90_000])
+	})
+
+	it("returns nothing for a browser recording", () => {
+		expect(extractVideoSegments([metaEvent(0), touchEvent(10)], T0)).toEqual([])
+	})
+})
+
+describe("segmentsTotalMs", () => {
+	it("measures to the end of the last segment, not its start", () => {
+		expect(segmentsTotalMs(extractVideoSegments([videoEvent(0, 30_000)], T0))).toBe(30_000)
+	})
+
+	it("is zero for an empty recording", () => {
+		expect(segmentsTotalMs([])).toBe(0)
+	})
+})
+
+describe("resolveSegment", () => {
+	// Two 30s segments with a 10s hole between them — the recorder stopped while
+	// the app was backgrounded.
+	const segments: ReadonlyArray<VideoSegment> = extractVideoSegments(
+		[videoEvent(0, 30_000), videoEvent(40_000, 30_000)],
+		T0,
+	)
+
+	it("maps an offset inside a segment to that segment's own clock", () => {
+		expect(resolveSegment(segments, 0)).toEqual({ index: 0, offsetSec: 0 })
+		expect(resolveSegment(segments, 12_500)).toEqual({ index: 0, offsetSec: 12.5 })
+		// Second segment starts at 40s, so 45s is 5s into its own media clock —
+		// not 45s, which is what a naive global-timeline seek would use.
+		expect(resolveSegment(segments, 45_000)).toEqual({ index: 1, offsetSec: 5 })
+	})
+
+	it("snaps a seek landing in a gap forward to the next segment", () => {
+		// 35s is dead air. Stalling there would look like frozen playback, so the
+		// playhead moves to the start of the next real footage.
+		expect(resolveSegment(segments, 35_000)).toEqual({ index: 1, offsetSec: 0 })
+	})
+
+	it("treats a segment boundary as the start of the next segment", () => {
+		expect(resolveSegment(segments, 30_000)).toEqual({ index: 1, offsetSec: 0 })
+	})
+
+	it("holds at the tail of the last segment past the end", () => {
+		// Reporting the end (rather than wrapping to 0) is what lets the engine
+		// fire `onFinish` instead of silently restarting the recording.
+		expect(resolveSegment(segments, 999_999)).toEqual({ index: 1, offsetSec: 30 })
+	})
+
+	it("clamps a negative offset to the first segment", () => {
+		expect(resolveSegment(segments, -5_000)).toEqual({ index: 0, offsetSec: 0 })
+	})
+
+	it("has nothing to resolve on an empty recording", () => {
+		expect(resolveSegment([], 1_000)).toBeUndefined()
+	})
+})