diff --git a/crates/editor/src/thumbnail.rs b/crates/editor/src/thumbnail.rs index 2c078355fec..9b1abe68a9f 100644 --- a/crates/editor/src/thumbnail.rs +++ b/crates/editor/src/thumbnail.rs @@ -1,6 +1,8 @@ use crate::{EditorInstance, editor::RendererTransitionInput}; use cap_project::{ProjectConfiguration, XY}; -use cap_rendering::{ProjectUniforms, RenderedFrame, ZoomTransformTimeline}; +use cap_rendering::{ + PrecomputedCursorTimeline, ProjectUniforms, RenderedFrame, ZoomTransformTimeline, +}; use std::{io::Write, path::Path, sync::Arc}; const THUMBNAIL_SIZE: XY = XY { x: 640, y: 640 }; @@ -85,7 +87,8 @@ impl EditorInstance { segment.recording_clip, ); zoom.ensure_precomputed_until(1.0 / THUMBNAIL_FPS as f32); - let uniforms = ProjectUniforms::new( + let cursor_timeline = PrecomputedCursorTimeline::from_project(&media.cursor, &project); + let uniforms = ProjectUniforms::new_with_precomputed_cursor( &self.render_constants, &project, 0, @@ -95,6 +98,7 @@ impl EditorInstance { &frames, duration, &zoom, + &cursor_timeline, ); let frame = self .renderer diff --git a/crates/export/src/preview.rs b/crates/export/src/preview.rs index 09e1172af4f..918bd05dd56 100644 --- a/crates/export/src/preview.rs +++ b/crates/export/src/preview.rs @@ -4,8 +4,9 @@ use base64::{Engine, engine::general_purpose::STANDARD}; use cap_editor::EditorInstance; use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, TimelineFrameMapping, XY}; use cap_rendering::{ - FrameRenderer, ProjectUniforms, RecordingSegmentDecoders, RenderVideoConstants, RenderedFrame, - RendererLayers, TransitionRenderInput, ZoomTransformTimeline, + FrameRenderer, PrecomputedCursorTimeline, ProjectUniforms, RecordingSegmentDecoders, + RenderVideoConstants, RenderedFrame, RendererLayers, TransitionRenderInput, + ZoomTransformTimeline, }; use image::{ Rgba, @@ -277,7 +278,9 @@ async fn render_preview_frame( ); zoom_timeline.ensure_precomputed_until((frame_number as f32 + 1.0) / settings.fps as f32); - let uniforms = ProjectUniforms::new( + let cursor_timeline = + PrecomputedCursorTimeline::from_project(segment_media.cursor, project_config); + let uniforms = ProjectUniforms::new_with_precomputed_cursor( render_constants, project_config, frame_number, @@ -287,6 +290,7 @@ async fn render_preview_frame( &segment_frames, total_duration, &zoom_timeline, + &cursor_timeline, ); let mut frame_renderer = FrameRenderer::new(render_constants); @@ -341,7 +345,16 @@ async fn render_preview_frame( outgoing.segment.recording_clip, ); outgoing_zoom.ensure_precomputed_until((frame_number as f32 + 1.0) / settings.fps as f32); - let outgoing_uniforms = ProjectUniforms::new( + let outgoing_cursor_timeline = if outgoing.segment.recording_clip == segment.recording_clip + { + None + } else { + Some(PrecomputedCursorTimeline::from_project( + outgoing_media.cursor, + project_config, + )) + }; + let outgoing_uniforms = ProjectUniforms::new_with_precomputed_cursor( render_constants, project_config, frame_number, @@ -351,6 +364,9 @@ async fn render_preview_frame( &outgoing_frames, total_duration, &outgoing_zoom, + outgoing_cursor_timeline + .as_ref() + .unwrap_or(&cursor_timeline), ); frame_renderer diff --git a/crates/rendering/src/cursor_interpolation.rs b/crates/rendering/src/cursor_interpolation.rs index 002283bb6e9..5696c2d47b7 100644 --- a/crates/rendering/src/cursor_interpolation.rs +++ b/crates/rendering/src/cursor_interpolation.rs @@ -3,7 +3,9 @@ use std::{ sync::{Arc, Mutex}, }; -use cap_project::{ClickSpringConfig, CursorClickEvent, CursorEvents, CursorMoveEvent, XY}; +use cap_project::{ + ClickSpringConfig, CursorClickEvent, CursorEvents, CursorMoveEvent, ProjectConfiguration, XY, +}; use crate::{ Coord, RawDisplayUVSpace, @@ -270,7 +272,7 @@ pub fn interpolate_cursor_with_click_spring( smoothing_config, click_spring, ); - interpolate_timeline(&timeline, time_ms) + interpolate_timeline(&timeline.events, &timeline.cursor_ids, time_ms) } else { interpolate_raw_cursor(cursor, time_ms, false) } @@ -348,6 +350,7 @@ pub(crate) struct CursorEventOrder { pub struct PrecomputedCursorTimeline { timeline: Vec, + cursor_ids: Vec, raw_cursor: CursorEvents, raw_moves_are_ordered: bool, clicks_are_ordered: bool, @@ -372,6 +375,7 @@ impl PrecomputedCursorTimeline { if cursor.moves.is_empty() || smoothing.is_none() { return Self { timeline: vec![], + cursor_ids: Vec::new(), raw_cursor: cursor.clone(), raw_moves_are_ordered, clicks_are_ordered, @@ -383,7 +387,7 @@ impl PrecomputedCursorTimeline { let smoothing_config = smoothing.unwrap(); let filtered_moves = filter_cursor_shake(&cursor.moves); let prepared_moves = decimate_cursor_moves(filtered_moves.as_ref()); - let timeline = build_smoothed_timeline( + let built = build_smoothed_timeline( cursor, prepared_moves.as_ref(), smoothing_config, @@ -391,7 +395,8 @@ impl PrecomputedCursorTimeline { ); Self { - timeline, + timeline: built.events, + cursor_ids: built.cursor_ids, raw_cursor: CursorEvents::default(), raw_moves_are_ordered, clicks_are_ordered, @@ -400,6 +405,21 @@ impl PrecomputedCursorTimeline { } } + /// Spring settings match playback and export: the base cursor, with style + /// overrides applied later by `ProjectUniforms::new_with_precomputed_cursor`. + pub fn from_project(cursor: &CursorEvents, project: &ProjectConfiguration) -> Self { + let smoothing = (!project.cursor.raw).then_some(SpringMassDamperSimulationConfig { + tension: project.cursor.tension, + mass: project.cursor.mass, + friction: project.cursor.friction, + }); + Self::new( + cursor, + smoothing, + Some(project.cursor.click_spring_config()), + ) + } + pub(crate) fn cached_variant( &self, cursor: &CursorEvents, @@ -443,7 +463,7 @@ impl PrecomputedCursorTimeline { pub fn interpolate(&self, time_secs: f32) -> Option { let time_ms = (time_secs * 1000.0) as f64; if self.has_smoothing { - interpolate_timeline(&self.timeline, time_ms) + interpolate_timeline(&self.timeline, &self.cursor_ids, time_ms) } else { interpolate_raw_cursor(&self.raw_cursor, time_ms, self.raw_moves_are_ordered) } @@ -463,14 +483,59 @@ impl PrecomputedCursorTimeline { } } +struct SmoothedTimeline { + events: Vec, + cursor_ids: Vec, +} + +/// Unique cursor images for one spring build. Steps store an index so a long +/// recording does not allocate a `String` at every 60 Hz sample. +struct CursorIdIndex { + ids: Vec, + last: Option, +} + +impl CursorIdIndex { + fn new() -> Self { + Self { + ids: Vec::new(), + last: None, + } + } + + fn intern(&mut self, id: &str) -> u32 { + if let Some(last) = self.last + && self.ids[last as usize] == id + { + return last; + } + if let Some(index) = self.ids.iter().position(|existing| existing == id) { + let index = index as u32; + self.last = Some(index); + return index; + } + let index = self.ids.len() as u32; + self.ids.push(id.to_owned()); + self.last = Some(index); + index + } +} + +fn cursor_id_string(cursor_ids: &[String], index: u32) -> String { + cursor_ids[index as usize].clone() +} + fn build_smoothed_timeline( cursor: &CursorEvents, moves: &[CursorMoveEvent], smoothing_config: SpringMassDamperSimulationConfig, click_spring: Option, -) -> Vec { +) -> SmoothedTimeline { if moves.is_empty() { - return vec![]; + return SmoothedTimeline { + events: vec![], + cursor_ids: Vec::new(), + }; } let moves_are_ordered = @@ -491,12 +556,13 @@ fn build_smoothed_timeline( let mut events = Vec::with_capacity(capacity); let mut target_hint: usize = 0; let mut cid_hint: usize = 0; + let mut cursor_ids = CursorIdIndex::new(); events.push(SmoothedCursorEvent { time: 0.0, position: start_pos, velocity: XY::new(0.0, 0.0), - cursor_id: moves[0].cursor_id.clone(), + cursor_id: cursor_ids.intern(&moves[0].cursor_id), }); let mut t_ms = SIMULATION_STEP_MS; @@ -516,7 +582,7 @@ fn build_smoothed_timeline( let lead_t = (clamped_t + lead_ms).min(end_time_ms); let (cx, cy) = position_at_time_hinted(moves, lead_t, &mut target_hint); let _ = position_at_time_hinted(moves, clamped_t, &mut cid_hint); - let cid = cursor_id_at_time(moves, clamped_t, cid_hint).to_string(); + let cursor_id = cursor_ids.intern(cursor_id_at_time(moves, clamped_t, cid_hint)); let target = if let Some(click) = next_click_within(&cursor.clicks, t_ms, CLICK_LOOKAHEAD_TARGET_MS) @@ -536,7 +602,7 @@ fn build_smoothed_timeline( time: t_ms as f32, position: sim.position, velocity: sim.velocity, - cursor_id: cid, + cursor_id, }); if t_ms >= end_time_ms && sim.is_at_rest() { @@ -546,11 +612,15 @@ fn build_smoothed_timeline( t_ms += SIMULATION_STEP_MS; } - events + SmoothedTimeline { + events, + cursor_ids: cursor_ids.ids, + } } fn interpolate_timeline( events: &[SmoothedCursorEvent], + cursor_ids: &[String], query_ms: f64, ) -> Option { if events.is_empty() { @@ -564,7 +634,7 @@ fn interpolate_timeline( return Some(InterpolatedCursorPosition { position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), velocity: e.velocity, - cursor_id: e.cursor_id.clone(), + cursor_id: cursor_id_string(cursor_ids, e.cursor_id), }); } @@ -573,7 +643,7 @@ fn interpolate_timeline( return Some(InterpolatedCursorPosition { position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), velocity: e.velocity, - cursor_id: e.cursor_id.clone(), + cursor_id: cursor_id_string(cursor_ids, e.cursor_id), }); } @@ -603,7 +673,7 @@ fn interpolate_timeline( return Some(InterpolatedCursorPosition { position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), velocity: e.velocity, - cursor_id: e.cursor_id.clone(), + cursor_id: cursor_id_string(cursor_ids, e.cursor_id), }); } } @@ -626,7 +696,7 @@ fn interpolate_timeline( a.velocity.x * inv + b.velocity.x * t, a.velocity.y * inv + b.velocity.y * t, ), - cursor_id: a.cursor_id.clone(), + cursor_id: cursor_id_string(cursor_ids, a.cursor_id), }) } @@ -732,7 +802,7 @@ struct SmoothedCursorEvent { time: f32, position: XY, velocity: XY, - cursor_id: String, + cursor_id: u32, } #[cfg(test)] @@ -1001,12 +1071,19 @@ mod tests { }) } + struct OracleCursorEvent { + time: f32, + position: XY, + velocity: XY, + cursor_id: String, + } + fn linear_build_smoothed_timeline( cursor: &CursorEvents, moves: &[CursorMoveEvent], smoothing_config: SpringMassDamperSimulationConfig, click_spring: Option, - ) -> Vec { + ) -> Vec { if moves.is_empty() { return vec![]; } @@ -1028,7 +1105,7 @@ mod tests { let mut target_hint: usize = 0; let mut cid_hint: usize = 0; - events.push(SmoothedCursorEvent { + events.push(OracleCursorEvent { time: 0.0, position: start_pos, velocity: XY::new(0.0, 0.0), @@ -1049,7 +1126,7 @@ mod tests { let lead_t = (clamped_t + lead_ms).min(end_time_ms); let (cx, cy) = position_at_time_hinted(moves, lead_t, &mut target_hint); let _ = position_at_time_hinted(moves, clamped_t, &mut cid_hint); - let cid = cursor_id_at_time(moves, clamped_t, cid_hint).to_string(); + let cursor_id = cursor_id_at_time(moves, clamped_t, cid_hint).to_string(); let target = if let Some(click) = next_click_within(&cursor.clicks, t_ms, CLICK_LOOKAHEAD_TARGET_MS) @@ -1064,11 +1141,11 @@ mod tests { sim.run(SIMULATION_STEP_MS as f32); - events.push(SmoothedCursorEvent { + events.push(OracleCursorEvent { time: t_ms as f32, position: sim.position, velocity: sim.velocity, - cursor_id: cid, + cursor_id, }); if t_ms >= end_time_ms && sim.is_at_rest() { @@ -1081,8 +1158,8 @@ mod tests { events } - fn linear_interpolate_timeline( - events: &[SmoothedCursorEvent], + fn oracle_interpolate_timeline( + events: &[OracleCursorEvent], query_ms: f64, ) -> Option { if events.is_empty() { @@ -1109,6 +1186,70 @@ mod tests { }); } + let end = events.partition_point(|event| event.time <= query); + let Some(window) = end + .checked_sub(1) + .and_then(|start| events.get(start..=end)) + .filter(|window| window[0].time <= query && query < window[1].time) + else { + let e = events.last().unwrap(); + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: e.cursor_id.clone(), + }); + }; + let (a, b) = (&window[0], &window[1]); + let dt = b.time - a.time; + let t = if dt.abs() < 1e-6 { + 0.0 + } else { + ((query - a.time) / dt).clamp(0.0, 1.0) + }; + let inv = 1.0 - t; + + Some(InterpolatedCursorPosition { + position: Coord::new(XY::new( + (a.position.x * inv + b.position.x * t) as f64, + (a.position.y * inv + b.position.y * t) as f64, + )), + velocity: XY::new( + a.velocity.x * inv + b.velocity.x * t, + a.velocity.y * inv + b.velocity.y * t, + ), + cursor_id: a.cursor_id.clone(), + }) + } + + fn linear_interpolate_timeline( + events: &[SmoothedCursorEvent], + cursor_ids: &[String], + query_ms: f64, + ) -> Option { + if events.is_empty() { + return None; + } + + let query = query_ms as f32; + + if query <= events[0].time { + let e = &events[0]; + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: cursor_id_string(cursor_ids, e.cursor_id), + }); + } + + if query >= events.last().unwrap().time { + let e = events.last().unwrap(); + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: cursor_id_string(cursor_ids, e.cursor_id), + }); + } + let first_time = events[0].time; let step = if events.len() > 1 { events[1].time - events[0].time @@ -1135,7 +1276,7 @@ mod tests { return Some(InterpolatedCursorPosition { position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), velocity: e.velocity, - cursor_id: e.cursor_id.clone(), + cursor_id: cursor_id_string(cursor_ids, e.cursor_id), }); } } @@ -1158,7 +1299,7 @@ mod tests { a.velocity.x * inv + b.velocity.x * t, a.velocity.y * inv + b.velocity.y * t, ), - cursor_id: a.cursor_id.clone(), + cursor_id: cursor_id_string(cursor_ids, a.cursor_id), }) } @@ -1185,7 +1326,11 @@ mod tests { } } - fn assert_timeline_bits(expected: &[SmoothedCursorEvent], actual: &[SmoothedCursorEvent]) { + fn assert_timeline_bits( + expected: &[OracleCursorEvent], + actual: &[SmoothedCursorEvent], + cursor_ids: &[String], + ) { assert_eq!(expected.len(), actual.len()); for (index, (expected, actual)) in expected.iter().zip(actual).enumerate() { assert_eq!( @@ -1213,16 +1358,23 @@ mod tests { actual.velocity.y.to_bits(), "velocity.y at {index}" ); - assert_eq!(expected.cursor_id, actual.cursor_id, "cursor_id at {index}"); + assert_eq!( + expected.cursor_id, cursor_ids[actual.cursor_id as usize], + "cursor_id at {index}" + ); } } + fn sample_cursor_ids() -> Vec { + vec!["0".into(), "1".into(), "2".into()] + } + fn sampled_event(time: f32, index: usize) -> SmoothedCursorEvent { SmoothedCursorEvent { time, position: XY::new(index as f32 * 0.013, index as f32 * -0.007), velocity: XY::new(index as f32 * -0.002, index as f32 * 0.003), - cursor_id: (index % 3).to_string(), + cursor_id: (index % 3) as u32, } } @@ -1265,7 +1417,9 @@ mod tests { assert!(timeline.has_smoothing); assert!(timeline.raw_cursor.moves.is_empty()); assert!(timeline.raw_cursor.clicks.is_empty()); - assert_timeline_bits(&expected, &timeline.timeline); + assert!(timeline.cursor_ids.len() <= 3); + assert!(timeline.timeline.len() > timeline.cursor_ids.len()); + assert_timeline_bits(&expected, &timeline.timeline, &timeline.cursor_ids); drop(moves); drop(filtered); drop(cursor); @@ -1273,7 +1427,7 @@ mod tests { for index in 0..=500 { let time_secs = index as f32 * 0.017_137; assert_position_bits( - linear_interpolate_timeline(&expected, (time_secs * 1000.0) as f64), + oracle_interpolate_timeline(&expected, (time_secs * 1000.0) as f64), timeline.interpolate(time_secs), ); } @@ -1676,12 +1830,12 @@ mod tests { let expected = linear_build_smoothed_timeline(&cursor, &moves, config, click_spring); let actual = build_smoothed_timeline(&cursor, &moves, config, click_spring); - assert_timeline_bits(&expected, &actual); + assert_timeline_bits(&expected, &actual.events, &actual.cursor_ids); for index in 0..=500 { let query = index as f64 * 17.137; assert_position_bits( - linear_interpolate_timeline(&expected, query), - interpolate_timeline(&actual, query), + oracle_interpolate_timeline(&expected, query), + interpolate_timeline(&actual.events, &actual.cursor_ids, query), ); } } @@ -1692,9 +1846,10 @@ mod tests { #[test] fn timeline_lookup_matches_linear_reference_at_long_times_and_duplicate_timestamps() { assert_position_bits( - linear_interpolate_timeline(&[], 0.0), - interpolate_timeline(&[], 0.0), + linear_interpolate_timeline(&[], &[], 0.0), + interpolate_timeline(&[], &[], 0.0), ); + let cursor_ids = sample_cursor_ids(); for duration_hours in [0.0, 1.0, 20.0] { for interval_ms in [1.0, SIMULATION_STEP_MS] { let origin_ms = duration_hours * 3_600_000.0; @@ -1717,8 +1872,8 @@ mod tests { origin_ms + 10_000.0, ] { assert_position_bits( - linear_interpolate_timeline(&events, query), - interpolate_timeline(&events, query), + linear_interpolate_timeline(&events, &cursor_ids, query), + interpolate_timeline(&events, &cursor_ids, query), ); } for event in &events { @@ -1729,8 +1884,8 @@ mod tests { event.time as f64 + interval_ms * 0.5, ] { assert_position_bits( - linear_interpolate_timeline(&events, query), - interpolate_timeline(&events, query), + linear_interpolate_timeline(&events, &cursor_ids, query), + interpolate_timeline(&events, &cursor_ids, query), ); } } @@ -1743,8 +1898,8 @@ mod tests { .collect(); for query in [-0.0, 0.0, 0.01, 8.0, 16.0, 16.01, 24.0, 32.0] { assert_position_bits( - linear_interpolate_timeline(&duplicate_origin, query), - interpolate_timeline(&duplicate_origin, query), + linear_interpolate_timeline(&duplicate_origin, &cursor_ids, query), + interpolate_timeline(&duplicate_origin, &cursor_ids, query), ); } } @@ -1765,12 +1920,16 @@ mod tests { let started = Instant::now(); let actual = build_smoothed_timeline(&cursor, &moves, DEFAULT_CLICK_SPRING, None); let optimized_time = started.elapsed(); - assert_timeline_bits(black_box(&expected), black_box(&actual)); + assert_timeline_bits( + black_box(&expected), + black_box(&actual.events), + black_box(&actual.cursor_ids), + ); println!( "cursor_precompute duration_secs={duration_secs} moves={} clicks={} samples={} baseline_ms={:.3} optimized_ms={:.3}", moves.len(), cursor.clicks.len(), - actual.len(), + actual.events.len(), baseline_time.as_secs_f64() * 1000.0, optimized_time.as_secs_f64() * 1000.0 ); @@ -1779,23 +1938,26 @@ mod tests { let events: Vec<_> = (0..=20 * 60 * 60 * 60) .map(|index| { let mut event = sampled_event((index as f64 * SIMULATION_STEP_MS) as f32, index); - event.cursor_id = String::new(); + event.cursor_id = 0; event }) .collect(); + let cursor_ids = [String::new()]; let queries: Vec<_> = (0..128) .map(|index| 72_000_000.0 - index as f64 * 171.13 - 1.0) .collect(); let started = Instant::now(); let expected: Vec<_> = queries .iter() - .map(|&query| linear_interpolate_timeline(black_box(&events), black_box(query))) + .map(|&query| { + linear_interpolate_timeline(black_box(&events), &cursor_ids, black_box(query)) + }) .collect(); let baseline_time = started.elapsed(); let started = Instant::now(); let actual: Vec<_> = queries .iter() - .map(|&query| interpolate_timeline(black_box(&events), black_box(query))) + .map(|&query| interpolate_timeline(black_box(&events), &cursor_ids, black_box(query))) .collect(); let optimized_time = started.elapsed(); for (expected, actual) in expected.into_iter().zip(actual) {