diff --git a/crates/chatd/src/media.rs b/crates/chatd/src/media.rs index 8a5aed1f..4674857c 100644 --- a/crates/chatd/src/media.rs +++ b/crates/chatd/src/media.rs @@ -230,6 +230,7 @@ fn new_call(conversation: Key, state: CallState) -> Live { security: None, picture: None, self_view: None, + share_view: None, }; Live { intent: NEXT_INTENT.fetch_add(1, Ordering::Relaxed), @@ -415,12 +416,14 @@ impl Live { } self.call.picture = picture; } - MediaResponse::SelfView { path, width, height, generation } => { - let picture = (!path.is_empty()).then(|| CallPicture { path, width, height, generation, user_id: String::new(), kind: PictureKind::Camera }); - if self.call.self_view == picture { + MediaResponse::SelfView { path, width, height, generation, share } => { + let kind = if share { PictureKind::Share } else { PictureKind::Camera }; + let picture = (!path.is_empty()).then(|| CallPicture { path, width, height, generation, user_id: String::new(), kind }); + let slot = if share { &mut self.call.share_view } else { &mut self.call.self_view }; + if *slot == picture { return false; } - self.call.self_view = picture; + *slot = picture; } MediaResponse::Sending { share, camera } => { let mut video: Vec = self.call.video.iter().filter(|stream| stream.direction != Direction::Encode).cloned().collect(); @@ -430,7 +433,9 @@ impl Live { if share { video.insert(0, VideoStream { direction: Direction::Encode, content: Content::Share, source: String::new() }); } - if video == self.call.video { + // My preview of a share that is no longer sent goes with it. + let dropped = !share && self.call.share_view.take().is_some(); + if video == self.call.video && !dropped { return false; } self.call.video = video; @@ -1741,6 +1746,34 @@ mod tests { } } + #[test] + fn my_camera_and_my_share_keep_their_own_self_views() { + let view = |path: &str, generation: u64, share: bool| MediaResponse::SelfView { path: path.into(), width: 640, height: 360, generation, share }; + let mut live = live(CallState::Joined); + assert!(live.apply(view("camera.jpg", 1, false))); + assert!(live.apply(view("share.jpg", 1, true))); + let camera = live.call.self_view.clone().unwrap(); + let share = live.call.share_view.clone().unwrap(); + assert_eq!((camera.path.as_str(), camera.kind), ("camera.jpg", PictureKind::Camera)); + assert_eq!((share.path.as_str(), share.kind), ("share.jpg", PictureKind::Share)); + assert!(!live.apply(view("share.jpg", 1, true)), "the same picture is no news"); + + // The share going off takes only its own view. + assert!(live.apply(view("", 0, true))); + assert!(live.call.share_view.is_none()); + assert!(live.call.self_view.is_some(), "the camera is still on"); + } + + #[test] + fn a_share_that_is_no_longer_sent_takes_its_view_with_it() { + let mut live = live(CallState::Joined); + assert!(live.apply(MediaResponse::SelfView { path: "share.jpg".into(), width: 640, height: 360, generation: 1, share: true })); + live.apply(MediaResponse::Sending { share: true, camera: false }); + assert!(live.call.share_view.is_some(), "still sending"); + assert!(live.apply(MediaResponse::Sending { share: false, camera: false })); + assert!(live.call.share_view.is_none()); + } + #[test] fn only_the_voice_connection_being_up_makes_a_call_joined() { let mut live = live(CallState::Connecting); diff --git a/crates/mediad/src/discord_video.rs b/crates/mediad/src/discord_video.rs index 7c132ac5..2772e2e7 100644 --- a/crates/mediad/src/discord_video.rs +++ b/crates/mediad/src/discord_video.rs @@ -241,15 +241,16 @@ impl Drive { match word { Word::Frame { payload, rtp, .. } => self.send_frame(dave, socket, rtp as u32, &payload), Word::Failed { words, .. } => out::send(&MediaResponse::Notice { detail: words }), - // The camera thread paints my own tile. A screen leg has no camera, so it sends none. - Word::SelfView(painted) => out::send(&match painted { + // The capture thread paints my own tile: my screen, from a screen leg. + Word::SelfView { share, painted } => out::send(&match painted { Some(painted) => MediaResponse::SelfView { path: painted.path.display().to_string(), width: painted.width, height: painted.height, generation: painted.generation, + share, }, - None => MediaResponse::SelfView { path: String::new(), width: 0, height: 0, generation: 0 }, + None => MediaResponse::SelfView { path: String::new(), width: 0, height: 0, generation: 0, share }, }), } } diff --git a/crates/mediad/src/picture.rs b/crates/mediad/src/picture.rs index a14b2d796..fe0033d1 100644 --- a/crates/mediad/src/picture.rs +++ b/crates/mediad/src/picture.rs @@ -413,6 +413,14 @@ impl Canvas { } } +/// A call that ends while a picture is on (the capture thread told to stop, with nothing turned +/// off first) leaves no picture of a screen or a face behind in the temp folder. +impl Drop for Canvas { + fn drop(&mut self) { + self.forget(); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/mediad/src/picture/desktop.rs b/crates/mediad/src/picture/desktop.rs index e0260fad..54d777e7 100644 --- a/crates/mediad/src/picture/desktop.rs +++ b/crates/mediad/src/picture/desktop.rs @@ -131,11 +131,47 @@ pub struct Outgoing { camera: Camera, cost: Cost, camera_cost: GrabCost, - /// My own camera, painted for me (`selfie`). - selfie_canvas: Canvas, - selfie_at: Option, - selfie_generation: u64, - selfie_painted: Option, + /// My own camera, painted for me. + selfie: Selfie, + /// My own share, painted for me. Apart from the camera's: both can be on at once. + screen_selfie: Selfie, +} + +/// One of my own pictures shrunk and written as a JPEG a few times a second, for my own tile. +#[derive(Default)] +struct Selfie { + canvas: Canvas, + at: Option, + generation: u64, + painted: Option, +} + +impl Selfie { + /// Paints this grab, unless the last was painted less than [`super::SELF_VIEW_EVERY`] ago. + fn shoot(&mut self, grabbed: &Rgba) { + if self.at.is_some_and(|at| at.elapsed() < super::SELF_VIEW_EVERY) { + return; + } + self.at = Some(Instant::now()); + let (width, height) = fit(grabbed.width, grabbed.height, super::SELF_VIEW.0, super::SELF_VIEW.1); + let rgb = super::screens::shrink(grabbed, width, height); + self.generation = self.generation.wrapping_add(1); + if let Some(painted) = self.canvas.paint(&rgb, width, height, self.generation) { + self.painted = Some(painted); + } + } + + /// The last picture painted and not yet taken. + fn take(&mut self) -> Option { + self.painted.take() + } + + /// The picture's file goes, and the next shot is due at once. + fn forget(&mut self) { + self.canvas.forget(); + self.painted = None; + self.at = None; + } } /// What a camera frame costs to read, in the log once a second or so, with the rate the camera @@ -209,29 +245,22 @@ impl Outgoing { let (width, height) = fit(grabbed.width, grabbed.height, quality.max.0, quality.max.1); let planes = i420(&grabbed.pixels, grabbed.width, grabbed.height, width, height, grabbed.bgra); let converted = Instant::now(); + // The grab is the whole screen, so the preview costs one shrink a tenth of a second. + self.screen_selfie.shoot(&grabbed); let encoded = coded(&mut self.share, width, height, quality.bps, quality.fps as f32, true)?.encode(&planes, &ask); self.cost.note(began, captured, converted, Instant::now(), (grabbed.width, grabbed.height), (width, height)); encoded } - /// My own camera for my own tile, a few times a second: the grab shrunk and written as a - /// JPEG, taken by [`Outgoing::self_view`]. - fn selfie(&mut self, grabbed: &Rgba) { - if self.selfie_at.is_some_and(|at| at.elapsed() < super::SELF_VIEW_EVERY) { - return; - } - self.selfie_at = Some(Instant::now()); - let (width, height) = fit(grabbed.width, grabbed.height, super::SELF_VIEW.0, super::SELF_VIEW.1); - let rgb = super::screens::shrink(grabbed, width, height); - self.selfie_generation = self.selfie_generation.wrapping_add(1); - if let Some(painted) = self.selfie_canvas.paint(&rgb, width, height, self.selfie_generation) { - self.selfie_painted = Some(painted); - } + /// My own camera as painted for me since last asked: a few times a second, the camera's + /// grab shrunk and written as a JPEG. + pub fn self_view(&mut self) -> Option { + self.selfie.take() } - /// The last self view painted and not yet taken. - pub fn self_view(&mut self) -> Option { - self.selfie_painted.take() + /// My own share as painted for me since last asked, on the same terms as [`Outgoing::self_view`]. + pub fn share_view(&mut self) -> Option { + self.screen_selfie.take() } /// The camera's two layers, high then low, or why the camera could not be read. A layer with @@ -243,7 +272,7 @@ impl Outgoing { let began = Instant::now(); let grabbed = self.camera.frame()?; self.camera_cost.note(began.elapsed(), (grabbed.width, grabbed.height)); - self.selfie(&grabbed); + self.selfie.shoot(&grabbed); let layer = |slot: &mut Option, ask: Option, most: (u32, u32), nominal: u32| match ask { Some(ask) => { let (width, height) = fit(grabbed.width, grabbed.height, most.0, most.1); @@ -259,14 +288,13 @@ impl Outgoing { pub fn close_camera(&mut self) { self.camera.close(); - self.selfie_canvas.forget(); - self.selfie_painted = None; - self.selfie_at = None; + self.selfie.forget(); } /// The share is off. Drop the encoder, and on Linux close the portal session so the recording indicator does not stay up. pub fn close_share(&mut self) { self.share = None; + self.screen_selfie.forget(); #[cfg(all(target_os = "linux", not(target_env = "ohos")))] super::portal::stop_screen(); } @@ -939,6 +967,45 @@ pub fn probe_camera() -> Result<(), String> { mod tests { use super::*; + fn screen() -> Rgba { + Rgba { width: 1920, height: 1080, pixels: vec![200; 1920 * 1080 * 4], bgra: true } + } + + #[test] + fn a_self_view_is_shrunk_to_the_corner_size_painted_at_a_steady_rate_and_let_go() { + let mut outgoing = Outgoing::default(); + outgoing.screen_selfie.shoot(&screen()); + let first = outgoing.share_view().expect("the first grab is painted at once"); + assert_eq!((first.width, first.height), super::super::SELF_VIEW); + assert!(first.path.exists()); + assert!(outgoing.share_view().is_none(), "taken once"); + assert!(outgoing.self_view().is_none(), "the camera's slot is a different one"); + + outgoing.screen_selfie.shoot(&screen()); + assert!(outgoing.share_view().is_none(), "a grab inside the interval is not painted"); + + outgoing.screen_selfie.at = None; + outgoing.screen_selfie.shoot(&screen()); + let next = outgoing.share_view().unwrap(); + assert_ne!(next.generation, first.generation); + assert!(!first.path.exists(), "the file before it is gone"); + + outgoing.screen_selfie.forget(); + assert!(!next.path.exists(), "and so is the last one when the share stops"); + } + + #[test] + fn a_call_that_ends_while_sharing_leaves_no_picture_of_the_screen_behind() { + let mut outgoing = Outgoing::default(); + outgoing.screen_selfie.shoot(&screen()); + outgoing.selfie.shoot(&screen()); + let files = [outgoing.share_view().unwrap().path, outgoing.self_view().unwrap().path]; + assert!(files.iter().all(|file| file.exists())); + // `Video::stop` ends the capture thread without turning the share off first. + drop(outgoing); + assert!(files.iter().all(|file| !file.exists()), "{files:?}"); + } + #[test] fn openh264_is_fetched_from_cisco() { let name = library_name().unwrap_or("openh264-2.6.0-win64.dll"); diff --git a/crates/mediad/src/picture/phone.rs b/crates/mediad/src/picture/phone.rs index bd3e0eed..eb746ec6 100644 --- a/crates/mediad/src/picture/phone.rs +++ b/crates/mediad/src/picture/phone.rs @@ -130,6 +130,16 @@ impl Outgoing { Ok((high, low)) } + /// A phone paints no preview of its own camera or share for its call screen, so `Source` has + /// nothing to take from it. + pub fn self_view(&mut self) -> Option { + None + } + + pub fn share_view(&mut self) -> Option { + None + } + /// The button is off: a phone's camera light goes off with it. pub fn close_camera(&mut self) { self.camera = None; diff --git a/crates/mediad/src/tiny.rs b/crates/mediad/src/tiny.rs index 26f8b125..52c9d571 100644 --- a/crates/mediad/src/tiny.rs +++ b/crates/mediad/src/tiny.rs @@ -1047,9 +1047,9 @@ impl Call { out::send(&MediaResponse::Sending { share: false, camera: self.sending_camera }); } } - Word::SelfView(painted) => out::send(&match painted { - Some(painted) => MediaResponse::SelfView { path: painted.path.display().to_string(), width: painted.width, height: painted.height, generation: painted.generation }, - None => MediaResponse::SelfView { path: String::new(), width: 0, height: 0, generation: 0 }, + Word::SelfView { share, painted } => out::send(&match painted { + Some(painted) => MediaResponse::SelfView { path: painted.path.display().to_string(), width: painted.width, height: painted.height, generation: painted.generation, share }, + None => MediaResponse::SelfView { path: String::new(), width: 0, height: 0, generation: 0, share }, }), Word::Failed { share: false, words } => { if !self.camera_noted { diff --git a/crates/mediad/src/video.rs b/crates/mediad/src/video.rs index f88de56f..14bd6dec 100644 --- a/crates/mediad/src/video.rs +++ b/crates/mediad/src/video.rs @@ -251,8 +251,8 @@ pub enum Word { }, /// The share or the camera cannot go on, in words. Failed { share: bool, words: String }, - /// My own camera, painted for my tile; `None` when the camera went off. - SelfView(Option), + /// My own camera, or with `share` my own share, painted for my tile; `None` when it went off. + SelfView { share: bool, painted: Option }, } /// The RTP time at 90 kHz of a capture `since` the first one. @@ -382,6 +382,10 @@ pub trait Source { fn self_view(&mut self) -> Option { None } + /// My own share painted for me since last asked, on the same terms. + fn share_view(&mut self) -> Option { + None + } } impl Source for Outgoing { @@ -389,6 +393,10 @@ impl Source for Outgoing { Outgoing::self_view(self) } + fn share_view(&mut self) -> Option { + Outgoing::share_view(self) + } + fn share(&mut self, ask: Ask) -> Result { Outgoing::share(self, ask) } @@ -526,6 +534,11 @@ impl Capture { return false; } } + if let Some(painted) = self.source.share_view() { + if self.tell.send(Word::SelfView { share: true, painted: Some(painted) }).is_err() { + return false; + } + } } if wants.camera { let high = self.ask(Track::High, wants.rates.high, since, at, fps); @@ -537,13 +550,11 @@ impl Capture { { return false; } - if wants.camera { - if let Some(painted) = self.source.self_view() { - if self.tell.send(Word::SelfView(Some(painted))).is_err() { - return false; - } - } - } + if let Some(painted) = self.source.self_view() { + if self.tell.send(Word::SelfView { share: false, painted: Some(painted) }).is_err() { + return false; + } + } } Err(words) => { if let Some(ask) = high { @@ -633,10 +644,11 @@ fn capture(mut work: Capture, orders: &Orders) { work.source.close_camera(); } if seen.camera && !wants.camera { - let _ = work.tell.send(Word::SelfView(None)); + let _ = work.tell.send(Word::SelfView { share: false, painted: None }); } if seen.share && !wants.share { work.source.close_share(); + let _ = work.tell.send(Word::SelfView { share: true, painted: None }); } schedule.rate(&wants); seen = wants; @@ -1395,6 +1407,12 @@ mod tests { skip_next: bool, refuse: Option, closed: u32, + /// Paints a picture of my camera and of my share with every round, as a desktop does. + painting: bool, + } + + fn painted(name: &str) -> Painted { + Painted { path: std::path::PathBuf::from(name), width: 4, height: 2, generation: 1 } } fn coded(ask: Ask, name: u8) -> Encoded { @@ -1430,6 +1448,14 @@ mod tests { fn close_camera(&mut self) { self.closed += 1; } + + fn self_view(&mut self) -> Option { + self.painting.then(|| painted("camera.jpg")) + } + + fn share_view(&mut self) -> Option { + self.painting.then(|| painted("share.jpg")) + } } fn work(controls: Arc) -> (Capture, Receiver) { @@ -1493,6 +1519,76 @@ mod tests { ); } + fn views(words: &Receiver) -> Vec<(bool, Option)> { + words + .try_iter() + .filter_map(|word| match word { + Word::SelfView { share, painted } => Some((share, painted.map(|painted| painted.path.display().to_string()))), + _ => None, + }) + .collect() + } + + #[test] + fn my_share_and_my_camera_are_painted_for_me_apart_when_both_are_on() { + let (mut capture, words) = work(Arc::default()); + capture.source.painting = true; + assert!(capture.round(&wanting(true, true), Instant::now(), 15.)); + assert_eq!( + views(&words), + [(true, Some("share.jpg".to_owned())), (false, Some("camera.jpg".to_owned()))], + "one slot each, so neither clobbers the other" + ); + } + + #[test] + fn only_what_is_on_is_painted_for_me() { + let (mut capture, words) = work(Arc::default()); + capture.source.painting = true; + assert!(capture.round(&wanting(true, false), Instant::now(), 15.)); + assert_eq!(views(&words), [(true, Some("share.jpg".to_owned()))], "a share alone has no camera view"); + assert!(capture.round(&wanting(false, true), Instant::now(), 15.)); + assert_eq!(views(&words), [(false, Some("camera.jpg".to_owned()))], "a camera alone has no share view"); + } + + #[test] + fn the_capture_thread_takes_my_share_view_down_when_the_share_stops() { + struct Viewing(Fake); + impl Source for Viewing { + fn share(&mut self, ask: Ask) -> Result { + self.0.share(ask) + } + fn camera(&mut self, high: Option, low: Option) -> Result<(Encoded, Encoded), String> { + self.0.camera(high, low) + } + fn close_camera(&mut self) {} + fn share_view(&mut self) -> Option { + self.0.share_view() + } + } + let sender = start_capture(|| Viewing(Fake { painting: true, ..Fake::default() }), Arc::default()).unwrap(); + let tell = |wants: Wants| { + sender.orders.order.lock().unwrap().wants = wants; + sender.orders.changed.notify_all(); + }; + tell(wanting(true, false)); + let seen = |word: &Word| matches!(word, Word::SelfView { share: true, painted: Some(_) }); + loop { + let word = sender.words.recv_timeout(Duration::from_secs(5)).unwrap(); + if seen(&word) { + break; + } + } + tell(Wants::default()); + let mut cleared = false; + while let Ok(word) = sender.words.recv_timeout(Duration::from_millis(300)) { + cleared |= matches!(word, Word::SelfView { share: true, painted: None }); + } + assert!(cleared, "the tile is told my share is gone"); + sender.orders.order.lock().unwrap().stop = true; + sender.orders.changed.notify_all(); + } + #[test] fn a_frame_sealing_drops_asks_again_and_a_skipped_keyframe_is_still_owed() { let controls: Arc = Arc::default(); @@ -1947,7 +2043,7 @@ mod tests { bytes += payload.len(); } Some(Word::Failed { words, .. }) => panic!("{words}"), - Some(Word::SelfView(_)) => {} + Some(Word::SelfView { .. }) => {} None => std::thread::yield_now(), } } @@ -1956,6 +2052,45 @@ mod tests { video.stop(); } + /// What `e2e/scenarios/share-self-view` runs: my own share, painted for me from the real + /// screen through the real capture thread, and taken down when the share stops. + #[test] + #[ignore = "needs a screen and the OpenH264 binary"] + fn the_capture_thread_paints_my_own_share_from_a_real_screen() { + let mut video = Video::new(Arc::default()); + video.want(Wants { share: true, rates: Rates::default(), ..Wants::default() }); + let started = Instant::now(); + let mut shown = None; + while shown.is_none() && started.elapsed() < Duration::from_secs(20) { + match video.next() { + Some(Word::SelfView { share: true, painted: Some(painted) }) => shown = Some(painted), + Some(Word::SelfView { share: false, .. }) => panic!("the camera is off, so no camera view"), + Some(Word::Failed { words, .. }) => panic!("{words}"), + Some(_) => {} + None => std::thread::sleep(Duration::from_millis(5)), + } + } + let shown = shown.expect("my own share was never painted for me"); + let jpeg = std::fs::read(&shown.path).expect("the picture is on disk"); + let size = image::load_from_memory_with_format(&jpeg, image::ImageFormat::Jpeg).expect("a JPEG the window can read"); + assert_eq!((size.width(), size.height()), (shown.width, shown.height)); + assert!(shown.width <= picture::SELF_VIEW.0 && shown.height <= picture::SELF_VIEW.1, "{}x{}", shown.width, shown.height); + eprintln!("share-self-view-proof: {}x{} JPEG of the real screen, {} bytes", shown.width, shown.height, jpeg.len()); + video.want(Wants::default()); + let stopped = Instant::now(); + let mut cleared = false; + while !cleared && stopped.elapsed() < Duration::from_secs(10) { + match video.next() { + Some(Word::SelfView { share: true, painted: None }) => cleared = true, + Some(_) => {} + None => std::thread::sleep(Duration::from_millis(5)), + } + } + assert!(cleared, "the window is told my share is gone"); + eprintln!("share-self-view-proof: cleared when the share stopped"); + video.stop(); + } + #[test] fn a_resume_starts_no_new_thread_and_hang_up_ends_them_all() { let mut video = Video::new(Arc::default()); diff --git a/crates/protocol/src/call.rs b/crates/protocol/src/call.rs index 52c07ba8..91ab574c 100644 --- a/crates/protocol/src/call.rs +++ b/crates/protocol/src/call.rs @@ -245,6 +245,10 @@ pub struct Call { /// My own camera as it is sent, for my tile. `None` while it is off. Its `user_id` is empty. #[serde(default, skip_serializing_if = "Option::is_none")] pub self_view: Option, + /// My own share as it is sent, for the stage while I am sharing. `None` while it is off. Its + /// `user_id` is empty. Apart from `self_view`: a camera and a share can be on together. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_view: Option, } /// A Tiny server call's protection (tinyserver's `docs/calls.md`, "End-to-end encryption"). @@ -890,9 +894,16 @@ pub enum MediaResponse { }, /// What our camera and share are actually sending, after an answer or a refusal. Sending { share: bool, camera: bool }, - /// Our own camera as it is being sent, a few times a second, so the person sees what - /// everyone else sees; an empty path when the camera went off. - SelfView { path: String, width: u32, height: u32, generation: u64 }, + /// Our own camera, or with `share` our own share, as it is being sent, a few times a second, + /// so the person sees what everyone else sees; an empty path when it went off. + SelfView { + path: String, + width: u32, + height: u32, + generation: u64, + #[serde(default)] + share: bool, + }, /// A picture that could not be captured, in words. The voice call stays. Notice { detail: String }, } @@ -945,6 +956,16 @@ impl MediaResponse { mod tests { use super::*; + #[test] + fn a_self_view_is_my_camera_unless_it_says_it_is_my_share() { + // A mediad from before shares had a preview sends none: that is my camera, never an error. + let older: MediaResponse = serde_json::from_str(r#"{"media":"self_view","path":"a.jpg","width":640,"height":360,"generation":3}"#).unwrap(); + assert_eq!(older, MediaResponse::SelfView { path: "a.jpg".into(), width: 640, height: 360, generation: 3, share: false }); + let mine = MediaResponse::SelfView { path: "b.jpg".into(), width: 640, height: 360, generation: 4, share: true }; + let back: MediaResponse = serde_json::from_str(&serde_json::to_string(&mine).unwrap()).unwrap(); + assert_eq!(back, mine); + } + #[test] fn a_requirement_is_sent_only_when_it_is_made_and_read_as_off_when_absent() { // An older chatd or mediad does not send it: that is "not required", never an error. diff --git a/crates/tinyclient/src/gui/call.rs b/crates/tinyclient/src/gui/call.rs index c8ad1659..d5a18e8a 100644 --- a/crates/tinyclient/src/gui/call.rs +++ b/crates/tinyclient/src/gui/call.rs @@ -223,6 +223,8 @@ pub struct State { pub(in crate::gui) feed: super::tiny::feed::Feed>, /// The frames of my own camera, for my tile. pub(in crate::gui) self_feed: super::tiny::feed::Feed>, + /// The frames of my own share, for my stream's tile and the stage. + pub(in crate::gui) screen_feed: super::tiny::feed::Feed>, /// Made the first time the dock is drawn. counter: OnceCell>, /// The same count, for a Tiny channel's stage: a second text, ticking on its own. @@ -1001,6 +1003,7 @@ impl Shell { security: None, picture: None, self_view: None, + share_view: None, }); } } diff --git a/crates/tinyclient/src/gui/call/dock.rs b/crates/tinyclient/src/gui/call/dock.rs index 2ccbdf8c..8b18ffca 100644 --- a/crates/tinyclient/src/gui/call/dock.rs +++ b/crates/tinyclient/src/gui/call/dock.rs @@ -1286,6 +1286,7 @@ mod tests { security: None, picture: None, self_view: None, + share_view: None, } } diff --git a/crates/tinyclient/src/gui/call/phone/screens.rs b/crates/tinyclient/src/gui/call/phone/screens.rs index 2c290d6d..82eb1c12 100644 --- a/crates/tinyclient/src/gui/call/phone/screens.rs +++ b/crates/tinyclient/src/gui/call/phone/screens.rs @@ -223,6 +223,7 @@ mod tests { security: None, picture: None, self_view: None, + share_view: None, } } diff --git a/crates/tinyclient/src/gui/call/security.rs b/crates/tinyclient/src/gui/call/security.rs index 762c55e9..4101d9b3 100644 --- a/crates/tinyclient/src/gui/call/security.rs +++ b/crates/tinyclient/src/gui/call/security.rs @@ -585,6 +585,7 @@ mod tests { security: Some(security), picture: None, self_view: None, + share_view: None, } } diff --git a/crates/tinyclient/src/gui/call/shot.rs b/crates/tinyclient/src/gui/call/shot.rs index a3eb6d42..5d2aaa85 100644 --- a/crates/tinyclient/src/gui/call/shot.rs +++ b/crates/tinyclient/src/gui/call/shot.rs @@ -18,6 +18,9 @@ //! faces; `tiny_security` opens the security panel from the channel's top bar, //! `tiny_security_bar` from the column's compact bar. `tiny_poor`'s numbers carry audio trouble //! too, for the health hints. +//! +//! `tiny_sharing` is a call where I share my screen and mediad has painted the first picture of +//! it for my own tile (a made-up one, the demo's schematic); `--tiny-stream` focuses my stream. use super::phone::{Sheet, Shown}; use super::security::Place; @@ -198,6 +201,7 @@ pub(super) fn call(snapshot: &Snapshot) -> Option { // The open conversation's, so the header pill reads Leave or Cancel as it would. let conversation = snapshot.open.as_ref().map_or_else(|| key("shot"), |open| open.key.clone()); let muted = matches!(name.as_str(), "joined_muted" | "muted_talking"); + let sharing = name == "sharing"; let participants = if tiny() { // A Tiny stage draws the room's seats, not the call's roster. Vec::new() @@ -257,13 +261,23 @@ pub(super) fn call(snapshot: &Snapshot) -> Option { input: "MacBook Pro Microphone".to_owned(), output: "MacBook Pro Speakers".to_owned(), engine_linked: true, - video: Vec::new(), + video: sharing.then(|| protocol::VideoStream { direction: protocol::Direction::Encode, content: protocol::Content::Share, source: String::new() }).into_iter().collect(), security: None, picture: None, self_view: None, + share_view: sharing.then(screen_picture).flatten(), }) } +/// A made-up picture of my own screen, as mediad paints it for my tile: the demo's schematic +/// shrunk to the self-view size and written as a JPEG. +fn screen_picture() -> Option { + let shrunk = image::open(photo("schematic.webp")).ok()?.resize(640, 360, image::imageops::FilterType::Triangle).to_rgb8(); + let path = std::env::temp_dir().join("tinyclient-shot-screen.jpg"); + shrunk.save_with_format(&path, image::ImageFormat::Jpeg).ok()?; + Some(protocol::CallPicture { path: path.display().to_string(), width: shrunk.width(), height: shrunk.height(), generation: 1, user_id: String::new(), kind: protocol::PictureKind::Share }) +} + fn mara() -> Subject { Subject { title: "Mara Okafor".to_owned(), kind: ConversationKind::Direct, avatar: None, space: None } } @@ -384,6 +398,7 @@ fn bare(conversation: Key, state: CallState, participants: Vec, joi security: None, picture: None, self_view: None, + share_view: None, } } diff --git a/crates/tinyclient/src/gui/mod.rs b/crates/tinyclient/src/gui/mod.rs index 90a51cd1..549f8926 100644 --- a/crates/tinyclient/src/gui/mod.rs +++ b/crates/tinyclient/src/gui/mod.rs @@ -301,7 +301,8 @@ fn shell_bindings() -> Vec { /// `--autoplay always|hover|never` (when GIFs play), `--frames N` (N frames 150 ms apart, as /// `out-1.png` …, to see moving pictures move). On a Tiny server (`--mode tiny`): `--tiny-room /// N` (the Nth channel), `--tiny-menu header|channel|person:N` (N: the Nth seat, `:sub` adds its -/// Move submenu), `--tiny-modal invite|code|kick|ban|profile`, `--tiny-settings`, `--tiny-card N` +/// Move submenu), `--tiny-stream` (my own stream filling the stage, with `TINYCLIENT_SHOT_CALL=tiny_sharing`), +/// `--tiny-modal invite|code|kick|ban|profile`, `--tiny-settings`, `--tiny-card N` /// (the Nth member's hover card), `--tiny-setup first|custom|refused|checklist|links|privacy|change|change-anyway|member` (first-run /// setup, or Server Settings › Privacy, over a made-up outline), `--tiny-confirm transfer|codes`. /// diff --git a/crates/tinyclient/src/gui/shot_demo.rs b/crates/tinyclient/src/gui/shot_demo.rs index f3633360..0ef91076 100644 --- a/crates/tinyclient/src/gui/shot_demo.rs +++ b/crates/tinyclient/src/gui/shot_demo.rs @@ -939,6 +939,7 @@ fn demo_call(conversation: &protocol::Key) -> protocol::Call { security: None, picture: None, self_view: None, + share_view: None, } } diff --git a/crates/tinyclient/src/gui/shot_demo_live.rs b/crates/tinyclient/src/gui/shot_demo_live.rs index 4a57d577..05ff09e2 100644 --- a/crates/tinyclient/src/gui/shot_demo_live.rs +++ b/crates/tinyclient/src/gui/shot_demo_live.rs @@ -106,6 +106,7 @@ fn bare(conversation: Key, state: CallState, participants: Vec) -> security: None, picture: None, self_view: None, + share_view: None, } } diff --git a/crates/tinyclient/src/gui/tiny/channel.rs b/crates/tinyclient/src/gui/tiny/channel.rs index 634112ef..1e8df564 100644 --- a/crates/tinyclient/src/gui/tiny/channel.rs +++ b/crates/tinyclient/src/gui/tiny/channel.rs @@ -444,14 +444,16 @@ impl Shell { self.call_state.feed.shown().cloned() } - /// My own camera as it is sent, while it is on. - fn tiny_self_frame(&self, call: Option<&Call>) -> Option> { - call?.self_view.as_ref().filter(|picture| !picture.path.is_empty())?; - self.call_state.self_feed.shown().cloned() + /// My own camera, or with `share` my own screen, as it is sent, while it is on. + fn tiny_self_frame(&self, call: Option<&Call>, share: bool) -> Option> { + let (picture, feed) = if share { (call?.share_view.as_ref(), &self.call_state.screen_feed) } else { (call?.self_view.as_ref(), &self.call_state.self_feed) }; + picture.filter(|picture| !picture.path.is_empty())?; + feed.shown().cloned() } - /// What a stream shows: its picture; mine, which is never sent back to me, said in words with - /// Stop; anyone else's, "Watch stream" until its first picture arrives. + /// What a stream shows: its picture; mine, the same small live view of it that I send, with + /// Stop over it (until the first one lands, words with Stop); anyone else's, "Watch stream" + /// until its first picture arrives. fn tiny_stream_body(&self, page: &Page, stream: &Stream, call: Option<&Call>, size: Size, cx: &mut Context) -> AnyElement { if let Some(frame) = self.tiny_frame(call, &stream.user_id) { return img(frame).size_full().object_fit(ObjectFit::Contain).into_any_element(); @@ -459,25 +461,35 @@ impl Shell { let small = size == Size::Strip; let large = size == Size::Big; let body = div().size_full().relative().flex().flex_col().items_center().justify_center().gap(px(if small { 4. } else { 10. })).p(px(8.)); - if stream.mine && !stream.share { - // My camera as everyone sees it, with Stop over it. - if let Some(frame) = self.tiny_self_frame(call) { - let stop = button_with_icon("tiny-stream-camera-off", ButtonKind::Danger, "icons/video-off.svg", "Turn camera off").on_click(cx.listener(|this, _, _, cx| { - cx.stop_propagation(); - this.call_ask(Request::Media(MediaRequest::Camera { on: false }), cx); - })); + if stream.mine { + // My camera or my screen as everyone sees it, with Stop over it. + if let Some(frame) = self.tiny_self_frame(call, stream.share) { + let stop = if stream.share { + button_with_icon("tiny-stream-stop", ButtonKind::Danger, "icons/monitor-off.svg", "Stop sharing").on_click(cx.listener(|this, _, _, cx| { + cx.stop_propagation(); + this.tiny_stop_share(cx); + })) + } else { + button_with_icon("tiny-stream-camera-off", ButtonKind::Danger, "icons/video-off.svg", "Turn camera off").on_click(cx.listener(|this, _, _, cx| { + cx.stop_propagation(); + this.call_ask(Request::Media(MediaRequest::Camera { on: false }), cx); + })) + }; return div() .size_full() .relative() .child(img(frame).size_full().object_fit(ObjectFit::Contain)) .when(!small, |tile| { - tile.child(div().absolute().bottom(px(10.)).left_0().right_0().flex().justify_center().child(stop.h(px(28.)).on_mouse_down(MouseButton::Left, |_, _, cx| cx.stop_propagation()))) + // A grid tile's caption sits at the bottom, so Stop goes along the top. + let row = div().absolute().left_0().right_0().flex().justify_center(); + let row = if size == Size::Grid { row.top(px(8.)) } else { row.bottom(px(10.)) }; + // On a picture that can be any colour, so it sits on the overlay as the strip's does. + tile.child(row.child(div().p(px(2.)).rounded(px(10.)).bg(theme::overlay()).child(stop.h(px(28.)).on_mouse_down(MouseButton::Left, |_, _, cx| cx.stop_propagation())))) }) .into_any_element(); } } if stream.mine { - let thumbnail = self.tiny_state.shared.as_ref().filter(|_| stream.share).map(|screen| screen.thumbnail.clone()).filter(|path| !path.is_empty()); let words = if stream.share { "You're sharing your screen" } else { "Your camera is on" }; let stop = if stream.share { button_with_icon("tiny-stream-stop", ButtonKind::Danger, "icons/monitor-off.svg", "Stop sharing").on_click(cx.listener(|this, _, _, cx| { @@ -491,11 +503,10 @@ impl Shell { })) }; let line = match &self.tiny_state.shared { - Some(screen) if stream.share => format!("Everyone in the call can watch {}. It isn't played back to you, so you don't see a mirror of your own screen.", screen.name), + Some(screen) if stream.share => format!("Everyone in the call can watch {}. A live view of it shows here as soon as the first picture is sent.", screen.name), _ => "Everyone in the call can watch it.".to_owned(), }; return body - .children(thumbnail.map(|path| img(std::path::PathBuf::from(path)).absolute().top_0().left_0().size_full().object_fit(ObjectFit::Cover).opacity(0.3))) // A grid tile has room for the words and Stop, not the icon as well. .when(small || large, |body| body.child(glyph(if stream.share { "icons/monitor-up.svg" } else { "icons/video.svg" }, if small { 18. } else { 36. }, theme::accent_text()))) .when(!small, |body| body.child(div().text_center().whitespace_normal().type_ui().font_weight(theme::SEMIBOLD).text_color(theme::text()).child(words))) @@ -1127,6 +1138,7 @@ mod tests { security: None, picture: None, self_view: None, + share_view: None, } } @@ -1134,6 +1146,22 @@ mod tests { TinySeat { user_id: user_id.into(), ..TinySeat::default() } } + #[test] + fn my_camera_and_my_share_are_two_feeds_that_do_not_clobber_each_other() { + use super::super::feed::{named, Which}; + let picture = |path: &str, generation, kind| protocol::CallPicture { path: path.into(), width: 640, height: 360, generation, user_id: String::new(), kind }; + let mut call = joined(false, false, false); + let of = |call: &Call, which: fn(&Which) -> bool| named(Some(call)).into_iter().find(|(feed, _)| which(feed)).and_then(|(_, frame)| frame); + let (mine, screen): (fn(&Which) -> bool, fn(&Which) -> bool) = (|which| matches!(which, Which::Mine), |which| matches!(which, Which::Screen)); + assert_eq!((of(&call, mine), of(&call, screen)), (None, None)); + call.share_view = Some(picture("share.jpg", 7, protocol::PictureKind::Share)); + assert_eq!((of(&call, mine), of(&call, screen)), (None, Some((7, "share.jpg".to_owned()))), "a share alone leaves the camera's feed empty"); + call.self_view = Some(picture("camera.jpg", 2, protocol::PictureKind::Camera)); + assert_eq!((of(&call, mine), of(&call, screen)), (Some((2, "camera.jpg".to_owned())), Some((7, "share.jpg".to_owned())))); + call.share_view = None; + assert_eq!((of(&call, mine), of(&call, screen)), (Some((2, "camera.jpg".to_owned())), None), "the share stopping lets go of its feed only"); + } + #[test] fn my_tile_shows_my_call_at_once_and_others_light_on_the_quicker_edge() { let mine = as_heard(&seat("robin"), true, Some(&joined(true, false, false))); diff --git a/crates/tinyclient/src/gui/tiny/feed.rs b/crates/tinyclient/src/gui/tiny/feed.rs index 08cc1bbe..ecdfc45f 100644 --- a/crates/tinyclient/src/gui/tiny/feed.rs +++ b/crates/tinyclient/src/gui/tiny/feed.rs @@ -101,11 +101,22 @@ fn frame(path: &Path) -> Option> { Some(Arc::new(RenderImage::new(vec![image::Frame::new(pixels)]))) } -/// Which of the two feeds: the picture being watched, or my own camera. +/// Which of the three feeds: the picture being watched, my own camera, or my own share. #[derive(Clone, Copy)] pub(in crate::gui) enum Which { Watched, Mine, + Screen, +} + +/// The frame a call names for each feed: its generation and file. +pub(in crate::gui) fn named(call: Option<&Call>) -> [(Which, Option<(u64, String)>); 3] { + let frame = |picture: Option<&protocol::CallPicture>| picture.map(|picture| (picture.generation, picture.path.clone())); + [ + (Which::Watched, frame(call.and_then(|call| call.picture.as_ref()))), + (Which::Mine, frame(call.and_then(|call| call.self_view.as_ref()))), + (Which::Screen, frame(call.and_then(|call| call.share_view.as_ref()))), + ] } impl Shell { @@ -113,18 +124,13 @@ impl Shell { match which { Which::Watched => &mut self.call_state.feed, Which::Mine => &mut self.call_state.self_feed, + Which::Screen => &mut self.call_state.screen_feed, } } /// The call changed: the frames it names are read, and ones no longer named are let go. pub(in crate::gui) fn feed_follow(&mut self, call: Option<&Call>, cx: &mut Context) { - let watched = call - .and_then(|call| call.picture.as_ref()) - .map(|picture| (picture.generation, picture.path.clone())); - let mine = call - .and_then(|call| call.self_view.as_ref()) - .map(|picture| (picture.generation, picture.path.clone())); - for (which, picture) in [(Which::Watched, watched), (Which::Mine, mine)] { + for (which, picture) in named(call) { let (read, gone) = self.feed_of(which).offer(picture); if let Some(gone) = gone { cx.drop_image(gone, None); diff --git a/crates/tinyclient/src/gui/tiny/mod.rs b/crates/tinyclient/src/gui/tiny/mod.rs index b38fd888..d438f37d 100644 --- a/crates/tinyclient/src/gui/tiny/mod.rs +++ b/crates/tinyclient/src/gui/tiny/mod.rs @@ -1518,6 +1518,16 @@ impl Shell { let room = self.tiny_state.channel.as_ref().and_then(|id| community.rooms.iter().find(|room| room.id == *id)); self.tiny_state.focus = room.and_then(|room| room.participants.get(ix)).map(|seat| stage::Focus::Person(seat.user_id.clone())); } + // My own stream, filling the stage (a made-up call that has me sharing, `tiny_sharing`). + if std::env::args().any(|arg| arg == "--tiny-stream") { + self.tiny_state.focus = Some(stage::Focus::Stream(community.me.clone())); + } + // The picture mediad would have painted for my tile is read as a call's change reads it. + if std::env::var("TINYCLIENT_SHOT_CALL").is_ok_and(|name| name == "tiny_sharing") { + if let Some(call) = self.snapshot.as_ref().and_then(|snapshot| self.call_shown(snapshot)) { + self.feed_follow(Some(&call), cx); + } + } match flag("--tiny-menu").as_deref() { Some("header") => self.tiny_open_float(Float::Header, cx), Some("stats") => self.tiny_open_float(Float::Stats, cx), diff --git a/e2e/scenarios/share-self-view/README.md b/e2e/scenarios/share-self-view/README.md new file mode 100644 index 00000000..5ea09264 --- /dev/null +++ b/e2e/scenarios/share-self-view/README.md @@ -0,0 +1,20 @@ +# My own share is painted for me + +Sharing a screen in a Tiny call used to leave my own stage with a text card and a blurred, stale +thumbnail from the start of the share. The share now paints a small live picture of what it sends, +the way the camera already did, and the stage draws that with Stop over it. + +This headless scenario runs the real capture thread (`mediad::video::Video`) on the real screen of +the host: it asks for a share, waits for the first picture painted for my own tile, and checks it +is a JPEG the window can read, no bigger than the camera's corner size, and that no camera +picture is made while the camera is off. It then stops the share and requires the capture thread to +say the picture is gone. The host needs a screen and the OpenH264 binary (fetched on first use), as +`the_capture_thread_shares_a_real_screen_through_the_real_encoder` does. + +Run `python e2e/scenarios/share-self-view/drive.py`. It requires the one ignored test and both +`share-self-view-proof:` lines, so a build from before the change (which has no such test) fails. + +This proves the production half on the platform where it runs. It does not open a window: the +stage drawing that picture, Stop over it, and the fallback card are covered by the window's own +tests, and the Tiny call that carries `share_view` from mediad through chatd is covered by chatd's +`media::tests`. diff --git a/e2e/scenarios/share-self-view/drive.linux.sh b/e2e/scenarios/share-self-view/drive.linux.sh new file mode 100644 index 00000000..f15187e5 --- /dev/null +++ b/e2e/scenarios/share-self-view/drive.linux.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +python3 "$(cd "$(dirname "$0")" && pwd)/drive.py" diff --git a/e2e/scenarios/share-self-view/drive.macos.sh b/e2e/scenarios/share-self-view/drive.macos.sh new file mode 100644 index 00000000..f15187e5 --- /dev/null +++ b/e2e/scenarios/share-self-view/drive.macos.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +python3 "$(cd "$(dirname "$0")" && pwd)/drive.py" diff --git a/e2e/scenarios/share-self-view/drive.py b/e2e/scenarios/share-self-view/drive.py new file mode 100644 index 00000000..864a850b --- /dev/null +++ b/e2e/scenarios/share-self-view/drive.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Share the real screen through mediad's capture thread and check my own share is painted for me.""" +import pathlib +import subprocess +import os +import sys + +root = pathlib.Path(__file__).resolve().parents[3] +test = "the_capture_thread_paints_my_own_share_from_a_real_screen" +command = [os.environ.get("CARGO", "cargo"), "test", "-p", "mediad", "--lib", test, "--", "--ignored", "--nocapture"] +result = subprocess.run(command, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") +print(result.stdout, end="") +# An older build has no such test: cargo runs none and the markers are absent, which fails. +proved = ( + result.returncode == 0 + and "share-self-view-proof:" in result.stdout + and "share-self-view-proof: cleared when the share stopped" in result.stdout + and "test result: ok. 1 passed" in result.stdout +) +sys.exit(0 if proved else 1) diff --git a/e2e/scenarios/share-self-view/drive.windows.ps1 b/e2e/scenarios/share-self-view/drive.windows.ps1 new file mode 100644 index 00000000..c3f4fecf --- /dev/null +++ b/e2e/scenarios/share-self-view/drive.windows.ps1 @@ -0,0 +1,3 @@ +$ErrorActionPreference = 'Stop' +& python "$PSScriptRoot/drive.py" +exit $LASTEXITCODE diff --git a/e2e/scenarios/share-self-view/scenario.json b/e2e/scenarios/share-self-view/scenario.json new file mode 100644 index 00000000..2c050b69 --- /dev/null +++ b/e2e/scenarios/share-self-view/scenario.json @@ -0,0 +1,12 @@ +{ + "id": "share-self-view", + "title": "Sharing a screen paints a small live picture of it for my own tile, and takes it down when the share stops", + "platforms": ["windows", "linux", "macos"], + "not_applicable": { + "android": "A phone shares through its own platform capture, which paints no preview for the call stage; this is the desktop capture path.", + "ios": "A phone shares through its own platform capture, which paints no preview for the call stage; this is the desktop capture path." + }, + "needs": [], + "release": false, + "max_still_seconds": 120 +}