diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cdb07255..db9cbd3f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,19 @@ released versions carry their date on the heading. ### Features +- 2026-10-08: **The desktop app downloads ffmpeg, ffprobe and wtsexporter + by itself.** Each time it starts, it downloads in the background whatever + is missing from its Tools Directory, with no button and nothing to wait + for. ffmpeg and ffprobe aren't downloaded when both are already installed + on the system path. Each program comes from a pinned release of its project, + wtsexporter from Message Crate's own copy of the WhatsApp Chat Exporter, + and every file is checked against a checksum the app carries. A file that + doesn't match is deleted. A program in the Tools Directory that isn't the + pinned one is replaced, but only once the new file has passed its check, + so with no internet the old one is still used. Settings → System shows + each download's progress, or why it failed, and a failed download is + tried again at the next start (#1053). + - 2026-10-08: **A phone number written without its country code can be given its country.** The import form has a **Phone's country** under **Processing Options (Advanced)**: every number the import writes without a diff --git a/CONTEXT.md b/CONTEXT.md index b1f9f5277..f6b280994 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -537,9 +537,10 @@ The directory inside the Message Crate Directory where the desktop app keeps the programs it downloads for itself: ffmpeg, ffprobe and wtsexporter. The app owns its contents and replaces them when a release needs a newer version. ffmpeg and ffprobe are looked for on `PATH` and then here, -wtsexporter only here, by the app and by the server it starts. The download -is decided, not built yet: #1053 tracks the work, and until it lands the app -downloads nothing. +wtsexporter only here, by the app and by the server it starts. The app +downloads what is missing here at every start, each file pinned to one +release and checksum, and keeps a record beside the programs of what it +wrote. _Avoid_: Tools Folder, Bin Directory, ffmpeg directory **Apple Messages Reader**: diff --git a/crates/libs/media/src/lib.rs b/crates/libs/media/src/lib.rs index 00d015418..c8ae7264e 100644 --- a/crates/libs/media/src/lib.rs +++ b/crates/libs/media/src/lib.rs @@ -27,8 +27,8 @@ pub use process::{ transcode_file, transcode_file_as, }; pub use tools::{ - FfmpegTools, ffmpeg_available, ffmpeg_path, ffmpeg_tools, ffprobe_path, require_ffmpeg, - set_tools_dir, tools_dir, + FfmpegTools, ffmpeg_available, ffmpeg_on_path, ffmpeg_path, ffmpeg_tools, ffprobe_path, + require_ffmpeg, set_tools_dir, tool_in_dir, tools_dir, }; pub use versions::{ THUMBNAIL_LONG_EDGE, browser_shows, make_preview, make_thumbnail, media_type_of, diff --git a/crates/libs/media/src/testutil.rs b/crates/libs/media/src/testutil.rs index ff26a8678..9fd89adf9 100644 --- a/crates/libs/media/src/testutil.rs +++ b/crates/libs/media/src/testutil.rs @@ -133,6 +133,17 @@ pub fn hide_ffmpeg() -> ToolsHidden { } } +/// Search `search_path` instead of the process's `PATH`, and `tools_dir` +/// as the Tools Directory, until the returned guard is dropped. Holds +/// [`tools_test_lock`], as [`hide_ffmpeg`] does. +#[must_use] +pub fn locate_tools(search_path: OsString, tools_dir: PathBuf) -> ToolsHidden { + let hidden = hide_ffmpeg(); + set_search_path(Some(search_path)); + set_tools_dir(Some(tools_dir)); + hidden +} + impl Drop for ToolsHidden { fn drop(&mut self) { set_search_path(self.previous_search_path.take()); diff --git a/crates/libs/media/src/tools.rs b/crates/libs/media/src/tools.rs index 0a8f03998..d7006085f 100644 --- a/crates/libs/media/src/tools.rs +++ b/crates/libs/media/src/tools.rs @@ -178,6 +178,27 @@ pub fn ffmpeg_available() -> bool { resolve_tools().is_ok_and(|tools| tools.missing().is_empty()) } +/// True when ffmpeg and ffprobe are both on `PATH` and answer `-version`. +/// The desktop app downloads neither then, because a person who installed +/// ffmpeg chose it (`docs/adr/0019`). +pub fn ffmpeg_on_path() -> bool { + let search_path = searched_path(); + ["ffmpeg", "ffprobe"] + .iter() + .all(|name| find_on_path(name, search_path.as_deref()).is_some()) +} + +/// The `PATH` the lookup searches: the one a test put in place +/// ([`set_search_path`]), else the process's own. +fn searched_path() -> Option { + tools_state() + .lock() + .expect("tools state lock") + .search_path + .clone() + .or_else(|| std::env::var_os("PATH")) +} + /// True when ffprobe is found. pub(crate) fn ffprobe_available() -> bool { ffprobe_path().is_some() @@ -227,16 +248,14 @@ fn command_runs(bin: &Path, args: &[&str]) -> bool { /// arrives, moves or leaves is seen at once without a restart; only the /// `-version` answer of a file is kept ([`ToolsState::answered`]). fn resolve_tools() -> Result { - let (search_path, tools_dir) = { - let state = tools_state().lock().expect("tools state lock"); - (state.search_path.clone(), state.tools_dir.clone()) - }; - let search_path = search_path.or_else(|| std::env::var_os("PATH")); - find_tools(search_path.as_deref(), tools_dir.as_deref()) + let search_path = searched_path(); + find_tools(search_path.as_deref(), tools_dir().as_deref()) } -/// The tool under `dir` if it is a file and runs. -fn find_tool_in_dir(dir: &Path, name: &str) -> Option { +/// The program `name` in `dir`, `.exe` added on Windows, when it is a file +/// that answers `-version`. The desktop app also asks this of a program in +/// the Tools Directory, to know that the file it keeps there runs. +pub fn tool_in_dir(dir: &Path, name: &str) -> Option { let candidate = dir.join(executable_name(name)); let metadata = std::fs::metadata(&candidate).ok()?; if !metadata.is_file() { @@ -273,7 +292,7 @@ fn find_on_path(name: &str, search_path: Option<&OsStr>) -> Option { .into_iter() .flat_map(std::env::split_paths) .filter(|dir| !dir.as_os_str().is_empty()) - .find_map(|dir| find_tool_in_dir(&dir, name)) + .find_map(|dir| tool_in_dir(&dir, name)) } /// Find ffmpeg and ffprobe on `search_path`, then in `tools_dir`, and @@ -296,8 +315,8 @@ fn find_tools(search_path: Option<&OsStr>, tools_dir: Option<&Path>) -> Result not used` and the reason. For ffmpeg and ffprobe, the reason is that they were found in two places, one only on `PATH` and the other only in the Tools Directory. For wtsexporter, the reason is that it has no permission to run. - -On macOS, an app opened from the Dock or Finder doesn't see the `PATH` a terminal sets, so ffmpeg and ffprobe installed with Homebrew are not found. -Linking both into the Tools Directory makes them found, for example `ln -s /opt/homebrew/bin/ffmpeg /opt/homebrew/bin/ffprobe ~/message-crate/tools/` (`/usr/local/bin` on an Intel Mac). +- A download arrow with `Downloading ` and how much has arrived, such as `12 MB of 29 MB (41%)`. + The line updates each second until the download ends. +- A cross with ` download failed` and the reason: no connection to the download's server, the status the server answered, a checksum that didn't match, a file that couldn't be written to the Tools Directory, or a program in the Tools Directory that doesn't run on this computer. + A failed download is tried again the next time the app starts. + A program that doesn't run is not downloaded again, because the download would be the same file. + For ffmpeg and ffprobe, the reason says to install them with a package manager, because the app uses the copy on `PATH`. + While an older copy is in place, the line shows that copy as found instead, because the old copy is still the one used. + +On macOS, an app opened from the Dock or Finder doesn't see the `PATH` Homebrew sets in a terminal, so it doesn't find ffmpeg and ffprobe installed with Homebrew. +It downloads its own copies into the Tools Directory instead, and nothing needs doing. [Attachments and media](/docs/user/features/messages/attachments-and-media/) covers what ffmpeg and ffprobe are used for. diff --git a/docs/src/content/docs/docs/user/import-sources/whatsapp.md b/docs/src/content/docs/docs/user/import-sources/whatsapp.md index a97f0d762..6fc82f57e 100644 --- a/docs/src/content/docs/docs/user/import-sources/whatsapp.md +++ b/docs/src/content/docs/docs/user/import-sources/whatsapp.md @@ -1,6 +1,6 @@ --- title: WhatsApp -description: Import WhatsApp from an Android phone or an iPhone backup. Advanced, because it needs a separate program and, on Android, a decryption key. +description: Import WhatsApp from an Android phone or an iPhone backup. Advanced, because it runs a separate program the desktop app downloads and, on Android, needs a decryption key. --- WhatsApp is the hardest source to import. @@ -8,35 +8,15 @@ It belongs after a first import of [iPhone](/docs/user/your-messages/back-up-an- Two things make it hard: -- The desktop app doesn't include the program that reads WhatsApp data. It must be installed separately ([issue 940](https://github.com/messagecrate/message-crate/issues/940)). +- The program that reads WhatsApp data, `wtsexporter`, isn't part of the desktop app. The app downloads it when it starts, so the first WhatsApp import needs the app to have started once with an internet connection. - On Android, WhatsApp's backup file is encrypted, and Import needs the key. -## Install wtsexporter +## wtsexporter -The desktop app runs `wtsexporter`, from the open-source [WhatsApp Chat Exporter](https://github.com/KnugiHK/WhatsApp-Chat-Exporter) project. -Install it from Message Crate's fork, [messagecrate/WhatsApp-Chat-Exporter](https://github.com/messagecrate/WhatsApp-Chat-Exporter), at release `0.13.0-mc.2`, because only the fork records the ids a quoted reply is linked by. -It needs Python and [`pipx`](https://pipx.pypa.io/). - -```bash title="Install wtsexporter" -pipx install --force "whatsapp-chat-exporter[android_backup,crypt15] @ https://github.com/messagecrate/WhatsApp-Chat-Exporter/archive/refs/tags/0.13.0-mc.2.tar.gz" -``` - -`wtsexporter --help` prints the program's options once it is installed. - -The desktop app runs `wtsexporter` only from its Tools Directory, which [**Settings → System**](/docs/user/features/settings/system/#media) shows, and never from `PATH`. -After the install, a link in the Tools Directory points to the `pipx` install of `wtsexporter`: - -```bash title="Linux and macOS" -mkdir -p ~/message-crate/tools -ln -sf "$(pipx environment --value PIPX_BIN_DIR)/wtsexporter" ~/message-crate/tools/ -``` - -On Windows, `wtsexporter.exe` is copied there instead: - -```powershell title="Windows" -New-Item -ItemType Directory -Force "$HOME\message-crate\tools" -Copy-Item "$(pipx environment --value PIPX_BIN_DIR)\wtsexporter.exe" "$HOME\message-crate\tools\" -``` +The desktop app runs `wtsexporter`, from Message Crate's fork of the open-source [WhatsApp Chat Exporter](https://github.com/KnugiHK/WhatsApp-Chat-Exporter), [messagecrate/WhatsApp-Chat-Exporter](https://github.com/messagecrate/WhatsApp-Chat-Exporter), because only the fork records the ids a quoted reply is linked by. +Each time it starts, the desktop app downloads the pinned release into its Tools Directory when it isn't there, in the background, and [**Settings → System**](/docs/user/features/settings/system/#media) shows the program or how its download stands. +The app runs `wtsexporter` only from the Tools Directory, never from `PATH`. +A program in the Tools Directory whose checksum is the pinned program's is kept, whoever put it there, and anything else under its name is replaced by the pinned program. ## WhatsApp on Android diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 7fdb4aba0..6dbf03c53 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -2342,7 +2342,9 @@ dependencies = [ "build-version", "chrono", "dirs 7.0.0", + "flate2", "go-sms-pro-exporter", + "hex", "httpmock", "imazing-exporter", "imessage-ir-exporter", @@ -2361,8 +2363,10 @@ dependencies = [ "message-staging", "open", "openextract-exporter", + "reqwest 0.12.28", "serde", "serde_json", + "sha2 0.11.0", "sms-backup-plus-exporter", "sms-backup-restore-exporter", "socket2", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 21d5a41c3..0c60c8f55 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -48,6 +48,14 @@ message-crate-serve-protocol = { path = "../crates/libs/serve-protocol" } # renamed onto the file the person chose (`commands::download`). tempfile = "3" +# The start-up download of ffmpeg, ffprobe and wtsexporter into the Tools +# Directory (`tool_downloads`): the HTTP client, the SHA-256 each file is +# checked against, and the gzip the ffmpeg-static files are published in. +reqwest = { version = "0.12.25", default-features = false, features = ["blocking", "rustls-tls", "rustls-tls-native-roots"] } +sha2 = "0.11.0" +hex = "0.4.3" +flate2 = "1.1.9" + # `killpg`, to kill the server the app started with every process the server # started (`local_server::process_tree`). [target.'cfg(unix)'.dependencies] diff --git a/src-tauri/src/commands/tools.rs b/src-tauri/src/commands/tools.rs index 2d5a5148c..c0a187ab3 100644 --- a/src-tauri/src/commands/tools.rs +++ b/src-tauri/src/commands/tools.rs @@ -1,5 +1,6 @@ //! What the Media part of Settings → System shows: where ffmpeg, ffprobe and -//! wtsexporter are, or that one is missing. +//! wtsexporter are, that one is missing, or how its download stands +//! ([`crate::tool_downloads`]). //! //! ffmpeg and ffprobe are looked for on `PATH`, then in the Tools Directory; //! wtsexporter only in the Tools Directory (#1053). The window names no @@ -8,6 +9,8 @@ use std::path::PathBuf; +use crate::tool_downloads::{DownloadState, Program, ToolDownloads}; + /// Where one program is, as Settings shows it. /// /// Tagged by `state`, one tag per state, so a state can be added beside the @@ -27,6 +30,19 @@ pub enum ToolStatus { /// Why it is not used. reason: String, }, + /// The program is downloading into the Tools Directory. + Downloading { + /// Bytes received so far. + received: u64, + /// The file's size, `None` when the server did not say. + total: Option, + }, + /// The program's download failed, for `reason`, and the program is not + /// found. It is tried again the next time the app starts. + DownloadFailed { + /// Why the download failed. + reason: String, + }, } impl ToolStatus { @@ -39,6 +55,23 @@ impl ToolStatus { None => Self::Missing, } } + + /// This status as `download` changes it. A download in progress is + /// shown whatever was found, because the file is about to change. A + /// failed one is shown only when the program is missing: a program found + /// is still used, and a program not used says why, which is the cause a + /// person must fix (two places, or no permission to run). + fn with_download(self, download: Option) -> Self { + match (download, self) { + (Some(DownloadState::Downloading { received, total }), _) => { + Self::Downloading { received, total } + } + (Some(DownloadState::Failed { reason }), Self::Missing) => { + Self::DownloadFailed { reason } + } + (_, status) => status, + } + } } /// Where each program the app runs is, and the Tools Directory it keeps them in. @@ -55,9 +88,15 @@ pub struct ToolsStatus { pub wtsexporter: ToolStatus, } -/// Look for ffmpeg, ffprobe and wtsexporter where this process runs them from. +/// Look for ffmpeg, ffprobe and wtsexporter where this process runs them from, +/// and say how each one's download stands. #[tauri::command] -pub fn tools_status() -> ToolsStatus { +pub fn tools_status(downloads: tauri::State<'_, ToolDownloads>) -> ToolsStatus { + tools_status_with(&downloads) +} + +/// [`tools_status`], with the downloads in `downloads`. +fn tools_status_with(downloads: &ToolDownloads) -> ToolsStatus { let (ffmpeg, ffprobe) = match media::ffmpeg_tools() { Ok(tools) => (ToolStatus::of(tools.ffmpeg), ToolStatus::of(tools.ffprobe)), // Found in two places: neither is used, and both say why. @@ -68,16 +107,17 @@ pub fn tools_status() -> ToolsStatus { (unusable.clone(), unusable) } }; + let wtsexporter = match whatsapp_exporter::wtsexporter_path() { + Ok(path) => ToolStatus::of(path), + Err(err) => ToolStatus::Unusable { + reason: err.to_string(), + }, + }; ToolsStatus { tools_dir: media::tools_dir().map(|dir| dir.display().to_string()), - ffmpeg, - ffprobe, - wtsexporter: match whatsapp_exporter::wtsexporter_path() { - Ok(path) => ToolStatus::of(path), - Err(err) => ToolStatus::Unusable { - reason: err.to_string(), - }, - }, + ffmpeg: ffmpeg.with_download(downloads.get(Program::Ffmpeg)), + ffprobe: ffprobe.with_download(downloads.get(Program::Ffprobe)), + wtsexporter: wtsexporter.with_download(downloads.get(Program::Wtsexporter)), } } @@ -129,9 +169,10 @@ mod tests { let tools = tempfile::tempdir().unwrap(); let name = whatsapp_exporter::wtsexporter_file_name(); media::set_tools_dir(Some(tools.path().to_path_buf())); - let missing = tools_status().wtsexporter; + let downloads = ToolDownloads::default(); + let missing = tools_status_with(&downloads).wtsexporter; media::testutil::write_with_mode(&tools.path().join(name), 0o755); - let found = tools_status().wtsexporter; + let found = tools_status_with(&downloads).wtsexporter; media::set_tools_dir(previous); assert_eq!(missing, ToolStatus::Missing); @@ -142,4 +183,77 @@ mod tests { } ); } + + /// A download in progress is sent with its bytes so far and its total, + /// and a failed one with its reason, in the shape the window reads + /// (`ToolStatus` in `web/src/lib/tauri.ts`). + #[test] + fn a_download_is_sent_as_its_progress_or_its_failure() { + let downloading = ToolStatus::Missing.with_download(Some(DownloadState::Downloading { + received: 1024, + total: Some(4096), + })); + assert_eq!( + serde_json::to_value(&downloading).unwrap(), + serde_json::json!({ "state": "downloading", "received": 1024, "total": 4096 }) + ); + let unknown_total = ToolStatus::Missing.with_download(Some(DownloadState::Downloading { + received: 10, + total: None, + })); + assert_eq!( + serde_json::to_value(&unknown_total).unwrap(), + serde_json::json!({ "state": "downloading", "received": 10, "total": null }) + ); + let failed = ToolStatus::Missing.with_download(Some(DownloadState::Failed { + reason: "The download's server answered 404 Not Found.".into(), + })); + assert_eq!( + serde_json::to_value(&failed).unwrap(), + serde_json::json!({ + "state": "downloadFailed", + "reason": "The download's server answered 404 Not Found.", + }) + ); + } + + /// A program not used keeps its reason when its download failed, because + /// that reason is what a person must fix. + #[test] + fn an_unusable_program_keeps_its_reason_over_a_failed_download() { + let unusable = ToolStatus::Unusable { + reason: "ffmpeg is on PATH at /usr/bin/ffmpeg and ffprobe is elsewhere.".into(), + }; + assert_eq!( + unusable.clone().with_download(Some(DownloadState::Failed { + reason: "no network".into() + })), + unusable + ); + } + + /// A program found stays found when its download failed, because it is + /// still the one used; a download in progress is shown over it. + #[test] + fn a_found_program_hides_a_failed_download_but_not_one_in_progress() { + let found = || ToolStatus::Found { + path: "/home/sam/message-crate/tools/wtsexporter".into(), + }; + assert_eq!( + found().with_download(Some(DownloadState::Failed { + reason: "no network".into() + })), + found() + ); + assert_eq!( + found().with_download(Some(DownloadState::Downloading { + received: 1, + total: None + })), + ToolStatus::Downloading { + received: 1, + total: None + } + ); + } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8382535fe..20ef8793b 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -20,3 +20,4 @@ pub mod local_server; pub mod run_directories; pub mod run_logs; pub mod state; +pub mod tool_downloads; diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index b230656af..ce5690ac0 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -17,6 +17,7 @@ mod local_server; mod run_directories; mod run_logs; mod state; +mod tool_downloads; use export_directories::ExportDirectories; use local_server::LocalServer; @@ -24,6 +25,7 @@ use run_directories::RunDirectories; use state::AppState; use std::sync::{Arc, Mutex}; use tauri::Manager; +use tool_downloads::ToolDownloads; /// Start the desktop window and wait until the user quits. fn main() { @@ -44,6 +46,9 @@ fn main() { .manage(app_state) // The Message Crate this app starts for itself, when asked to. .manage(LocalServer::default()) + // Where each download into the Tools Directory stands, which + // Settings → System → Media shows. + .manage(ToolDownloads::default()) // The Staging Directory and the run directories made under it, kept // in the app-data directory; the Export Directory; and the start-up // sweep of the Scratch and Export Directories. @@ -51,8 +56,11 @@ fn main() { // ffmpeg and ffprobe are looked for on PATH, then in the Tools // Directory; wtsexporter only there (#1053). Named before the // server is started, which is told it too. + // What is missing there is downloaded in the background, and + // nothing waits for it (`tool_downloads`). if let Some(home) = dirs::home_dir() { - app_directories::use_tools_dir_in(&home); + let tools = app_directories::use_tools_dir_in(&home); + tool_downloads::start(tools, app.state::().inner().clone()); } let app_data_dir = app.path().app_data_dir()?; let record = app_data_dir.join(run_directories::RECORD_FILE); diff --git a/src-tauri/src/tool_downloads.rs b/src-tauri/src/tool_downloads.rs new file mode 100644 index 000000000..dacd6fb1f --- /dev/null +++ b/src-tauri/src/tool_downloads.rs @@ -0,0 +1,725 @@ +//! The download of ffmpeg, ffprobe and wtsexporter into the Tools Directory +//! (#1053, `docs/adr/0019`). +//! +//! At every start the app checks the Tools Directory in the background and +//! downloads what is missing or older than the release pinned here. Nothing +//! waits for it, and a failed download is tried again at the next start. +//! +//! - **ffmpeg and ffprobe** are not downloaded when both are on `PATH`, +//! because a person who installed ffmpeg chose it. +//! - **Each file is pinned** to one release and one SHA-256 carried in +//! [`pins`]. A download that does not match is deleted and the failure +//! says the checksum did not match, because nobody clicked to start it. +//! - **One check at a time.** The check holds a lock on [`LOCK_FILE`] in +//! the Tools Directory while it runs, and a second app open at the same +//! time skips its check, so neither deletes the other's temporary file +//! nor writes over the other's record. +//! - **A file is replaced only after its successor passed.** The new file +//! is written under a temporary name in the Tools Directory, checked, and +//! then renamed over the old one, so a failed download leaves the old one +//! in use. +//! +//! Which file in the Tools Directory the app wrote is kept in +//! [`MANIFEST_FILE`] beside the programs: the release, the asset and its +//! checksum, the program's checksum, and the file's [`Stamp`]. One rule +//! holds for all three programs, because the Tools Directory belongs to the +//! app: +//! +//! - **The pinned one stays.** A file is the pinned one when its entry +//! names the pinned release and its stamp is the one recorded, so no file +//! is read again at start-up. One that does not run is not downloaded +//! again, because the download would be the same file: Settings says it +//! doesn't run until a newer pin or another file changes that. +//! - **A pinned file with no entry is adopted.** A file with no entry, or +//! whose stamp changed, whose SHA-256 is the pinned program's gets an +//! entry and stays. The pinned program's SHA-256 is carried beside the +//! asset's, because ffmpeg's and ffprobe's assets are gzipped. +//! - **Anything else is replaced** after a good download, whoever put it +//! there: a file from an older release, or any other file. Until the +//! download passes, the old file stays in use, so with no internet a file +//! put there by hand is still the one used. + +use std::collections::{BTreeMap, HashMap}; +use std::fmt; +use std::fs::File; +use std::io::{self, Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use sha2::{Digest, Sha256}; + +mod pins; +pub use pins::pinned_for; + +/// Where the pinned releases are downloaded from. Tests point the download +/// at a server of their own instead. +pub const GITHUB: &str = "https://github.com"; + +/// The record of the files the app wrote, in the Tools Directory. +pub const MANIFEST_FILE: &str = "manifest.json"; + +/// The file the check holds a lock on while it runs, in the Tools Directory. +pub const LOCK_FILE: &str = ".check.lock"; + +/// The start of the temporary file the record is written to. +const MANIFEST_TEMP_PREFIX: &str = ".manifest-"; + +/// The steps a program's download writes a temporary file for: the file as +/// it arrives, and the program it unpacks to. +const TEMP_STEPS: [&str; 2] = ["download", "unpack"]; + +/// The start of the temporary file `program`'s download writes at `step`. +fn temp_prefix(program: Program, step: &str) -> String { + format!(".{}.{step}-", program.name()) +} + +/// Delete the temporary files an interrupted check left in `dir`. A +/// temporary file is deleted when the check that wrote it ends, but an app +/// closed or killed mid-download runs no clean-up, and each one left is up +/// to about 110 MB. +fn delete_leftovers(dir: &Path) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let prefixes: Vec = [Program::Ffmpeg, Program::Ffprobe, Program::Wtsexporter] + .into_iter() + .flat_map(|program| TEMP_STEPS.map(|step| temp_prefix(program, step))) + .chain([MANIFEST_TEMP_PREFIX.to_string()]) + .collect(); + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if prefixes + .iter() + .any(|prefix| name.starts_with(prefix.as_str())) + { + let _ = std::fs::remove_file(entry.path()); + } + } +} + +/// A program the app keeps in the Tools Directory. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] +#[serde(rename_all = "lowercase")] +pub enum Program { + /// ffmpeg. + Ffmpeg, + /// ffprobe. + Ffprobe, + /// wtsexporter. + Wtsexporter, +} + +impl Program { + /// The program's name, as it is looked for. + pub fn name(self) -> &'static str { + match self { + Self::Ffmpeg => "ffmpeg", + Self::Ffprobe => "ffprobe", + Self::Wtsexporter => "wtsexporter", + } + } + + /// The program's file name in the Tools Directory, with `.exe` on + /// Windows. + pub fn file_name(self) -> String { + if cfg!(windows) { + format!("{}.exe", self.name()) + } else { + self.name().to_string() + } + } +} + +/// One program's file as a pinned release publishes it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Pinned { + /// The program the file is. + pub program: Program, + /// The GitHub project, `owner/name`. + pub repo: &'static str, + /// The release tag. + pub release: &'static str, + /// The release asset's name. + pub asset: &'static str, + /// The asset's SHA-256, in lowercase hex. + pub sha256: &'static str, + /// The program's SHA-256, in lowercase hex: the asset's when it is not + /// gzipped, and what it unpacks to when it is. + pub program_sha256: &'static str, + /// Whether the asset is gzipped: the checksum is over the gzipped + /// file, and the program is what it unpacks to. + pub gzip: bool, +} + +impl Pinned { + /// The asset's address under `base`, which is [`GITHUB`] outside tests. + pub fn url(&self, base: &str) -> String { + format!( + "{base}/{}/releases/download/{}/{}", + self.repo, self.release, self.asset + ) + } +} + +/// One file the app wrote in the Tools Directory, as [`MANIFEST_FILE`] +/// records it. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Written { + /// The release it came from. + pub release: String, + /// The release asset it came from. + pub asset: String, + /// The asset's SHA-256. + pub sha256: String, + /// The program's SHA-256, which is the asset's when it is not gzipped. + pub program_sha256: String, + /// The program file as it was written, which tells the app's file from + /// one put over it since. + pub stamp: Stamp, +} + +impl Written { + /// What the app records after writing `pinned`'s program, as `stamp`. + fn of(pinned: &Pinned, stamp: Stamp) -> Self { + Self { + release: pinned.release.to_string(), + asset: pinned.asset.to_string(), + sha256: pinned.sha256.to_string(), + program_sha256: pinned.program_sha256.to_string(), + stamp, + } + } + + /// Whether this is the file `pinned` publishes. + fn is(&self, pinned: &Pinned) -> bool { + self.release == pinned.release + && self.asset == pinned.asset + && self.sha256 == pinned.sha256 + && self.program_sha256 == pinned.program_sha256 + } +} + +/// What tells the file the app wrote from another put in its place, without +/// reading it: a copy of the same size made since has a different modified +/// time, and on Unix a different inode. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Stamp { + /// The file's size in bytes. + pub size: u64, + /// When the file was last modified, in nanoseconds since 1970, if the + /// platform says. + pub modified_ns: Option, + /// The file's inode, on Unix. + pub inode: Option, +} + +impl Stamp { + /// The stamp of the file `metadata` describes. + fn of(metadata: &std::fs::Metadata) -> Self { + #[cfg(unix)] + let inode = { + use std::os::unix::fs::MetadataExt; + Some(metadata.ino()) + }; + #[cfg(not(unix))] + let inode = None; + Self { + size: metadata.len(), + modified_ns: metadata + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .and_then(|since| u64::try_from(since.as_nanos()).ok()), + inode, + } + } + + /// The stamp of the file at `path`. + fn at(path: &Path) -> io::Result { + std::fs::metadata(path).map(|metadata| Self::of(&metadata)) + } +} + +/// The files the app wrote, by program. +type Manifest = BTreeMap; + +/// The record in `dir`. One that is missing or can't be read is empty, and +/// every file is then taken as put there by someone else. +fn read_manifest(dir: &Path) -> Manifest { + std::fs::read(dir.join(MANIFEST_FILE)) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_default() +} + +/// Write the record in `dir`, under a temporary name and then renamed, so a +/// half-written record is never read. +fn write_manifest(dir: &Path, manifest: &Manifest) -> io::Result<()> { + let mut file = tempfile::Builder::new() + .prefix(MANIFEST_TEMP_PREFIX) + .tempfile_in(dir)?; + serde_json::to_writer_pretty(&mut file, manifest)?; + file.as_file().sync_all()?; + file.persist(dir.join(MANIFEST_FILE)) + .map(|_| ()) + .map_err(|err| err.error) +} + +/// What the start-up check does with one pinned program. +#[derive(Debug, PartialEq, Eq)] +enum Need { + /// The file there is the pinned one and runs: it stays. + Keep, + /// The file there is the pinned one and does not run: it stays, and + /// Settings says so, since downloading it again gives the same file. + DoesNotRun, + /// The file there is the pinned program with no entry, or with a stamp + /// that changed: it stays and goes into the record. + Adopt, + /// The program is downloaded, and replaces whatever is there once the + /// download passed. + Download, +} + +/// What to do about `pinned` in `dir`, given the record. +fn need(pinned: &Pinned, dir: &Path, manifest: &Manifest) -> Need { + let path = dir.join(pinned.program.file_name()); + let Ok(stamp) = Stamp::at(&path) else { + return Need::Download; + }; + if let Some(written) = manifest.get(&pinned.program) + && written.is(pinned) + && written.stamp == stamp + { + return if runs(pinned.program, dir) { + Need::Keep + } else { + Need::DoesNotRun + }; + } + if file_sha256(&path).is_ok_and(|sha256| sha256 == pinned.program_sha256) { + Need::Adopt + } else { + Need::Download + } +} + +/// Whether `program` in `dir` runs, as the lookup that uses it decides: +/// ffmpeg and ffprobe answer `-version` (an answer `media` keeps until the +/// file changes), and wtsexporter is a file with an executable bit, which +/// is all its lookup asks because a `pipx` shim is slow to start. +fn runs(program: Program, dir: &Path) -> bool { + match program { + Program::Ffmpeg | Program::Ffprobe => media::tool_in_dir(dir, program.name()).is_some(), + Program::Wtsexporter => is_executable(&dir.join(program.file_name())), + } +} + +/// Whether the file at `path` is a file with an executable bit. +#[cfg(unix)] +fn is_executable(path: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(path) + .is_ok_and(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0) +} + +/// Windows runs any `.exe`, so a file is enough. +#[cfg(not(unix))] +fn is_executable(path: &Path) -> bool { + path.is_file() +} + +/// The SHA-256 of the file at `path`, in lowercase hex. +fn file_sha256(path: &Path) -> io::Result { + let mut file = File::open(path)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 64 * 1024]; + loop { + match file.read(&mut buffer) { + Ok(0) => break, + Ok(read) => hasher.update(&buffer[..read]), + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) => return Err(err), + } + } + Ok(hex::encode(hasher.finalize())) +} + +/// Why a download failed, as Settings shows it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DownloadError { + /// No answer from the server: no network, a name that does not + /// resolve, a connection cut off or timed out. + NoNetwork(String), + /// The server answered with a status other than success. + Http(u16), + /// The file arrived, and its SHA-256 is not the pinned one. + ChecksumMismatch { + /// The SHA-256 the app carries. + expected: String, + /// The SHA-256 of what arrived. + actual: String, + }, + /// The file could not be written to the Tools Directory. + CouldNotWrite(String), + /// The program passed its checksum and is in place, and does not run. + DoesNotRun(Program), +} + +impl fmt::Display for DownloadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NoNetwork(why) => write!(f, "No connection to the download's server: {why}."), + Self::Http(status) => { + let reason = reqwest::StatusCode::from_u16(*status) + .ok() + .and_then(|status| status.canonical_reason()) + .map(|reason| format!(" {reason}")) + .unwrap_or_default(); + write!(f, "The download's server answered {status}{reason}.") + } + Self::ChecksumMismatch { expected, actual } => write!( + f, + "The downloaded file's checksum did not match the one this app carries, so the file was deleted. \ + SHA-256 expected {expected}, got {actual}." + ), + Self::CouldNotWrite(why) => { + write!( + f, + "The file could not be written to the Tools Directory: {why}." + ) + } + Self::DoesNotRun(program @ (Program::Ffmpeg | Program::Ffprobe)) => write!( + f, + "{program} in the Tools Directory doesn't run on this computer. \ + Install it with your package manager instead; the app uses the copy on PATH.", + program = program.name() + ), + Self::DoesNotRun(Program::Wtsexporter) => write!( + f, + "wtsexporter in the Tools Directory doesn't run on this computer. \ + See \"Import can't find wtsexporter\" in Troubleshooting at messagecrate.app." + ), + } + } +} + +/// An error and the errors under it, as one line. +fn error_chain(err: &dyn std::error::Error) -> String { + let mut line = err.to_string(); + let mut source = err.source(); + while let Some(cause) = source { + line.push_str(": "); + line.push_str(&cause.to_string()); + source = cause.source(); + } + line +} + +/// The client the downloads use: rustls, a `User-Agent` naming the app, 30 +/// seconds to connect, and 60 seconds for any one read, so a stalled +/// download fails rather than hanging the check. +fn download_client() -> reqwest::Result { + reqwest::blocking::Client::builder() + .user_agent(concat!( + "MessageCrate/", + env!("MESSAGE_CRATE_BUILD"), + " (desktop app tool download)" + )) + .connect_timeout(Duration::from_secs(30)) + .timeout(Duration::from_secs(60)) + .build() +} + +/// Download `pinned` from `url` into `dir` and put it in place. +/// +/// The file is streamed to a temporary file in `dir` and its SHA-256 +/// computed on the way. On a match it is unpacked when gzipped and the +/// program's SHA-256 checked, then made executable and renamed over the +/// program; on a mismatch, or any failure, +/// the temporary file is deleted and the program there is left as it was. +/// `progress` hears the bytes received so far and the total when the +/// server says it. +fn download( + client: &reqwest::blocking::Client, + url: &str, + pinned: &Pinned, + dir: &Path, + progress: &dyn Fn(u64, Option), +) -> Result { + let write_error = |err: io::Error| DownloadError::CouldNotWrite(err.to_string()); + let mut response = client + .get(url) + .send() + .map_err(|err| DownloadError::NoNetwork(error_chain(&err)))?; + let status = response.status(); + if !status.is_success() { + return Err(DownloadError::Http(status.as_u16())); + } + let total = response.content_length(); + let mut arrived = tempfile::Builder::new() + .prefix(&temp_prefix(pinned.program, "download")) + .tempfile_in(dir) + .map_err(write_error)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 64 * 1024]; + let mut received = 0u64; + progress(received, total); + loop { + let read = match response.read(&mut buffer) { + Ok(0) => break, + Ok(read) => read, + Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, + Err(err) => return Err(DownloadError::NoNetwork(error_chain(&err))), + }; + hasher.update(&buffer[..read]); + arrived.write_all(&buffer[..read]).map_err(write_error)?; + received += read as u64; + progress(received, total); + } + let actual = hex::encode(hasher.finalize()); + if actual != pinned.sha256 { + return Err(DownloadError::ChecksumMismatch { + expected: pinned.sha256.to_string(), + actual, + }); + } + let mut program = if pinned.gzip { + let mut unpacked = tempfile::Builder::new() + .prefix(&temp_prefix(pinned.program, "unpack")) + .tempfile_in(dir) + .map_err(write_error)?; + let packed = arrived.as_file_mut(); + packed.rewind().map_err(write_error)?; + io::copy(&mut flate2::read::GzDecoder::new(packed), &mut unpacked).map_err(write_error)?; + unpacked.flush().map_err(write_error)?; + let actual = file_sha256(unpacked.path()).map_err(write_error)?; + if actual != pinned.program_sha256 { + return Err(DownloadError::ChecksumMismatch { + expected: pinned.program_sha256.to_string(), + actual, + }); + } + unpacked + } else { + arrived + }; + program.flush().map_err(write_error)?; + program.as_file().sync_all().map_err(write_error)?; + make_executable(program.path()).map_err(write_error)?; + let target = dir.join(pinned.program.file_name()); + program + .persist(&target) + .map_err(|err| write_error(err.error))?; + // Renaming keeps the size, modified time and inode, so the stamp taken + // now is the one the next start sees. + let stamp = Stamp::at(&target).map_err(write_error)?; + Ok(Written::of(pinned, stamp)) +} + +/// Give the file at `path` its executable bits. +#[cfg(unix)] +fn make_executable(path: &Path) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)) +} + +/// Windows runs any `.exe`, so there is no bit to set. +#[cfg(not(unix))] +fn make_executable(_path: &Path) -> io::Result<()> { + Ok(()) +} + +/// Where one program's download stands, as Settings shows it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DownloadState { + /// Downloading: `received` bytes so far, of `total` when the server + /// said. + Downloading { + /// Bytes received so far. + received: u64, + /// The file's size, when the server said. + total: Option, + }, + /// The last download failed, for `reason`. + Failed { + /// Why, as [`DownloadError`] says it. + reason: String, + }, +} + +/// The download state of each program, shared by the start-up check and +/// `tools_status`. A program with no entry is not downloading and has not +/// failed since the app started. +#[derive(Debug, Clone, Default)] +pub struct ToolDownloads(Arc>>); + +impl ToolDownloads { + /// Where `program`'s download stands, if anywhere. + pub fn get(&self, program: Program) -> Option { + self.0 + .lock() + .expect("tool downloads lock") + .get(&program) + .cloned() + } + + fn set(&self, program: Program, state: DownloadState) { + self.0 + .lock() + .expect("tool downloads lock") + .insert(program, state); + } + + fn clear(&self, program: Program) { + self.0.lock().expect("tool downloads lock").remove(&program); + } +} + +/// The state of `program` in place that does not run. +fn does_not_run(program: Program) -> DownloadState { + DownloadState::Failed { + reason: DownloadError::DoesNotRun(program).to_string(), + } +} + +/// Take the lock on [`LOCK_FILE`] in `dir`, making `dir` first. `None` when +/// another check holds it. The lock is let go when the file is closed. +fn lock_check(dir: &Path) -> io::Result> { + std::fs::create_dir_all(dir)?; + let file = File::options() + .create(true) + .truncate(false) + .write(true) + .open(dir.join(LOCK_FILE))?; + match file.try_lock() { + Ok(()) => Ok(Some(file)), + Err(std::fs::TryLockError::WouldBlock) => Ok(None), + Err(std::fs::TryLockError::Error(err)) => Err(err), + } +} + +/// Check `dir` against `pinned` and download, one after another, what is +/// missing or not the pinned release, from `base`. ffmpeg and ffprobe are +/// passed over when both are on `PATH`. Each download's progress and +/// failure go to `downloads`, and each file written goes into the record. +/// +/// Another app's check running on `dir` already does the work, so this one +/// returns at once. When the lock can't be taken because the Tools Directory +/// can't be written, the check goes on and its downloads say why. +pub fn download_missing(dir: &Path, base: &str, pinned: &[Pinned], downloads: &ToolDownloads) { + let _lock = match lock_check(dir) { + Ok(Some(lock)) => Some(lock), + Ok(None) => return, + Err(_) => None, + }; + delete_leftovers(dir); + let ffmpeg_on_path = media::ffmpeg_on_path(); + let mut manifest = read_manifest(dir); + let mut wanted = Vec::new(); + for pin in pinned { + if ffmpeg_on_path && matches!(pin.program, Program::Ffmpeg | Program::Ffprobe) { + continue; + } + match need(pin, dir, &manifest) { + Need::Keep => {} + Need::Adopt => { + let path = dir.join(pin.program.file_name()); + let _ = make_executable(&path); + if let Ok(stamp) = Stamp::at(&path) { + manifest.insert(pin.program, Written::of(pin, stamp)); + let _ = write_manifest(dir, &manifest); + } + if !runs(pin.program, dir) { + downloads.set(pin.program, does_not_run(pin.program)); + } + } + Need::DoesNotRun => downloads.set(pin.program, does_not_run(pin.program)), + Need::Download => wanted.push(pin), + } + } + if wanted.is_empty() { + return; + } + // Every program wanted shows as downloading from now on, before the + // first request, so Settings opened while a connection is made, or + // between two downloads, keeps asking until the last one ends. + for pin in &wanted { + downloads.set( + pin.program, + DownloadState::Downloading { + received: 0, + total: None, + }, + ); + } + let fail_all = |err: DownloadError| { + for pin in &wanted { + downloads.set( + pin.program, + DownloadState::Failed { + reason: err.to_string(), + }, + ); + } + }; + let client = match download_client() { + Ok(client) => client, + Err(err) => return fail_all(DownloadError::NoNetwork(error_chain(&err))), + }; + if let Err(err) = std::fs::create_dir_all(dir) { + return fail_all(DownloadError::CouldNotWrite(err.to_string())); + } + for pin in wanted { + let program = pin.program; + let progress = |received, total| { + downloads.set(program, DownloadState::Downloading { received, total }); + }; + match download(&client, &pin.url(base), pin, dir, &progress) { + Ok(written) => { + manifest.insert(program, written); + if runs(program, dir) { + downloads.clear(program); + } else { + downloads.set(program, does_not_run(program)); + } + if let Err(err) = write_manifest(dir, &manifest) { + // The program is in place and works; without the record + // the next start takes it as someone else's. + downloads.set( + program, + DownloadState::Failed { + reason: DownloadError::CouldNotWrite(format!( + "{} could not be written: {err}", + MANIFEST_FILE + )) + .to_string(), + }, + ); + } + } + Err(err) => downloads.set( + program, + DownloadState::Failed { + reason: err.to_string(), + }, + ), + } + } +} + +/// Start the check of the Tools Directory `dir` on a thread of its own, for +/// this computer's platform, so login and browsing don't wait for it. +pub fn start(dir: PathBuf, downloads: ToolDownloads) { + let pinned = pinned_for(std::env::consts::OS, std::env::consts::ARCH); + let _ = std::thread::Builder::new() + .name("tool-downloads".into()) + .spawn(move || download_missing(&dir, GITHUB, &pinned, &downloads)); +} + +#[cfg(test)] +mod tests; diff --git a/src-tauri/src/tool_downloads/pins.rs b/src-tauri/src/tool_downloads/pins.rs new file mode 100644 index 000000000..04484b5be --- /dev/null +++ b/src-tauri/src/tool_downloads/pins.rs @@ -0,0 +1,214 @@ +//! The releases the app downloads ffmpeg, ffprobe and wtsexporter from, and +//! the SHA-256 of each file, by platform. A newer pinned ffmpeg release is a +//! change to this file alone (`docs/adr/0019`). A newer wtsexporter release +//! is a change to this file, to the install hints in +//! `crates/exporters/whatsapp-exporter/src/wtsexporter.rs`, and to the +//! developer install in `AGENTS.md` and +//! `docs/src/content/docs/docs/developer/contributing.md`, which name it. + +use super::{Pinned, Program}; + +/// The project the ffmpeg and ffprobe files come from. +const FFMPEG_STATIC: &str = "eugeneware/ffmpeg-static"; + +/// The `ffmpeg-static` release pinned: its files report ffmpeg 7.0.2, a GPL +/// build with `libx264` and `libx265`. +const FFMPEG_RELEASE: &str = "b6.1.1"; + +/// The project the wtsexporter files come from: Message Crate's fork, whose +/// JSON says who sent each message and who is in each group. +const WTSEXPORTER_REPO: &str = "messagecrate/WhatsApp-Chat-Exporter"; + +/// The fork release pinned, the first with `full_key_id` and `reply_key_id`. +const WTSEXPORTER_RELEASE: &str = "0.13.0-mc.2"; + +/// The `.gz` file of ffmpeg and of ffprobe for one platform, with the +/// SHA-256 of each `.gz` and of the program it unpacks to. +struct FfmpegFiles { + /// `std::env::consts::OS`. + os: &'static str, + /// `std::env::consts::ARCH`. + arch: &'static str, + /// ffmpeg's asset, its SHA-256, and the unpacked program's SHA-256. + ffmpeg: (&'static str, &'static str, &'static str), + /// ffprobe's asset, its SHA-256, and the unpacked program's SHA-256. + ffprobe: (&'static str, &'static str, &'static str), +} + +/// The files of the pinned `ffmpeg-static` release, by platform. +/// +/// The project publishes no checksum file. Each SHA-256 here was computed on +/// 2026-10-08 by downloading the asset with +/// `curl -L https://github.com/eugeneware/ffmpeg-static/releases/download//` +/// and running `sha256sum` on it; `gzip -t` passed on each. The program's +/// SHA-256 was computed the same day by `gunzip -c | sha256sum`, so +/// the app knows a copy put in the Tools Directory by hand for the pinned one. +/// +/// The `.gz` files are downloaded rather than the plain ones because they +/// are a third of the size (19 to 30 MB against 45 to 83 MB). The file is +/// checked as it arrives, before it is unpacked, and the program it unpacks +/// to is checked again. +/// +/// Windows on ARM gets the x64 files, which Windows runs under emulation. +/// Whether they run there is not checked yet (#1053). +const FFMPEG_FILES: &[FfmpegFiles] = &[ + FfmpegFiles { + os: "linux", + arch: "x86_64", + ffmpeg: ( + "ffmpeg-linux-x64.gz", + "bfe8a8fc511530457b528c48d77b5737527b504a3797a9bc4866aeca69c2dffa", + "e7e7fb30477f717e6f55f9180a70386c62677ef8a4d4d1a5d948f4098aa3eb99", + ), + ffprobe: ( + "ffprobe-linux-x64.gz", + "25d9b6ccb05e3d9de9e04e31e2506d8dd7f9f0418981965ac6df12e8d3afd067", + "4f231a1960d83e403d08f7971e271707bec278a9ae18e21b8b5b03186668450d", + ), + }, + FfmpegFiles { + os: "linux", + arch: "aarch64", + ffmpeg: ( + "ffmpeg-linux-arm64.gz", + "754a678672298bc68156adff58aa7385a592c2b30b1d0ae8750c45c915c4bac0", + "6bb182d0d75d23028db82e9e4f723ca69b853d055698486e6984ddb2c06fb8ce", + ), + ffprobe: ( + "ffprobe-linux-arm64.gz", + "2ab6aba60ee84412dff9188720703376cb4e7aaf7e0b5e43aa8249f2acae5bf8", + "d17ae9b4c297d48e2521ba14e417bb0537c6ff77c584cdbcd6bb0d8d0307a2e8", + ), + }, + FfmpegFiles { + os: "macos", + arch: "x86_64", + ffmpeg: ( + "ffmpeg-darwin-x64.gz", + "929b375c1182d956c51f7ac25e0b2b0411fb01f6f407aa15c9758efeb4242106", + "ebdddc936f61e14049a2d4b549a412b8a40deeff6540e58a9f2a2da9e6b18894", + ), + ffprobe: ( + "ffprobe-darwin-x64.gz", + "d4da574d6e2e197bd259b47d69cf262df9e312af24ad960444f6d806d3d4c186", + "fa3add0ce901f7241abe0dfc0155d958fc834aca3f8ce61f87cc712ae669c1e0", + ), + }, + FfmpegFiles { + os: "macos", + arch: "aarch64", + ffmpeg: ( + "ffmpeg-darwin-arm64.gz", + "8923876afa8db5585022d7860ec7e589af192f441c56793971276d450ed3bbfa", + "a90e3db6a3fd35f6074b013f948b1aa45b31c6375489d39e572bea3f18336584", + ), + ffprobe: ( + "ffprobe-darwin-arm64.gz", + "d986a8ec7b030899fe66a8a288ed809a3543338705a3ce178cfb85869c5d80be", + "bb2db6f5d8cef919da12fbf592119a987202a8c060a886f3cab091f9cab90b64", + ), + }, + FfmpegFiles { + os: "windows", + arch: "x86_64", + ffmpeg: ( + "ffmpeg-win32-x64.gz", + "8883a3dffbd0a16cf4ef95206ea05283f78908dbfb118f73c83f4951dcc06d77", + "04e1307997530f9cf2fe35cba2ca7e8875ca91da02f89d6c7243df819c94ad00", + ), + ffprobe: ( + "ffprobe-win32-x64.gz", + "f309e6223ad89d2fe54bccd420a7709b66fd27540674e92309578ed491a43c8d", + "3a7e2dc003dc2cd1472827e4c7c4f056ae1ae0ae7c5bbc580c99b49827351ba4", + ), + }, + FfmpegFiles { + os: "windows", + arch: "aarch64", + ffmpeg: ( + "ffmpeg-win32-x64.gz", + "8883a3dffbd0a16cf4ef95206ea05283f78908dbfb118f73c83f4951dcc06d77", + "04e1307997530f9cf2fe35cba2ca7e8875ca91da02f89d6c7243df819c94ad00", + ), + ffprobe: ( + "ffprobe-win32-x64.gz", + "f309e6223ad89d2fe54bccd420a7709b66fd27540674e92309578ed491a43c8d", + "3a7e2dc003dc2cd1472827e4c7c4f056ae1ae0ae7c5bbc580c99b49827351ba4", + ), + }, +]; + +/// The wtsexporter file for each platform the fork builds for, as +/// `(os, arch, asset, sha256)`. The checksums are copied from the release's +/// `SHA256SUMS` asset, read on 2026-10-08; the app never fetches that file. +/// Linux on ARM has no file. +const WTSEXPORTER_FILES: &[(&str, &str, &str, &str)] = &[ + ( + "linux", + "x86_64", + "wtsexporter_linux_x64", + "2756dfbcc7e320390e34244a8fc7160770d99d6fef17dea6a2d6ce269b62f71b", + ), + ( + "macos", + "x86_64", + "wtsexporter_macos_x64", + "b0e885474108d810fa7ba31360275494fc93ebadae117859ba7e90fa705f45e3", + ), + ( + "macos", + "aarch64", + "wtsexporter_macos_arm64", + "03f90f32e956a5df1cbfbc1a1a87524ab05450f645703df0c16d630671eda9bf", + ), + ( + "windows", + "x86_64", + "wtsexporter_win_x64.exe", + "4f4c91ab7829fa4a78673b5124c354b7be0c6d7446689ac58c56461f7988707f", + ), + ( + "windows", + "aarch64", + "wtsexporter_win_arm64.exe", + "aa154042bec5b96953dda8d3c6daabac4fbbd9bd51c28fd350ba416317d9ec10", + ), +]; + +/// The pinned files for a computer whose `std::env::consts` are `os` and +/// `arch`: ffmpeg and ffprobe, then wtsexporter. A platform with no file +/// for a program gets nothing for it. +pub fn pinned_for(os: &str, arch: &str) -> Vec { + let mut pinned = Vec::new(); + if let Some(files) = FFMPEG_FILES.iter().find(|f| f.os == os && f.arch == arch) { + for (program, (asset, sha256, program_sha256)) in [ + (Program::Ffmpeg, files.ffmpeg), + (Program::Ffprobe, files.ffprobe), + ] { + pinned.push(Pinned { + program, + repo: FFMPEG_STATIC, + release: FFMPEG_RELEASE, + asset, + sha256, + program_sha256, + gzip: true, + }); + } + } + if let Some(&(_, _, asset, sha256)) = WTSEXPORTER_FILES + .iter() + .find(|(file_os, file_arch, _, _)| *file_os == os && *file_arch == arch) + { + pinned.push(Pinned { + program: Program::Wtsexporter, + repo: WTSEXPORTER_REPO, + release: WTSEXPORTER_RELEASE, + asset, + sha256, + program_sha256: sha256, + gzip: false, + }); + } + pinned +} diff --git a/src-tauri/src/tool_downloads/tests.rs b/src-tauri/src/tool_downloads/tests.rs new file mode 100644 index 000000000..c1b2fd227 --- /dev/null +++ b/src-tauri/src/tool_downloads/tests.rs @@ -0,0 +1,806 @@ +use super::*; + +use std::ffi::OsString; + +use httpmock::prelude::*; + +/// The SHA-256 of `bytes`, in lowercase hex. +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +/// `bytes` gzipped, as `ffmpeg-static` publishes its files. +fn gzipped(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} + +fn leak(text: String) -> &'static str { + Box::leak(text.into_boxed_str()) +} + +/// `program` pinned to release `release`, whose asset is `published` with +/// the checksum `sha256`. +fn pin_with( + program: Program, + release: &'static str, + published: &[u8], + gzip: bool, + sha256: &str, +) -> (Pinned, Vec) { + let pinned = Pinned { + program, + repo: "owner/project", + release, + asset: leak(format!( + "{}-test{}", + program.name(), + if gzip { ".gz" } else { "" } + )), + sha256: leak(sha256.to_string()), + program_sha256: leak(if gzip { + let mut program = Vec::new(); + flate2::read::GzDecoder::new(published) + .read_to_end(&mut program) + .unwrap(); + sha256_hex(&program) + } else { + sha256.to_string() + }), + gzip, + }; + (pinned, published.to_vec()) +} + +/// `program` pinned to the asset `published`, with its true checksum. +fn pin(program: Program, release: &'static str, published: &[u8], gzip: bool) -> (Pinned, Vec) { + pin_with(program, release, published, gzip, &sha256_hex(published)) +} + +/// Serve `body` at `pinned`'s path on `server`. +fn serve<'a>(server: &'a MockServer, pinned: &Pinned, body: &[u8]) -> httpmock::Mock<'a> { + let path = format!( + "/{}/releases/download/{}/{}", + pinned.repo, pinned.release, pinned.asset + ); + let body = body.to_vec(); + server.mock(move |when, then| { + when.method(GET).path(path); + then.status(200).body(body); + }) +} + +/// The lookup pointed at an empty `PATH` and the Tools Directory `dir`, so +/// ffmpeg on this machine's `PATH` changes nothing. +fn no_ffmpeg_on_path(dir: &Path) -> media::testutil::ToolsHidden { + media::testutil::locate_tools(OsString::new(), dir.to_path_buf()) +} + +/// The names in `dir` other than `keep` and the lock file, to see that no +/// temporary file was left behind. +fn other_files(dir: &Path, keep: &[&str]) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name != LOCK_FILE && !keep.contains(&name.as_str())) + .collect(); + names.sort(); + names +} + +/// The file that arrives is refused when its checksum is not the pinned one: +/// it is deleted, nothing is put in place, and the failure says the +/// checksum did not match. +#[test] +fn a_download_whose_checksum_does_not_match_is_refused() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let server = MockServer::start(); + let (pinned, _) = pin_with( + Program::Wtsexporter, + "r1", + b"", + false, + &sha256_hex(b"the pinned file"), + ); + serve(&server, &pinned, b"a file changed on the way"); + let downloads = ToolDownloads::default(); + + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&pinned), + &downloads, + ); + + assert!(!tools.path().join(Program::Wtsexporter.file_name()).exists()); + assert_eq!(other_files(tools.path(), &[]), Vec::::new()); + let Some(DownloadState::Failed { reason }) = downloads.get(Program::Wtsexporter) else { + panic!( + "the download did not fail: {:?}", + downloads.get(Program::Wtsexporter) + ); + }; + assert!( + reason.contains("checksum did not match"), + "the reason does not name the checksum: {reason}" + ); + assert!(reason.contains(&sha256_hex(b"a file changed on the way"))); +} + +/// A program the app wrote from an older release stays in use while the +/// newer one fails to download or does not match, and is replaced only by +/// a download that passed. +#[test] +fn an_older_program_is_replaced_only_after_a_good_download() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let target = tools.path().join(Program::Wtsexporter.file_name()); + std::fs::write(&target, b"old release").unwrap(); + let (old, _) = pin(Program::Wtsexporter, "r1", b"old release", false); + make_executable(&target).unwrap(); + let mut manifest = Manifest::new(); + manifest.insert( + Program::Wtsexporter, + Written::of(&old, Stamp::at(&target).unwrap()), + ); + write_manifest(tools.path(), &manifest).unwrap(); + let (new, new_bytes) = pin(Program::Wtsexporter, "r2", b"new release", false); + let downloads = ToolDownloads::default(); + + // 404 Not Found: the old file stays. + let server = MockServer::start(); + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&new), + &downloads, + ); + assert_eq!(std::fs::read(&target).unwrap(), b"old release"); + let Some(DownloadState::Failed { reason }) = downloads.get(Program::Wtsexporter) else { + panic!("the download did not fail"); + }; + assert!(reason.contains("404 Not Found"), "{reason}"); + + // A file that does not match: the old file stays. + let mut wrong = serve(&server, &new, b"not the new release"); + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&new), + &downloads, + ); + assert_eq!(std::fs::read(&target).unwrap(), b"old release"); + assert_eq!(read_manifest(tools.path()), manifest); + wrong.delete(); + + // A good download replaces it, and the record names the new release. + serve(&server, &new, &new_bytes); + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&new), + &downloads, + ); + assert_eq!(std::fs::read(&target).unwrap(), b"new release"); + assert_eq!(downloads.get(Program::Wtsexporter), None); + assert_eq!( + read_manifest(tools.path()).get(&Program::Wtsexporter), + Some(&Written::of(&new, Stamp::at(&target).unwrap())) + ); + assert_eq!( + other_files( + tools.path(), + &[MANIFEST_FILE, &Program::Wtsexporter.file_name()] + ), + Vec::::new() + ); +} + +/// A gzipped asset is checked as it arrives, then unpacked into the program, +/// which can run. +#[test] +fn a_gzipped_program_is_unpacked_and_made_executable() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let program = b"#!/bin/sh\nexit 0\n"; + let (pinned, published) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(program), true); + let server = MockServer::start(); + serve(&server, &pinned, &published); + + download_missing( + tools.path(), + &server.base_url(), + &[pinned], + &ToolDownloads::default(), + ); + + let path = tools.path().join(Program::Ffmpeg.file_name()); + assert_eq!(std::fs::read(&path).unwrap(), program); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o111, 0o111, "not executable: {mode:o}"); + } +} + +/// A gzipped asset that passes its own checksum is still refused when the +/// program it unpacks to is not the pinned program. +#[test] +fn a_gzipped_program_that_is_not_the_pinned_one_is_refused() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let (mut pinned, published) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(b"ffmpeg"), true); + pinned.program_sha256 = leak(sha256_hex(b"the pinned program")); + let server = MockServer::start(); + serve(&server, &pinned, &published); + let downloads = ToolDownloads::default(); + + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&pinned), + &downloads, + ); + + assert!(!tools.path().join(Program::Ffmpeg.file_name()).exists()); + assert_eq!(other_files(tools.path(), &[]), Vec::::new()); + let Some(DownloadState::Failed { reason }) = downloads.get(Program::Ffmpeg) else { + panic!( + "the download did not fail: {:?}", + downloads.get(Program::Ffmpeg) + ); + }; + assert!(reason.contains(&sha256_hex(b"ffmpeg")), "{reason}"); +} + +/// Write a program at `path` that answers `-version`. +#[cfg(unix)] +fn write_runnable(path: &Path) { + use std::os::unix::fs::PermissionsExt; + std::fs::write(path, "#!/bin/sh\nexit 0\n").unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); +} + +/// ffmpeg and ffprobe on `PATH` are used as they are, and neither is +/// downloaded; wtsexporter still is. +#[cfg(unix)] +#[test] +fn ffmpeg_on_path_is_not_downloaded() { + let bin = tempfile::tempdir().unwrap(); + write_runnable(&bin.path().join("ffmpeg")); + write_runnable(&bin.path().join("ffprobe")); + let tools = tempfile::tempdir().unwrap(); + let _tools = media::testutil::locate_tools( + bin.path().as_os_str().to_os_string(), + tools.path().to_path_buf(), + ); + let server = MockServer::start(); + let (ffmpeg, ffmpeg_gz) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(b"ffmpeg"), true); + let (ffprobe, ffprobe_gz) = pin(Program::Ffprobe, "b6.1.1", &gzipped(b"ffprobe"), true); + let (wts, wts_bytes) = pin(Program::Wtsexporter, "r1", b"wtsexporter", false); + let ffmpeg_asked = serve(&server, &ffmpeg, &ffmpeg_gz); + let ffprobe_asked = serve(&server, &ffprobe, &ffprobe_gz); + let wts_asked = serve(&server, &wts, &wts_bytes); + + download_missing( + tools.path(), + &server.base_url(), + &[ffmpeg, ffprobe, wts], + &ToolDownloads::default(), + ); + + ffmpeg_asked.assert_calls(0); + ffprobe_asked.assert_calls(0); + wts_asked.assert_calls(1); + assert!(!tools.path().join("ffmpeg").exists()); + assert!(!tools.path().join("ffprobe").exists()); +} + +/// ffmpeg on `PATH` without ffprobe is not enough: both are downloaded. +#[cfg(unix)] +#[test] +fn ffmpeg_on_path_without_ffprobe_is_downloaded() { + let bin = tempfile::tempdir().unwrap(); + write_runnable(&bin.path().join("ffmpeg")); + let tools = tempfile::tempdir().unwrap(); + let _tools = media::testutil::locate_tools( + bin.path().as_os_str().to_os_string(), + tools.path().to_path_buf(), + ); + let server = MockServer::start(); + let (ffmpeg, ffmpeg_gz) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(b"ffmpeg"), true); + let (ffprobe, ffprobe_gz) = pin(Program::Ffprobe, "b6.1.1", &gzipped(b"ffprobe"), true); + serve(&server, &ffmpeg, &ffmpeg_gz); + serve(&server, &ffprobe, &ffprobe_gz); + + download_missing( + tools.path(), + &server.base_url(), + &[ffmpeg, ffprobe], + &ToolDownloads::default(), + ); + + assert_eq!( + std::fs::read(tools.path().join("ffmpeg")).unwrap(), + b"ffmpeg" + ); + assert_eq!( + std::fs::read(tools.path().join("ffprobe")).unwrap(), + b"ffprobe" + ); +} + +/// A program the app wrote from the pinned release is not downloaded again, +/// and its file is not read to know it. +#[test] +fn the_pinned_program_the_app_wrote_is_kept() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let (pinned, published) = pin(Program::Wtsexporter, "r1", b"pinned", false); + let target = tools.path().join(Program::Wtsexporter.file_name()); + std::fs::write(&target, b"pinned").unwrap(); + make_executable(&target).unwrap(); + let mut manifest = Manifest::new(); + manifest.insert( + Program::Wtsexporter, + Written::of(&pinned, Stamp::at(&target).unwrap()), + ); + write_manifest(tools.path(), &manifest).unwrap(); + let server = MockServer::start(); + let asked = serve(&server, &pinned, &published); + + download_missing( + tools.path(), + &server.base_url(), + &[pinned], + &ToolDownloads::default(), + ); + + asked.assert_calls(0); +} + +/// A file put over the one the app wrote, the same size but not the same +/// file, is not the app's: its stamp differs, and it is replaced. +#[test] +fn a_file_put_over_the_apps_own_is_replaced() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let (pinned, published) = pin(Program::Wtsexporter, "r1", b"pinned", false); + let target = tools.path().join(Program::Wtsexporter.file_name()); + std::fs::write(&target, b"pinned").unwrap(); + make_executable(&target).unwrap(); + let mut manifest = Manifest::new(); + manifest.insert( + Program::Wtsexporter, + Written::of(&pinned, Stamp::at(&target).unwrap()), + ); + write_manifest(tools.path(), &manifest).unwrap(); + // Six other bytes, modified at another time. + std::fs::write(&target, b"theirs").unwrap(); + File::options() + .write(true) + .open(&target) + .unwrap() + .set_modified(std::time::UNIX_EPOCH + Duration::from_secs(1_000_000)) + .unwrap(); + let server = MockServer::start(); + let asked = serve(&server, &pinned, &published); + + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&pinned), + &ToolDownloads::default(), + ); + + asked.assert_calls(1); + assert_eq!(std::fs::read(&target).unwrap(), b"pinned"); +} + +/// ffmpeg and ffprobe put in the Tools Directory by hand are replaced by a +/// good download, even when they run, because the Tools Directory belongs +/// to the app. With no download to replace them they stay in use. +#[cfg(unix)] +#[test] +fn ffmpeg_put_there_by_hand_is_replaced_after_a_good_download() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let theirs = b"#!/bin/sh\n# theirs\nexit 0\n"; + let ours = b"#!/bin/sh\n# pinned\nexit 0\n"; + for name in ["ffmpeg", "ffprobe"] { + let path = tools.path().join(name); + std::fs::write(&path, theirs).unwrap(); + make_executable(&path).unwrap(); + } + let (ffmpeg, ffmpeg_gz) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(ours), true); + let (ffprobe, ffprobe_gz) = pin(Program::Ffprobe, "b6.1.1", &gzipped(ours), true); + let downloads = ToolDownloads::default(); + + // No network: what is there stays. + download_missing( + tools.path(), + "http://127.0.0.1:9", + &[ffmpeg.clone(), ffprobe.clone()], + &downloads, + ); + for name in ["ffmpeg", "ffprobe"] { + assert_eq!(std::fs::read(tools.path().join(name)).unwrap(), theirs); + } + + let server = MockServer::start(); + serve(&server, &ffmpeg, &ffmpeg_gz); + serve(&server, &ffprobe, &ffprobe_gz); + download_missing( + tools.path(), + &server.base_url(), + &[ffmpeg, ffprobe], + &downloads, + ); + for (name, program) in [("ffmpeg", Program::Ffmpeg), ("ffprobe", Program::Ffprobe)] { + assert_eq!(std::fs::read(tools.path().join(name)).unwrap(), ours); + assert_eq!(downloads.get(program), None); + } +} + +/// A pinned file the app recorded that does not run is not downloaded +/// again, start after start, because the download would be the same file; +/// the failure says it doesn't run. +#[cfg(unix)] +#[test] +fn a_pinned_program_that_does_not_run_is_not_downloaded_again() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let not_a_program = b"not a program"; + let (pinned, published) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(not_a_program), true); + let server = MockServer::start(); + let asked = serve(&server, &pinned, &published); + + for _start in 0..2 { + let downloads = ToolDownloads::default(); + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&pinned), + &downloads, + ); + let Some(DownloadState::Failed { reason }) = downloads.get(Program::Ffmpeg) else { + panic!( + "the program was not said not to run: {:?}", + downloads.get(Program::Ffmpeg) + ); + }; + assert_eq!( + reason, + "ffmpeg in the Tools Directory doesn't run on this computer. \ + Install it with your package manager instead; the app uses the copy on PATH." + ); + } + + asked.assert_calls(1); +} + +/// The pinned program put there by hand that does not run is kept and +/// recorded, and not downloaded, because the download would be the same file. +#[cfg(unix)] +#[test] +fn a_pinned_program_put_there_by_hand_that_does_not_run_is_not_downloaded() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let not_a_program = b"not a program"; + let (pinned, published) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(not_a_program), true); + std::fs::write(tools.path().join("ffmpeg"), not_a_program).unwrap(); + let server = MockServer::start(); + let asked = serve(&server, &pinned, &published); + let downloads = ToolDownloads::default(); + + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&pinned), + &downloads, + ); + + asked.assert_calls(0); + assert!(read_manifest(tools.path()).contains_key(&Program::Ffmpeg)); + assert!(matches!( + downloads.get(Program::Ffmpeg), + Some(DownloadState::Failed { .. }) + )); +} + +/// ffmpeg is kept and recorded again when its record is lost, because the +/// file is the pinned program, though the pin is of the `.gz`. +#[cfg(unix)] +#[test] +fn ffmpeg_with_no_record_is_adopted_when_it_is_the_pinned_program() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let program = b"#!/bin/sh\n# pinned\nexit 0\n"; + let (pinned, published) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(program), true); + let target = tools.path().join(Program::Ffmpeg.file_name()); + std::fs::write(&target, program).unwrap(); + make_executable(&target).unwrap(); + let server = MockServer::start(); + let asked = serve(&server, &pinned, &published); + let downloads = ToolDownloads::default(); + + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&pinned), + &downloads, + ); + + asked.assert_calls(0); + assert_eq!(downloads.get(Program::Ffmpeg), None); + assert_eq!( + read_manifest(tools.path()).get(&Program::Ffmpeg), + Some(&Written::of(&pinned, Stamp::at(&target).unwrap())) + ); +} + +/// The app's own file whose stamp stopped matching, as after a sync or a +/// remount, is recorded again with its new stamp and not downloaded, because +/// its content is still the pinned program. +#[cfg(unix)] +#[test] +fn a_changed_stamp_on_the_pinned_program_is_recorded_again() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let program = b"#!/bin/sh\n# pinned\nexit 0\n"; + let (pinned, published) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(program), true); + let target = tools.path().join(Program::Ffmpeg.file_name()); + std::fs::write(&target, program).unwrap(); + make_executable(&target).unwrap(); + let mut manifest = Manifest::new(); + manifest.insert( + Program::Ffmpeg, + Written::of(&pinned, Stamp::at(&target).unwrap()), + ); + write_manifest(tools.path(), &manifest).unwrap(); + File::options() + .write(true) + .open(&target) + .unwrap() + .set_modified(std::time::UNIX_EPOCH + Duration::from_secs(1_000_000)) + .unwrap(); + let server = MockServer::start(); + let asked = serve(&server, &pinned, &published); + + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&pinned), + &ToolDownloads::default(), + ); + + asked.assert_calls(0); + let recorded = read_manifest(tools.path()); + assert_eq!( + recorded.get(&Program::Ffmpeg), + Some(&Written::of(&pinned, Stamp::at(&target).unwrap())) + ); + assert_ne!(recorded, manifest); +} + +/// A program put there by hand is kept and recorded when it is the pinned +/// file, and replaced when it is anything else, because its release can't +/// be told from the file. +#[test] +fn a_program_put_there_by_hand_is_kept_only_when_it_is_the_pinned_file() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let target = tools.path().join(Program::Wtsexporter.file_name()); + let (pinned, published) = pin(Program::Wtsexporter, "r2", b"pinned", false); + let server = MockServer::start(); + let mut asked = serve(&server, &pinned, &published); + + std::fs::write(&target, b"pinned").unwrap(); + download_missing( + tools.path(), + &server.base_url(), + std::slice::from_ref(&pinned), + &ToolDownloads::default(), + ); + asked.assert_calls(0); + assert_eq!( + read_manifest(tools.path()).get(&Program::Wtsexporter), + Some(&Written::of(&pinned, Stamp::at(&target).unwrap())) + ); + asked.delete(); + + std::fs::write(&target, b"a pipx shim, or an older release").unwrap(); + let asked = serve(&server, &pinned, &published); + download_missing( + tools.path(), + &server.base_url(), + &[pinned], + &ToolDownloads::default(), + ); + asked.assert_calls(1); + assert_eq!(std::fs::read(&target).unwrap(), b"pinned"); +} + +/// The temporary files an interrupted check left are deleted at the next +/// start, and nothing else in the Tools Directory is. +#[test] +fn an_interrupted_downloads_leftovers_are_deleted() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + for name in [ + ".ffmpeg.download-a1b2c3", + ".ffprobe.unpack-d4e5f6", + ".wtsexporter.download-g7h8i9", + ".manifest-j0k1l2", + "notes.txt", + ] { + std::fs::write(tools.path().join(name), b"left").unwrap(); + } + + download_missing( + tools.path(), + "http://127.0.0.1:9", + &[], + &ToolDownloads::default(), + ); + + assert_eq!( + other_files(tools.path(), &[]), + vec!["notes.txt".to_string()] + ); +} + +/// Every program to be downloaded shows as downloading before its first +/// request, so Settings keeps asking through a slow connect and the gap +/// between two downloads. +#[test] +fn every_program_wanted_shows_as_downloading_before_its_request() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let (ffmpeg, ffmpeg_gz) = pin(Program::Ffmpeg, "b6.1.1", &gzipped(b"ffmpeg"), true); + let (wts, wts_bytes) = pin(Program::Wtsexporter, "r1", b"wtsexporter", false); + let server = MockServer::start(); + let ffmpeg_path = format!( + "/{}/releases/download/{}/{}", + ffmpeg.repo, ffmpeg.release, ffmpeg.asset + ); + server.mock(|when, then| { + when.method(GET).path(ffmpeg_path); + then.status(200) + .delay(Duration::from_millis(500)) + .body(ffmpeg_gz); + }); + serve(&server, &wts, &wts_bytes); + let downloads = ToolDownloads::default(); + let base = server.base_url(); + let dir = tools.path().to_path_buf(); + + let mut seen_waiting = false; + std::thread::scope(|scope| { + let check = scope.spawn(|| download_missing(&dir, &base, &[ffmpeg, wts], &downloads)); + while !check.is_finished() { + if downloads.get(Program::Wtsexporter) + == Some(DownloadState::Downloading { + received: 0, + total: None, + }) + { + seen_waiting = true; + } + std::thread::sleep(Duration::from_millis(5)); + } + }); + + assert!( + seen_waiting, + "wtsexporter was not downloading while it waited" + ); + assert_eq!(downloads.get(Program::Wtsexporter), None); +} + +/// A second app's check, started while the first one runs, does nothing: +/// it neither downloads nor deletes the first one's temporary files. +#[test] +fn a_second_check_at_the_same_time_does_nothing() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let (pinned, published) = pin(Program::Wtsexporter, "r1", b"wtsexporter", false); + let server = MockServer::start(); + let path = format!( + "/{}/releases/download/{}/{}", + pinned.repo, pinned.release, pinned.asset + ); + let asked = server.mock(|when, then| { + when.method(GET).path(path); + then.status(200) + .delay(Duration::from_millis(500)) + .body(published); + }); + let first = ToolDownloads::default(); + let second = ToolDownloads::default(); + let base = server.base_url(); + let dir = tools.path().to_path_buf(); + + std::thread::scope(|scope| { + let check = + scope.spawn(|| download_missing(&dir, &base, std::slice::from_ref(&pinned), &first)); + // The first check holds the lock once it shows a download. + while first.get(Program::Wtsexporter).is_none() { + assert!(!check.is_finished(), "the first check ended at once"); + std::thread::sleep(Duration::from_millis(1)); + } + let other = + scope.spawn(|| download_missing(&dir, &base, std::slice::from_ref(&pinned), &second)); + other.join().unwrap(); + assert!( + !check.is_finished(), + "the first check ended before the second" + ); + }); + + asked.assert_calls(1); + assert_eq!(second.get(Program::Wtsexporter), None); + assert_eq!(first.get(Program::Wtsexporter), None); + assert_eq!( + std::fs::read(tools.path().join(Program::Wtsexporter.file_name())).unwrap(), + b"wtsexporter" + ); +} + +/// With no network the check fails quietly, saying so, and leaves nothing. +#[test] +fn no_network_is_a_failed_download_that_says_so() { + let tools = tempfile::tempdir().unwrap(); + let _tools = no_ffmpeg_on_path(tools.path()); + let (pinned, _) = pin(Program::Wtsexporter, "r1", b"pinned", false); + let downloads = ToolDownloads::default(); + + // Port 9 on this computer, where nothing listens. + download_missing(tools.path(), "http://127.0.0.1:9", &[pinned], &downloads); + + let Some(DownloadState::Failed { reason }) = downloads.get(Program::Wtsexporter) else { + panic!("the download did not fail"); + }; + assert!(reason.starts_with("No connection"), "{reason}"); + assert_eq!(other_files(tools.path(), &[]), Vec::::new()); +} + +/// Each supported platform pins ffmpeg, ffprobe and, except Linux on ARM, +/// wtsexporter, each with a whole SHA-256 and a GitHub release address. +#[test] +fn every_platform_is_pinned() { + for (os, arch) in [ + ("linux", "x86_64"), + ("linux", "aarch64"), + ("macos", "x86_64"), + ("macos", "aarch64"), + ("windows", "x86_64"), + ("windows", "aarch64"), + ] { + let pinned = pinned_for(os, arch); + let programs: Vec = pinned.iter().map(|pin| pin.program).collect(); + let expected = if (os, arch) == ("linux", "aarch64") { + vec![Program::Ffmpeg, Program::Ffprobe] + } else { + vec![Program::Ffmpeg, Program::Ffprobe, Program::Wtsexporter] + }; + assert_eq!(programs, expected, "{os} {arch}"); + for pin in &pinned { + assert!( + pin.sha256.len() == 64 && pin.sha256.chars().all(|c| c.is_ascii_hexdigit()), + "{}", + pin.asset + ); + assert_eq!( + pin.url(GITHUB), + format!( + "https://github.com/{}/releases/download/{}/{}", + pin.repo, pin.release, pin.asset + ) + ); + } + } +} diff --git a/web/src/lib/queryKeys.ts b/web/src/lib/queryKeys.ts index 4d4a77755..e12d4abbe 100644 --- a/web/src/lib/queryKeys.ts +++ b/web/src/lib/queryKeys.ts @@ -168,4 +168,10 @@ export const keys = { */ noSelection: ["trash", "no-selection"] as const, }, + /** + * Where ffmpeg, ffprobe and wtsexporter are and how their downloads stand, + * from the desktop process. It belongs to this computer, not to an account, + * so it is used as it is, without `routeQueryKey` in front (ADR 0002). + */ + desktopToolsStatus: { all: ["desktop", "tools-status"] as const }, }; diff --git a/web/src/lib/tauri.ts b/web/src/lib/tauri.ts index 54f9ec3c0..bea9e02e4 100644 --- a/web/src/lib/tauri.ts +++ b/web/src/lib/tauri.ts @@ -492,7 +492,11 @@ export async function invokeFormat(config: { export type ToolStatus = | { state: "found"; path: string } | { state: "missing" } - | { state: "unusable"; reason: string }; + | { state: "unusable"; reason: string } + /** Downloading into the Tools Directory: bytes so far, and the total when the server said. */ + | { state: "downloading"; received: number; total: number | null } + /** The download failed and the program is not found. Tried again at the next start. */ + | { state: "downloadFailed"; reason: string }; /** * Where ffmpeg, ffprobe and wtsexporter are. ffmpeg and ffprobe are looked @@ -511,6 +515,13 @@ export async function invokeToolsStatus(): Promise { return invoke("tools_status"); } +/** Whether any of ffmpeg, ffprobe and wtsexporter is downloading now. */ +export function toolsDownloading(status: ToolsStatus): boolean { + return [status.ffmpeg, status.ffprobe, status.wtsexporter].some( + (tool) => tool.state === "downloading", + ); +} + /** The programs Convert and Compress run. */ export type MediaToolName = "ffmpeg" | "ffprobe"; diff --git a/web/src/screens/settings/SystemSection.test.tsx b/web/src/screens/settings/SystemSection.test.tsx index 30eeb7236..01b141987 100644 --- a/web/src/screens/settings/SystemSection.test.tsx +++ b/web/src/screens/settings/SystemSection.test.tsx @@ -1,11 +1,12 @@ /** @vitest-environment jsdom */ -import { cleanup, render, screen, waitFor } from "@testing-library/react"; +import { cleanup, screen, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { setBaseUrl } from "../../lib/api"; import { APP_BUILD } from "../../lib/build"; import { getOpenToNetwork } from "../../lib/localServer"; import { readerLicenseUrl, readerSourceUrl } from "../../lib/thirdPartySoftware"; +import { renderWithProviders as render } from "../../test/providers"; import { fill, setupUser } from "../../test/user"; import { SystemSection } from "./SystemSection"; @@ -30,7 +31,8 @@ vi.mock("../../lib/tauri-check", () => ({ isTauri: () => tauriState.isTauri, })); -vi.mock("../../lib/tauri", () => ({ +vi.mock("../../lib/tauri", async (importOriginal) => ({ + toolsDownloading: (await importOriginal()).toolsDownloading, invokeToolsStatus: () => toolsStatus(), invokeStagingRoot: async () => ({ root: desktopStaging.root || desktopStaging.defaultRoot, @@ -362,6 +364,37 @@ describe("SystemSection", () => { expect(media?.querySelector("input")).toBeNull(); }); + it("shows a download's progress, and asks again until it ends", async () => { + toolsStatus.mockResolvedValueOnce({ + toolsDir: "/home/demo/message-crate/tools", + ffmpeg: { state: "downloading", received: 12 * 1024 * 1024, total: 29 * 1024 * 1024 }, + ffprobe: { state: "downloading", received: 0, total: null }, + wtsexporter: { state: "found", path: "/home/demo/message-crate/tools/wtsexporter" }, + }); + render(); + expect(await screen.findByLabelText("Downloading ffmpeg - 12 MB of 29 MB (41%)")).toBeTruthy(); + expect(screen.getByLabelText("Downloading ffprobe - 0 B so far")).toBeTruthy(); + // The next answer, a second later, has both in place. + expect(await screen.findByLabelText(/Found ffmpeg/i, {}, { timeout: 3000 })).toBeTruthy(); + expect(screen.getByLabelText(/Found ffprobe/i)).toBeTruthy(); + }); + + it("says why a download failed", async () => { + const reason = "No connection to the download's server: error sending request."; + toolsStatus.mockResolvedValue({ + toolsDir: "/home/demo/message-crate/tools", + ffmpeg: { state: "found", path: "/usr/bin/ffmpeg" }, + ffprobe: { state: "found", path: "/usr/bin/ffprobe" }, + wtsexporter: { state: "downloadFailed", reason }, + }); + render(); + expect( + await screen.findByLabelText( + `wtsexporter download failed. ${reason} It is tried again the next time the app starts.`, + ), + ).toBeTruthy(); + }); + it("says so when the desktop process cannot be asked", async () => { toolsStatus.mockRejectedValue(new Error("no desktop process")); render(); diff --git a/web/src/screens/settings/SystemSection.tsx b/web/src/screens/settings/SystemSection.tsx index bab9b6b43..1510aedda 100644 --- a/web/src/screens/settings/SystemSection.tsx +++ b/web/src/screens/settings/SystemSection.tsx @@ -1,10 +1,12 @@ +import { useQuery } from "@tanstack/react-query"; import { useEffect, useId, useRef, useState } from "react"; import Checkbox from "../../components/Checkbox"; -import { CheckIcon, XIcon } from "../../components/icons"; +import { CheckIcon, DownloadIcon, XIcon } from "../../components/icons"; import OpenPathButton from "../../components/OpenPathButton"; import PathPicker from "../../components/PathPicker"; import PlainButton from "../../components/PlainButton"; import { getBaseUrl } from "../../lib/api"; +import { formatBytes } from "../../lib/attachmentProgressCopy"; import { APP_BUILD } from "../../lib/build"; import { getOpenToNetwork, @@ -14,6 +16,7 @@ import { setLocalServerOpenToNetwork, setOpenToNetwork, } from "../../lib/localServer"; +import { keys } from "../../lib/queryKeys"; import { getRememberImporterPaths, setRememberImporterPaths } from "../../lib/system-settings"; import { invokeExportDirectory, @@ -22,6 +25,7 @@ import { invokeToolsStatus, type ToolStatus, type ToolsStatus, + toolsDownloading, } from "../../lib/tauri"; import { isTauri } from "../../lib/tauri-check"; import { readerLicenseUrl, readerSourceUrl } from "../../lib/thirdPartySoftware"; @@ -54,6 +58,19 @@ function stagingHelpExample(stagingDir: string, defaultDir: string): string { type ToolName = "ffmpeg" | "ffprobe" | "wtsexporter"; +/** A download's progress: "12 MB of 29 MB (41%)", or "12 MB so far" with no total. */ +function downloadProgress(received: number, total: number | null): string { + if (total == null || total <= 0) return `${formatBytes(received)} so far`; + const percent = Math.min(100, Math.floor((received / total) * 100)); + return `${formatBytes(received)} of ${formatBytes(total)} (${percent}%)`; +} + +/** Said once on every failed download's line, whatever the reason: each is retried. */ +const DOWNLOAD_RETRIED = "It is tried again the next time the app starts."; + +/** How often Settings asks the desktop process again while a download runs. */ +const DOWNLOAD_POLL_MS = 1000; + /** * One program's status line: the path it was found at, that it is missing, * or why it is not used. One case per `state` the desktop process sends. @@ -147,6 +164,32 @@ function ToolStatusRow({ ); } + case "downloading": { + const progress = downloadProgress(status.received, status.total); + const label = `Downloading ${name} - ${progress}`; + return ( +
  • + + + Downloading {name} + {" - "} + {progress} + +
    • + ); + } + case "downloadFailed": { + const label = `${name} download failed. ${status.reason} ${DOWNLOAD_RETRIED}`; + return ( +
  • + + + {name} download failed. {status.reason}{" "} + {DOWNLOAD_RETRIED} + +
    • + ); + } } } @@ -378,28 +421,26 @@ export function SystemSection() { /** The Staging Directory the desktop process holds now. */ const [stagingError, setStagingError] = useState(null); const [rememberPaths, setRememberPaths] = useState(false); - const [tools, setTools] = useState(null); - const [toolsError, setToolsError] = useState(null); + // Where the programs are belongs to this computer, not to an account, so + // this is a plain `useQuery`, asked again each second while a download runs. + const toolsQuery = useQuery({ + queryKey: keys.desktopToolsStatus.all, + queryFn: invokeToolsStatus, + enabled: isTauri(), + retry: false, + refetchInterval: (query) => + query.state.data && toolsDownloading(query.state.data) ? DOWNLOAD_POLL_MS : false, + }); + const tools: ToolsStatus | null = toolsQuery.data ?? null; + const toolsError = toolsQuery.error + ? `Could not look for the media tools. ${toolsQuery.error instanceof Error ? toolsQuery.error.message : String(toolsQuery.error)}` + : null; useEffect(() => { if (!isTauri()) return; setRememberPaths(getRememberImporterPaths()); - let live = true; - invokeToolsStatus().then( - (status) => { - if (live) setTools(status); - }, - (caught: unknown) => { - if (live) { - setToolsError( - `Could not look for the media tools. ${caught instanceof Error ? caught.message : String(caught)}`, - ); - } - }, - ); - void (async () => { try { const staging = await invokeStagingRoot(); @@ -409,10 +450,6 @@ export function SystemSection() { setStagingError(caught instanceof Error ? caught.message : String(caught)); } })(); - - return () => { - live = false; - }; }, []); // The desktop process keeps the setting and decides what it takes: every