From 67086e10f24c26924ee678e0f949288900cfff11 Mon Sep 17 00:00:00 2001 From: wgqqqqq Date: Tue, 21 Jul 2026 20:50:02 +0800 Subject: [PATCH 1/3] Add voice input ASR model configuration --- .gitignore | 1 + Cargo.toml | 3 + scripts/core-boundaries/rules/crate-rules.mjs | 12 + .../core-boundaries/rules/feature-rules.mjs | 21 +- .../rules/source/forbidden-rules.mjs | 17 + scripts/core-boundaries/self-test.mjs | 48 + src/apps/desktop/Cargo.toml | 3 +- src/apps/desktop/src/api/app_state.rs | 8 + src/apps/desktop/src/api/mod.rs | 1 + src/apps/desktop/src/api/speech_api.rs | 142 +++ src/apps/desktop/src/lib.rs | 10 + .../infrastructure/app_paths/path_manager.rs | 29 + .../assembly/core/src/service/config/types.rs | 28 + .../assembly/product-capabilities/src/lib.rs | 92 +- .../tests/product_capabilities.rs | 43 +- src/crates/contracts/core-types/src/lib.rs | 2 + src/crates/contracts/core-types/src/speech.rs | 136 +++ src/crates/contracts/events/src/lib.rs | 2 + src/crates/contracts/events/src/speech.rs | 2 + .../services/services-integrations/Cargo.toml | 20 + .../services/services-integrations/src/lib.rs | 3 + .../services-integrations/src/speech/audio.rs | 23 + .../src/speech/downloader.rs | 382 ++++++++ .../services-integrations/src/speech/error.rs | 31 + .../services-integrations/src/speech/mod.rs | 419 ++++++++ .../src/speech/model_catalog.rs | 173 ++++ .../src/speech/model_store.rs | 411 ++++++++ .../src/speech/qwen3_asr_int8.rs | 185 ++++ .../src/speech/recognizer.rs | 23 + .../src/speech/recognizer_router.rs | 47 + .../src/speech/sensevoice_int8.rs | 169 ++++ .../services-integrations/src/speech/types.rs | 110 +++ .../src/app/scenes/settings/SettingsScene.tsx | 2 + .../src/app/scenes/settings/settingsConfig.ts | 7 + .../settings/settingsTabSearchContent.ts | 10 + .../src/flow_chat/components/ChatInput.scss | 263 ++++- .../src/flow_chat/components/ChatInput.tsx | 61 +- .../voice/ComposerVoiceInputButton.tsx | 211 ++++ .../voice/useComposerVoiceInput.test.tsx | 224 +++++ .../components/voice/useComposerVoiceInput.ts | 657 +++++++++++++ .../components/voice/voiceInputAudio.ts | 1 + .../api/adapters/peer-device-adapter.test.ts | 10 + .../api/adapters/peer-device-adapter.ts | 9 + src/web-ui/src/infrastructure/api/index.ts | 5 +- .../api/service-api/SpeechAPI.ts | 121 +++ .../config/components/AIModelConfig.tsx | 4 +- .../config/components/DefaultModelConfig.tsx | 31 +- .../config/components/VoiceInputConfig.scss | 321 ++++++ .../config/components/VoiceInputConfig.tsx | 917 ++++++++++++++++++ .../components/VoiceInputDiagnostics.tsx | 345 +++++++ .../config/components/common/index.ts | 4 +- .../src/infrastructure/config/hooks/index.ts | 1 + .../config/hooks/useAIExperienceSettings.ts | 33 + src/web-ui/src/infrastructure/config/index.ts | 1 + .../services/AIExperienceConfigService.ts | 20 +- .../config/services/modelCategory.ts | 21 + .../config/services/modelConfigs.ts | 2 +- .../src/infrastructure/config/types/index.ts | 25 +- .../i18n/presets/namespaceRegistry.ts | 1 + .../infrastructure/speech/voiceInputAudio.ts | 204 ++++ src/web-ui/src/locales/en-US/flow-chat.json | 3 +- src/web-ui/src/locales/en-US/settings.json | 2 + .../locales/en-US/settings/voice-input.json | 198 ++++ src/web-ui/src/locales/zh-CN/flow-chat.json | 3 +- src/web-ui/src/locales/zh-CN/settings.json | 2 + .../locales/zh-CN/settings/voice-input.json | 198 ++++ src/web-ui/src/locales/zh-TW/flow-chat.json | 3 +- src/web-ui/src/locales/zh-TW/settings.json | 6 +- .../locales/zh-TW/settings/voice-input.json | 47 + src/web-ui/src/shared/types/chat.ts | 4 +- 70 files changed, 6482 insertions(+), 91 deletions(-) create mode 100644 src/apps/desktop/src/api/speech_api.rs create mode 100644 src/crates/contracts/core-types/src/speech.rs create mode 100644 src/crates/contracts/events/src/speech.rs create mode 100644 src/crates/services/services-integrations/src/speech/audio.rs create mode 100644 src/crates/services/services-integrations/src/speech/downloader.rs create mode 100644 src/crates/services/services-integrations/src/speech/error.rs create mode 100644 src/crates/services/services-integrations/src/speech/mod.rs create mode 100644 src/crates/services/services-integrations/src/speech/model_catalog.rs create mode 100644 src/crates/services/services-integrations/src/speech/model_store.rs create mode 100644 src/crates/services/services-integrations/src/speech/qwen3_asr_int8.rs create mode 100644 src/crates/services/services-integrations/src/speech/recognizer.rs create mode 100644 src/crates/services/services-integrations/src/speech/recognizer_router.rs create mode 100644 src/crates/services/services-integrations/src/speech/sensevoice_int8.rs create mode 100644 src/crates/services/services-integrations/src/speech/types.rs create mode 100644 src/web-ui/src/flow_chat/components/voice/ComposerVoiceInputButton.tsx create mode 100644 src/web-ui/src/flow_chat/components/voice/useComposerVoiceInput.test.tsx create mode 100644 src/web-ui/src/flow_chat/components/voice/useComposerVoiceInput.ts create mode 100644 src/web-ui/src/flow_chat/components/voice/voiceInputAudio.ts create mode 100644 src/web-ui/src/infrastructure/api/service-api/SpeechAPI.ts create mode 100644 src/web-ui/src/infrastructure/config/components/VoiceInputConfig.scss create mode 100644 src/web-ui/src/infrastructure/config/components/VoiceInputConfig.tsx create mode 100644 src/web-ui/src/infrastructure/config/components/VoiceInputDiagnostics.tsx create mode 100644 src/web-ui/src/infrastructure/config/hooks/index.ts create mode 100644 src/web-ui/src/infrastructure/config/hooks/useAIExperienceSettings.ts create mode 100644 src/web-ui/src/infrastructure/speech/voiceInputAudio.ts create mode 100644 src/web-ui/src/locales/en-US/settings/voice-input.json create mode 100644 src/web-ui/src/locales/zh-CN/settings/voice-input.json create mode 100644 src/web-ui/src/locales/zh-TW/settings/voice-input.json diff --git a/.gitignore b/.gitignore index ca5075bb42..5bc2199530 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,4 @@ ASSETS_LICENSES.md external/ /.bitfun/search/flashgrep-index/ .agents/ +/.flashgrep-index-engine/ diff --git a/Cargo.toml b/Cargo.toml index 1deab41539..a566a862d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -226,6 +226,9 @@ qrcode = "0.14" # WebSocket client tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] } +# Local speech recognition +sherpa-onnx = "1.13.4" + # MCP and remote runtimes rmcp = { version = "1.7", default-features = false, features = [ "auth", diff --git a/scripts/core-boundaries/rules/crate-rules.mjs b/scripts/core-boundaries/rules/crate-rules.mjs index 94e8a8dadd..4803b3a5a5 100644 --- a/scripts/core-boundaries/rules/crate-rules.mjs +++ b/scripts/core-boundaries/rules/crate-rules.mjs @@ -25,6 +25,18 @@ export const noCoreDependencyCrates = [ ]; export const forbiddenManifestDependencyRules = [ + { + dependencyNames: ['sherpa-onnx'], + scanRoots: ['src/apps', 'src/crates', 'BitFun-Installer/src-tauri'], + forbidWorkspaceAliases: false, + allowManifestPaths: [ + 'src/crates/services/services-integrations/Cargo.toml', + ], + reason: + 'speech recognition engines are concrete integration service dependencies', + message: + 'sherpa-onnx must stay in services-integrations and be injected by a product composition root', + }, { dependencyNames: ['bitfun-opencode-adapter'], scanRoots: ['src/apps', 'src/crates', 'BitFun-Installer/src-tauri'], diff --git a/scripts/core-boundaries/rules/feature-rules.mjs b/scripts/core-boundaries/rules/feature-rules.mjs index 2938d6f605..101cbbd973 100644 --- a/scripts/core-boundaries/rules/feature-rules.mjs +++ b/scripts/core-boundaries/rules/feature-rules.mjs @@ -48,24 +48,27 @@ export const optionalDependencyFeatureOwnerRules = [ { depName: 'anyhow', ownerFeatures: ['browser-control', 'debug-log', 'mcp', 'remote-connect', 'remote-ssh', 'remote-ssh-concrete'] }, { depName: 'async-trait', - ownerFeatures: ['mcp', 'remote-connect', 'remote-ssh', 'remote-ssh-concrete', 'review-platform', 'workspace-search'], + ownerFeatures: ['mcp', 'remote-connect', 'remote-ssh', 'remote-ssh-concrete', 'review-platform', 'speech', 'workspace-search'], }, { depName: 'base64', - ownerFeatures: ['mcp', 'miniapp-runtime', 'remote-connect', 'remote-ssh-concrete'], + ownerFeatures: ['mcp', 'miniapp-runtime', 'remote-connect', 'remote-ssh-concrete', 'speech'], }, { depName: 'bitfun-agent-runtime', ownerFeatures: ['deep-research'] }, + { depName: 'bitfun-core-types', ownerFeatures: ['speech'] }, { depName: 'bitfun-product-domains', ownerFeatures: ['canvas-runtime', 'function-agents', 'miniapp-runtime', 'plugin-source'] }, { depName: 'bitfun-runtime-ports', ownerFeatures: ['remote-connect', 'remote-ssh', 'remote-ssh-concrete'] }, { depName: 'bitfun-services-core', ownerFeatures: ['browser-control', 'git', 'mcp', 'miniapp-runtime', 'remote-connect', 'review-platform', 'workspace-search'], }, - { depName: 'chrono', ownerFeatures: ['debug-log', 'git', 'remote-connect', 'remote-ssh-concrete', 'review-platform'] }, + { depName: 'bzip2', ownerFeatures: ['speech'] }, + { depName: 'chrono', ownerFeatures: ['debug-log', 'git', 'remote-connect', 'remote-ssh-concrete', 'review-platform', 'speech'] }, { depName: 'dirs', ownerFeatures: ['browser-control', 'miniapp-runtime', 'remote-connect', 'remote-ssh-concrete'] }, { depName: 'dunce', ownerFeatures: ['plugin-source', 'remote-ssh', 'workspace-search'] }, { depName: 'fs2', ownerFeatures: ['plugin-source'] }, { depName: 'futures', ownerFeatures: ['mcp', 'remote-connect', 'review-platform'] }, + { depName: 'futures-util', ownerFeatures: ['speech'] }, { depName: 'git2', ownerFeatures: ['git'] }, { depName: 'hex', ownerFeatures: ['plugin-source', 'remote-connect'] }, { depName: 'hostname', ownerFeatures: ['remote-connect'] }, @@ -78,7 +81,7 @@ export const optionalDependencyFeatureOwnerRules = [ { depName: 'oxc', ownerFeatures: ['canvas-runtime'] }, { depName: 'qrcode', ownerFeatures: ['remote-connect'] }, { depName: 'rand', ownerFeatures: ['mcp', 'remote-connect', 'remote-ssh-concrete'] }, - { depName: 'reqwest', ownerFeatures: ['announcement', 'browser-control', 'debug-log', 'mcp', 'miniapp-runtime', 'remote-connect', 'review-platform', 'web-tools'] }, + { depName: 'reqwest', ownerFeatures: ['announcement', 'browser-control', 'debug-log', 'mcp', 'miniapp-runtime', 'remote-connect', 'review-platform', 'speech', 'web-tools'] }, { depName: 'rmcp', ownerFeatures: ['mcp'] }, { depName: 'russh', ownerFeatures: ['remote-ssh-concrete'] }, { depName: 'russh-keys', ownerFeatures: ['remote-ssh-concrete'] }, @@ -86,16 +89,18 @@ export const optionalDependencyFeatureOwnerRules = [ { depName: 'rustls', ownerFeatures: ['remote-connect'] }, { depName: 'rustls-native-certs', ownerFeatures: ['remote-connect'] }, { depName: 'schannel', ownerFeatures: ['remote-connect'] }, - { depName: 'sha2', ownerFeatures: ['canvas-runtime', 'plugin-source', 'remote-connect', 'remote-ssh', 'review-platform'] }, + { depName: 'sha2', ownerFeatures: ['canvas-runtime', 'plugin-source', 'remote-connect', 'remote-ssh', 'review-platform', 'speech'] }, + { depName: 'sherpa-onnx', ownerFeatures: ['speech'] }, { depName: 'shellexpand', ownerFeatures: ['remote-ssh-concrete'] }, { depName: 'sse-stream', ownerFeatures: ['mcp'] }, { depName: 'ssh_config', ownerFeatures: ['remote-ssh-concrete', 'ssh_config'] }, { depName: 'terminal-core', ownerFeatures: ['remote-ssh', 'remote-ssh-concrete'] }, - { depName: 'thiserror', ownerFeatures: ['browser-control', 'git', 'plugin-source', 'remote-ssh', 'remote-ssh-concrete', 'review-platform', 'web-tools', 'workspace-search'] }, + { depName: 'tar', ownerFeatures: ['speech'] }, + { depName: 'thiserror', ownerFeatures: ['browser-control', 'git', 'plugin-source', 'remote-ssh', 'remote-ssh-concrete', 'review-platform', 'speech', 'web-tools', 'workspace-search'] }, { depName: 'tokio-tungstenite', ownerFeatures: ['remote-connect'] }, - { depName: 'tokio-util', ownerFeatures: ['remote-ssh'] }, + { depName: 'tokio-util', ownerFeatures: ['remote-ssh', 'speech'] }, { depName: 'urlencoding', ownerFeatures: ['canvas-runtime', 'remote-connect', 'review-platform'] }, - { depName: 'uuid', ownerFeatures: ['canvas-runtime', 'debug-log', 'miniapp-runtime', 'plugin-source', 'remote-connect', 'remote-ssh-concrete'] }, + { depName: 'uuid', ownerFeatures: ['canvas-runtime', 'debug-log', 'miniapp-runtime', 'plugin-source', 'remote-connect', 'remote-ssh-concrete', 'speech'] }, { depName: 'which', ownerFeatures: ['miniapp-runtime', 'remote-connect', 'workspace-search'] }, { depName: 'windows', ownerFeatures: ['plugin-source', 'review-platform'] }, { depName: 'x25519-dalek', ownerFeatures: ['remote-connect'] }, diff --git a/scripts/core-boundaries/rules/source/forbidden-rules.mjs b/scripts/core-boundaries/rules/source/forbidden-rules.mjs index b58626d113..6b7cd024e9 100644 --- a/scripts/core-boundaries/rules/source/forbidden-rules.mjs +++ b/scripts/core-boundaries/rules/source/forbidden-rules.mjs @@ -4150,6 +4150,23 @@ export const forbiddenContentUnderRules = [ }, ], }, + { + path: 'src/crates/assembly/core/src/service', + reason: + 'concrete speech runtime ownership belongs in services-integrations', + patterns: [ + { + regex: /\bpub\s+mod\s+speech\s*;/, + message: + 'core must not declare a speech service owner; use stable core-types contracts and compose the services-integrations provider at the app boundary', + }, + { + regex: /\bpub\s+use\s+(?:self::)?speech(?:::|\s*::)/, + message: + 'core must not re-export the concrete speech service provider', + }, + ], + }, { path: 'src/crates/assembly/core/src', reason: diff --git a/scripts/core-boundaries/self-test.mjs b/scripts/core-boundaries/self-test.mjs index f2214f4efb..7447d440ba 100644 --- a/scripts/core-boundaries/self-test.mjs +++ b/scripts/core-boundaries/self-test.mjs @@ -706,6 +706,28 @@ export function runManifestParserSelfTest({ throw new Error(`services-integrations review-platform must own optional dependency ${dep}`); } } + for (const dep of [ + 'async-trait', + 'base64', + 'bitfun-core-types', + 'bzip2', + 'chrono', + 'futures-util', + 'reqwest', + 'sha2', + 'sherpa-onnx', + 'tar', + 'thiserror', + 'tokio-util', + 'uuid', + ]) { + const owner = servicesOptionalOwnerRule?.dependencies.find( + (dependency) => dependency.depName === dep, + ); + if (!owner?.ownerFeatures.includes('speech')) { + throw new Error(`services-integrations speech must own optional dependency ${dep}`); + } + } const productDomainsOptionalOwnerRule = optionalDependencyFeatureOwnerRules.find( (rule) => rule.crateName === 'product-domains', ); @@ -1007,6 +1029,32 @@ export function runManifestParserSelfTest({ if (!opencodeManifestRule) { throw new Error('OpenCode adapter must have a forbidden manifest dependency rule'); } + const speechEngineManifestRule = forbiddenManifestDependencyRules.find((rule) => + rule.dependencyNames?.includes('sherpa-onnx'), + ); + if (!speechEngineManifestRule) { + throw new Error('speech engine must have a forbidden manifest dependency rule'); + } + if ( + !speechEngineManifestRule.allowManifestPaths?.includes( + 'src/crates/services/services-integrations/Cargo.toml', + ) + ) { + throw new Error('speech engine manifest guard must allow only its integration service owner'); + } + const coreSpeechOwnerRule = forbiddenContentUnderRules.find( + (rule) => rule.path === 'src/crates/assembly/core/src/service', + ); + if (!coreSpeechOwnerRule) { + throw new Error('core service tree must have a speech ownership guard'); + } + if ( + !coreSpeechOwnerRule.patterns.some((pattern) => + pattern.regex.source.includes('mod\\s+speech'), + ) + ) { + throw new Error('core speech ownership guard must forbid a speech service module'); + } for (const scanRoot of ['src/apps', 'src/crates', 'BitFun-Installer/src-tauri']) { if (!opencodeManifestRule.scanRoots?.includes(scanRoot)) { throw new Error(`OpenCode adapter manifest guard must scan ${scanRoot}`); diff --git a/src/apps/desktop/Cargo.toml b/src/apps/desktop/Cargo.toml index 1d83639afc..9f82876f3c 100644 --- a/src/apps/desktop/Cargo.toml +++ b/src/apps/desktop/Cargo.toml @@ -21,7 +21,8 @@ serde_json = { workspace = true } # Internal crates bitfun-core = { path = "../../crates/assembly/core", default-features = false, features = ["product-full"] } bitfun-product-domains = { path = "../../crates/contracts/product-domains", default-features = false } -bitfun-services-integrations = { path = "../../crates/services/services-integrations", default-features = false, features = ["canvas-runtime"] } +bitfun-services-integrations = { path = "../../crates/services/services-integrations", default-features = false, features = ["canvas-runtime", "speech"] } +bitfun-core-types = { path = "../../crates/contracts/core-types" } bitfun-agent-tools = { path = "../../crates/execution/tool-contracts" } bitfun-transport = { path = "../../crates/adapters/transport", features = ["tauri-adapter"] } bitfun-events = { path = "../../crates/contracts/events" } diff --git a/src/apps/desktop/src/api/app_state.rs b/src/apps/desktop/src/api/app_state.rs index 55aac1fe20..16b3a18baa 100644 --- a/src/apps/desktop/src/api/app_state.rs +++ b/src/apps/desktop/src/api/app_state.rs @@ -12,6 +12,7 @@ use bitfun_core::service::remote_ssh::{ }; use bitfun_core::service::{announcement, config, filesystem, mcp, search, token_usage, workspace}; use bitfun_core::util::errors::*; +use bitfun_services_integrations::speech::{SpeechService, SpeechStoragePaths}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -69,6 +70,7 @@ pub struct AppState { pub config_service: Arc, pub filesystem_service: Arc, pub workspace_search_service: Arc, + pub speech_service: Arc, pub agent_registry: Arc, pub mcp_service: Option>, pub acp_client_service: Option>, @@ -186,6 +188,11 @@ impl AppState { std::path::PathBuf::from("worker_host.js") } }; + let speech_service = Arc::new(SpeechService::new(SpeechStoragePaths::new( + path_manager.speech_models_dir(), + path_manager.speech_model_downloads_dir(), + path_manager.speech_input_temp_dir(), + ))); let js_worker_pool = JsWorkerPool::new(path_manager, worker_host_path) .ok() .map(Arc::new); @@ -294,6 +301,7 @@ impl AppState { config_service, filesystem_service, workspace_search_service, + speech_service, agent_registry, mcp_service, acp_client_service, diff --git a/src/apps/desktop/src/api/mod.rs b/src/apps/desktop/src/api/mod.rs index 3625688372..26b0bf55a4 100644 --- a/src/apps/desktop/src/api/mod.rs +++ b/src/apps/desktop/src/api/mod.rs @@ -39,6 +39,7 @@ pub mod session_api; pub mod session_storage_path; pub mod skill_api; pub mod snapshot_service; +pub mod speech_api; pub mod ssh_api; pub mod startchat_agent_api; pub mod storage_commands; diff --git a/src/apps/desktop/src/api/speech_api.rs b/src/apps/desktop/src/api/speech_api.rs new file mode 100644 index 0000000000..9edc88ff9c --- /dev/null +++ b/src/apps/desktop/src/api/speech_api.rs @@ -0,0 +1,142 @@ +//! Desktop adapter for local speech input. + +use crate::api::AppState; +use bitfun_core_types::speech::{ + SpeechAppendAudioChunkRequest, SpeechAppendAudioChunkResponse, SpeechCancelInputSessionRequest, + SpeechCancelModelDownloadRequest, SpeechDeleteModelRequest, SpeechDownloadModelRequest, + SpeechFinishInputSessionRequest, SpeechInputSession, SpeechListModelsResponse, + SpeechModelProgressEvent, SpeechModelStatus, SpeechStartInputSessionRequest, + SpeechTranscriptionResult, SpeechVerifyModelRequest, +}; +use bitfun_events::{SPEECH_MODEL_PROGRESS_EVENT, SPEECH_MODEL_STATUS_CHANGED_EVENT}; +use tauri::{AppHandle, Emitter, State}; + +#[tauri::command] +pub async fn speech_list_models( + state: State<'_, AppState>, +) -> Result { + state + .speech_service + .list_models() + .await + .map_err(|error| format!("Failed to list speech models: {error}")) +} + +#[tauri::command] +pub async fn speech_download_model( + state: State<'_, AppState>, + app: AppHandle, + request: SpeechDownloadModelRequest, +) -> Result { + let progress_app = app.clone(); + let status = state + .speech_service + .download_model(request, move |event: SpeechModelProgressEvent| { + if let Err(error) = progress_app.emit(SPEECH_MODEL_PROGRESS_EVENT, &event) { + log::warn!("Failed to emit speech model progress event: {error}"); + } + }) + .await + .map_err(|error| format!("Failed to download speech model: {error}"))?; + emit_status(&app, &status); + Ok(status) +} + +#[tauri::command] +pub async fn speech_cancel_model_download( + state: State<'_, AppState>, + app: AppHandle, + request: SpeechCancelModelDownloadRequest, +) -> Result { + let status = state + .speech_service + .cancel_model_download(request) + .await + .map_err(|error| format!("Failed to cancel speech model download: {error}"))?; + emit_status(&app, &status); + Ok(status) +} + +#[tauri::command] +pub async fn speech_delete_model( + state: State<'_, AppState>, + app: AppHandle, + request: SpeechDeleteModelRequest, +) -> Result { + let status = state + .speech_service + .delete_model(request) + .await + .map_err(|error| format!("Failed to delete speech model: {error}"))?; + emit_status(&app, &status); + Ok(status) +} + +#[tauri::command] +pub async fn speech_verify_model( + state: State<'_, AppState>, + app: AppHandle, + request: SpeechVerifyModelRequest, +) -> Result { + let status = state + .speech_service + .verify_model(request) + .await + .map_err(|error| format!("Failed to verify speech model: {error}"))?; + emit_status(&app, &status); + Ok(status) +} + +#[tauri::command] +pub async fn speech_start_input_session( + state: State<'_, AppState>, + request: SpeechStartInputSessionRequest, +) -> Result { + state + .speech_service + .start_input_session(request) + .await + .map_err(|error| format!("Failed to start speech input session: {error}")) +} + +#[tauri::command] +pub async fn speech_append_audio_chunk( + state: State<'_, AppState>, + request: SpeechAppendAudioChunkRequest, +) -> Result { + state + .speech_service + .append_audio_chunk(request) + .await + .map_err(|error| format!("Failed to append speech audio chunk: {error}")) +} + +#[tauri::command] +pub async fn speech_finish_input_session( + state: State<'_, AppState>, + request: SpeechFinishInputSessionRequest, +) -> Result { + state + .speech_service + .finish_input_session(request) + .await + .map_err(|error| format!("Failed to transcribe speech input: {error}")) +} + +#[tauri::command] +pub async fn speech_cancel_input_session( + state: State<'_, AppState>, + request: SpeechCancelInputSessionRequest, +) -> Result<(), String> { + state + .speech_service + .cancel_input_session(request) + .await + .map_err(|error| format!("Failed to cancel speech input session: {error}")) +} + +fn emit_status(app: &AppHandle, status: &SpeechModelStatus) { + if let Err(error) = app.emit(SPEECH_MODEL_STATUS_CHANGED_EVENT, status) { + log::warn!("Failed to emit speech model status event: {error}"); + } +} diff --git a/src/apps/desktop/src/lib.rs b/src/apps/desktop/src/lib.rs index 3894d36f0c..02e092aa12 100644 --- a/src/apps/desktop/src/lib.rs +++ b/src/apps/desktop/src/lib.rs @@ -53,6 +53,7 @@ use api::search_api::*; use api::session_api::*; use api::skill_api::*; use api::snapshot_service::*; +use api::speech_api::*; use api::startchat_agent_api::*; use api::storage_commands::*; use api::subagent_api::*; @@ -971,6 +972,15 @@ pub async fn run() { get_runtime_logging_info, export_diagnostics_bundle, get_runtime_capabilities, + speech_list_models, + speech_download_model, + speech_cancel_model_download, + speech_delete_model, + speech_verify_model, + speech_start_input_session, + speech_append_audio_chunk, + speech_finish_input_session, + speech_cancel_input_session, get_agent_profile_configs, get_agent_profile_config, set_agent_profile_config, diff --git a/src/crates/assembly/core/src/infrastructure/app_paths/path_manager.rs b/src/crates/assembly/core/src/infrastructure/app_paths/path_manager.rs index 95c9333ff2..3c829ea9cf 100644 --- a/src/crates/assembly/core/src/infrastructure/app_paths/path_manager.rs +++ b/src/crates/assembly/core/src/infrastructure/app_paths/path_manager.rs @@ -272,6 +272,31 @@ impl PathManager { self.user_root.join("data") } + /// User-level managed model resources shared across workspaces. + pub fn user_models_dir(&self) -> PathBuf { + self.user_data_dir().join("models") + } + + /// User-level speech recognition model resources shared across workspaces. + pub fn speech_models_dir(&self) -> PathBuf { + self.user_models_dir().join("speech") + } + + /// Versioned speech model resource directory. + pub fn speech_model_dir(&self, model_id: &str, version: &str) -> PathBuf { + self.speech_models_dir().join(model_id).join(version) + } + + /// Temporary download workspace for managed speech model resources. + pub fn speech_model_downloads_dir(&self) -> PathBuf { + self.cache_root().join("model-downloads").join("speech") + } + + /// Temporary audio chunks for local voice input sessions. + pub fn speech_input_temp_dir(&self) -> PathBuf { + self.temp_dir().join("speech-input") + } + /// Get user memory database file: ~/.config/bitfun/data/memories/memories.sqlite pub fn memories_database_file(&self) -> PathBuf { self.user_data_dir() @@ -534,11 +559,15 @@ impl PathManager { self.user_agents_dir(), self.cache_root(), self.user_data_dir(), + self.user_models_dir(), + self.speech_models_dir(), + self.speech_model_downloads_dir(), self.user_cron_dir(), self.user_rules_dir(), self.miniapps_dir(), self.logs_dir(), self.temp_dir(), + self.speech_input_temp_dir(), ]; for dir in dirs { diff --git a/src/crates/assembly/core/src/service/config/types.rs b/src/crates/assembly/core/src/service/config/types.rs index 2abab38623..463b5f41c8 100644 --- a/src/crates/assembly/core/src/service/config/types.rs +++ b/src/crates/assembly/core/src/service/config/types.rs @@ -173,6 +173,31 @@ pub struct AiExperienceQuickAction { pub enabled: bool, } +/// Local voice input preferences for the chat composer. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct VoiceInputConfig { + pub enabled: bool, + pub provider: String, + pub model_id: String, + pub default_language: String, + pub max_recording_seconds: u32, + pub microphone_device_id: String, +} + +impl Default for VoiceInputConfig { + fn default() -> Self { + Self { + enabled: true, + provider: "local".to_string(), + model_id: "sensevoice-small-int8".to_string(), + default_language: "auto".to_string(), + max_recording_seconds: 60, + microphone_device_id: String::new(), + } + } +} + /// AI experience configuration. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] @@ -195,6 +220,8 @@ pub struct AIExperienceConfig { pub agent_companion_pet: Option, /// Whether to enable flashgrep-backed accelerated workspace search. pub enable_workspace_search: bool, + /// Local speech-to-text settings for the chat composer. + pub voice_input: VoiceInputConfig, /// User-defined quick actions (post-coding menu); persisted for the web UI. #[serde(default)] pub quick_actions: Vec, @@ -1398,6 +1425,7 @@ impl Default for AIExperienceConfig { agent_companion_display_mode: "desktop".to_string(), agent_companion_pet: default_agent_companion_pet(), enable_workspace_search: false, + voice_input: VoiceInputConfig::default(), quick_actions: Vec::new(), } } diff --git a/src/crates/assembly/product-capabilities/src/lib.rs b/src/crates/assembly/product-capabilities/src/lib.rs index 7b577ed2e2..4441e9cf15 100644 --- a/src/crates/assembly/product-capabilities/src/lib.rs +++ b/src/crates/assembly/product-capabilities/src/lib.rs @@ -27,6 +27,7 @@ pub enum ProductCapabilityId { DeepResearch, MiniApp, Canvas, + VoiceInput, } impl ProductCapabilityId { @@ -37,6 +38,7 @@ impl ProductCapabilityId { Self::DeepResearch => "deep-research", Self::MiniApp => "miniapp", Self::Canvas => "canvas", + Self::VoiceInput => "voice-input", } } } @@ -1020,37 +1022,57 @@ const DEEP_RESEARCH_HARNESS_PROVIDERS: &[HarnessProviderDescriptor] = &[DEEP_RESEARCH_HARNESS_PROVIDER]; const MINIAPP_HARNESS_PROVIDERS: &[HarnessProviderDescriptor] = &[MINIAPP_HARNESS_PROVIDER]; +const CODE_AGENT_CAPABILITY_PACK: ProductCapabilityPack = ProductCapabilityPack::new( + ProductCapabilityId::CodeAgent, + CODE_AGENT_SERVICES, + CODE_AGENT_TOOL_GROUPS, + NO_HARNESS_PROVIDERS, +); +const DEEP_REVIEW_CAPABILITY_PACK: ProductCapabilityPack = ProductCapabilityPack::new( + ProductCapabilityId::DeepReview, + DEEP_REVIEW_SERVICES, + INTEGRATION_TOOL_GROUPS, + DEEP_REVIEW_HARNESS_PROVIDERS, +); +const DEEP_RESEARCH_CAPABILITY_PACK: ProductCapabilityPack = ProductCapabilityPack::new( + ProductCapabilityId::DeepResearch, + DEEP_RESEARCH_SERVICES, + INTEGRATION_TOOL_GROUPS, + DEEP_RESEARCH_HARNESS_PROVIDERS, +); +const MINIAPP_CAPABILITY_PACK: ProductCapabilityPack = ProductCapabilityPack::new( + ProductCapabilityId::MiniApp, + MINIAPP_SERVICES, + INTEGRATION_TOOL_GROUPS, + MINIAPP_HARNESS_PROVIDERS, +); +const CANVAS_CAPABILITY_PACK: ProductCapabilityPack = ProductCapabilityPack::new( + ProductCapabilityId::Canvas, + CANVAS_SERVICES, + CANVAS_TOOL_GROUPS, + NO_HARNESS_PROVIDERS, +); +const VOICE_INPUT_CAPABILITY_PACK: ProductCapabilityPack = ProductCapabilityPack::new( + ProductCapabilityId::VoiceInput, + &[], + &[], + NO_HARNESS_PROVIDERS, +); + +const CORE_COMPATIBILITY_CAPABILITY_PACKS: &[ProductCapabilityPack] = &[ + CODE_AGENT_CAPABILITY_PACK, + DEEP_REVIEW_CAPABILITY_PACK, + DEEP_RESEARCH_CAPABILITY_PACK, + MINIAPP_CAPABILITY_PACK, + CANVAS_CAPABILITY_PACK, +]; const DEFAULT_PRODUCT_CAPABILITY_PACKS: &[ProductCapabilityPack] = &[ - ProductCapabilityPack::new( - ProductCapabilityId::CodeAgent, - CODE_AGENT_SERVICES, - CODE_AGENT_TOOL_GROUPS, - NO_HARNESS_PROVIDERS, - ), - ProductCapabilityPack::new( - ProductCapabilityId::DeepReview, - DEEP_REVIEW_SERVICES, - INTEGRATION_TOOL_GROUPS, - DEEP_REVIEW_HARNESS_PROVIDERS, - ), - ProductCapabilityPack::new( - ProductCapabilityId::DeepResearch, - DEEP_RESEARCH_SERVICES, - INTEGRATION_TOOL_GROUPS, - DEEP_RESEARCH_HARNESS_PROVIDERS, - ), - ProductCapabilityPack::new( - ProductCapabilityId::MiniApp, - MINIAPP_SERVICES, - INTEGRATION_TOOL_GROUPS, - MINIAPP_HARNESS_PROVIDERS, - ), - ProductCapabilityPack::new( - ProductCapabilityId::Canvas, - CANVAS_SERVICES, - CANVAS_TOOL_GROUPS, - NO_HARNESS_PROVIDERS, - ), + CODE_AGENT_CAPABILITY_PACK, + DEEP_REVIEW_CAPABILITY_PACK, + DEEP_RESEARCH_CAPABILITY_PACK, + MINIAPP_CAPABILITY_PACK, + CANVAS_CAPABILITY_PACK, + VOICE_INPUT_CAPABILITY_PACK, ]; const EMPTY_PRODUCT_CAPABILITY_PACKS: &[ProductCapabilityPack] = &[]; @@ -1082,10 +1104,12 @@ pub fn default_product_harness_registry() -> Result ProductCapabilityRegistry { match profile { - DeliveryProfile::ProductFull - | DeliveryProfile::Desktop - | DeliveryProfile::Cli - | DeliveryProfile::Acp => default_product_capability_registry(), + DeliveryProfile::ProductFull | DeliveryProfile::Desktop => { + default_product_capability_registry() + } + DeliveryProfile::Cli | DeliveryProfile::Acp => { + ProductCapabilityRegistry::new(CORE_COMPATIBILITY_CAPABILITY_PACKS) + } DeliveryProfile::Server | DeliveryProfile::Remote | DeliveryProfile::Web diff --git a/src/crates/assembly/product-capabilities/tests/product_capabilities.rs b/src/crates/assembly/product-capabilities/tests/product_capabilities.rs index 784d2362c6..abf41e68b8 100644 --- a/src/crates/assembly/product-capabilities/tests/product_capabilities.rs +++ b/src/crates/assembly/product-capabilities/tests/product_capabilities.rs @@ -173,7 +173,8 @@ fn capability_packs_describe_service_tool_and_harness_requirements() { "deep-review", "deep-research", "miniapp", - "canvas" + "canvas", + "voice-input" ] ); @@ -223,7 +224,7 @@ fn capability_packs_describe_service_tool_and_harness_requirements() { #[test] fn product_assembly_plan_keeps_full_capabilities_only_for_core_compatibility_profiles() { - let expected_capabilities = vec![ + let shared_capabilities = vec![ "code-agent", "deep-review", "deep-research", @@ -238,13 +239,10 @@ fn product_assembly_plan_keeps_full_capabilities_only_for_core_compatibility_pro "core.integration", ]; - for profile in [ - DeliveryProfile::ProductFull, - DeliveryProfile::Desktop, - DeliveryProfile::Cli, - DeliveryProfile::Acp, - ] { + for profile in [DeliveryProfile::ProductFull, DeliveryProfile::Desktop] { let plan = product_assembly_plan_for_profile(profile); + let mut expected_capabilities = shared_capabilities.clone(); + expected_capabilities.push("voice-input"); assert_eq!(plan.profile(), profile); assert_eq!( @@ -254,7 +252,31 @@ fn product_assembly_plan_keeps_full_capabilities_only_for_core_compatibility_pro .map(|capability_id| capability_id.id()) .collect::>(), expected_capabilities, - "{profile} must preserve the current product-full capability set until explicit trimming is proven" + "{profile} must include the desktop voice-input capability" + ); + assert_eq!( + plan.capability_assembly() + .tool_provider_group_plan() + .iter() + .map(|group| group.provider_id()) + .collect::>(), + expected_tool_groups, + "{profile} must preserve current tool provider groups" + ); + } + + for profile in [DeliveryProfile::Cli, DeliveryProfile::Acp] { + let plan = product_assembly_plan_for_profile(profile); + + assert_eq!(plan.profile(), profile); + assert_eq!( + plan.capability_set() + .ids() + .iter() + .map(|capability_id| capability_id.id()) + .collect::>(), + shared_capabilities, + "{profile} must not select desktop-only voice input" ); assert_eq!( plan.capability_assembly() @@ -759,7 +781,8 @@ fn default_capability_assembly_keeps_service_tool_and_harness_facts_together() { "deep-review", "deep-research", "miniapp", - "canvas" + "canvas", + "voice-input" ] ); diff --git a/src/crates/contracts/core-types/src/lib.rs b/src/crates/contracts/core-types/src/lib.rs index 1f21a9966d..d3ce98b04d 100644 --- a/src/crates/contracts/core-types/src/lib.rs +++ b/src/crates/contracts/core-types/src/lib.rs @@ -7,6 +7,7 @@ pub mod ai; pub mod errors; pub mod lsp; pub mod session; +pub mod speech; pub mod surface; pub mod tool_image_attachment; @@ -17,6 +18,7 @@ pub use ai::{ }; pub use errors::{AiErrorDetail, ErrorCategory}; pub use session::SessionKind; +pub use speech::*; pub use surface::{ ApprovalSource, CapabilityRequest, CapabilityRequestKind, PermissionDecision, PermissionScope, RuntimeArtifactKind, RuntimeArtifactRef, SurfaceKind, ThreadEnvironment, ThreadEnvironmentKind, diff --git a/src/crates/contracts/core-types/src/speech.rs b/src/crates/contracts/core-types/src/speech.rs new file mode 100644 index 0000000000..1c08c8e6d8 --- /dev/null +++ b/src/crates/contracts/core-types/src/speech.rs @@ -0,0 +1,136 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SpeechModelInstallState { + NotInstalled, + Downloading, + Installed, + Verifying, + Corrupt, + Deleting, + Error, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechModelProgress { + pub model_id: String, + pub downloaded_bytes: u64, + pub total_bytes: u64, + pub percent: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechModelStatus { + pub model_id: String, + pub display_name: String, + pub provider: String, + pub version: String, + pub description: String, + pub languages: Vec, + pub state: SpeechModelInstallState, + pub installed_path: Option, + pub installed_bytes: u64, + pub expected_bytes: u64, + pub progress: Option, + pub error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechListModelsResponse { + pub models: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechDownloadModelRequest { + pub model_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechCancelModelDownloadRequest { + pub model_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechDeleteModelRequest { + pub model_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechVerifyModelRequest { + pub model_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechModelProgressEvent { + pub status: SpeechModelStatus, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechStartInputSessionRequest { + #[serde(default)] + pub model_id: Option, + #[serde(default)] + pub language: Option, + #[serde(default)] + pub sample_rate: Option, + #[serde(default)] + pub max_recording_seconds: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechInputSession { + pub session_id: String, + pub model_id: String, + pub language: String, + pub sample_rate: u32, + pub max_recording_seconds: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechAppendAudioChunkRequest { + pub session_id: String, + /// Base64-encoded PCM16 little-endian mono audio. + pub pcm16_base64: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechAppendAudioChunkResponse { + pub received_bytes: u64, + pub received_seconds: f64, + pub limit_reached: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechFinishInputSessionRequest { + pub session_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechCancelInputSessionRequest { + pub session_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechTranscriptionResult { + pub text: String, + pub language: String, + pub duration_ms: u64, + pub audio_duration_seconds: f64, +} diff --git a/src/crates/contracts/events/src/lib.rs b/src/crates/contracts/events/src/lib.rs index 80efb64911..691de38645 100644 --- a/src/crates/contracts/events/src/lib.rs +++ b/src/crates/contracts/events/src/lib.rs @@ -9,6 +9,7 @@ pub mod agentic_projection_manifest; pub mod backend; pub mod emitter; pub mod frontend_projection; +pub mod speech; pub mod types; pub use agentic::{ @@ -28,4 +29,5 @@ pub use backend::{ }; pub use emitter::EventEmitter; pub use frontend_projection::{project_agentic_frontend_event, AgenticFrontendEvent}; +pub use speech::{SPEECH_MODEL_PROGRESS_EVENT, SPEECH_MODEL_STATUS_CHANGED_EVENT}; pub use types::*; diff --git a/src/crates/contracts/events/src/speech.rs b/src/crates/contracts/events/src/speech.rs new file mode 100644 index 0000000000..e8bec2e376 --- /dev/null +++ b/src/crates/contracts/events/src/speech.rs @@ -0,0 +1,2 @@ +pub const SPEECH_MODEL_PROGRESS_EVENT: &str = "speech://model-download-progress"; +pub const SPEECH_MODEL_STATUS_CHANGED_EVENT: &str = "speech://model-status-changed"; diff --git a/src/crates/services/services-integrations/Cargo.toml b/src/crates/services/services-integrations/Cargo.toml index 68c5ca9eb9..abb38bef3a 100644 --- a/src/crates/services/services-integrations/Cargo.toml +++ b/src/crates/services/services-integrations/Cargo.toml @@ -17,6 +17,7 @@ log = { workspace = true } bitfun-agent-runtime = { path = "../../execution/agent-runtime", optional = true } bitfun-agent-tools = { path = "../../execution/tool-contracts", optional = true } bitfun-events = { path = "../../contracts/events" } +bitfun-core-types = { path = "../../contracts/core-types", optional = true } bitfun-product-domains = { path = "../../contracts/product-domains", default-features = false, optional = true } bitfun-runtime-ports = { path = "../../contracts/runtime-ports", optional = true } aes-gcm = { workspace = true, optional = true } @@ -26,9 +27,11 @@ argon2 = { workspace = true, optional = true } async-trait = { workspace = true, optional = true } base64 = { workspace = true, optional = true } bitfun-services-core = { path = "../services-core", default-features = false, optional = true } +bzip2 = { workspace = true, optional = true } chrono = { workspace = true, optional = true } dunce = { workspace = true, optional = true } futures = { workspace = true, optional = true } +futures-util = { workspace = true, optional = true } fs2 = { workspace = true, optional = true } git2 = { workspace = true, optional = true } hostname = { workspace = true, optional = true } @@ -47,8 +50,10 @@ rmcp = { workspace = true, optional = true } rustls = { workspace = true, optional = true } rustls-native-certs = { version = "0.8", optional = true } sha2 = { workspace = true, optional = true } +sherpa-onnx = { workspace = true, optional = true } sse-stream = { workspace = true, optional = true } thiserror = { workspace = true, optional = true } +tar = { workspace = true, optional = true } tokio-util = { workspace = true, optional = true } tokio-tungstenite = { workspace = true, optional = true } urlencoding = { workspace = true, optional = true } @@ -197,6 +202,21 @@ review-platform = [ "urlencoding", "windows", ] +speech = [ + "async-trait", + "base64", + "bzip2", + "chrono", + "dep:bitfun-core-types", + "futures-util", + "reqwest", + "sha2", + "sherpa-onnx", + "tar", + "thiserror", + "tokio-util", + "uuid", +] workspace-search = [ "async-trait", "bitfun-services-core", diff --git a/src/crates/services/services-integrations/src/lib.rs b/src/crates/services/services-integrations/src/lib.rs index 7a76f38766..53e3dd2a11 100644 --- a/src/crates/services/services-integrations/src/lib.rs +++ b/src/crates/services/services-integrations/src/lib.rs @@ -48,6 +48,9 @@ pub mod review_platform; #[cfg(feature = "review-platform")] pub(crate) mod review_platform_http; +#[cfg(feature = "speech")] +pub mod speech; + #[cfg(feature = "workspace-search")] pub mod workspace_search; diff --git a/src/crates/services/services-integrations/src/speech/audio.rs b/src/crates/services/services-integrations/src/speech/audio.rs new file mode 100644 index 0000000000..70f62c912a --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/audio.rs @@ -0,0 +1,23 @@ +use super::{BitFunError, BitFunResult}; + +pub(super) fn pcm16_le_to_f32_samples(bytes: &[u8]) -> BitFunResult> { + if bytes.len() % 2 != 0 { + return Err(BitFunError::validation( + "PCM16 audio payload must have an even number of bytes", + )); + } + + let mut samples = Vec::with_capacity(bytes.len() / 2); + for chunk in bytes.chunks_exact(2) { + let sample = i16::from_le_bytes([chunk[0], chunk[1]]); + samples.push(sample as f32 / i16::MAX as f32); + } + Ok(samples) +} + +pub(super) fn pcm16_duration_seconds(byte_len: u64, sample_rate: u32) -> f64 { + if sample_rate == 0 { + return 0.0; + } + byte_len as f64 / 2.0 / sample_rate as f64 +} diff --git a/src/crates/services/services-integrations/src/speech/downloader.rs b/src/crates/services/services-integrations/src/speech/downloader.rs new file mode 100644 index 0000000000..601b2d1831 --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/downloader.rs @@ -0,0 +1,382 @@ +use super::model_store::{validate_relative_archive_path, SpeechModelStore}; +use super::types::{ + SpeechModelArtifact, SpeechModelArtifactKind, SpeechModelManifest, SpeechModelProgress, + SpeechModelStatus, +}; +use super::{BitFunError, BitFunResult}; +use bzip2::read::BzDecoder; +use futures_util::StreamExt; +use sha2::{Digest, Sha256}; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tar::Archive; +use tokio::fs; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +pub(super) async fn download_and_install_model( + store: &SpeechModelStore, + manifest: &SpeechModelManifest, + cancel: CancellationToken, + on_progress: F, +) -> BitFunResult +where + F: Fn(SpeechModelProgress) + Send + Sync, +{ + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(30 * 60)) + .build() + .map_err(|error| BitFunError::Http(error.to_string()))?; + let total_bytes = manifest.expected_bytes(); + let mut completed_bytes = 0u64; + let mut downloaded_artifacts = Vec::with_capacity(manifest.artifacts.len()); + + for artifact in &manifest.artifacts { + let artifact_path = ensure_artifact_downloaded( + store, + manifest, + artifact, + &client, + completed_bytes, + total_bytes, + cancel.clone(), + &on_progress, + ) + .await?; + completed_bytes = completed_bytes.saturating_add(artifact.size_bytes); + downloaded_artifacts.push((artifact.clone(), artifact_path)); + } + + on_progress(SpeechModelProgress { + model_id: manifest.id.clone(), + downloaded_bytes: total_bytes, + total_bytes, + percent: 100.0, + }); + install_artifacts(store, manifest, &downloaded_artifacts).await?; + store.status_for_manifest(manifest).await +} + +async fn ensure_artifact_downloaded( + store: &SpeechModelStore, + manifest: &SpeechModelManifest, + artifact: &SpeechModelArtifact, + client: &reqwest::Client, + completed_bytes: u64, + total_bytes: u64, + cancel: CancellationToken, + on_progress: &F, +) -> BitFunResult +where + F: Fn(SpeechModelProgress) + Send + Sync, +{ + let artifact_path = store.artifact_download_path(manifest, artifact); + let partial_path = store.artifact_partial_path(manifest, artifact); + if let Some(parent) = partial_path.parent() { + fs::create_dir_all(parent).await?; + } + + if artifact_path.exists() { + let actual_hash = sha256_file(&artifact_path).await?; + if actual_hash == artifact.sha256 { + return Ok(artifact_path); + } + fs::remove_file(&artifact_path).await?; + } + + let sources = std::iter::once(&artifact.source_url) + .chain(artifact.fallback_source_urls.iter()) + .collect::>(); + let mut source_errors = Vec::with_capacity(sources.len()); + + for (source_index, source_url) in sources.iter().enumerate() { + if cancel.is_cancelled() { + let _ = fs::remove_file(&partial_path).await; + return Err(download_cancelled_error(manifest)); + } + let _ = fs::remove_file(&partial_path).await; + on_progress(SpeechModelProgress { + model_id: manifest.id.clone(), + downloaded_bytes: completed_bytes, + total_bytes, + percent: progress_percent(completed_bytes, total_bytes), + }); + + match download_source( + client, + source_url, + &partial_path, + manifest, + artifact, + completed_bytes, + total_bytes, + &cancel, + on_progress, + ) + .await + { + Ok(()) => { + if artifact_path.exists() { + fs::remove_file(&artifact_path).await?; + } + fs::rename(&partial_path, &artifact_path).await?; + return Ok(artifact_path); + } + Err(error) => { + let _ = fs::remove_file(&partial_path).await; + if cancel.is_cancelled() { + return Err(download_cancelled_error(manifest)); + } + log::warn!( + "Speech model download source failed: model_id={}, artifact_id={}, source_index={}, error={}", + manifest.id, + artifact.id, + source_index, + error + ); + source_errors.push(error.to_string()); + } + } + } + Err(BitFunError::Http(format!( + "Speech model artifact download failed from all {} configured sources: {}", + sources.len(), + source_errors.join("; ") + ))) +} + +async fn download_source( + client: &reqwest::Client, + source_url: &str, + partial_path: &Path, + manifest: &SpeechModelManifest, + artifact: &SpeechModelArtifact, + completed_bytes: u64, + model_total_bytes: u64, + cancel: &CancellationToken, + on_progress: &F, +) -> BitFunResult<()> +where + F: Fn(SpeechModelProgress) + Send + Sync, +{ + let response_request = client + .get(source_url) + .header(reqwest::header::USER_AGENT, "BitFun") + .send(); + let response = tokio::select! { + _ = cancel.cancelled() => { + return Err(download_cancelled_error(manifest)); + } + response = response_request => response, + } + .map_err(|error| BitFunError::Http(error.to_string()))? + .error_for_status() + .map_err(|error| BitFunError::Http(error.to_string()))?; + + let total_bytes = response.content_length().unwrap_or(artifact.size_bytes); + let mut stream = response.bytes_stream(); + let mut file = fs::File::create(&partial_path).await?; + let mut hasher = Sha256::new(); + let mut downloaded = 0u64; + + loop { + let next_chunk = tokio::select! { + _ = cancel.cancelled() => { + drop(file); + let _ = fs::remove_file(&partial_path).await; + return Err(download_cancelled_error(manifest)); + } + chunk = stream.next() => chunk, + }; + let Some(chunk) = next_chunk else { + break; + }; + + let chunk = chunk.map_err(|error| BitFunError::Http(error.to_string()))?; + file.write_all(&chunk).await?; + hasher.update(&chunk); + downloaded += chunk.len() as u64; + let percent = if total_bytes > 0 { + progress_percent( + completed_bytes.saturating_add(downloaded), + model_total_bytes, + ) + } else { + 0.0 + }; + on_progress(SpeechModelProgress { + model_id: manifest.id.clone(), + downloaded_bytes: completed_bytes.saturating_add(downloaded), + total_bytes: model_total_bytes, + percent, + }); + } + file.flush().await?; + drop(file); + + if cancel.is_cancelled() { + let _ = fs::remove_file(&partial_path).await; + return Err(download_cancelled_error(manifest)); + } + + let actual_hash = format!("{:x}", hasher.finalize()); + if actual_hash != artifact.sha256 { + let _ = fs::remove_file(&partial_path).await; + return Err(BitFunError::validation(format!( + "Speech model checksum mismatch: expected={}, actual={}", + artifact.sha256, actual_hash + ))); + } + + Ok(()) +} + +fn progress_percent(downloaded_bytes: u64, total_bytes: u64) -> f64 { + if total_bytes > 0 { + downloaded_bytes as f64 / total_bytes as f64 * 100.0 + } else { + 0.0 + } +} + +fn download_cancelled_error(manifest: &SpeechModelManifest) -> BitFunError { + BitFunError::Cancelled(format!("Speech model download cancelled: {}", manifest.id)) +} + +async fn install_artifacts( + store: &SpeechModelStore, + manifest: &SpeechModelManifest, + artifacts: &[(SpeechModelArtifact, PathBuf)], +) -> BitFunResult<()> { + let final_dir = store.model_dir(manifest); + let parent = final_dir.parent().ok_or_else(|| { + BitFunError::service(format!( + "Speech model path has no parent: {}", + final_dir.display() + )) + })?; + fs::create_dir_all(parent).await?; + + let staging = parent.join(format!(".installing-{}", Uuid::new_v4().simple())); + if staging.exists() { + fs::remove_dir_all(&staging).await?; + } + fs::create_dir_all(&staging).await?; + + let install_result = + install_artifacts_into_staging(store, manifest, artifacts, &staging, &final_dir).await; + if install_result.is_err() && staging.exists() { + let _ = fs::remove_dir_all(&staging).await; + } + install_result +} + +async fn install_artifacts_into_staging( + store: &SpeechModelStore, + manifest: &SpeechModelManifest, + artifacts: &[(SpeechModelArtifact, PathBuf)], + staging: &Path, + final_dir: &Path, +) -> BitFunResult<()> { + for (artifact, path) in artifacts { + match artifact.kind { + SpeechModelArtifactKind::TarBz2 => { + let archive_path = path.to_path_buf(); + let staging_for_extract = staging.to_path_buf(); + tokio::task::spawn_blocking(move || { + extract_tar_bz2(&archive_path, &staging_for_extract) + }) + .await + .map_err(|e| { + BitFunError::service(format!("Speech model extraction task failed: {e}")) + })??; + } + SpeechModelArtifactKind::File => { + let target_relative = artifact + .install_path + .as_deref() + .unwrap_or(&artifact.file_name); + let target = + staging.join(validate_relative_archive_path(Path::new(target_relative))?); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).await?; + } + fs::copy(path, target).await?; + } + } + } + + let payload_dir = find_payload_dir(&staging, &manifest.required_files).await?; + if final_dir.exists() { + fs::remove_dir_all(&final_dir).await?; + } + + if payload_dir == staging { + fs::rename(&staging, &final_dir).await?; + } else { + fs::rename(&payload_dir, &final_dir).await?; + if staging.exists() { + fs::remove_dir_all(&staging).await?; + } + } + + store.write_install_record(manifest, &final_dir).await?; + Ok(()) +} + +async fn sha256_file(path: &Path) -> BitFunResult { + let mut file = fs::File::open(path).await?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 1024 * 1024]; + loop { + let read = file.read(&mut buffer).await?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn extract_tar_bz2(archive_path: &Path, destination: &Path) -> BitFunResult<()> { + let file = File::open(archive_path)?; + let decoder = BzDecoder::new(file); + let mut archive = Archive::new(decoder); + for entry in archive.entries()? { + let mut entry = entry?; + let relative = validate_relative_archive_path(&entry.path()?)?; + let target = destination.join(relative); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent)?; + } + entry.unpack(&target)?; + } + Ok(()) +} + +async fn find_payload_dir(staging: &Path, required_files: &[String]) -> BitFunResult { + if has_required_files_at(staging, required_files) { + return Ok(staging.to_path_buf()); + } + + let mut entries = fs::read_dir(staging).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.is_dir() && has_required_files_at(&path, required_files) { + return Ok(path); + } + } + + Err(BitFunError::validation( + "Downloaded speech model archive does not contain the required model files", + )) +} + +fn has_required_files_at(path: &Path, required_files: &[String]) -> bool { + required_files + .iter() + .all(|relative| path.join(relative).is_file()) +} diff --git a/src/crates/services/services-integrations/src/speech/error.rs b/src/crates/services/services-integrations/src/speech/error.rs new file mode 100644 index 0000000000..9327eb532c --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/error.rs @@ -0,0 +1,31 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum SpeechError { + #[error("{0}")] + Validation(String), + #[error("{0}")] + NotFound(String), + #[error("{0}")] + Cancelled(String), + #[error("{0}")] + Http(String), + #[error("{0}")] + Service(String), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Serialization(#[from] serde_json::Error), +} + +impl SpeechError { + pub fn validation(message: impl Into) -> Self { + Self::Validation(message.into()) + } + + pub fn service(message: impl Into) -> Self { + Self::Service(message.into()) + } +} + +pub type SpeechResult = Result; diff --git a/src/crates/services/services-integrations/src/speech/mod.rs b/src/crates/services/services-integrations/src/speech/mod.rs new file mode 100644 index 0000000000..e5dca07949 --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/mod.rs @@ -0,0 +1,419 @@ +//! Local speech input services. + +mod audio; +mod downloader; +mod error; +mod model_catalog; +mod model_store; +mod qwen3_asr_int8; +mod recognizer; +mod recognizer_router; +mod sensevoice_int8; +mod types; + +use self::downloader::download_and_install_model; +use self::model_catalog::get_builtin_speech_model_manifest; +use self::model_store::SpeechModelStore; +use self::recognizer::{SpeechRecognizer, SpeechRecognizerWarmupRequest}; +use self::recognizer_router::SpeechRecognizerRouter; +use self::types::SpeechTranscribeRequest; +pub use self::types::{ + DEFAULT_MAX_RECORDING_SECONDS, DEFAULT_SPEECH_SAMPLE_RATE, LOCAL_QWEN3_ASR_0_6B_INT8_MODEL_ID, + LOCAL_QWEN3_ASR_0_6B_INT8_MODEL_REF, LOCAL_SENSEVOICE_SMALL_INT8_MODEL_ID, + LOCAL_SENSEVOICE_SMALL_INT8_MODEL_REF, +}; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64::Engine; +pub use bitfun_core_types::speech::*; +pub use error::{SpeechError, SpeechResult}; +use error::{SpeechError as BitFunError, SpeechResult as BitFunResult}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::fs; +use tokio::io::AsyncWriteExt; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +#[derive(Debug, Clone)] +pub struct SpeechStoragePaths { + models_dir: PathBuf, + downloads_dir: PathBuf, + input_temp_dir: PathBuf, +} + +impl SpeechStoragePaths { + pub fn new(models_dir: PathBuf, downloads_dir: PathBuf, input_temp_dir: PathBuf) -> Self { + Self { + models_dir, + downloads_dir, + input_temp_dir, + } + } + + pub fn models_dir(&self) -> &std::path::Path { + &self.models_dir + } + + pub fn downloads_dir(&self) -> &std::path::Path { + &self.downloads_dir + } + + pub fn input_temp_dir(&self) -> &std::path::Path { + &self.input_temp_dir + } +} + +#[derive(Clone)] +pub struct SpeechService { + store: SpeechModelStore, + recognizer: Arc, + downloads: Arc>>>, + sessions: Arc>>, +} + +#[derive(Debug)] +struct SpeechModelDownloadControl { + cancel: CancellationToken, + finished: CancellationToken, +} + +#[derive(Debug)] +struct SpeechInputSessionState { + session: SpeechInputSession, + audio_path: PathBuf, + received_bytes: u64, +} + +impl SpeechService { + pub fn new(paths: SpeechStoragePaths) -> Self { + Self { + store: SpeechModelStore::new(paths), + recognizer: Arc::new(SpeechRecognizerRouter::new()), + downloads: Arc::new(Mutex::new(HashMap::new())), + sessions: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub async fn list_models(&self) -> BitFunResult { + Ok(SpeechListModelsResponse { + models: self.store.list_statuses().await?, + }) + } + + pub async fn model_status(&self, model_id: &str) -> BitFunResult { + let manifest = get_builtin_speech_model_manifest(model_id)?; + self.store.status_for_manifest(&manifest).await + } + + pub async fn download_model( + &self, + request: SpeechDownloadModelRequest, + on_progress: F, + ) -> BitFunResult + where + F: Fn(SpeechModelProgressEvent) + Send + Sync + 'static, + { + let manifest = get_builtin_speech_model_manifest(&request.model_id)?; + let control = Arc::new(SpeechModelDownloadControl { + cancel: CancellationToken::new(), + finished: CancellationToken::new(), + }); + { + let mut downloads = self.downloads.lock().await; + if downloads.contains_key(&manifest.id) { + return Err(BitFunError::validation(format!( + "Speech model is already downloading: {}", + manifest.id + ))); + } + downloads.insert(manifest.id.clone(), Arc::clone(&control)); + } + + let store = self.store.clone(); + let downloads = Arc::clone(&self.downloads); + let task_control = Arc::clone(&control); + let model_id = manifest.id.clone(); + let task = tokio::spawn(async move { + let result = download_and_install_model( + &store, + &manifest, + task_control.cancel.clone(), + |progress| { + let status = SpeechModelStatus { + model_id: manifest.id.clone(), + display_name: manifest.display_name.clone(), + provider: manifest.provider.clone(), + version: manifest.version.clone(), + description: manifest.description.clone(), + languages: manifest.languages.clone(), + state: SpeechModelInstallState::Downloading, + installed_path: None, + installed_bytes: progress.downloaded_bytes, + expected_bytes: manifest.expected_bytes(), + progress: Some(progress), + error: None, + }; + on_progress(SpeechModelProgressEvent { status }); + }, + ) + .await; + + task_control.finished.cancel(); + let mut active_downloads = downloads.lock().await; + let is_current = active_downloads + .get(&model_id) + .map(|active| Arc::ptr_eq(active, &task_control)) + .unwrap_or(false); + if is_current { + active_downloads.remove(&model_id); + } + result + }); + + task.await.map_err(|error| { + BitFunError::service(format!("Speech model download task failed: {error}")) + })? + } + + async fn cancel_active_download(&self, model_id: &str) { + let control = self.downloads.lock().await.get(model_id).cloned(); + if let Some(control) = control { + control.cancel.cancel(); + control.finished.cancelled().await; + } + } + + pub async fn cancel_model_download( + &self, + request: SpeechCancelModelDownloadRequest, + ) -> BitFunResult { + let manifest = get_builtin_speech_model_manifest(&request.model_id)?; + self.cancel_active_download(&manifest.id).await; + self.store.status_for_manifest(&manifest).await + } + + pub async fn delete_model( + &self, + request: SpeechDeleteModelRequest, + ) -> BitFunResult { + let manifest = get_builtin_speech_model_manifest(&request.model_id)?; + self.cancel_active_download(&manifest.id).await; + self.recognizer.unload().await?; + self.store.delete_model(&manifest).await + } + + pub async fn verify_model( + &self, + request: SpeechVerifyModelRequest, + ) -> BitFunResult { + let manifest = get_builtin_speech_model_manifest(&request.model_id)?; + self.store.verify_model(&manifest).await + } + + pub async fn start_input_session( + &self, + request: SpeechStartInputSessionRequest, + ) -> BitFunResult { + let model_id = request + .model_id + .unwrap_or_else(|| LOCAL_SENSEVOICE_SMALL_INT8_MODEL_ID.to_string()); + let manifest = get_builtin_speech_model_manifest(&model_id)?; + if !self.store.has_required_files(&manifest).await { + return Err(BitFunError::NotFound( + "Speech model is not installed; download it before starting voice input" + .to_string(), + )); + } + + let sample_rate = request.sample_rate.unwrap_or(DEFAULT_SPEECH_SAMPLE_RATE); + if sample_rate == 0 { + return Err(BitFunError::validation( + "Sample rate must be greater than zero", + )); + } + let max_recording_seconds = request + .max_recording_seconds + .unwrap_or(DEFAULT_MAX_RECORDING_SECONDS); + if max_recording_seconds == 0 { + return Err(BitFunError::validation( + "Recording limit must be greater than zero", + )); + } + let language = request.language.unwrap_or_else(|| "auto".to_string()); + let model_dir = self.store.model_dir(&manifest); + let recognizer = Arc::clone(&self.recognizer); + let warmup_language = language.clone(); + let warmup_model_id = model_id.clone(); + let warmup_recognizer = manifest.recognizer; + tokio::spawn(async move { + if let Err(error) = recognizer + .warmup(SpeechRecognizerWarmupRequest { + model_dir, + recognizer: warmup_recognizer, + language: warmup_language, + }) + .await + { + log::warn!( + "Failed to warm up speech recognizer: model_id={}, error={}", + warmup_model_id, + error + ); + } + }); + + let session_id = Uuid::new_v4().to_string(); + let temp_dir = self.store.paths().input_temp_dir().to_path_buf(); + fs::create_dir_all(&temp_dir).await?; + let audio_path = temp_dir.join(format!("{session_id}.pcm")); + fs::File::create(&audio_path).await?; + + let session = SpeechInputSession { + session_id: session_id.clone(), + model_id, + language, + sample_rate, + max_recording_seconds, + }; + self.sessions.lock().await.insert( + session_id, + SpeechInputSessionState { + session: session.clone(), + audio_path, + received_bytes: 0, + }, + ); + Ok(session) + } + + pub async fn append_audio_chunk( + &self, + request: SpeechAppendAudioChunkRequest, + ) -> BitFunResult { + let bytes = BASE64_STANDARD + .decode(request.pcm16_base64.as_bytes()) + .map_err(|e| BitFunError::validation(format!("Invalid base64 audio chunk: {e}")))?; + if bytes.len() % 2 != 0 { + return Err(BitFunError::validation( + "PCM16 audio chunks must contain complete samples", + )); + } + + let mut sessions = self.sessions.lock().await; + let state = sessions + .get_mut(&request.session_id) + .ok_or_else(|| BitFunError::NotFound("Speech input session not found".to_string()))?; + let max_bytes = + state.session.sample_rate as u64 * state.session.max_recording_seconds as u64 * 2; + let remaining_bytes = max_bytes.saturating_sub(state.received_bytes); + let accepted_bytes = bytes.len().min(remaining_bytes as usize) & !1; + + if accepted_bytes > 0 { + let mut file = fs::OpenOptions::new() + .append(true) + .open(&state.audio_path) + .await?; + file.write_all(&bytes[..accepted_bytes]).await?; + state.received_bytes += accepted_bytes as u64; + } + Ok(SpeechAppendAudioChunkResponse { + received_bytes: state.received_bytes, + received_seconds: audio::pcm16_duration_seconds( + state.received_bytes, + state.session.sample_rate, + ), + limit_reached: state.received_bytes >= max_bytes, + }) + } + + pub async fn finish_input_session( + &self, + request: SpeechFinishInputSessionRequest, + ) -> BitFunResult { + let state = self + .sessions + .lock() + .await + .remove(&request.session_id) + .ok_or_else(|| BitFunError::NotFound("Speech input session not found".to_string()))?; + let manifest = get_builtin_speech_model_manifest(&state.session.model_id)?; + let pcm16_le = fs::read(&state.audio_path).await?; + let _ = fs::remove_file(&state.audio_path).await; + if pcm16_le.is_empty() { + return Err(BitFunError::validation("No speech audio was captured")); + } + + self.recognizer + .transcribe(SpeechTranscribeRequest { + model_dir: self.store.model_dir(&manifest), + recognizer: manifest.recognizer, + pcm16_le, + sample_rate: state.session.sample_rate, + language: state.session.language, + }) + .await + } + + pub async fn cancel_input_session( + &self, + request: SpeechCancelInputSessionRequest, + ) -> BitFunResult<()> { + if let Some(state) = self.sessions.lock().await.remove(&request.session_id) { + let _ = fs::remove_file(state.audio_path).await; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn append_audio_chunk_truncates_at_recording_limit() { + let root = std::env::temp_dir().join(format!( + "bitfun-speech-limit-test-{}", + Uuid::new_v4().simple() + )); + let service = SpeechService::new(SpeechStoragePaths::new( + root.join("models"), + root.join("downloads"), + root.join("input"), + )); + fs::create_dir_all(&root).await.unwrap(); + let audio_path = root.join("session.pcm"); + fs::File::create(&audio_path).await.unwrap(); + let session_id = "limit-test-session".to_string(); + service.sessions.lock().await.insert( + session_id.clone(), + SpeechInputSessionState { + session: SpeechInputSession { + session_id: session_id.clone(), + model_id: LOCAL_SENSEVOICE_SMALL_INT8_MODEL_ID.to_string(), + language: "auto".to_string(), + sample_rate: 2, + max_recording_seconds: 1, + }, + audio_path: audio_path.clone(), + received_bytes: 0, + }, + ); + + let response = service + .append_audio_chunk(SpeechAppendAudioChunkRequest { + session_id, + pcm16_base64: BASE64_STANDARD.encode([1_u8, 2, 3, 4, 5, 6]), + }) + .await + .unwrap(); + + assert_eq!(response.received_bytes, 4); + assert_eq!(response.received_seconds, 1.0); + assert!(response.limit_reached); + assert_eq!(fs::read(&audio_path).await.unwrap(), vec![1, 2, 3, 4]); + + let _ = fs::remove_dir_all(root).await; + } +} diff --git a/src/crates/services/services-integrations/src/speech/model_catalog.rs b/src/crates/services/services-integrations/src/speech/model_catalog.rs new file mode 100644 index 0000000000..b2dbb8b111 --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/model_catalog.rs @@ -0,0 +1,173 @@ +use super::types::{ + SpeechModelArtifact, SpeechModelArtifactKind, SpeechModelManifest, SpeechRecognizerKind, + LOCAL_QWEN3_ASR_0_6B_INT8_MODEL_ID, LOCAL_SENSEVOICE_SMALL_INT8_MODEL_ID, +}; +use super::{BitFunError, BitFunResult}; + +pub(super) fn builtin_speech_model_manifests() -> Vec { + vec![ + sensevoice_small_int8_manifest(), + qwen3_asr_0_6b_int8_manifest(), + ] +} + +pub(super) fn get_builtin_speech_model_manifest( + model_id: &str, +) -> BitFunResult { + builtin_speech_model_manifests() + .into_iter() + .find(|manifest| manifest.id == model_id) + .ok_or_else(|| BitFunError::NotFound(format!("Unknown speech model: {model_id}"))) +} + +pub(super) fn sensevoice_small_int8_manifest() -> SpeechModelManifest { + SpeechModelManifest { + id: LOCAL_SENSEVOICE_SMALL_INT8_MODEL_ID.to_string(), + display_name: "SenseVoice Small INT8".to_string(), + provider: "k2-fsa/sherpa-onnx".to_string(), + version: "2025-09-09".to_string(), + variant: "int8".to_string(), + description: "Local multilingual speech recognition for Mandarin, Cantonese, English, Japanese, and Korean.".to_string(), + source_page_url: "https://k2-fsa.github.io/sherpa/onnx/sense-voice/index.html".to_string(), + license_name: Some("Apache-2.0".to_string()), + languages: vec![ + "auto".to_string(), + "zh".to_string(), + "yue".to_string(), + "en".to_string(), + "ja".to_string(), + "ko".to_string(), + ], + required_files: vec![ + "model.int8.onnx".to_string(), + "tokens.txt".to_string(), + "README.md".to_string(), + ], + recognizer: SpeechRecognizerKind::SenseVoiceInt8, + artifacts: vec![SpeechModelArtifact { + id: "model-archive".to_string(), + file_name: "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2".to_string(), + kind: SpeechModelArtifactKind::TarBz2, + source_url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2".to_string(), + fallback_source_urls: vec![ + "https://gh-proxy.com/https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2".to_string(), + ], + size_bytes: 165_783_878, + sha256: "7305f7905bfcf77fa0b39388a313f3da35c68d971661a65475b56fb2162c8e63".to_string(), + install_path: None, + }], + } +} + +pub(super) fn qwen3_asr_0_6b_int8_manifest() -> SpeechModelManifest { + SpeechModelManifest { + id: LOCAL_QWEN3_ASR_0_6B_INT8_MODEL_ID.to_string(), + display_name: "Qwen3-ASR 0.6B INT8".to_string(), + provider: "k2-fsa/sherpa-onnx".to_string(), + version: "2026-03-25".to_string(), + variant: "int8".to_string(), + description: "Higher-quality local multilingual speech recognition based on Qwen3-ASR.".to_string(), + source_page_url: "https://k2-fsa.github.io/sherpa/onnx/qwen3-asr/pretrained.html".to_string(), + license_name: Some("Apache-2.0".to_string()), + languages: vec![ + "auto".to_string(), + "zh".to_string(), + "yue".to_string(), + "en".to_string(), + "ar".to_string(), + "de".to_string(), + "fr".to_string(), + "es".to_string(), + "pt".to_string(), + "id".to_string(), + "it".to_string(), + "ko".to_string(), + "ru".to_string(), + "th".to_string(), + "vi".to_string(), + "ja".to_string(), + "tr".to_string(), + "hi".to_string(), + "ms".to_string(), + "nl".to_string(), + "sv".to_string(), + "da".to_string(), + "fi".to_string(), + "pl".to_string(), + "cs".to_string(), + "fil".to_string(), + "fa".to_string(), + "el".to_string(), + "hu".to_string(), + "mk".to_string(), + "ro".to_string(), + ], + required_files: vec![ + "conv_frontend.onnx".to_string(), + "encoder.int8.onnx".to_string(), + "decoder.int8.onnx".to_string(), + "tokenizer/merges.txt".to_string(), + "tokenizer/tokenizer_config.json".to_string(), + "tokenizer/vocab.json".to_string(), + "README.md".to_string(), + ], + recognizer: SpeechRecognizerKind::Qwen3AsrInt8, + artifacts: vec![SpeechModelArtifact { + id: "model-archive".to_string(), + file_name: "sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2".to_string(), + kind: SpeechModelArtifactKind::TarBz2, + source_url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2".to_string(), + fallback_source_urls: vec![ + "https://gh-proxy.com/https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2".to_string(), + ], + size_bytes: 878_702_423, + sha256: "393f8a14e2f5fb96746aaab342997a40641001fbd5bf9592a080a8329178ee96".to_string(), + install_path: None, + }], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sensevoice_manifest_has_distinct_fallback_sources() { + let manifest = sensevoice_small_int8_manifest(); + + let artifact = manifest.artifacts.first().unwrap(); + assert!(!artifact.fallback_source_urls.is_empty()); + assert!(artifact + .fallback_source_urls + .iter() + .all(|source| source != &artifact.source_url)); + assert_eq!(artifact.sha256.len(), 64); + } + + #[test] + fn qwen3_manifest_declares_required_runtime_files() { + let manifest = qwen3_asr_0_6b_int8_manifest(); + + assert_eq!(manifest.recognizer, SpeechRecognizerKind::Qwen3AsrInt8); + assert!(manifest + .required_files + .contains(&"conv_frontend.onnx".to_string())); + assert!(manifest + .required_files + .contains(&"encoder.int8.onnx".to_string())); + assert!(manifest + .required_files + .contains(&"decoder.int8.onnx".to_string())); + assert!(manifest + .required_files + .contains(&"tokenizer/vocab.json".to_string())); + assert_eq!( + manifest + .artifacts + .iter() + .map(|artifact| artifact.size_bytes) + .sum::(), + 878_702_423 + ); + } +} diff --git a/src/crates/services/services-integrations/src/speech/model_store.rs b/src/crates/services/services-integrations/src/speech/model_store.rs new file mode 100644 index 0000000000..0ec03188cc --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/model_store.rs @@ -0,0 +1,411 @@ +use super::model_catalog::builtin_speech_model_manifests; +use super::types::{ + InstalledSpeechModelArtifactRecord, InstalledSpeechModelFileRecord, InstalledSpeechModelRecord, + SpeechModelArtifact, SpeechModelInstallState, SpeechModelManifest, SpeechModelStatus, +}; +use super::{BitFunError, BitFunResult, SpeechStoragePaths}; +use chrono::Utc; +use serde::Deserialize; +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::path::{Component, Path, PathBuf}; +use tokio::fs; +use tokio::io::AsyncReadExt; + +const INSTALL_RECORD_FILE: &str = "bitfun-model-install.json"; + +#[derive(Deserialize)] +struct SpeechModelInstallEnvelope { + install: InstalledSpeechModelRecord, +} + +#[derive(Clone)] +pub(super) struct SpeechModelStore { + paths: SpeechStoragePaths, +} + +impl SpeechModelStore { + pub(super) fn new(paths: SpeechStoragePaths) -> Self { + Self { paths } + } + + pub(super) fn paths(&self) -> &SpeechStoragePaths { + &self.paths + } + + pub(super) fn model_dir(&self, manifest: &SpeechModelManifest) -> PathBuf { + self.paths + .models_dir() + .join(&manifest.id) + .join(&manifest.version) + } + + pub(super) fn artifact_download_path( + &self, + manifest: &SpeechModelManifest, + artifact: &SpeechModelArtifact, + ) -> PathBuf { + self.paths + .downloads_dir() + .join(&manifest.id) + .join(&manifest.version) + .join(&artifact.file_name) + } + + pub(super) fn artifact_partial_path( + &self, + manifest: &SpeechModelManifest, + artifact: &SpeechModelArtifact, + ) -> PathBuf { + self.artifact_download_path(manifest, artifact) + .with_file_name(format!("{}.partial", artifact.file_name)) + } + + pub(super) async fn list_statuses(&self) -> BitFunResult> { + let mut statuses = Vec::new(); + for manifest in builtin_speech_model_manifests() { + statuses.push(self.status_for_manifest(&manifest).await?); + } + Ok(statuses) + } + + pub(super) async fn status_for_manifest( + &self, + manifest: &SpeechModelManifest, + ) -> BitFunResult { + let model_dir = self.model_dir(manifest); + let installed_bytes = dir_size(&model_dir).await?; + let installed = self.has_required_files(manifest).await; + let state = if installed { + SpeechModelInstallState::Installed + } else if model_dir.exists() { + SpeechModelInstallState::Corrupt + } else { + SpeechModelInstallState::NotInstalled + }; + + Ok(SpeechModelStatus { + model_id: manifest.id.clone(), + display_name: manifest.display_name.clone(), + version: manifest.version.clone(), + state, + installed_path: installed.then_some(model_dir), + installed_bytes, + expected_bytes: manifest.expected_bytes(), + progress: None, + error: None, + provider: manifest.provider.clone(), + description: manifest.description.clone(), + languages: manifest.languages.clone(), + }) + } + + pub(super) async fn has_required_files(&self, manifest: &SpeechModelManifest) -> bool { + let model_dir = self.model_dir(manifest); + if !model_dir.is_dir() { + return false; + } + + manifest + .required_files + .iter() + .all(|relative| model_dir.join(relative).is_file()) + } + + pub(super) async fn verify_model( + &self, + manifest: &SpeechModelManifest, + ) -> BitFunResult { + let mut status = self.status_for_manifest(manifest).await?; + if !self.has_required_files(manifest).await { + status.state = SpeechModelInstallState::Corrupt; + status.error = Some("Required model files are missing".to_string()); + return Ok(status); + } + + if let Err(error) = self.verify_install_record(manifest).await { + status.state = SpeechModelInstallState::Corrupt; + status.error = Some(error); + } + Ok(status) + } + + async fn verify_install_record(&self, manifest: &SpeechModelManifest) -> Result<(), String> { + let model_dir = self.model_dir(manifest); + let record_path = model_dir.join(INSTALL_RECORD_FILE); + let payload = fs::read(&record_path) + .await + .map_err(|error| format!("Failed to read speech model install record: {error}"))?; + let envelope: SpeechModelInstallEnvelope = serde_json::from_slice(&payload) + .map_err(|error| format!("Failed to parse speech model install record: {error}"))?; + let record = envelope.install; + + if record.id != manifest.id || record.version != manifest.version { + return Err( + "Speech model install record does not match the model manifest".to_string(), + ); + } + if record.artifacts.is_empty() { + let Some(first_artifact) = manifest.artifacts.first() else { + return Err("Speech model manifest does not contain artifacts".to_string()); + }; + if record.archive_sha256 != first_artifact.sha256 { + return Err( + "Speech model install record does not match the model artifact".to_string(), + ); + } + } else { + for artifact in &manifest.artifacts { + let expected = record + .artifacts + .iter() + .find(|installed| installed.id == artifact.id) + .ok_or_else(|| { + format!("Missing install data for model artifact: {}", artifact.id) + })?; + if expected.file_name != artifact.file_name + || expected.size_bytes != artifact.size_bytes + || expected.sha256 != artifact.sha256 + { + return Err(format!( + "Speech model install record does not match artifact: {}", + artifact.id + )); + } + } + } + if record.files.is_empty() { + return Err( + "Speech model install record does not contain file integrity data".to_string(), + ); + } + + for relative in &manifest.required_files { + let expected = record + .files + .iter() + .find(|file| file.path == *relative) + .ok_or_else(|| format!("Missing integrity data for required file: {relative}"))?; + let path = model_dir.join(relative); + let metadata = fs::metadata(&path).await.map_err(|error| { + format!("Failed to inspect required model file {relative}: {error}") + })?; + if metadata.len() != expected.size_bytes { + return Err(format!("Speech model file size mismatch: {relative}")); + } + let actual_hash = sha256_file(&path).await.map_err(|error| { + format!("Failed to hash required model file {relative}: {error}") + })?; + if actual_hash != expected.sha256 { + return Err(format!("Speech model file checksum mismatch: {relative}")); + } + } + Ok(()) + } + + pub(super) async fn write_install_record( + &self, + manifest: &SpeechModelManifest, + model_dir: &Path, + ) -> BitFunResult<()> { + let mut files = Vec::with_capacity(manifest.required_files.len()); + for relative in &manifest.required_files { + let path = model_dir.join(relative); + let metadata = fs::metadata(&path).await?; + files.push(InstalledSpeechModelFileRecord { + path: relative.clone(), + size_bytes: metadata.len(), + sha256: sha256_file(&path).await?, + }); + } + let artifacts = manifest + .artifacts + .iter() + .map(|artifact| InstalledSpeechModelArtifactRecord { + id: artifact.id.clone(), + file_name: artifact.file_name.clone(), + size_bytes: artifact.size_bytes, + sha256: artifact.sha256.clone(), + }) + .collect::>(); + let first_artifact = manifest.artifacts.first(); + let record = InstalledSpeechModelRecord { + id: manifest.id.clone(), + version: manifest.version.clone(), + installed_at_ms: Utc::now().timestamp_millis(), + source_url: first_artifact + .map(|artifact| artifact.source_url.clone()) + .unwrap_or_default(), + archive_sha256: first_artifact + .map(|artifact| artifact.sha256.clone()) + .unwrap_or_default(), + artifacts, + files, + }; + let payload = serde_json::to_vec_pretty(&json!({ + "model": manifest, + "install": record, + }))?; + fs::write(model_dir.join(INSTALL_RECORD_FILE), payload).await?; + Ok(()) + } + + pub(super) async fn delete_model( + &self, + manifest: &SpeechModelManifest, + ) -> BitFunResult { + let root = self.paths.models_dir().to_path_buf(); + let target = self.model_dir(manifest); + if !target.exists() { + return self.status_for_manifest(manifest).await; + } + + let root = canonical_or_create(&root).await?; + let resolved = target.canonicalize().map_err(|e| { + BitFunError::service(format!("Failed to resolve speech model path: {e}")) + })?; + if !resolved.starts_with(&root) { + return Err(BitFunError::validation( + "Refusing to delete path outside managed speech models directory", + )); + } + + fs::remove_dir_all(&resolved).await?; + self.cleanup_download(manifest).await?; + self.status_for_manifest(manifest).await + } + + pub(super) async fn cleanup_download( + &self, + manifest: &SpeechModelManifest, + ) -> BitFunResult<()> { + let dir = self + .paths + .downloads_dir() + .join(&manifest.id) + .join(&manifest.version); + if dir.exists() { + fs::remove_dir_all(dir).await?; + } + Ok(()) + } +} + +async fn sha256_file(path: &Path) -> BitFunResult { + let mut file = fs::File::open(path).await?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 1024 * 1024]; + loop { + let read = file.read(&mut buffer).await?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +pub(super) fn validate_relative_archive_path(path: &Path) -> BitFunResult { + let mut sanitized = PathBuf::new(); + for component in path.components() { + match component { + Component::Normal(part) => sanitized.push(part), + Component::CurDir => {} + _ => { + return Err(BitFunError::validation(format!( + "Archive entry contains unsafe path: {}", + path.display() + ))); + } + } + } + if sanitized.as_os_str().is_empty() { + return Err(BitFunError::validation("Archive entry path is empty")); + } + Ok(sanitized) +} + +pub(super) async fn dir_size(path: &Path) -> BitFunResult { + fn inner( + path: &Path, + ) -> std::pin::Pin> + Send + '_>> { + Box::pin(async move { + if !path.exists() { + return Ok(0); + } + let metadata = fs::metadata(path).await?; + if metadata.is_file() { + return Ok(metadata.len()); + } + + let mut total = 0u64; + let mut entries = fs::read_dir(path).await?; + while let Some(entry) = entries.next_entry().await? { + total += inner(&entry.path()).await?; + } + Ok(total) + }) + } + + inner(path).await +} + +async fn canonical_or_create(path: &Path) -> BitFunResult { + fs::create_dir_all(path).await?; + path.canonicalize() + .map_err(|e| BitFunError::service(format!("Failed to resolve directory: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::speech::model_catalog::sensevoice_small_int8_manifest; + use uuid::Uuid; + + #[tokio::test] + async fn verify_model_detects_same_size_file_corruption() { + let root = std::env::temp_dir().join(format!( + "bitfun-speech-model-verify-test-{}", + Uuid::new_v4().simple() + )); + let paths = SpeechStoragePaths::new( + root.join("models"), + root.join("downloads"), + root.join("input"), + ); + let store = SpeechModelStore::new(paths); + let mut manifest = sensevoice_small_int8_manifest(); + manifest.id = "verify-test-model".to_string(); + manifest.version = "test-version".to_string(); + manifest.required_files = vec!["model.onnx".to_string(), "tokens.txt".to_string()]; + + let model_dir = store.model_dir(&manifest); + fs::create_dir_all(&model_dir).await.unwrap(); + fs::write(model_dir.join("model.onnx"), b"model-good") + .await + .unwrap(); + fs::write(model_dir.join("tokens.txt"), b"token-good") + .await + .unwrap(); + store + .write_install_record(&manifest, &model_dir) + .await + .unwrap(); + + let verified = store.verify_model(&manifest).await.unwrap(); + assert_eq!(verified.state, SpeechModelInstallState::Installed); + + fs::write(model_dir.join("model.onnx"), b"model-baad") + .await + .unwrap(); + let corrupt = store.verify_model(&manifest).await.unwrap(); + assert_eq!(corrupt.state, SpeechModelInstallState::Corrupt); + assert!(corrupt + .error + .as_deref() + .unwrap_or_default() + .contains("checksum mismatch")); + + let _ = fs::remove_dir_all(root).await; + } +} diff --git a/src/crates/services/services-integrations/src/speech/qwen3_asr_int8.rs b/src/crates/services/services-integrations/src/speech/qwen3_asr_int8.rs new file mode 100644 index 0000000000..9c5ea9eed5 --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/qwen3_asr_int8.rs @@ -0,0 +1,185 @@ +use super::audio::pcm16_le_to_f32_samples; +use super::recognizer::{SpeechRecognizer, SpeechRecognizerWarmupRequest}; +use super::types::{SpeechTranscribeRequest, SpeechTranscriptionResult}; +use super::{BitFunError, BitFunResult}; +use async_trait::async_trait; +use sherpa_onnx::{OfflineQwen3ASRModelConfig, OfflineRecognizer, OfflineRecognizerConfig}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +#[derive(Clone, Default)] +pub(super) struct Qwen3AsrInt8Recognizer { + cache: Arc>>, +} + +struct CachedQwen3AsrRecognizer { + conv_frontend_path: PathBuf, + encoder_path: PathBuf, + decoder_path: PathBuf, + tokenizer_path: PathBuf, + recognizer: OfflineRecognizer, +} + +impl Qwen3AsrInt8Recognizer { + pub(super) fn new() -> Self { + Self::default() + } +} + +#[async_trait] +impl SpeechRecognizer for Qwen3AsrInt8Recognizer { + async fn warmup(&self, request: SpeechRecognizerWarmupRequest) -> BitFunResult<()> { + let cache = Arc::clone(&self.cache); + tokio::task::spawn_blocking(move || { + let paths = qwen3_paths(&request.model_dir); + ensure_model_files(&paths)?; + let mut cache = cache + .lock() + .map_err(|_| BitFunError::service("Speech recognizer cache lock is poisoned"))?; + ensure_cached_recognizer(&mut cache, paths)?; + Ok(()) + }) + .await + .map_err(|e| BitFunError::service(format!("Speech recognizer warmup task failed: {e}")))? + } + + async fn unload(&self) -> BitFunResult<()> { + let cache = Arc::clone(&self.cache); + tokio::task::spawn_blocking(move || { + let mut cache = cache + .lock() + .map_err(|_| BitFunError::service("Speech recognizer cache lock is poisoned"))?; + *cache = None; + Ok(()) + }) + .await + .map_err(|e| BitFunError::service(format!("Speech recognizer unload task failed: {e}")))? + } + + async fn transcribe( + &self, + request: SpeechTranscribeRequest, + ) -> BitFunResult { + let cache = Arc::clone(&self.cache); + tokio::task::spawn_blocking(move || transcribe_blocking(request, cache)) + .await + .map_err(|e| BitFunError::service(format!("Speech transcription task failed: {e}")))? + } +} + +#[derive(Debug)] +struct Qwen3Paths { + conv_frontend: PathBuf, + encoder: PathBuf, + decoder: PathBuf, + tokenizer: PathBuf, +} + +fn qwen3_paths(model_dir: &Path) -> Qwen3Paths { + Qwen3Paths { + conv_frontend: model_dir.join("conv_frontend.onnx"), + encoder: model_dir.join("encoder.int8.onnx"), + decoder: model_dir.join("decoder.int8.onnx"), + tokenizer: model_dir.join("tokenizer"), + } +} + +fn transcribe_blocking( + request: SpeechTranscribeRequest, + cache: Arc>>, +) -> BitFunResult { + let started = Instant::now(); + let paths = qwen3_paths(&request.model_dir); + ensure_model_files(&paths)?; + + let samples = pcm16_le_to_f32_samples(&request.pcm16_le)?; + if samples.is_empty() { + return Err(BitFunError::validation("No audio samples were provided")); + } + + let text = { + let mut cache = cache + .lock() + .map_err(|_| BitFunError::service("Speech recognizer cache lock is poisoned"))?; + let cached = ensure_cached_recognizer(&mut cache, paths)?; + let stream = cached.recognizer.create_stream(); + stream.accept_waveform(request.sample_rate as i32, &samples); + cached.recognizer.decode(&stream); + + stream + .get_result() + .ok_or_else(|| BitFunError::service("Failed to read speech result"))? + .text + .trim() + .to_string() + }; + + let audio_duration_seconds = samples.len() as f64 / request.sample_rate as f64; + Ok(SpeechTranscriptionResult { + text, + language: request.language, + duration_ms: started.elapsed().as_millis() as u64, + audio_duration_seconds, + }) +} + +fn ensure_model_files(paths: &Qwen3Paths) -> BitFunResult<()> { + if !paths.conv_frontend.is_file() + || !paths.encoder.is_file() + || !paths.decoder.is_file() + || !paths.tokenizer.is_dir() + || !paths.tokenizer.join("merges.txt").is_file() + || !paths.tokenizer.join("tokenizer_config.json").is_file() + || !paths.tokenizer.join("vocab.json").is_file() + { + return Err(BitFunError::NotFound( + "Qwen3-ASR model files are missing; download or repair the model first".to_string(), + )); + } + Ok(()) +} + +fn ensure_cached_recognizer( + cache: &mut Option, + paths: Qwen3Paths, +) -> BitFunResult<&mut CachedQwen3AsrRecognizer> { + let should_reload = cache.as_ref().is_none_or(|cached| { + cached.conv_frontend_path != paths.conv_frontend + || cached.encoder_path != paths.encoder + || cached.decoder_path != paths.decoder + || cached.tokenizer_path != paths.tokenizer + }); + + if should_reload { + let recognizer = create_recognizer(&paths)?; + *cache = Some(CachedQwen3AsrRecognizer { + conv_frontend_path: paths.conv_frontend, + encoder_path: paths.encoder, + decoder_path: paths.decoder, + tokenizer_path: paths.tokenizer, + recognizer, + }); + } + + cache + .as_mut() + .ok_or_else(|| BitFunError::service("Speech recognizer cache is empty")) +} + +fn create_recognizer(paths: &Qwen3Paths) -> BitFunResult { + let mut config = OfflineRecognizerConfig::default(); + config.model_config.qwen3_asr = OfflineQwen3ASRModelConfig { + conv_frontend: Some(paths.conv_frontend.to_string_lossy().to_string()), + encoder: Some(paths.encoder.to_string_lossy().to_string()), + decoder: Some(paths.decoder.to_string_lossy().to_string()), + tokenizer: Some(paths.tokenizer.to_string_lossy().to_string()), + max_new_tokens: 512, + ..OfflineQwen3ASRModelConfig::default() + }; + config.model_config.num_threads = 3; + config.decoding_method = Some("greedy_search".to_string()); + + OfflineRecognizer::create(&config) + .ok_or_else(|| BitFunError::service("Failed to create Qwen3-ASR speech recognizer")) +} diff --git a/src/crates/services/services-integrations/src/speech/recognizer.rs b/src/crates/services/services-integrations/src/speech/recognizer.rs new file mode 100644 index 0000000000..ed0232c8e2 --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/recognizer.rs @@ -0,0 +1,23 @@ +use super::types::{SpeechRecognizerKind, SpeechTranscribeRequest, SpeechTranscriptionResult}; +use super::BitFunResult; +use async_trait::async_trait; +use std::path::PathBuf; + +#[derive(Debug, Clone)] +pub(super) struct SpeechRecognizerWarmupRequest { + pub model_dir: PathBuf, + pub recognizer: SpeechRecognizerKind, + pub language: String, +} + +#[async_trait] +pub(super) trait SpeechRecognizer: Send + Sync { + async fn warmup(&self, request: SpeechRecognizerWarmupRequest) -> BitFunResult<()>; + + async fn unload(&self) -> BitFunResult<()>; + + async fn transcribe( + &self, + request: SpeechTranscribeRequest, + ) -> BitFunResult; +} diff --git a/src/crates/services/services-integrations/src/speech/recognizer_router.rs b/src/crates/services/services-integrations/src/speech/recognizer_router.rs new file mode 100644 index 0000000000..0b12bf5a76 --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/recognizer_router.rs @@ -0,0 +1,47 @@ +use super::qwen3_asr_int8::Qwen3AsrInt8Recognizer; +use super::recognizer::{SpeechRecognizer, SpeechRecognizerWarmupRequest}; +use super::sensevoice_int8::SenseVoiceInt8Recognizer; +use super::types::{SpeechRecognizerKind, SpeechTranscribeRequest, SpeechTranscriptionResult}; +use super::BitFunResult; +use async_trait::async_trait; + +#[derive(Clone)] +pub(super) struct SpeechRecognizerRouter { + sensevoice: SenseVoiceInt8Recognizer, + qwen3_asr: Qwen3AsrInt8Recognizer, +} + +impl SpeechRecognizerRouter { + pub(super) fn new() -> Self { + Self { + sensevoice: SenseVoiceInt8Recognizer::new(), + qwen3_asr: Qwen3AsrInt8Recognizer::new(), + } + } +} + +#[async_trait] +impl SpeechRecognizer for SpeechRecognizerRouter { + async fn warmup(&self, request: SpeechRecognizerWarmupRequest) -> BitFunResult<()> { + match request.recognizer { + SpeechRecognizerKind::SenseVoiceInt8 => self.sensevoice.warmup(request).await, + SpeechRecognizerKind::Qwen3AsrInt8 => self.qwen3_asr.warmup(request).await, + } + } + + async fn unload(&self) -> BitFunResult<()> { + self.sensevoice.unload().await?; + self.qwen3_asr.unload().await?; + Ok(()) + } + + async fn transcribe( + &self, + request: SpeechTranscribeRequest, + ) -> BitFunResult { + match request.recognizer { + SpeechRecognizerKind::SenseVoiceInt8 => self.sensevoice.transcribe(request).await, + SpeechRecognizerKind::Qwen3AsrInt8 => self.qwen3_asr.transcribe(request).await, + } + } +} diff --git a/src/crates/services/services-integrations/src/speech/sensevoice_int8.rs b/src/crates/services/services-integrations/src/speech/sensevoice_int8.rs new file mode 100644 index 0000000000..68a338fb51 --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/sensevoice_int8.rs @@ -0,0 +1,169 @@ +use super::audio::pcm16_le_to_f32_samples; +use super::recognizer::{SpeechRecognizer, SpeechRecognizerWarmupRequest}; +use super::types::{SpeechTranscribeRequest, SpeechTranscriptionResult}; +use super::{BitFunError, BitFunResult}; +use async_trait::async_trait; +use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig, OfflineSenseVoiceModelConfig}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +#[derive(Clone, Default)] +pub(super) struct SenseVoiceInt8Recognizer { + cache: Arc>>, +} + +struct CachedSenseVoiceRecognizer { + model_path: PathBuf, + tokens_path: PathBuf, + language: String, + recognizer: OfflineRecognizer, +} + +impl SenseVoiceInt8Recognizer { + pub(super) fn new() -> Self { + Self::default() + } +} + +#[async_trait] +impl SpeechRecognizer for SenseVoiceInt8Recognizer { + async fn warmup(&self, request: SpeechRecognizerWarmupRequest) -> BitFunResult<()> { + let cache = Arc::clone(&self.cache); + tokio::task::spawn_blocking(move || { + let model_dir = request.model_dir; + let language = request.language; + let model_path = model_dir.join("model.int8.onnx"); + let tokens_path = model_dir.join("tokens.txt"); + ensure_model_files(&model_path, &tokens_path)?; + let mut cache = cache + .lock() + .map_err(|_| BitFunError::service("Speech recognizer cache lock is poisoned"))?; + ensure_cached_recognizer(&mut cache, model_path, tokens_path, language)?; + Ok(()) + }) + .await + .map_err(|e| BitFunError::service(format!("Speech recognizer warmup task failed: {e}")))? + } + + async fn unload(&self) -> BitFunResult<()> { + let cache = Arc::clone(&self.cache); + tokio::task::spawn_blocking(move || { + let mut cache = cache + .lock() + .map_err(|_| BitFunError::service("Speech recognizer cache lock is poisoned"))?; + *cache = None; + Ok(()) + }) + .await + .map_err(|e| BitFunError::service(format!("Speech recognizer unload task failed: {e}")))? + } + + async fn transcribe( + &self, + request: SpeechTranscribeRequest, + ) -> BitFunResult { + let cache = Arc::clone(&self.cache); + tokio::task::spawn_blocking(move || transcribe_blocking(request, cache)) + .await + .map_err(|e| BitFunError::service(format!("Speech transcription task failed: {e}")))? + } +} + +fn transcribe_blocking( + request: SpeechTranscribeRequest, + cache: Arc>>, +) -> BitFunResult { + let started = Instant::now(); + let model_path = request.model_dir.join("model.int8.onnx"); + let tokens_path = request.model_dir.join("tokens.txt"); + ensure_model_files(&model_path, &tokens_path)?; + + let samples = pcm16_le_to_f32_samples(&request.pcm16_le)?; + if samples.is_empty() { + return Err(BitFunError::validation("No audio samples were provided")); + } + + let text = { + let mut cache = cache + .lock() + .map_err(|_| BitFunError::service("Speech recognizer cache lock is poisoned"))?; + let cached = ensure_cached_recognizer( + &mut cache, + model_path, + tokens_path, + request.language.clone(), + )?; + let stream = cached.recognizer.create_stream(); + stream.accept_waveform(request.sample_rate as i32, &samples); + cached.recognizer.decode(&stream); + + stream + .get_result() + .ok_or_else(|| BitFunError::service("Failed to read speech result"))? + .text + .trim() + .to_string() + }; + + let audio_duration_seconds = samples.len() as f64 / request.sample_rate as f64; + Ok(SpeechTranscriptionResult { + text, + language: request.language, + duration_ms: started.elapsed().as_millis() as u64, + audio_duration_seconds, + }) +} + +fn ensure_model_files(model_path: &Path, tokens_path: &Path) -> BitFunResult<()> { + if !model_path.is_file() || !tokens_path.is_file() { + return Err(BitFunError::NotFound( + "SenseVoice model files are missing; download or repair the model first".to_string(), + )); + } + Ok(()) +} + +fn ensure_cached_recognizer( + cache: &mut Option, + model_path: PathBuf, + tokens_path: PathBuf, + language: String, +) -> BitFunResult<&mut CachedSenseVoiceRecognizer> { + let should_reload = cache.as_ref().is_none_or(|cached| { + cached.model_path != model_path + || cached.tokens_path != tokens_path + || cached.language != language + }); + + if should_reload { + let recognizer = create_recognizer(&model_path, &tokens_path, &language)?; + *cache = Some(CachedSenseVoiceRecognizer { + model_path, + tokens_path, + language, + recognizer, + }); + } + + cache + .as_mut() + .ok_or_else(|| BitFunError::service("Speech recognizer cache is empty")) +} + +fn create_recognizer( + model_path: &Path, + tokens_path: &Path, + language: &str, +) -> BitFunResult { + let mut config = OfflineRecognizerConfig::default(); + config.model_config.sense_voice = OfflineSenseVoiceModelConfig { + model: Some(model_path.to_string_lossy().to_string()), + language: Some(language.to_string()), + use_itn: true, + }; + config.model_config.tokens = Some(tokens_path.to_string_lossy().to_string()); + + OfflineRecognizer::create(&config) + .ok_or_else(|| BitFunError::service("Failed to create speech recognizer")) +} diff --git a/src/crates/services/services-integrations/src/speech/types.rs b/src/crates/services/services-integrations/src/speech/types.rs new file mode 100644 index 0000000000..732f1145ec --- /dev/null +++ b/src/crates/services/services-integrations/src/speech/types.rs @@ -0,0 +1,110 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +pub const LOCAL_SENSEVOICE_SMALL_INT8_MODEL_ID: &str = "sensevoice-small-int8"; +pub const LOCAL_SENSEVOICE_SMALL_INT8_MODEL_REF: &str = "local:sensevoice-small-int8"; +pub const LOCAL_QWEN3_ASR_0_6B_INT8_MODEL_ID: &str = "qwen3-asr-0.6b-int8"; +pub const LOCAL_QWEN3_ASR_0_6B_INT8_MODEL_REF: &str = "local:qwen3-asr-0.6b-int8"; +pub(super) use bitfun_core_types::speech::{ + SpeechModelInstallState, SpeechModelProgress, SpeechModelStatus, SpeechTranscriptionResult, +}; + +pub const DEFAULT_SPEECH_SAMPLE_RATE: u32 = 16_000; +pub const DEFAULT_MAX_RECORDING_SECONDS: u32 = 60; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct SpeechModelManifest { + pub id: String, + pub display_name: String, + pub provider: String, + pub version: String, + pub variant: String, + pub description: String, + pub source_page_url: String, + pub license_name: Option, + pub languages: Vec, + pub required_files: Vec, + pub recognizer: SpeechRecognizerKind, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum SpeechRecognizerKind { + SenseVoiceInt8, + Qwen3AsrInt8, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct SpeechModelArtifact { + pub id: String, + pub file_name: String, + pub kind: SpeechModelArtifactKind, + pub source_url: String, + #[serde(default)] + pub fallback_source_urls: Vec, + pub size_bytes: u64, + pub sha256: String, + #[serde(default)] + pub install_path: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum SpeechModelArtifactKind { + TarBz2, + File, +} + +impl SpeechModelManifest { + pub(super) fn expected_bytes(&self) -> u64 { + self.artifacts + .iter() + .map(|artifact| artifact.size_bytes) + .sum() + } +} + +#[derive(Debug, Clone)] +pub(super) struct SpeechTranscribeRequest { + pub model_dir: PathBuf, + pub recognizer: SpeechRecognizerKind, + pub pcm16_le: Vec, + pub sample_rate: u32, + pub language: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct InstalledSpeechModelRecord { + pub id: String, + pub version: String, + pub installed_at_ms: i64, + #[serde(default)] + pub source_url: String, + #[serde(default)] + pub archive_sha256: String, + #[serde(default)] + pub artifacts: Vec, + #[serde(default)] + pub files: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct InstalledSpeechModelArtifactRecord { + pub id: String, + pub file_name: String, + pub size_bytes: u64, + pub sha256: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct InstalledSpeechModelFileRecord { + pub path: String, + pub size_bytes: u64, + pub sha256: String, +} diff --git a/src/web-ui/src/app/scenes/settings/SettingsScene.tsx b/src/web-ui/src/app/scenes/settings/SettingsScene.tsx index 67614c3bfe..0d5a105ee9 100644 --- a/src/web-ui/src/app/scenes/settings/SettingsScene.tsx +++ b/src/web-ui/src/app/scenes/settings/SettingsScene.tsx @@ -19,6 +19,7 @@ const AppearanceConfig = lazy(() => import('../../../infrastructure/config/compo const ReviewConfig = lazy(() => import('../../../infrastructure/config/components/ReviewConfig')); const MemoriesConfig = lazy(() => import('../../../infrastructure/config/components/MemoriesConfig')); const QuickActionsConfig = lazy(() => import('../../../infrastructure/config/components/QuickActionsConfig')); +const VoiceInputConfig = lazy(() => import('../../../infrastructure/config/components/VoiceInputConfig')); const ArchivedSessionsConfig = lazy(() => import('./components/ArchivedSessionsConfig')); const KeyboardShortcutsTab = lazy(() => import('./components/KeyboardShortcutsTab')); const SessionPersonalizationConfig = lazy(() => @@ -67,6 +68,7 @@ const SettingsScene: React.FC = () => { case 'session-personalization': Content = SessionPersonalizationConfig; break; case 'session-permissions': Content = SessionPermissionsConfig; break; case 'quick-actions': Content = QuickActionsConfig; break; + case 'voice-input': Content = VoiceInputConfig; break; case 'review': Content = ReviewConfig; break; case 'memories': Content = MemoriesConfig; break; case 'mcp-tools': Content = McpToolsConfig; break; diff --git a/src/web-ui/src/app/scenes/settings/settingsConfig.ts b/src/web-ui/src/app/scenes/settings/settingsConfig.ts index dc3c6eac57..608a5e7ed9 100644 --- a/src/web-ui/src/app/scenes/settings/settingsConfig.ts +++ b/src/web-ui/src/app/scenes/settings/settingsConfig.ts @@ -13,6 +13,7 @@ export type ConfigTab = | 'session-personalization' | 'session-permissions' | 'quick-actions' + | 'voice-input' | 'review' | 'memories' | 'mcp-tools' @@ -180,6 +181,12 @@ export const SETTINGS_CATEGORIES: ConfigCategoryDef[] = [ 'shortcut', ], }, + { + id: 'voice-input', + labelKey: 'configCenter.tabs.voiceInput', + descriptionKey: 'configCenter.tabDescriptions.voiceInput', + keywords: ['voice', 'speech', 'microphone', 'dictation', 'transcription', 'audio'], + }, { id: 'review', labelKey: 'configCenter.tabs.review', diff --git a/src/web-ui/src/app/scenes/settings/settingsTabSearchContent.ts b/src/web-ui/src/app/scenes/settings/settingsTabSearchContent.ts index 8b47940e62..f0f71d822f 100644 --- a/src/web-ui/src/app/scenes/settings/settingsTabSearchContent.ts +++ b/src/web-ui/src/app/scenes/settings/settingsTabSearchContent.ts @@ -169,5 +169,15 @@ export const SETTINGS_TAB_SEARCH_CONTENT: Record = ({ setIsModelSwitching(loading); }, []); - const handleSendOrCancel = useCallback(async () => { + const handleSendOrCancel = useCallback(async (messageOverride?: string) => { if (!derivedState) return; const { sendButtonMode } = derivedState; - const draftTrimmed = inputState.value.trim(); + const draftTrimmed = (messageOverride ?? inputState.value).trim(); // While generating, an empty control in `cancel` mode means stop. If the user has typed a follow-up, // never treat this path as cancel — that would call cancel_dialog_turn and abort the current round early. @@ -2719,7 +2721,8 @@ export const ChatInput: React.FC = ({ const originalPendingLargePastes = { ...pendingLargePastesRef.current }; const message = expandComposerSpecialTokens(originalMessage); const messageCharCount = getCharacterCount(message); - const localSlashCommandsEnabled = !isAcpInputSession; + // Voice transcripts are always message content; they must not accidentally execute local commands. + const localSlashCommandsEnabled = !isAcpInputSession && messageOverride === undefined; if (localSlashCommandsEnabled && isSlashCommand(message, '/btw')) { // When idle, /btw can be sent via the normal send button. @@ -3489,6 +3492,25 @@ export const ChatInput: React.FC = ({ }, []); + const voiceInput = useComposerVoiceInput({ + activateInput: () => dispatchInput({ type: 'ACTIVATE' }), + focusInputSoon: () => { + window.requestAnimationFrame(() => richTextInputRef.current?.focus()); + }, + insertText: (text) => { + const current = inputState.value.trim(); + const mergedText = current ? `${inputState.value.trimEnd()} ${text}` : text; + dispatchInput({ + type: 'SET_VALUE', + payload: mergedText, + }); + return mergedText; + }, + submitText: async (text) => { + await handleSendOrCancel(text); + }, + }); + const renderActionButton = () => { if (!derivedState) return ; @@ -3499,7 +3521,7 @@ export const ChatInput: React.FC = ({
void handleSendOrCancel()} data-testid="chat-input-cancel-btn" >
@@ -3513,7 +3535,7 @@ export const ChatInput: React.FC = ({ return ( void handleSendOrCancel()} disabled={isModelSwitching} tooltip={t('input.retry')} size="small" @@ -3539,7 +3561,7 @@ export const ChatInput: React.FC = ({ void handleSendOrCancel()} disabled={!inputState.value.trim() || isModelSwitching} data-testid="chat-input-send-btn" tooltip={t('input.sendShortcut')} @@ -3554,7 +3576,7 @@ export const ChatInput: React.FC = ({ return ( void handleSendOrCancel()} disabled={!inputState.value.trim() || isModelSwitching} data-testid="chat-input-send-btn" tooltip={t('input.sendShortcut')} @@ -4190,18 +4212,21 @@ export const ChatInput: React.FC = ({
-
- -
+ {voiceInput.phase === 'idle' ? ( +
+ +
+ ) : null} - {renderActionButton()} + + {voiceInput.phase === 'idle' ? renderActionButton() : null}
diff --git a/src/web-ui/src/flow_chat/components/voice/ComposerVoiceInputButton.tsx b/src/web-ui/src/flow_chat/components/voice/ComposerVoiceInputButton.tsx new file mode 100644 index 0000000000..4a57ea5be0 --- /dev/null +++ b/src/web-ui/src/flow_chat/components/voice/ComposerVoiceInputButton.tsx @@ -0,0 +1,211 @@ +import { useEffect, useRef, useState } from 'react'; +import { ArrowUp, Check, Loader2, Mic, VolumeX, X } from 'lucide-react'; +import { IconButton } from '@/component-library'; +import type { ComposerVoiceInputController } from './useComposerVoiceInput'; + +const VOICE_TIMELINE_SAMPLE_COUNT = 32; +const VOICE_TIMELINE_TICK_MS = 86; +const VOICE_SILENCE_THRESHOLD = 0.035; + +function createFlatTimelineSamples(): number[] { + return Array.from({ length: VOICE_TIMELINE_SAMPLE_COUNT }, () => 0); +} + +function formatElapsedTime(totalSeconds: number): string { + const minutes = Math.floor(totalSeconds / 60).toString().padStart(2, '0'); + const seconds = (totalSeconds % 60).toString().padStart(2, '0'); + return `${minutes}:${seconds}`; +} + +interface ComposerVoiceInputButtonProps { + controller: ComposerVoiceInputController; +} + +export function ComposerVoiceInputButton({ controller }: ComposerVoiceInputButtonProps) { + const [timelineSamples, setTimelineSamples] = useState(createFlatTimelineSamples); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + const currentLevelRef = useRef(0); + + useEffect(() => { + currentLevelRef.current = controller.audioLevel; + }, [controller.audioLevel]); + + useEffect(() => { + if (controller.phase === 'idle' || controller.phase === 'preparing') { + setTimelineSamples(createFlatTimelineSamples()); + return undefined; + } + if (controller.phase === 'transcribing') { + return undefined; + } + + setTimelineSamples(createFlatTimelineSamples()); + const timerId = window.setInterval(() => { + const level = currentLevelRef.current < VOICE_SILENCE_THRESHOLD + ? 0 + : Math.min(1, currentLevelRef.current); + setTimelineSamples(previous => [...previous.slice(1), level]); + }, VOICE_TIMELINE_TICK_MS); + + return () => window.clearInterval(timerId); + }, [controller.phase]); + + useEffect(() => { + if (controller.phase === 'idle' || controller.phase === 'preparing') { + setElapsedSeconds(0); + return undefined; + } + if (controller.phase !== 'recording') { + return undefined; + } + + const timerId = window.setInterval(() => { + setElapsedSeconds(previous => previous + 1); + }, 1000); + return () => window.clearInterval(timerId); + }, [controller.phase]); + + if (!controller.enabled) { + return null; + } + + const preparing = controller.phase === 'preparing'; + const transcribing = controller.phase === 'transcribing'; + const recording = controller.phase === 'recording'; + const activeVoicePill = preparing || recording || transcribing; + + if (activeVoicePill) { + const currentSample = !recording || controller.audioLevel < VOICE_SILENCE_THRESHOLD + ? 0 + : Math.min(1, controller.audioLevel); + const visibleTimelineSamples = recording + ? [...timelineSamples.slice(0, -1), currentSample] + : timelineSamples; + const controlsDisabled = preparing || transcribing; + + return ( + + +