Multimodel

Cargo.toml

@@ -11,13 +11,17 @@ crate-type = ["cdylib", "staticlib"]
 default = ["native-transcription"]
 native-transcription = ["dep:ort", "dep:transcribe-rs"]
 ios-onnx = ["native-transcription"]
+ios-dual-backend = ["ios-onnx", "transcribe-rs/whisper"]
+android-dual-backend = ["native-transcription", "transcribe-rs/whisper"]
+whisper-mobile-spike = ["dep:sha2", "dep:transcribe-rs", "transcribe-rs/whisper"]
 
 [dependencies]
 log = "0.4"
 once_cell = "1.19"
 serde_json = "1"
-transcribe-rs = { path = "transcribe-rs", optional = true }
+transcribe-rs = { path = "transcribe-rs", optional = true, default-features = false, features = ["parakeet"] }
 ort = { version = "=2.0.0-rc.10", optional = true }
+sha2 = { version = "0.10", optional = true }
 
 [target.'cfg(target_os = "android")'.dependencies]
 android_logger = "0.13"

app/build.gradle.kts

@@ -15,6 +15,9 @@ val hasReleaseSigning = listOf(
     releaseKeyAlias,
     releaseKeyPassword,
 ).all { !it.isNullOrBlank() }
+val cargoExecutable = System.getenv("CARGO")
+    ?: File(System.getProperty("user.home"), ".cargo/bin/cargo").takeIf(File::isFile)?.absolutePath
+    ?: "cargo"
 
 android {
     namespace = "me.maxistar.voiceinbox"
@@ -122,17 +125,39 @@ val cargoNdkBuild by tasks.registering(Exec::class) {
         ?: System.getenv("ANDROID_NDK")
         ?: android.ndkDirectory.absolutePath
     environment("ANDROID_NDK_HOME", ndkDir)
+    val ndkPrebuiltDir = File(ndkDir, "toolchains/llvm/prebuilt").listFiles()
+        ?.firstOrNull(File::isDirectory) ?: throw GradleException("NDK LLVM prebuilt directory was not found")
+    val whisperCompatDir = layout.buildDirectory.dir("whisper-android-compat").get().asFile.apply { mkdirs() }
+    val whisperCompatArchive = File(whisperCompatDir, "libggml-blas.a")
+    if (!whisperCompatArchive.exists()) {
+        exec { commandLine(File(ndkPrebuiltDir, "bin/llvm-ar").absolutePath, "crs", whisperCompatArchive.absolutePath) }
+    }
+    val cmake = File(android.sdkDirectory, "cmake").listFiles()
+        ?.sortedByDescending(File::getName)
+        ?.map { File(it, "bin/cmake") }
+        ?.firstOrNull(File::isFile)
+        ?: throw GradleException("Android SDK CMake is required for the Whisper build")
+    environment("CMAKE", cmake.absolutePath)
+    environment("CMAKE_GENERATOR", "Ninja")
+    environment("CMAKE_MAKE_PROGRAM", File(cmake.parentFile, "ninja").absolutePath)
+    environment("CMAKE_ANDROID_ARCH_ABI", "arm64-v8a")
+    environment("RUSTFLAGS", "-Lnative=${whisperCompatDir.absolutePath}")
+    environment(
+        "CMAKE_TOOLCHAIN_FILE",
+        rootProject.file("scripts/whisper-android-cmake/android.toolchain.cmake").absolutePath,
+    )
 
     val extractDir = layout.buildDirectory.dir("ort-extracted").get().asFile
     environment("ORT_LIB_LOCATION", File(extractDir, "jni/arm64-v8a").absolutePath)
     environment("ORT_INCLUDE_DIR", File(extractDir, "headers").absolutePath)
 
     val jniLibsDir = project.file("src/main/jniLibs")
     commandLine(
-        "cargo", "ndk",
+        cargoExecutable, "ndk",
         "-t", "arm64-v8a",
         "-o", jniLibsDir.absolutePath,
         "build", "--release",
+        "--features", "android-dual-backend",
     )
 
     doLast {

app/src/androidTest/java/me/maxistar/voiceinbox/EndToEndTranscriptionInstrumentedTest.kt

@@ -25,7 +25,10 @@ class EndToEndTranscriptionInstrumentedTest {
 
     @Test
     fun wavAndM4aBatchAppendSeparateTranscriptEntries() {
-        val model = SpeechModelRepository(targetContext.noBackupFilesDir.resolve("models")).inspect()
+        val model = SpeechModelRepository(
+            targetContext.noBackupFilesDir.resolve("models"),
+            SpeechModelCatalog.defaultModel.manifest,
+        ).inspect()
         assumeTrue(model is InstalledSpeechModelState.Ready)
         targetContext.deleteDatabase(AndroidSqlDelightAudioCatalogFactory.DATABASE_NAME)
 

app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt

@@ -61,6 +61,7 @@ class MainActivityInstrumentedTest {
 
             openActionBarOverflowOrOptionsMenu(InstrumentationRegistry.getInstrumentation().targetContext)
             onView(withText(R.string.menu_settings)).check(matches(isDisplayed()))
+            onView(withText(R.string.menu_documentation)).check(matches(isDisplayed()))
         }
     }
 

app/src/androidTest/java/me/maxistar/voiceinbox/SpeechModelLocalImporterInstrumentedTest.kt

@@ -29,7 +29,16 @@ class SpeechModelLocalImporterInstrumentedTest {
             )
             val progress = mutableListOf<SpeechModelImportProgress>()
 
-            val installed = SpeechModelLocalImporter(context.contentResolver, repository)
+            val installed = SpeechModelLocalImporter(
+                repository = repository,
+                requiredDocuments = { _, _ ->
+                    mapOf(
+                        "recording.wav" to DocumentsContract.buildDocumentUriUsingTree(tree, "wav").toString(),
+                        "recording.m4a" to DocumentsContract.buildDocumentUriUsingTree(tree, "m4a").toString(),
+                    )
+                },
+                openInputStream = { context.contentResolver.openInputStream(Uri.parse(it)) },
+            )
                 .import(tree.toString()) { progress += it }
                 .getOrThrow()
 

app/src/androidTest/java/me/maxistar/voiceinbox/TranscriptionWorkerInstrumentedTest.kt

@@ -241,7 +241,10 @@ class TranscriptionWorkerInstrumentedTest {
 
     private fun requireModel() {
         assumeTrue(
-            SpeechModelRepository(targetContext.noBackupFilesDir.resolve("models")).inspect()
+            SpeechModelRepository(
+                targetContext.noBackupFilesDir.resolve("models"),
+                SpeechModelCatalog.defaultModel.manifest,
+            ).inspect()
                 is InstalledSpeechModelState.Ready,
         )
     }

app/src/main/java/me/maxistar/voiceinbox/AndroidMainScreenStateHost.kt

@@ -69,6 +69,7 @@ data class AndroidMainScreenState(
     val importEnabled: Boolean,
     val folderSync: AndroidFolderSyncPresentation,
     val onboardingHint: AndroidOnboardingHintPresentation,
+    val transcriptionActive: Boolean,
 ) {
     val refreshFolderVisible: Boolean get() = folderSync.visible
     val refreshFolderEnabled: Boolean get() = folderSync.enabled
@@ -120,6 +121,7 @@ object AndroidTaskListSnapshotMapper {
             entriesById = input.entries.associateBy(AudioCatalogEntry::id),
             importEnabled = input.importEnabled,
             folderSync = input.folderSync,
+            transcriptionActive = input.transcription.active,
             onboardingHint = AndroidOnboardingHintPresenter.present(
                 lifecycle = input.onboardingLifecycle,
                 filter = input.filter,
@@ -137,6 +139,10 @@ object AndroidTaskListSnapshotMapper {
 
 }
 
+internal object AndroidTaskListAnimationPolicy {
+    fun suppressStructuralAnimations(transcriptionActive: Boolean): Boolean = transcriptionActive
+}
+
 class AndroidMainScreenStateHost(
     private val savedStateHandle: SavedStateHandle,
 ) : ViewModel() {