sherpa: P2P lexicon generation for espeak-orphaned models (#66)

.github/workflows/sherpaonnx-live.yml

@@ -44,14 +44,15 @@ jobs:
         uses: actions/cache@v4
         with:
           path: |
-            ~/.rust-tts-wrapper/sherpaonnx/piper-nl_BE-rdh-low
+            ~/.rust-tts-wrapper/sherpaonnx/piper-en_US-amy-low
             ~/.rust-tts-wrapper/sherpaonnx/icefall-en-ljspeech
             ~/.rust-tts-wrapper/sherpaonnx/kokoro-zh_en-int8
             ~/.rust-tts-wrapper/sherpaonnx/supertonic-3-multilingual
             ~/.rust-tts-wrapper/sherpaonnx/zipvoice-zh_en-emilia-distill-int8
             ~/.rust-tts-wrapper/sherpaonnx/kyutai-en-pocket-tts-int8
             ~/.rust-tts-wrapper/sherpaonnx/hifigan_v2.onnx
             ~/.rust-tts-wrapper/sherpaonnx/vocos_24khz.onnx
+            ~/.voicegarden/lexicons/eng-US
           key: sherpa-models-${{ matrix.os }}-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: |
             sherpa-models-${{ matrix.os }}-
@@ -76,23 +77,45 @@ jobs:
           echo "MODEL_DIR=$HOME/.rust-tts-wrapper/sherpaonnx" >> $GITHUB_ENV
           echo "$GITHUB_WORKSPACE/scripts" >> $GITHUB_PATH
 
-      - name: Download VITS test model (piper-nl_BE-rdh-low)
+      - name: Download VITS test model (piper-en_US-amy-low)
+        # English piper: espeak left sherpa-onnx 2.x, so this voice
+        # speaks via the auto-generated lexicon from the voicegarden
+        # eng-US bundle (downloaded in the next step).
         shell: bash
         run: |
           set -e
           mkdir -p "$MODEL_DIR"
           cd "$MODEL_DIR"
-          if [ -d "piper-nl_BE-rdh-low" ] && [ -n "$(ls -A piper-nl_BE-rdh-low 2>/dev/null)" ]; then
-            echo "piper-nl_BE-rdh-low already present, skipping download"
+          if [ -d "piper-en_US-amy-low" ] && [ -n "$(ls -A piper-en_US-amy-low 2>/dev/null)" ]; then
+            echo "piper-en_US-amy-low already present, skipping download"
             exit 0
           fi
-          URL=$(get-model-url.py piper-nl_BE-rdh-low)
+          URL=$(get-model-url.py piper-en_US-amy-low)
           echo "Downloading $URL"
           curl -fsSL -o piper.tar.bz2 "$URL"
-          mkdir -p piper-nl_BE-rdh-low
-          tar -xjf piper.tar.bz2 -C piper-nl_BE-rdh-low --strip-components=1
+          mkdir -p piper-en_US-amy-low
+          tar -xjf piper.tar.bz2 -C piper-en_US-amy-low --strip-components=1
           rm piper.tar.bz2
-          ls -la piper-nl_BE-rdh-low/
+          ls -la piper-en_US-amy-low/
+
+      - name: Download voicegarden eng-US lexicon assets
+        # gruut lexicon + P2P convention pairs — the engine
+        # auto-generates lexicon.txt for espeak-orphaned English models
+        # (piper, matcha-en, kokoro-en, kitten) from these (#66/#67).
+        shell: bash
+        run: |
+          set -e
+          DEST="$HOME/.voicegarden/lexicons/eng-US"
+          if [ -f "$DEST/lexicon.txt" ] && [ -f "$DEST/p2p-espeak.tsv" ]; then
+            echo "voicegarden eng-US assets already present"
+            exit 0
+          fi
+          mkdir -p "$DEST"
+          curl -fsSL -o eng-US.tar.gz \
+            https://github.com/AACTools/voicegarden-lexicons/releases/latest/download/eng-US.tar.gz
+          tar -xzf eng-US.tar.gz -C "$DEST"
+          rm eng-US.tar.gz
+          ls -la "$DEST"
 
       - name: Download Matcha test model (icefall-en-ljspeech)
         shell: bash
@@ -222,6 +245,23 @@ jobs:
           rm pocket.tar.bz2
           ls -la kyutai-en-pocket-tts-int8/ | head -5
 
+      - name: Ensure sherpa-onnx prebuilt libs are complete
+        # sherpa-onnx-sys caches its downloaded static libs under
+        # target/sherpa-onnx-prebuilt and SKIPS the download when the
+        # directory exists. A partially-restored rust-cache then fails
+        # at link time ("could not find native static library
+        # sherpa-onnx-c-api") on every rerun. Delete the cache when it
+        # lacks the lib so the build script re-downloads.
+        shell: bash
+        run: |
+          DIR="target/sherpa-onnx-prebuilt"
+          if [ -d "$DIR" ] && [ -z "$(find "$DIR" -name 'libsherpa-onnx-c-api.a' -o -name 'sherpa-onnx-c-api.lib' 2>/dev/null | head -1)" ]; then
+            echo "cached sherpa-onnx prebuilt libs incomplete — forcing re-download"
+            rm -rf "$DIR"
+          else
+            echo "sherpa-onnx prebuilt cache OK (or absent)"
+          fi
+
       - name: Run live SherpaOnnx tests
         shell: bash
         run: |

Cargo.toml

@@ -207,3 +207,18 @@ strip = true
 all = "warn"
 pedantic = "warn"
 cargo = "warn"
+
+[[example]]
+name = "lexicon-convert"
+path = "examples/lexicon_convert.rs"
+required-features = ["sherpaonnx"]
+
+[[example]]
+name = "kokoro-ab"
+path = "examples/kokoro_ab.rs"
+required-features = ["sherpaonnx"]
+
+[[example]]
+name = "sherpa-debug"
+path = "examples/sherpa_debug.rs"
+required-features = ["sherpaonnx"]

SHERPA2_TODO.md

@@ -122,31 +122,169 @@ Progress:
       not this PR's scope.
 - [x] CI green on the loop-2 commit (28/28) → READY TO MERGE
 
-## Phase 2 — lexicon generation PR (tracking issue #66)
+## Phase 2 — lexicon generation PR (tracking issue #66) — PR #67 OPEN
+
+Branch: feat/sherpa-lexicons (off main, after #65 merged).
+
+**STATUS: misaki half done. PR #67 open after 4 review loops (6→6→7→8/10,
+merge). Engine auto-generation validated end-to-end on kokoro-en.
+floravox side pushed as feat/p2p-transducer (PhonemeTransducer hybrid +
+trainer --p2p + prepare_p2p_data.py).**
 
 Scope locked by Phase 1 results: the ONLY viable front-end for
 espeak-orphaned families (piper 536 voices, matcha-en, kokoro-en,
 kitten) on 2.x is a per-model lexicon.txt.
 
-- [ ] voicegarden-lexicons (already an optional dep!) → per-model
-      lexicon.txt generator:
-      - read the model's tokens.txt vocabulary
-      - map gruut/CMUDict source phonemes onto in-vocab symbols
-        (kokoro's misaki conventions ≠ piper's espeak conventions —
-        the mapping is per-family)
-      - OOV via the Phonetisaurus FST
-      - cache beside the model dir, named lexicon.txt (auto-picked-up
-        by the collectors wired in Phase 1)
-- [ ] Phoneme-convention mapping audit (gruut → kokoro tokens /
-      piper tokens / kitten tokens)
-- [ ] A/B: espeak-1.13 output vs lexicon-2.x output, whisper
-      round-trip comparison per family
-- [ ] Un-guard the 12 orphaned live tests (8 with reason strings
-      + 4 bare-ignore: sherpa_streams, speechmarkdown_passthrough,
-      stop_safety, pitch_shift) — search `is_sherpa2_orphan` to find
-      them all; close #66
+### Phase 2 findings (gruut vs model conventions — the hard part)
+
+The gruut→vocab symbol mapping is SOLVED (all 40 gruut IPA symbols map
+or decompose into piper's 130-token vocab; 100% coverage, zero skips,
+124k entries generated). But **symbol mapping ≠ phonological equivalence**:
+each model family was trained on a different phonemizer's output and
+expects that phonemizer's transcription CONVENTIONS:
+
+| family | trained on | gruut matches? | evidence |
+|---|---|---|---|
+| kokoro-en | misaki | ❌ different | kokoro multi-lang lexicon uses uppercase `O` `A` `I` as internal phoneme symbols; gruut says `oʊ` `eɪ` — same sound, different encoding |
+| piper | espeak-ng | ⚠️ close but wrong | gruut lexicon → audio but garbled ("No, this is Aster"); gruut marks stress on monosyllables, uses `ɛ` where espeak uses `ə` in unstressed syllables, etc. |
+| matcha-en | espeak-ng (same as piper) | ⚠️ same issues | |
+| kitten | misaki (like kokoro) | ❌ | |
+
+**Key discovery: the kokoro multi-lang release ships `lexicon-us-en.txt`
+and `lexicon-gb-en.txt` that WORK for kokoro-en** (proven in Phase 1).
+That file uses the misaki convention and matches kokoro-en's tokens.
+
+### Revised Phase 2 approach: per-family lexicon source
+- [ ] kokoro-en / kitten: use the kokoro multi-lang's bundled
+      lexicon-us-en.txt (already proven to work — just document the
+      download path; or ship a copy in voicegarden-lexicons)
+- [ ] piper / matcha-en: gruut→espeak phonological convention mapping
+      (not just symbol mapping — stress placement, unstressed vowel
+      reduction, rhotic unwinding). This is real linguistic work:
+      - espeak reduces unstressed vowels to schwa more aggressively
+      - espeak doesn't stress monosyllabic function words
+      - espeak represents some diphthongs differently
+      - Alternative: find/generate an espeak-compatible English lexicon
+        (CMUdict with espeak-style phoneme labels, or phonemize the
+        gruut lexicon through espeak once and cache)
+- [ ] A/B: espeak-1.13 output vs lexicon-2.x output, whisper round-trip
+- [ ] Un-guard the 12 orphaned live tests; close #66
 - [ ] PR + loop-review + CI green
 
+### Code written so far (sherpa_lexicon.rs + sherpa_convention.rs + sherpa_p2p.rs, 23 tests green)
+- PhonemeMap::for_english() — builds gruut→target symbol mapping
+- generate_lexicon_txt() — gruut lexicon → sherpa lexicon.txt format
+- Convention::Misaki / Convention::Espeak — rule-based convention
+  conversion: symbol substitution + function-word destressing + rhotic
+  splitting + unstressed vowel reduction
+- convert_lexicon_file() — layered generation: P2P table → rules →
+  vocab filter, returns (written, skipped, via_table)
+- P2pTable (sherpa_p2p.rs) — aligned-pairs TSV → exact lookup
+- ensure_generated_lexicon() + discover_model_assets() — engine-side
+  auto-generation with convention detection from tokens.txt
+
+### Phase 2 hard finding: word-specific phonological differences
+
+The rules-based conversion handles the SYMBOL layer and the systematic
+PHONOLOGICAL layer, but a long tail of word-specific differences makes
+the output garbled in practice:
+
+| word | gruut (after rules) | misaki (correct) | issue |
+|---|---|---|---|
+| hello | `h ə l ˈ O` | `h ə l ˈ O` | ✓ identical |
+| world | `w ˈ ɜ ɹ l d` | `w ˈ ɜ ɹ l d` | ✓ identical |
+| go | `ɡ ˈ O` | `ɡ ˌ O` | stress level: primary vs secondary |
+| the | `ð ə` | `ð i` | vowel quality: schwa vs /i/ |
+| a | `ə` | `A` | strong form (letter name) vs weak form |
+| converted | `k ə n v ˈ ɜ ɹ t ɪ d` | `k ə n v ˈ ɜ ɹ ɾ ᵻ d` | flap + barred-i allophones |
+
+Most words (~90% of the tested 100) produce IDENTICAL output after the
+rules. The ~10% that differ are enough to garble Whisper transcription.
+
+### P2P transducer experiment (Phonetisaurus phoneme→phoneme)
+
+**Architecture**: train a Phonetisaurus WFST on aligned phoneme pairs
+(gruut → kokoro bundled), then at runtime: gruut phonemize →
+transduce → model-ready phonemes. Single G2P source, per-convention
+transducer, no per-model bundles. This is the RIGHT architecture.
+
+**What we built**:
+- `--p2p` mode in floravox-train-phonetisaurus (atomic pipe-separated units,
+  identity-pair boost ×50 after co-occurrence counting)
+- `PhonetisaurusG2p::transduce()` method (token-level search, same Dijkstra)
+- 49k aligned pairs from gruut↔kokoro overlap, model trains in seconds
+- `PhonemeTransducer` (floravox-g2p `p2p` module): direct-lookup HashMap +
+  WFST fallback hybrid — lookup is exact for all in-vocabulary words
+
+**What works**:
+- Tiny model (10 examples): PERFECT transduction
+- Symbol coverage: 100%, zero vocabulary violations
+- The training + decode pipeline is sound
+- The **lookup layer** of the hybrid: 100% exact on in-vocabulary words
+
+**The WFST fallback limitation (accepted)**:
+- Large models (49k pairs, 100k states): the Dijkstra decode takes
+  epsilon-heavy paths through n-gram backoff arcs and drops vowels
+  systematically; epsilon penalties (up to 100.0) and epsilon-closure
+  both fail to fix it. The model structure — not the search — lacks
+  reachable output arcs for those vowels.
+- Root cause: M2M alignment optimises corpus likelihood, decomposing
+  compound phonemes across the corpus; the n-gram WFST then generalises
+  that decomposition.
+- Pragmatic resolution: **hybrid** — direct HashMap lookup for
+  in-vocabulary words (100% correct), WFST fallback only for OOV
+  sequences (approximate). For lexicon GENERATION the rules layer is
+  the OOV fallback instead (see below); the WFST matters only for
+  runtime phonemization of unseen words, where approximation is
+  acceptable.
+
+### Phase 2 SOLUTION: P2P lookup table + rules fallback (VALIDATED)
+
+`src/sherpa_p2p.rs` — std-only `P2pTable`: aligned-pairs TSV
+(`gruut|ph1|ph2\ttgt1 tgt2`) → HashMap. Layered into
+`convert_lexicon_file` as: table (exact, word-specific) → rules
+(systematic) → vocab filter.
+
+**Validation against the kokoro-en ground truth** (the model's own
+bundled `lexicon-us-en.txt`, aligned via
+`prepare_p2p_data.py`):
+- 133,637 entries generated from the gruut lexicon, **0 skipped**
+  (100% vocab coverage), 53,213 via table / 80,424 via rules
+- All six garbling-causing words from the rules-only experiment now
+  match ground truth EXACTLY: `go ɡ ˌ O` (stress level), `the ð i`
+  (weak form), `a A` (strong form), `converted ... ɾ ᵻ d` (allophones),
+  `hello`, `world`
+- 88.8% of the 53,146 overlap words are byte-identical to ground truth;
+  the residual 11.2% differs only in sub-phonemic allophone choices
+  (`ə` vs `ᵊ`, plain vs flapped `t`/`d` — both schwa/flap class, no
+  garbling) and homophone collisions inherent to phoneme-sequence
+  keying (first-wins, stable)
+
+**Engine wiring done**: `sherpaonnx_engine` calls
+`ensure_generated_lexicon` at model load — when an espeak-orphaned
+model ships no lexicon and (gruut lexicon + aligned pairs) are locally
+available, one is generated into the model dir. Convention is detected
+from the model's own tokens.txt (uppercase `A I O W Y` ⇒ misaki).
+Asset discovery: `SHERPA_GRUUT_LEXICON`/`SHERPA_P2P_PAIRS` env vars →
+pairs beside the model or gruut lexicon → voicegarden cache. Pairs are
+REQUIRED (rules alone garble ~10% of words); no pairs → no generation.
+A/B wavs generated (`examples/kokoro-ab`).
+
+### Phase 2 remaining
+- [ ] PR #67 review + merge; then un-guard the 12 orphaned live tests
+      (search `is_sherpa2_orphan`); close #66
+- [ ] Ship the aligned pairs as a voicegarden-lexicons release asset
+      (`p2p-misaki.tsv` alongside `lexicon.txt` in eng-US) so
+      auto-generation works out of the box
+- [ ] piper / matcha-en (espeak convention): need an espeak-convention
+      reference lexicon to align against (CMUdict is BSD; generate
+      `p2p-espeak.tsv` from gruut↔CMUdict-IPA alignment)
+- [ ] floravox PR: review feat/p2p-transducer, publish 0.8.10+ with
+      the `p2p` module (PhonemeTransducer) so the WFST fallback is
+      available downstream
+- [ ] A/B: whisper round-trip comparison per family (wavs generated,
+      whisper not installed locally — CI or manual listen)
+
 ## Phase 3 — Hume cloud engine (separate quiet PR)
 - [ ] cloud_engine: hume provider following cartesia/deepgram patterns
 - [ ] Mocked-HTTP tests; PR + loop-review + CI green

examples/kokoro_ab.rs

@@ -0,0 +1,77 @@
+// A/B: synthesize the same sentence with whatever lexicon.txt currently
+// sits in the model dir; save a wav. Swap lexicons between runs to compare.
+// usage: cargo run --features sherpaonnx --example kokoro-ab -- <model-id> <out.wav> [sentence]
+fn main() {
+    use rust_tts_wrapper::create_engine;
+    use std::sync::{Arc, Mutex};
+
+    let model_id = std::env::args()
+        .nth(1)
+        .expect("model id, e.g. kokoro-en-v0_19");
+    let out_path = std::env::args()
+        .nth(2)
+        .unwrap_or_else(|| "/tmp/kokoro_ab.wav".into());
+    let sentence = std::env::args()
+        .nth(3)
+        .unwrap_or_else(|| "Hello world, this is a test of the converted lexicon.".into());
+
+    let creds = format!(r#"{{"modelId":"{model_id}"}}"#);
+    let engine = create_engine("sherpaonnx", &creds).expect("engine");
+    let audio = Arc::new(Mutex::new(Vec::<u8>::new()));
+    let a2 = audio.clone();
+    let mut cb = move |c: &[u8]| a2.lock().unwrap().extend_from_slice(c);
+    engine
+        .speak(&sentence, None, 1.0, 1.0, 1.0, Some(&mut cb), None, None)
+        .expect("speak");
+    let data = audio.lock().unwrap().clone();
+
+    // model sample rate from its .onnx.json config (`audio.sample_rate`)
+    let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
+    let model_dir = std::path::Path::new(&home)
+        .join(".rust-tts-wrapper")
+        .join("sherpaonnx")
+        .join(&model_id);
+    let sample_rate = std::fs::read_dir(&model_dir)
+        .ok()
+        .and_then(|entries| {
+            entries.filter_map(Result::ok).map(|e| e.path()).find(|p| {
+                p.extension().is_some_and(|e| e == "json")
+                    && p.file_name()
+                        .and_then(|n| n.to_str())
+                        .is_some_and(|n| n.ends_with(".onnx.json"))
+            })
+        })
+        .and_then(|cfg| std::fs::read_to_string(cfg).ok())
+        .and_then(|text| {
+            text.split("\"sample_rate\"")
+                .nth(1)?
+                .split(':')
+                .nth(1)?
+                .split(|c: char| !c.is_ascii_digit())
+                .find(|s| !s.is_empty())
+                .and_then(|s| s.parse::<u32>().ok())
+        })
+        .unwrap_or(24000);
+
+    // the callback emits i16le mono PCM — wrap in a matching WAV header
+    let mut wav: Vec<u8> = Vec::with_capacity(44 + data.len());
+    let n = u32::try_from(data.len()).expect("wav under 4 GiB");
+    wav.extend_from_slice(b"RIFF");
+    wav.extend_from_slice(&(36 + n).to_le_bytes());
+    wav.extend_from_slice(b"WAVEfmt ");
+    wav.extend_from_slice(&16u32.to_le_bytes());
+    wav.extend_from_slice(&1u16.to_le_bytes()); // PCM16
+    wav.extend_from_slice(&1u16.to_le_bytes()); // mono
+    wav.extend_from_slice(&sample_rate.to_le_bytes());
+    wav.extend_from_slice(&(sample_rate * 2).to_le_bytes()); // byte rate
+    wav.extend_from_slice(&2u16.to_le_bytes()); // block align
+    wav.extend_from_slice(&16u16.to_le_bytes()); // bits
+    wav.extend_from_slice(b"data");
+    wav.extend_from_slice(&n.to_le_bytes());
+    wav.extend_from_slice(&data);
+    std::fs::write(&out_path, wav).unwrap();
+    println!(
+        "wrote {out_path} ({} bytes pcm @ {sample_rate} Hz)",
+        data.len()
+    );
+}