Speed up voxtral_realtime streaming and stop EOS from ending a stream

CMakeLists.txt

@@ -875,6 +875,14 @@ if (ENGINE_BUILD_TESTS)
         COMMAND audio_chunking_test
     )
 
+    add_engine_unittest(voxtral_realtime_stream_chunking_test tests/unittests/test_voxtral_realtime_stream_chunking.cpp)
+    target_include_directories(voxtral_realtime_stream_chunking_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests)
+
+    add_test(
+        NAME voxtral_realtime_stream_chunking_test
+        COMMAND voxtral_realtime_stream_chunking_test
+    )
+
     add_engine_unittest(chinese_normalization_test tests/unittests/test_chinese_normalization.cpp)
 
     add_test(

docs/asr.md

@@ -286,6 +286,31 @@ Streaming CLI:
 audiocpp_cli --task asr --family voxtral_realtime --model models/Voxtral-Mini-4B-Realtime-2602-GGUF/voxtral-mini-4b-realtime-2602-q8_0.gguf --backend cuda --threads 8 --mode streaming --audio assets/resources/sample.wav --text-out transcript.txt
 ```
 
+> **Throughput.** A streaming step always advances 80 ms of audio, so a step has to cost under
+> 80 ms to keep up with a realtime source. Measured on an Apple M3 Air (Metal, q8_0):
+>
+> | Config | short clip, cool | sustained 7 min |
+> |---|---:|---:|
+> | default | 78 ms/step (0.98x) | 88 ms/step (1.10x) |
+> | `stream_batch_tokens=4` | 60 ms/step (0.76x) | 74 ms/step (0.92x) |
+>
+> The default splits roughly 48 ms for the text decoder and 30 ms for the audio encoder; batching
+> takes the encoder to ~13 ms. The second column is what a long session actually gets on a fanless
+> machine: a short clip run immediately after the 7-minute one still measured 88 ms/step, so the
+> gap is the machine staying warm rather than anything that resets between sessions. Budget for the
+> sustained column, and prefer `stream_batch_tokens=4` if the source is realtime.
+>
+> The decoder runs one step per 80 ms whether the audio holds speech or silence, so a session that
+> does fall behind stays behind — the lag is monotonic and does not recover during pauses. Measure
+> your own hardware before relying on a live source.
+
+Streaming session options:
+
+| Option | Default | Meaning |
+|---|---:|---|
+| `--session-option voxtral_realtime.stream_batch_tokens=<n>` | `1` | Audio tokens per encoder forward. The decoder still runs one step per 80 ms; batching only amortizes the encoder's fixed per-forward cost, which dominates it. `4` takes the encoder from ~30 to ~13 ms/step, at the price of delaying every partial by up to `n * 80 ms`. |
+| `--session-option voxtral_realtime.stream_decode_cache_steps=<n>` | `1024` | Decoder KV cache size in 80 ms steps (~82 s of context). Built once when the stream starts, so a long session never stalls on a cache-growth rebuild; the cache ring then wraps in place, and a 7-minute stream stays coherent across five wraparounds. Lower values trade context for memory, not for speed. |
+
 Streaming server config:
 
 ```json
@@ -328,11 +353,16 @@ curl -N http://127.0.0.1:8080/v1/audio/transcriptions \
 | `--top-k` | integer | `50` | Top-k sampling limit; `0` disables top-k. |
 | `--seed` | integer | `1234` | Sampling seed. |
 | `--text-out` | TXT path | not set | Transcript output. The transcript is also printed to stdout. |
-| `--session-option voxtral_realtime.weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Shared matmul weight storage type. |
-| `--session-option voxtral_realtime.audio_encoder_weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q8_0` | shared setting | Audio encoder matmul weight storage type. |
-| `--session-option voxtral_realtime.text_decoder_weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q8_0` | shared setting | Text decoder matmul weight storage type. |
+| `--session-option voxtral_realtime.weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q4_0`, `q4_k`, `q5_k`, `q6_k`, `q8_0` | `native` | Shared matmul weight storage type. |
+| `--session-option voxtral_realtime.audio_encoder_weight_type=<type>` | same as above | shared setting | Audio encoder matmul weight storage type. Leave at `native` for streaming: the encoder is not bandwidth-bound there, so quantizing it makes it slower. |
+| `--session-option voxtral_realtime.text_decoder_weight_type=<type>` | same as above | shared setting | Text decoder matmul weight storage type. `q4_k` roughly halves the streaming decoder step cost. |
 | `--session-option voxtral_realtime.audio_encoder_graph_arena_mb=<n>` | MB | `512` | Audio encoder graph arena size. |
 | `--session-option voxtral_realtime.text_decoder_prefill_graph_arena_mb=<n>` | MB | `512` | Text decoder prefill graph arena size. |
 | `--session-option voxtral_realtime.text_decoder_decode_graph_arena_mb=<n>` | MB | `512` | Text decoder cached-step graph arena size. |
 
+Weight storage types are applied when the model loads, so asking for one the GGUF does not already
+hold means requantizing on the CPU before the first token appears — around three minutes for
+`q4_k` from the shipped q8_0 package. Prefer the published `q4_k` GGUF variant, which needs no
+load-time conversion. See [GGUF](gguf.md).
+
 For backend weight-type controls, use `audiocpp_cli --inspect --model <model-dir> --family <family>`.

include/engine/models/voxtral_realtime/audio_encoder.h

@@ -27,6 +27,8 @@ class VoxtralRealtimeAudioEncoderRuntime {
     VoxtralRealtimeAudioEmbeddings encode_stream_chunk(
         const VoxtralRealtimeFeatures & features,
         VoxtralRealtimeAudioEncoderStreamState & state);
+    // Emits the summed upload/compute/readback split of every streaming chunk seen so far.
+    void log_stream_timings() const;
 
 private:
     std::unique_ptr<Impl> impl_;

include/engine/models/voxtral_realtime/frontend.h

@@ -21,15 +21,18 @@ class VoxtralRealtimeFrontend {
     explicit VoxtralRealtimeFrontend(std::shared_ptr<const VoxtralRealtimeAssets> assets);
 
     VoxtralRealtimeFeatures extract(const runtime::AudioBuffer & audio, bool first_chunk) const;
+    // A steady chunk carries `steady_tokens` audio tokens of 80 ms each; batching them into one
+    // extraction yields 8 * steady_tokens feature frames from a single STFT pass.
     VoxtralRealtimeFeatures extract_stream_chunk(
         const runtime::AudioBuffer & audio,
         bool first_chunk,
-        VoxtralRealtimeFrontendStreamState & state) const;
+        VoxtralRealtimeFrontendStreamState & state,
+        int64_t steady_tokens = 1) const;
 
     int64_t first_stream_chunk_samples() const;
-    int64_t steady_stream_chunk_samples() const;
+    int64_t steady_stream_chunk_samples(int64_t steady_tokens = 1) const;
     int64_t first_stream_chunk_advance_samples() const;
-    int64_t steady_stream_chunk_advance_samples() const;
+    int64_t steady_stream_chunk_advance_samples(int64_t steady_tokens = 1) const;
 
 private:
     std::shared_ptr<const VoxtralRealtimeAssets> assets_;

include/engine/models/voxtral_realtime/session.h

@@ -44,6 +44,9 @@ class VoxtralRealtimeSession final
     runtime::TaskResult run_single(const VoxtralRealtimeRequest & request, bool first_chunk);
     runtime::StreamEvent process_available_stream_chunks();
     runtime::StreamEvent process_one_stream_chunk(const runtime::AudioBuffer & audio);
+    // Feeds a freshly decoded token back as the next stream input and, when it carries text,
+    // appends it to the transcript and refreshes the event's partial.
+    void record_stream_token(int32_t token, runtime::StreamEvent & event);
 
     runtime::TaskSpec task_;
     std::shared_ptr<const VoxtralRealtimeAssets> assets_;
@@ -54,6 +57,8 @@ class VoxtralRealtimeSession final
     size_t text_decoder_weight_context_bytes_ = 128ull * 1024ull * 1024ull;
     assets::TensorStorageType audio_encoder_weight_storage_type_ = assets::TensorStorageType::Native;
     assets::TensorStorageType text_decoder_weight_storage_type_ = assets::TensorStorageType::Native;
+    int64_t stream_decode_cache_steps_ = 1024;
+    int64_t stream_batch_tokens_ = 1;
     VoxtralRealtimeTokenizer tokenizer_;
     VoxtralRealtimeFrontend frontend_;
     VoxtralRealtimeAudioEncoderRuntime audio_encoder_;
@@ -62,6 +67,10 @@ class VoxtralRealtimeSession final
     runtime::AudioBuffer streaming_audio_;
     size_t streaming_audio_offset_values_ = 0;
     int64_t streaming_steps_processed_ = 0;
+    // Per-stage wall time summed over every steady step, reported by finalize().
+    double stream_frontend_ms_ = 0.0;
+    double stream_encoder_ms_ = 0.0;
+    double stream_decoder_ms_ = 0.0;
     VoxtralRealtimeGenerationOptions streaming_generation_;
     runtime::StreamEventCallback stream_event_sink_;
     VoxtralRealtimeFrontendStreamState frontend_stream_state_;
@@ -71,7 +80,6 @@ class VoxtralRealtimeSession final
     bool stream_started_ = false;
     bool first_stream_chunk_ = true;
     bool have_previous_stream_token_ = false;
-    bool stream_reached_eos_ = false;
     std::chrono::steady_clock::time_point stream_wall_start_{};
 };
 

include/engine/models/voxtral_realtime/text_decoder.h

@@ -20,7 +20,8 @@ class VoxtralRealtimeTextDecoderRuntime {
         size_t prefill_graph_arena_bytes,
         size_t decode_graph_arena_bytes,
         size_t weight_context_bytes,
-        assets::TensorStorageType weight_storage_type);
+        assets::TensorStorageType weight_storage_type,
+        int64_t stream_decode_cache_steps);
     ~VoxtralRealtimeTextDecoderRuntime();
 
     VoxtralRealtimeGeneratedTokens generate(
@@ -31,12 +32,17 @@ class VoxtralRealtimeTextDecoderRuntime {
         const VoxtralRealtimePrompt & prompt,
         const VoxtralRealtimeAudioEmbeddings & audio_embeddings,
         const VoxtralRealtimeGenerationOptions & options);
+    // `audio_row` selects one row of a possibly batched encoder pass; each call still advances
+    // the decoder by exactly one 80 ms step.
     int32_t stream_step(
         int32_t previous_token,
         const VoxtralRealtimeAudioEmbeddings & audio_embeddings,
+        int64_t audio_row,
         int64_t num_delay_tokens,
         const VoxtralRealtimeGenerationOptions & options);
     bool is_eos(int32_t token) const;
+    // Emits the summed upload/compute/readback/sample split of every streaming decode step.
+    void log_stream_timings() const;
 
 private:
     std::unique_ptr<Impl> impl_;

src/models/voxtral_realtime/audio_encoder.cpp

@@ -93,10 +93,16 @@ assets::TensorStorageType normalize_weight_storage(assets::TensorStorageType sto
         case assets::TensorStorageType::F16:
         case assets::TensorStorageType::BF16:
         case assets::TensorStorageType::Q8_0:
+        // Requantized from the source GGUF at load time. Tensors whose rows are not a whole
+        // number of blocks (the 3-wide conv kernels) fall back to F32 in the weight store.
+        case assets::TensorStorageType::Q4_0:
+        case assets::TensorStorageType::Q4_K:
+        case assets::TensorStorageType::Q5_K:
+        case assets::TensorStorageType::Q6_K:
             return storage_type;
         default:
             throw std::runtime_error(
-                "VoxTral audio_encoder_weight_type supports only native, f32, f16, bf16, and q8_0");
+                "VoxTral audio_encoder_weight_type supports only native, f32, f16, bf16, q4_0, q4_k, q5_k, q6_k, and q8_0");
     }
 }
 
@@ -219,8 +225,9 @@ core::TensorValue set_audio_kv_rows(
     const core::TensorValue & row_indices) {
     core::validate_rank_between(cache, 4, 4, "cache");
     core::validate_rank_between(rows, 4, 4, "rows");
-    if (cache.type != GGML_TYPE_F32 || rows.type != GGML_TYPE_F32) {
-        throw std::runtime_error("VoxTral audio streaming cache update requires f32 tensors");
+    // ggml_set_rows converts on write, so an f16 cache still takes f32 rows straight from RoPE.
+    if ((cache.type != GGML_TYPE_F32 && cache.type != GGML_TYPE_F16) || rows.type != GGML_TYPE_F32) {
+        throw std::runtime_error("VoxTral audio streaming cache update requires f32 rows and an f32 or f16 cache");
     }
     if (row_indices.type != GGML_TYPE_I32 || row_indices.shape.rank != 1 ||
         row_indices.shape.dims[0] != rows.shape.dims[1]) {
@@ -277,17 +284,18 @@ core::TensorValue build_audio_attention_streaming_static(
     q = rope.build(ctx, q, positions);
     k = rope.build(ctx, k, positions);
     q = modules::TransposeModule({{0, 2, 1, 3}, q.shape.rank}).build(ctx, q);
+    q = core::ensure_backend_addressable_layout(ctx, q);
     k = core::ensure_backend_addressable_layout(ctx, k);
     v = core::ensure_backend_addressable_layout(ctx, v);
     auto updated_key = set_audio_kv_rows(ctx, key_cache, k, cache_slots);
     auto updated_value = set_audio_kv_rows(ctx, value_cache, v, cache_slots);
+    // Feed the sliding-window cache to flash attention as a strided view. Materializing the
+    // transpose would copy the full 750-step K and V caches once per layer per 80 ms chunk.
     auto k_heads = modules::TransposeModule({{0, 2, 1, 3}, updated_key.shape.rank}).build(ctx, updated_key);
     auto v_heads = modules::TransposeModule({{0, 2, 1, 3}, updated_value.shape.rank}).build(ctx, updated_value);
-    k_heads = core::wrap_tensor(ggml_cont(ctx.ggml, k_heads.tensor), k_heads.shape, k_heads.type);
-    v_heads = core::wrap_tensor(ggml_cont(ctx.ggml, v_heads.tensor), v_heads.shape, v_heads.type);
     auto context = modules::ScaledDotProductAttentionModule({
         config.head_dim,
-        modules::ScaledDotProductAttentionLowering::Flash,
+        modules::ScaledDotProductAttentionLowering::FlashPreserveViews,
         GGML_PREC_F32,
     }).build(ctx, q, k_heads, v_heads, attention_mask);
     context = core::ensure_backend_addressable_layout(ctx, context);
@@ -502,27 +510,30 @@ class StreamingAudioCache {
         conv2_cache_ = ggml_new_tensor_3d(ctx_.get(), GGML_TYPE_F32, 1, config.audio.hidden_size, 1);
         key_caches_.reserve(static_cast<size_t>(config.audio.num_hidden_layers));
         value_caches_.reserve(static_cast<size_t>(config.audio.num_hidden_layers));
+        // The sliding-window cache is read in full by flash attention on every 80 ms chunk and is
+        // never exported, so it is held in F16: half the read traffic, and the type Metal's flash
+        // attention kernels prefer.
         for (int64_t layer = 0; layer < config.audio.num_hidden_layers; ++layer) {
             key_caches_.push_back(core::wrap_tensor(
                 ggml_new_tensor_4d(
                     ctx_.get(),
-                    GGML_TYPE_F32,
+                    GGML_TYPE_F16,
                     config.audio.head_dim,
                     config.audio.num_key_value_heads,
                     cache_steps_,
                     1),
                 core::TensorShape::from_dims({1, cache_steps_, config.audio.num_key_value_heads, config.audio.head_dim}),
-                GGML_TYPE_F32));
+                GGML_TYPE_F16));
             value_caches_.push_back(core::wrap_tensor(
                 ggml_new_tensor_4d(
                     ctx_.get(),
-                    GGML_TYPE_F32,
+                    GGML_TYPE_F16,
                     config.audio.head_dim,
                     config.audio.num_key_value_heads,
                     cache_steps_,
                     1),
                 core::TensorShape::from_dims({1, cache_steps_, config.audio.num_key_value_heads, config.audio.head_dim}),
-                GGML_TYPE_F32));
+                GGML_TYPE_F16));
         }
         state_buffer_.reset(ggml_backend_alloc_ctx_tensors(ctx_.get(), backend_));
         if (state_buffer_ == nullptr) {
@@ -695,12 +706,17 @@ class StreamingAudioGraph {
                 .build(ctx, x, {weights_->projector2_weight, std::nullopt});
         output_ = x.tensor;
         ggml_set_output(output_);
-        ggml_set_output(next_conv1_cache_);
-        ggml_set_output(next_conv2_cache_);
         graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false);
+        // Expand the encoder body first so the conv-cache writes land after every node that reads
+        // the caches: the backend runs nodes in graph order, so this ordering is what keeps the
+        // in-graph update from clobbering the current chunk's own left context.
         ggml_build_forward_expand(graph_, output_);
-        ggml_build_forward_expand(graph_, next_conv1_cache_);
-        ggml_build_forward_expand(graph_, next_conv2_cache_);
+        ggml_build_forward_expand(
+            graph_,
+            ggml_cpy(ctx_.get(), next_conv1_cache_, cache_.conv1_cache()));
+        ggml_build_forward_expand(
+            graph_,
+            ggml_cpy(ctx_.get(), next_conv2_cache_, cache_.conv2_cache()));
         gallocr_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_)));
         if (gallocr_ == nullptr ||
             !ggml_gallocr_reserve(gallocr_.get(), graph_) ||
@@ -728,6 +744,7 @@ class StreamingAudioGraph {
         if (features.frames != feature_frames_ || features.mel_bins != config.frontend.feature_size) {
             throw std::runtime_error("VoxTral streaming audio encoder feature shape mismatch");
         }
+        const auto upload_start = Clock::now();
         auto input_value = core::wrap_tensor(input_, core::TensorShape::from_dims({1, config.frontend.feature_size, feature_frames_}), GGML_TYPE_F32);
         core::write_tensor_f32(input_value, features.values);
         if (state.seen_encoder_steps == 0) {
@@ -736,25 +753,36 @@ class StreamingAudioGraph {
         write_positions(state.seen_encoder_steps);
         write_cache_slots(state.seen_encoder_steps);
         write_mask(state.seen_encoder_steps);
+        upload_ms_ += engine::debug::elapsed_ms(upload_start);
         core::set_backend_threads(backend_, threads_);
+        const auto compute_start = Clock::now();
         const ggml_status status = core::compute_backend_graph(backend_, graph_);
         ggml_backend_synchronize(backend_);
+        compute_ms_ += engine::debug::elapsed_ms(compute_start);
         if (status != GGML_STATUS_SUCCESS) {
             throw std::runtime_error("VoxTral streaming audio encoder graph compute failed");
         }
         state.first_chunk = false;
         state.seen_encoder_steps += current_steps_;
         state.cached_encoder_steps = std::min<int64_t>(cache_steps_, state.cached_encoder_steps + current_steps_);
-        ggml_backend_tensor_copy_async(backend_, backend_, next_conv1_cache_, cache_.conv1_cache());
-        ggml_backend_tensor_copy_async(backend_, backend_, next_conv2_cache_, cache_.conv2_cache());
-        ggml_backend_synchronize(backend_);
         VoxtralRealtimeAudioEmbeddings out;
+        const auto readback_start = Clock::now();
         out.values = core::read_tensor_f32(output_);
+        readback_ms_ += engine::debug::elapsed_ms(readback_start);
         out.tokens = tokens_;
         out.hidden_size = config.hidden_size;
+        ++runs_;
         return out;
     }
 
+    void log_timings(const char * prefix) const {
+        const std::string base = std::string("voxtral_realtime.audio_encoder.stream.") + prefix;
+        engine::debug::timing_log_scalar(base + ".runs", runs_);
+        engine::debug::timing_log_scalar(base + ".upload_ms", upload_ms_);
+        engine::debug::timing_log_scalar(base + ".compute_ms", compute_ms_);
+        engine::debug::timing_log_scalar(base + ".readback_ms", readback_ms_);
+    }
+
 private:
     void write_positions(int64_t start) {
         std::vector<int32_t> positions(static_cast<size_t>(current_steps_));
@@ -810,6 +838,10 @@ class StreamingAudioGraph {
     ggml_tensor * output_ = nullptr;
     std::vector<ggml_fp16_t> mask_values_;
     ggml_cgraph * graph_ = nullptr;
+    int64_t runs_ = 0;
+    double upload_ms_ = 0.0;
+    double compute_ms_ = 0.0;
+    double readback_ms_ = 0.0;
 };
 
 }  // namespace
@@ -884,6 +916,15 @@ struct VoxtralRealtimeAudioEncoderRuntime::Impl {
         return graph_slot->run(features, state);
     }
 
+    void log_stream_timings() const {
+        if (first_stream_graph != nullptr) {
+            first_stream_graph->log_timings("first");
+        }
+        if (steady_stream_graph != nullptr) {
+            steady_stream_graph->log_timings("steady");
+        }
+    }
+
     std::shared_ptr<const VoxtralRealtimeAssets> assets;
     ggml_backend_t backend = nullptr;
     core::BackendType backend_type = core::BackendType::Cpu;
@@ -925,4 +966,8 @@ VoxtralRealtimeAudioEmbeddings VoxtralRealtimeAudioEncoderRuntime::encode_stream
     return impl_->encode_stream_chunk(features, state);
 }
 
+void VoxtralRealtimeAudioEncoderRuntime::log_stream_timings() const {
+    impl_->log_stream_timings();
+}
+
 }  // namespace engine::models::voxtral_realtime