From 9c5549323fff5646f430d0776d69fd9fbf785200 Mon Sep 17 00:00:00 2001 From: ryysel Date: Mon, 2 Nov 2026 06:36:02 +0300 Subject: [PATCH] agent: keep the command that follows the wake word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wake spotter verifies a candidate before reporting it, so on a loaded host the server asks the speaker to listen about a second after the wake word. A command said without a pause after it («Маруся, который час») is over by then: the turn is transcribed as empty and the speaker stays silent. A pause after the wake word hid the problem. Keep the last five seconds of idle microphone audio and replay it through the endpointer when listening starts. The buffered frames already passed the noise floor and the high-pass filter, so the buffered speech goes out with the turn as usual. The high-pass filter is new as well: room rumble can sit only a few dB below speech in a quiet room, and the endpointer compares speech with the floor. Filtering below 180 Hz gives quiet speech more headroom, and the «no speech detected» log now includes the peak and the floor in dBFS, so a muted microphone is not mistaken for a quiet one. --- internal/agent/session.go | 78 ++++++++++++++++++++++++++++----- internal/audio/highpass.go | 56 +++++++++++++++++++++++ internal/audio/highpass_test.go | 45 +++++++++++++++++++ 3 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 internal/audio/highpass.go create mode 100644 internal/audio/highpass_test.go diff --git a/internal/agent/session.go b/internal/agent/session.go index b7a8463..8cc9b13 100644 --- a/internal/agent/session.go +++ b/internal/agent/session.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "log/slog" + "math" "strings" "time" @@ -61,6 +62,11 @@ func (s state) indicator() indicator.State { const sendTimeout = 5 * time.Second +// idleBuffered is how much audio before the wake is kept for the replay. A +// command said right after the wake word may be completely over by the time +// the server reports the detection. +const idleBuffered = 5 * time.Second + // session is the per-connection state machine. All fields are owned by the // run goroutine. type session struct { @@ -75,18 +81,23 @@ type session struct { since time.Time turnSeq int turnID string - speech bool // turn_start was sent for the current turn - remote bool // the current turn was started by the server (remote_turn) - ending bool // the model ended the conversation: no follow-up - announce bool // the current turn plays an announcement (TypeAnnounce) - askListen bool // ...after which an answer is expected - replyTo string // the turn being recorded answers this announcement - wakeFeed bool // stream idle microphone audio for the server's wake word spotter - deafUntil time.Time // ignore the mic while our own earcon plays + speech bool // turn_start was sent for the current turn + maxDB float64 // loudest frame of the current listening window + hp *audio.HighPass // removes room rumble before the VAD + remote bool // the current turn was started by the server (remote_turn) + ending bool // the model ended the conversation: no follow-up + announce bool // the current turn plays an announcement (TypeAnnounce) + askListen bool // ...after which an answer is expected + replyTo string // the turn being recorded answers this announcement + wakeFeed bool // stream idle microphone audio for the server's wake word spotter + deafUntil time.Time // ignore the mic while our own earcon plays preroll [][]byte - failed bool // the current turn produced an error - replied bool // the current turn produced audio - ducked bool // media was paused by us and must be resumed + idle [][]byte // idle microphone audio, replayed after a wake + idleBytes int + idleAt time.Time // when the last idle frame arrived + failed bool // the current turn produced an error + replied bool // the current turn produced audio + ducked bool // media was paused by us and must be resumed heard strings.Builder said strings.Builder @@ -235,11 +246,15 @@ func (s *session) startListening(ctx context.Context, timeout time.Duration) { s.turnSeq++ s.turnID = fmt.Sprintf("%s-%d-%d", s.a.cfg.DeviceID, time.Now().Unix(), s.turnSeq) s.speech, s.failed, s.replied, s.remote, s.ending = false, false, false, false, false + s.maxDB = -120 s.announce, s.askListen, s.replyTo = false, false, "" s.preroll = s.preroll[:0] s.heard.Reset() s.said.Reset() + if s.hp == nil { + s.hp = audio.NewHighPass(180, float64(s.inFormat.SampleRateHertz)) + } s.ep.Reset(timeout) s.duck(ctx) @@ -248,11 +263,14 @@ func (s *session) startListening(ctx context.Context, timeout time.Duration) { s.deafUntil = time.Now().Add(earcon + 80*time.Millisecond) } s.setState(stateListening) + s.replayIdle(ctx) } // onFrame handles 20 ms of microphone audio. func (s *session) onFrame(ctx context.Context, f []byte) { + s.hp.Process(f) if s.state == stateIdle { + s.rememberIdle(f) s.ep.Observe(f) if s.wakeFeed && !s.a.muted.Load() { s.sendAudio(ctx, f) @@ -262,6 +280,42 @@ func (s *session) onFrame(ctx context.Context, f []byte) { if s.state != stateListening || time.Now().Before(s.deafUntil) { return } + s.listenFrame(ctx, f) +} + +// rememberIdle keeps the last idleBuffered seconds of filtered microphone +// audio, so startListening can replay what was said while the wake spotter +// was still confirming the wake word. +func (s *session) rememberIdle(f []byte) { + s.idleAt = time.Now() + limit := s.inFormat.FrameSize(idleBuffered) + s.idle = append(s.idle, append([]byte(nil), f...)) + s.idleBytes += len(f) + for s.idleBytes > limit && len(s.idle) > 1 { + s.idleBytes -= len(s.idle[0]) + s.idle = s.idle[1:] + } +} + +// replayIdle feeds the buffered idle audio to the endpointer and forgets it. +func (s *session) replayIdle(ctx context.Context) { + frames := s.idle + s.idle, s.idleBytes = nil, 0 + if time.Since(s.idleAt) > 500*time.Millisecond { + return // a reply or announcement played since; the buffer is stale + } + for _, f := range frames { + if s.state != stateListening { + return // the buffered audio already completed the utterance + } + s.listenFrame(ctx, f) + } +} + +func (s *session) listenFrame(ctx context.Context, f []byte) { + if db := audio.DBFS(audio.RMS(f)); db > s.maxDB { + s.maxDB = db + } dec := s.ep.Process(f, s.inFormat.Duration(len(f))) switch dec { @@ -283,7 +337,7 @@ func (s *session) onFrame(ctx context.Context, f []byte) { s.a.log.Info("utterance captured", "turn", s.turnID, "by", "device", "length", time.Since(s.since).Round(100*time.Millisecond)) s.setState(stateThinking) case vad.NoSpeech: - s.a.log.Info("no speech detected") + s.a.log.Info("no speech detected", "max_db", math.Round(s.maxDB), "floor_db", math.Round(s.ep.NoiseFloor())) s.declineAnswer(ctx) s.a.playEarcon(ctx, audio.EarconDone) s.goIdle(ctx) diff --git a/internal/audio/highpass.go b/internal/audio/highpass.go new file mode 100644 index 0000000..03eedb6 --- /dev/null +++ b/internal/audio/highpass.go @@ -0,0 +1,56 @@ +// Copyright 2026 The Marusya-Gemini Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package audio + +import ( + "encoding/binary" + "math" +) + +// HighPass is a one-pole high-pass filter for 16-bit little-endian mono PCM. +// Rooms bring fan and mains rumble below ~200 Hz, and the endpointer compares +// speech with the noise floor: without the filter quiet speech can sit only a +// few dB above the rumble and never count as speech. +type HighPass struct { + a float64 + x1, y1 float64 +} + +// NewHighPass returns a filter with the given cutoff in Hz. +func NewHighPass(cutoffHz, sampleRate float64) *HighPass { + if cutoffHz <= 0 || sampleRate <= 0 || cutoffHz >= sampleRate/2 { + return &HighPass{a: 1} + } + return &HighPass{a: 1 / (1 + 2*math.Pi*cutoffHz/sampleRate)} +} + +// Process filters pcm in place. +func (h *HighPass) Process(pcm []byte) { + if h == nil { + return + } + for i := 0; i+1 < len(pcm); i += 2 { + x := float64(int16(binary.LittleEndian.Uint16(pcm[i:]))) + y := h.a * (h.y1 + x - h.x1) + h.x1, h.y1 = x, y + switch { + case y > math.MaxInt16: + y = math.MaxInt16 + case y < math.MinInt16: + y = math.MinInt16 + } + binary.LittleEndian.PutUint16(pcm[i:], uint16(int16(math.Round(y)))) + } +} diff --git a/internal/audio/highpass_test.go b/internal/audio/highpass_test.go new file mode 100644 index 0000000..24fa1fb --- /dev/null +++ b/internal/audio/highpass_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 The Marusya-Gemini Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package audio + +import ( + "encoding/binary" + "math" + "testing" +) + +func tone(freq, rate float64, n int) []byte { + pcm := make([]byte, n*2) + for i := range n { + v := int16(12000 * math.Sin(2*math.Pi*freq*float64(i)/rate)) + binary.LittleEndian.PutUint16(pcm[i*2:], uint16(v)) + } + return pcm +} + +func TestHighPassKeepsSpeechAndDropsRumble(t *testing.T) { + const rate = 16000 + rumble := NewHighPass(180, rate) + speech := NewHighPass(180, rate) + low, high := tone(50, rate, 1600), tone(1000, rate, 1600) + rumble.Process(low) + speech.Process(high) + if got, want := RMS(low), 12000.0/32768.0/math.Sqrt2; got > want*0.5 { + t.Errorf("50 Hz survived the filter: rms %.3f, want well below %.3f", got, want*0.5) + } + if got, want := RMS(high), 12000.0/32768.0/math.Sqrt2; got < want*0.85 { + t.Errorf("1 kHz was attenuated: rms %.3f, want about %.3f", got, want) + } +}