diff --git a/internal/agent/session.go b/internal/agent/session.go index b7a8463..8cc9b13 100644 --- a/internal/agent/session.go +++ b/internal/agent/session.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "log/slog" + "math" "strings" "time" @@ -61,6 +62,11 @@ func (s state) indicator() indicator.State { const sendTimeout = 5 * time.Second +// idleBuffered is how much audio before the wake is kept for the replay. A +// command said right after the wake word may be completely over by the time +// the server reports the detection. +const idleBuffered = 5 * time.Second + // session is the per-connection state machine. All fields are owned by the // run goroutine. type session struct { @@ -75,18 +81,23 @@ type session struct { since time.Time turnSeq int turnID string - speech bool // turn_start was sent for the current turn - remote bool // the current turn was started by the server (remote_turn) - ending bool // the model ended the conversation: no follow-up - announce bool // the current turn plays an announcement (TypeAnnounce) - askListen bool // ...after which an answer is expected - replyTo string // the turn being recorded answers this announcement - wakeFeed bool // stream idle microphone audio for the server's wake word spotter - deafUntil time.Time // ignore the mic while our own earcon plays + speech bool // turn_start was sent for the current turn + maxDB float64 // loudest frame of the current listening window + hp *audio.HighPass // removes room rumble before the VAD + remote bool // the current turn was started by the server (remote_turn) + ending bool // the model ended the conversation: no follow-up + announce bool // the current turn plays an announcement (TypeAnnounce) + askListen bool // ...after which an answer is expected + replyTo string // the turn being recorded answers this announcement + wakeFeed bool // stream idle microphone audio for the server's wake word spotter + deafUntil time.Time // ignore the mic while our own earcon plays preroll [][]byte - failed bool // the current turn produced an error - replied bool // the current turn produced audio - ducked bool // media was paused by us and must be resumed + idle [][]byte // idle microphone audio, replayed after a wake + idleBytes int + idleAt time.Time // when the last idle frame arrived + failed bool // the current turn produced an error + replied bool // the current turn produced audio + ducked bool // media was paused by us and must be resumed heard strings.Builder said strings.Builder @@ -235,11 +246,15 @@ func (s *session) startListening(ctx context.Context, timeout time.Duration) { s.turnSeq++ s.turnID = fmt.Sprintf("%s-%d-%d", s.a.cfg.DeviceID, time.Now().Unix(), s.turnSeq) s.speech, s.failed, s.replied, s.remote, s.ending = false, false, false, false, false + s.maxDB = -120 s.announce, s.askListen, s.replyTo = false, false, "" s.preroll = s.preroll[:0] s.heard.Reset() s.said.Reset() + if s.hp == nil { + s.hp = audio.NewHighPass(180, float64(s.inFormat.SampleRateHertz)) + } s.ep.Reset(timeout) s.duck(ctx) @@ -248,11 +263,14 @@ func (s *session) startListening(ctx context.Context, timeout time.Duration) { s.deafUntil = time.Now().Add(earcon + 80*time.Millisecond) } s.setState(stateListening) + s.replayIdle(ctx) } // onFrame handles 20 ms of microphone audio. func (s *session) onFrame(ctx context.Context, f []byte) { + s.hp.Process(f) if s.state == stateIdle { + s.rememberIdle(f) s.ep.Observe(f) if s.wakeFeed && !s.a.muted.Load() { s.sendAudio(ctx, f) @@ -262,6 +280,42 @@ func (s *session) onFrame(ctx context.Context, f []byte) { if s.state != stateListening || time.Now().Before(s.deafUntil) { return } + s.listenFrame(ctx, f) +} + +// rememberIdle keeps the last idleBuffered seconds of filtered microphone +// audio, so startListening can replay what was said while the wake spotter +// was still confirming the wake word. +func (s *session) rememberIdle(f []byte) { + s.idleAt = time.Now() + limit := s.inFormat.FrameSize(idleBuffered) + s.idle = append(s.idle, append([]byte(nil), f...)) + s.idleBytes += len(f) + for s.idleBytes > limit && len(s.idle) > 1 { + s.idleBytes -= len(s.idle[0]) + s.idle = s.idle[1:] + } +} + +// replayIdle feeds the buffered idle audio to the endpointer and forgets it. +func (s *session) replayIdle(ctx context.Context) { + frames := s.idle + s.idle, s.idleBytes = nil, 0 + if time.Since(s.idleAt) > 500*time.Millisecond { + return // a reply or announcement played since; the buffer is stale + } + for _, f := range frames { + if s.state != stateListening { + return // the buffered audio already completed the utterance + } + s.listenFrame(ctx, f) + } +} + +func (s *session) listenFrame(ctx context.Context, f []byte) { + if db := audio.DBFS(audio.RMS(f)); db > s.maxDB { + s.maxDB = db + } dec := s.ep.Process(f, s.inFormat.Duration(len(f))) switch dec { @@ -283,7 +337,7 @@ func (s *session) onFrame(ctx context.Context, f []byte) { s.a.log.Info("utterance captured", "turn", s.turnID, "by", "device", "length", time.Since(s.since).Round(100*time.Millisecond)) s.setState(stateThinking) case vad.NoSpeech: - s.a.log.Info("no speech detected") + s.a.log.Info("no speech detected", "max_db", math.Round(s.maxDB), "floor_db", math.Round(s.ep.NoiseFloor())) s.declineAnswer(ctx) s.a.playEarcon(ctx, audio.EarconDone) s.goIdle(ctx) diff --git a/internal/audio/highpass.go b/internal/audio/highpass.go new file mode 100644 index 0000000..03eedb6 --- /dev/null +++ b/internal/audio/highpass.go @@ -0,0 +1,56 @@ +// Copyright 2026 The Marusya-Gemini Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package audio + +import ( + "encoding/binary" + "math" +) + +// HighPass is a one-pole high-pass filter for 16-bit little-endian mono PCM. +// Rooms bring fan and mains rumble below ~200 Hz, and the endpointer compares +// speech with the noise floor: without the filter quiet speech can sit only a +// few dB above the rumble and never count as speech. +type HighPass struct { + a float64 + x1, y1 float64 +} + +// NewHighPass returns a filter with the given cutoff in Hz. +func NewHighPass(cutoffHz, sampleRate float64) *HighPass { + if cutoffHz <= 0 || sampleRate <= 0 || cutoffHz >= sampleRate/2 { + return &HighPass{a: 1} + } + return &HighPass{a: 1 / (1 + 2*math.Pi*cutoffHz/sampleRate)} +} + +// Process filters pcm in place. +func (h *HighPass) Process(pcm []byte) { + if h == nil { + return + } + for i := 0; i+1 < len(pcm); i += 2 { + x := float64(int16(binary.LittleEndian.Uint16(pcm[i:]))) + y := h.a * (h.y1 + x - h.x1) + h.x1, h.y1 = x, y + switch { + case y > math.MaxInt16: + y = math.MaxInt16 + case y < math.MinInt16: + y = math.MinInt16 + } + binary.LittleEndian.PutUint16(pcm[i:], uint16(int16(math.Round(y)))) + } +} diff --git a/internal/audio/highpass_test.go b/internal/audio/highpass_test.go new file mode 100644 index 0000000..24fa1fb --- /dev/null +++ b/internal/audio/highpass_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 The Marusya-Gemini Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package audio + +import ( + "encoding/binary" + "math" + "testing" +) + +func tone(freq, rate float64, n int) []byte { + pcm := make([]byte, n*2) + for i := range n { + v := int16(12000 * math.Sin(2*math.Pi*freq*float64(i)/rate)) + binary.LittleEndian.PutUint16(pcm[i*2:], uint16(v)) + } + return pcm +} + +func TestHighPassKeepsSpeechAndDropsRumble(t *testing.T) { + const rate = 16000 + rumble := NewHighPass(180, rate) + speech := NewHighPass(180, rate) + low, high := tone(50, rate, 1600), tone(1000, rate, 1600) + rumble.Process(low) + speech.Process(high) + if got, want := RMS(low), 12000.0/32768.0/math.Sqrt2; got > want*0.5 { + t.Errorf("50 Hz survived the filter: rms %.3f, want well below %.3f", got, want*0.5) + } + if got, want := RMS(high), 12000.0/32768.0/math.Sqrt2; got < want*0.85 { + t.Errorf("1 kHz was attenuated: rms %.3f, want about %.3f", got, want) + } +}