From 743cca3466632d2e1b62b248abe378f0da7cb619 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Thu, 29 Oct 2026 09:50:45 +0800 Subject: [PATCH] feat(twilio): keep a reply the caller cut off as the part they heard ConversationRelay stops playback where the caller cuts in, but the whole reply still went into the conversation history, so the next turn believed the question had been asked in full and never came back to it. The relay session now gets the last say over what the history keeps: for a reply cut off while it was still being streamed it returns the text that actually went out, with the rest named in a mark. The dialog is appended after that callback, so the stored message is the marked one, while what is spoken is unchanged. Only a reply cut off mid-stream can be marked; once the whole reply is out, Twilio holds the text it has yet to speak and a cut-in there leaves nothing of ours to go by. Co-Authored-By: Claude Opus 5 --- .../Models/AgentTurnCallbacks.cs | 6 +++ .../Services/ConversationRelaySession.cs | 40 ++++++++++++++++++- .../Services/TwilioConversationProcessor.cs | 30 +++++++++++++- 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs index fcbb782b7..3cf122a73 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs @@ -27,4 +27,10 @@ public class AgentTurnCallbacks /// and the tool may take a while before anything else is said /// public Func? OnStreamEndedWithToolCall { get; init; } + + /// + /// The last say over what the conversation history keeps of the reply: called with the reply the agent wrote, + /// returns what is stored in its place. Lets a flow record a reply the caller cut off as the part they heard. + /// + public Func? RewriteReply { get; init; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 5f21e14a9..719178350 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -51,6 +51,7 @@ public class ConversationRelaySession private bool _utteranceOpen; private bool _streamedThisTurn; private bool _dropStream; + private bool _cutInMidReply; // The current turn's reply is in; the response watchdog says nothing more once it is set private bool _turnReplied; @@ -235,6 +236,7 @@ private async Task HandleMessageAsync(string json) if (_isProcessing && _streamedThisTurn) { _dropStream = true; + _cutInMidReply = true; } } var interruptedAfter = GetString(root, "durationUntilInterruptMs"); @@ -316,6 +318,7 @@ private async Task ProcessPendingAsync() _utteranceOpen = false; _streamedThisTurn = false; _dropStream = false; + _cutInMidReply = false; _turnReplied = false; _turnStartedUtc = DateTime.UtcNow; } @@ -333,7 +336,8 @@ private async Task ProcessPendingAsync() OnStreamStarted = stream ? CloseUtteranceAsync : null, // A line said before a slow tool ("Okay, that's everything I need") is spoken now, and the watchdog // counts the silence after it instead of waiting for the next response to close it - OnStreamEndedWithToolCall = stream ? CloseUtteranceAsync : null + OnStreamEndedWithToolCall = stream ? CloseUtteranceAsync : null, + RewriteReply = stream ? MarkWhatTheCallerCutOff : null }; AssistantMessage? reply; @@ -587,6 +591,40 @@ await SendIfAsync(() => }, new { type = "text", token = string.Empty, last = true }); } + /// + /// What the conversation history keeps of a reply the caller cut in on: the part that went out, and after it a + /// mark naming the rest, so the next turn knows the caller never heard it. Every other reply is kept as it is. + /// + /// + /// Only a reply cut off while it was still being streamed can be marked: once the whole reply is out, Twilio + /// holds the text it has yet to speak and a cut-in there leaves nothing of ours to go by. + /// + private string MarkWhatTheCallerCutOff(string content) + { + var final = content?.Trim() ?? string.Empty; + string sent; + lock (_sync) + { + if (!_cutInMidReply || final.Length == 0) + { + return content; + } + sent = (_utteranceOpen ? _streamSegment.ToString() : _lastStreamSegment).Trim(); + } + + // The reply is not what was streamed (text set by a tool, or a reply rewritten after the model wrote it), + // so where the caller cut in is not known + if (sent.Length == 0 || !final.StartsWith(sent, StringComparison.Ordinal)) + { + return content; + } + + var unheard = final.Substring(sent.Length).Trim(); + return unheard.Length == 0 + ? content + : $"{sent} [the caller cut in here; they did not hear: \"{unheard}\"]"; + } + /// /// The part of the final reply that was not streamed. Normally nothing; the whole reply when it is not /// what the model streamed last, e.g. text set by a tool, or a reply rewritten after the model wrote it. diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs index 6ae5ce0b9..6b14a69d2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs @@ -99,11 +99,14 @@ await conv.SendMessage(message.AgentId, replyMessage: await BuildPostbackMessageModel(conv, message), async msg => { + // What the caller is told, which a rewrite below must not change + var spoken = msg.Content; + RewriteForHistory(msg, callbacks); reply = new AssistantMessage() { ConversationEnd = msg.Instruction?.ConversationEnd ?? false, HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName), - Content = msg.Content, + Content = spoken, MessageId = msg.MessageId }; } @@ -112,6 +115,31 @@ await conv.SendMessage(message.AgentId, return reply; } + /// + /// Lets the flow change what the history keeps of the reply. The dialog is appended after this callback runs, + /// so the message changed here is the one that is stored. + /// + private static void RewriteForHistory(RoleDialogModel msg, AgentTurnCallbacks? callbacks) + { + if (callbacks?.RewriteReply == null || string.IsNullOrEmpty(msg.Content)) + { + return; + } + + var stored = callbacks.RewriteReply(msg.Content); + if (string.IsNullOrEmpty(stored) || stored == msg.Content) + { + return; + } + + // Kept in step only where it is the reply itself; a rich message the model refined is left alone + if (msg.RichContent?.Message != null && msg.RichContent.Message.Text == msg.Content) + { + msg.RichContent.Message.Text = stored; + } + msg.Content = stored; + } + private static async Task BuildPostbackMessageModel( IConversationService conv, CallerMessage message) {