From a76fb184975ba204297467ed8df9c987b8dccf98 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Thu, 29 Oct 2026 17:29:11 +0800 Subject: [PATCH] feat(twilio): let a phone flow keep what the agent says from being cut off With interruptible set for the call, anything the caller's line picks up stops playback, the caller or their phone picking up the agent's own voice alike, and Twilio drops what it had yet to speak. A line the caller has to hear, such as a confirmation number, could be lost that way with no one aware of it. A flow now sets the twilio_relay_keep_uninterrupted state around such a line, and the text frames sent while it is on go out with "interruptible": false, which Twilio applies to that message over the call's setting. The turn reads the state live through AgentTurnCallbacks.KeepUninterrupted, so a tool that turns it on part way through covers what the agent says after it; text said once the turn is over goes out as the state stood when the reply came in (AssistantMessage), and an utterance's closing frame as its pieces did. Co-Authored-By: Claude Opus 5.5 --- .../Models/AgentTurnCallbacks.cs | 10 +++++++ .../Models/AssistantMessage.cs | 5 ++++ .../Models/ConversationRelayStates.cs | 13 +++++++++ .../Services/ConversationRelaySession.cs | 29 +++++++++++++++---- .../Services/TwilioConversationProcessor.cs | 10 ++++++- 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs index 3cf122a73..1a81dc577 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs @@ -33,4 +33,14 @@ public class AgentTurnCallbacks /// returns what is stored in its place. Lets a flow record a reply the caller cut off as the part they heard. /// public Func? RewriteReply { get; init; } + + /// + /// Set by the turn once it can read the conversation's states: whether what the agent says now must play to the end + /// (). + /// + /// + /// Read before each piece of the reply goes out, so a flow can turn it on part way through a turn, e.g. once a + /// confirmation number is coming up. Only valid while the turn runs. + /// + public Func? KeepUninterrupted { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs index 547b09d93..87f3e4377 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -8,5 +8,10 @@ public class AssistantMessage public string MessageId { get; set; } public string? SpeechFileName { get; set; } public string Hints { get; set; } + + /// + /// What of the reply is said once the turn is over must play to the end, as the flow asked when the reply came in + /// + public bool KeepUninterrupted { get; set; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs new file mode 100644 index 000000000..82c145a60 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Plugin.Twilio.Models; + +/// +/// Conversation states a phone flow sets to steer the ConversationRelay session +/// +public static class ConversationRelayStates +{ + /// + /// "true" while what the agent says must play to the end, whoever speaks over it: a flow sets it around a line the + /// caller has to hear, such as a confirmation number, and clears it once that line is said + /// + public const string KeepUninterrupted = "twilio_relay_keep_uninterrupted"; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 719178350..08c2708bb 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -53,6 +53,12 @@ public class ConversationRelaySession private bool _dropStream; private bool _cutInMidReply; + // The current turn's callbacks, for what the flow wants of the reply as it goes out + private AgentTurnCallbacks? _turnCallbacks; + + // How the utterance being streamed went out, so its closing frame goes out the same way + private bool _utteranceKeptUninterrupted; + // The current turn's reply is in; the response watchdog says nothing more once it is set private bool _turnReplied; private DateTime _turnStartedUtc; @@ -339,6 +345,7 @@ private async Task ProcessPendingAsync() OnStreamEndedWithToolCall = stream ? CloseUtteranceAsync : null, RewriteReply = stream ? MarkWhatTheCallerCutOff : null }; + _turnCallbacks = callbacks; AssistantMessage? reply; try @@ -384,7 +391,7 @@ private async Task ProcessPendingAsync() continue; } - await SpeakAsync(GetUnstreamedText(reply.Content)); + await SpeakAsync(GetUnstreamedText(reply.Content), keepUninterrupted: reply.KeepUninterrupted); if (reply.HumanIntervationNeeded) { await EndAsync(ConversationRelayHandoff.Transfer); @@ -546,6 +553,8 @@ private async Task FailAsync() /// private async Task StreamTextAsync(string text) { + var keep = _turnCallbacks?.KeepUninterrupted?.Invoke() == true; + // Decided once the send lock is held: a piece queued behind another send must not go out after the // caller cut in (_dropStream) or the session ended meanwhile await SendIfAsync(() => @@ -557,21 +566,30 @@ await SendIfAsync(() => _streamSegment.Append(text); _utteranceOpen = true; _streamedThisTurn = true; + _utteranceKeptUninterrupted = keep; return true; - }, new { type = "text", token = text, last = false }); + }, TextFrame(text, last: false, keep)); } + // A line the caller must hear goes out marked so: Twilio then plays it to the end, whoever speaks over it, which + // overrides the interruptible attribute set for the call + private static object TextFrame(string token, bool last, bool keepUninterrupted) => keepUninterrupted + ? new { type = "text", token, last, interruptible = false } + : new { type = "text", token, last }; + /// /// Ends the utterance being streamed, so what follows (an indication, the next response) is spoken on its own. /// private async Task CloseUtteranceAsync() { + bool keep; lock (_sync) { if (!_utteranceOpen) { return; } + keep = _utteranceKeptUninterrupted; } // Closed only once the send lock is held, so nothing else (a filler) can go out between the utterance being @@ -588,7 +606,7 @@ await SendIfAsync(() => _streamSegment.Clear(); MarkSpeaking(EstimateSpeech(_lastStreamSegment.Length)); return true; - }, new { type = "text", token = string.Empty, last = true }); + }, TextFrame(string.Empty, last: true, keep)); } /// @@ -704,7 +722,8 @@ await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), } /// Extra condition, checked under once the send lock is held - private async Task SpeakAsync(string? text, Func? canSpeak = null) + /// The caller must hear all of it, so it plays to the end whoever speaks over it + private async Task SpeakAsync(string? text, Func? canSpeak = null, bool keepUninterrupted = false) { if (string.IsNullOrWhiteSpace(text)) { @@ -712,7 +731,7 @@ private async Task SpeakAsync(string? text, Func? canSpeak = null) } await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), - new { type = "text", token = text, last = true }, EstimateSpeech(text.Length)); + TextFrame(text, last: true, keepUninterrupted), EstimateSpeech(text.Length)); } /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs index 6b14a69d2..80ba8dae8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs @@ -94,6 +94,13 @@ public static class TwilioConversationProcessor } } + if (callbacks != null) + { + // Read live while the turn runs, so a tool that turns it on (a confirmation number coming up) covers what + // the agent says after it in the same turn + callbacks.KeepUninterrupted = () => conv.States.GetState(ConversationRelayStates.KeepUninterrupted) == "true"; + } + await conv.SendMessage(message.AgentId, inputMsg, replyMessage: await BuildPostbackMessageModel(conv, message), @@ -107,7 +114,8 @@ await conv.SendMessage(message.AgentId, ConversationEnd = msg.Instruction?.ConversationEnd ?? false, HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName), Content = spoken, - MessageId = msg.MessageId + MessageId = msg.MessageId, + KeepUninterrupted = callbacks?.KeepUninterrupted?.Invoke() ?? false }; } );