diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs index 3cf122a73..1a81dc577 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs @@ -33,4 +33,14 @@ public class AgentTurnCallbacks /// returns what is stored in its place. Lets a flow record a reply the caller cut off as the part they heard. /// public Func? RewriteReply { get; init; } + + /// + /// Set by the turn once it can read the conversation's states: whether what the agent says now must play to the end + /// (). + /// + /// + /// Read before each piece of the reply goes out, so a flow can turn it on part way through a turn, e.g. once a + /// confirmation number is coming up. Only valid while the turn runs. + /// + public Func? KeepUninterrupted { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs index 547b09d93..87f3e4377 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -8,5 +8,10 @@ public class AssistantMessage public string MessageId { get; set; } public string? SpeechFileName { get; set; } public string Hints { get; set; } + + /// + /// What of the reply is said once the turn is over must play to the end, as the flow asked when the reply came in + /// + public bool KeepUninterrupted { get; set; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs new file mode 100644 index 000000000..82c145a60 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationRelayStates.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Plugin.Twilio.Models; + +/// +/// Conversation states a phone flow sets to steer the ConversationRelay session +/// +public static class ConversationRelayStates +{ + /// + /// "true" while what the agent says must play to the end, whoever speaks over it: a flow sets it around a line the + /// caller has to hear, such as a confirmation number, and clears it once that line is said + /// + public const string KeepUninterrupted = "twilio_relay_keep_uninterrupted"; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 719178350..08c2708bb 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -53,6 +53,12 @@ public class ConversationRelaySession private bool _dropStream; private bool _cutInMidReply; + // The current turn's callbacks, for what the flow wants of the reply as it goes out + private AgentTurnCallbacks? _turnCallbacks; + + // How the utterance being streamed went out, so its closing frame goes out the same way + private bool _utteranceKeptUninterrupted; + // The current turn's reply is in; the response watchdog says nothing more once it is set private bool _turnReplied; private DateTime _turnStartedUtc; @@ -339,6 +345,7 @@ private async Task ProcessPendingAsync() OnStreamEndedWithToolCall = stream ? CloseUtteranceAsync : null, RewriteReply = stream ? MarkWhatTheCallerCutOff : null }; + _turnCallbacks = callbacks; AssistantMessage? reply; try @@ -384,7 +391,7 @@ private async Task ProcessPendingAsync() continue; } - await SpeakAsync(GetUnstreamedText(reply.Content)); + await SpeakAsync(GetUnstreamedText(reply.Content), keepUninterrupted: reply.KeepUninterrupted); if (reply.HumanIntervationNeeded) { await EndAsync(ConversationRelayHandoff.Transfer); @@ -546,6 +553,8 @@ private async Task FailAsync() /// private async Task StreamTextAsync(string text) { + var keep = _turnCallbacks?.KeepUninterrupted?.Invoke() == true; + // Decided once the send lock is held: a piece queued behind another send must not go out after the // caller cut in (_dropStream) or the session ended meanwhile await SendIfAsync(() => @@ -557,21 +566,30 @@ await SendIfAsync(() => _streamSegment.Append(text); _utteranceOpen = true; _streamedThisTurn = true; + _utteranceKeptUninterrupted = keep; return true; - }, new { type = "text", token = text, last = false }); + }, TextFrame(text, last: false, keep)); } + // A line the caller must hear goes out marked so: Twilio then plays it to the end, whoever speaks over it, which + // overrides the interruptible attribute set for the call + private static object TextFrame(string token, bool last, bool keepUninterrupted) => keepUninterrupted + ? new { type = "text", token, last, interruptible = false } + : new { type = "text", token, last }; + /// /// Ends the utterance being streamed, so what follows (an indication, the next response) is spoken on its own. /// private async Task CloseUtteranceAsync() { + bool keep; lock (_sync) { if (!_utteranceOpen) { return; } + keep = _utteranceKeptUninterrupted; } // Closed only once the send lock is held, so nothing else (a filler) can go out between the utterance being @@ -588,7 +606,7 @@ await SendIfAsync(() => _streamSegment.Clear(); MarkSpeaking(EstimateSpeech(_lastStreamSegment.Length)); return true; - }, new { type = "text", token = string.Empty, last = true }); + }, TextFrame(string.Empty, last: true, keep)); } /// @@ -704,7 +722,8 @@ await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), } /// Extra condition, checked under once the send lock is held - private async Task SpeakAsync(string? text, Func? canSpeak = null) + /// The caller must hear all of it, so it plays to the end whoever speaks over it + private async Task SpeakAsync(string? text, Func? canSpeak = null, bool keepUninterrupted = false) { if (string.IsNullOrWhiteSpace(text)) { @@ -712,7 +731,7 @@ private async Task SpeakAsync(string? text, Func? canSpeak = null) } await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), - new { type = "text", token = text, last = true }, EstimateSpeech(text.Length)); + TextFrame(text, last: true, keepUninterrupted), EstimateSpeech(text.Length)); } /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs index 6b14a69d2..80ba8dae8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs @@ -94,6 +94,13 @@ public static class TwilioConversationProcessor } } + if (callbacks != null) + { + // Read live while the turn runs, so a tool that turns it on (a confirmation number coming up) covers what + // the agent says after it in the same turn + callbacks.KeepUninterrupted = () => conv.States.GetState(ConversationRelayStates.KeepUninterrupted) == "true"; + } + await conv.SendMessage(message.AgentId, inputMsg, replyMessage: await BuildPostbackMessageModel(conv, message), @@ -107,7 +114,8 @@ await conv.SendMessage(message.AgentId, ConversationEnd = msg.Instruction?.ConversationEnd ?? false, HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName), Content = spoken, - MessageId = msg.MessageId + MessageId = msg.MessageId, + KeepUninterrupted = callbacks?.KeepUninterrupted?.Invoke() ?? false }; } );