diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.Agents.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.Agents.cs new file mode 100644 index 000000000..095de2328 --- /dev/null +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.Agents.cs @@ -0,0 +1,90 @@ +using AwesomeAssertions; +using Bunit; +using Microsoft.AspNetCore.Components; +using PanoramicData.Blazor.Interfaces; +using PanoramicData.Blazor.Models; + +namespace PanoramicData.Blazor.Test; + +/// +/// The agent picker: offered in the input area only when the service lists two or more agents, and choosing one sets +/// . +/// +public partial class PDChatTests +{ + private static readonly PDChatAgentOption _merlin = new("merlin", "Merlin", "Answers about the product", "/merlin.png"); + private static readonly PDChatAgentOption _alice = new("alice", "Alice", "Development help"); + + /// Verifies that no picker is shown with no agents, or with only one to choose. + [Theory] + [InlineData(0)] + [InlineData(1)] + public void The_agent_picker_is_hidden_with_fewer_than_two_agents(int count) + { + var component = RenderChat(new FakeChatService { Agents = [.. new[] { _merlin, _alice }.Take(count)] }); + + component.FindAll(".pdchat-agent-picker").Should().BeEmpty(); + component.FindAll(".chat-input-accessories").Should().BeEmpty("there is nothing else to show there"); + } + + /// Verifies that no picker is shown when the service does not list agents at all. + [Fact] + public void The_agent_picker_is_hidden_without_agents() + => RenderChat(new FakeChatService()).FindAll(".pdchat-agent-picker").Should().BeEmpty(); + + /// Verifies that two or more agents are offered in the input area, the first selected by default. + [Fact] + public void The_agent_picker_is_shown_in_the_input_area_with_two_agents() + { + var component = RenderChat(new FakeChatService { Agents = [_merlin, _alice] }); + + var select = component.Find(".chat-input-container .chat-input-accessories .pdchat-agent-picker select"); + select.GetAttribute("title").Should().Be("Who you are talking to"); + var options = select.QuerySelectorAll("option"); + options.Select(option => option.TextContent).Should().Equal("Merlin", "Alice"); + options.Select(option => option.GetAttribute("value")).Should().Equal("merlin", "alice"); + options[0].GetAttribute("title").Should().Be("Answers about the product"); + options[0].HasAttribute("selected").Should().BeTrue(); + component.Find(".pdchat-agent-icon").GetAttribute("src").Should().Be("/merlin.png"); + } + + /// Verifies that choosing an agent sets the service's selected agent, and the picker follows it. + [Fact] + public async Task Choosing_an_agent_sets_the_selected_agent() + { + var service = new FakeChatService { Agents = [_merlin, _alice] }; + var component = RenderChat(service); + ((IChatService)service).SelectedAgentId.Should().BeNull(); + + await component.Find(".pdchat-agent-picker select").ChangeAsync(new ChangeEventArgs { Value = "alice" }); + + ((IChatService)service).SelectedAgentId.Should().Be("alice"); + component.WaitForAssertion(() => component.Find("option[value='alice']").HasAttribute("selected").Should().BeTrue(), Patience); + component.FindAll(".pdchat-agent-icon").Should().BeEmpty("Alice has no icon"); + } + + /// Verifies that a value that is not one of the agents is ignored. + [Fact] + public async Task An_unknown_agent_is_not_selected() + { + var service = new FakeChatService { Agents = [_merlin, _alice] }; + var component = RenderChat(service); + + await component.Find(".pdchat-agent-picker select").ChangeAsync(new ChangeEventArgs { Value = "mallory" }); + + ((IChatService)service).SelectedAgentId.Should().BeNull(); + } + + /// Verifies that the picker and the Voice control share the input area. + [Fact] + public void The_agent_picker_sits_beside_the_Voice_control() + { + var service = VoiceService(); + service.Agents = [_merlin, _alice]; + + var accessories = RenderChat(service).Find(".chat-input-accessories"); + + accessories.QuerySelector(".pdchat-agent-picker").Should().NotBeNull(); + accessories.QuerySelector(".pdchat-voice-toggle").Should().NotBeNull(); + } +} diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.Fakes.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.Fakes.cs index 34a56d816..319da5a7a 100644 --- a/PanoramicData.Blazor.Test/Components/PDChatTests.Fakes.cs +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.Fakes.cs @@ -55,6 +55,7 @@ private sealed partial class FakeChatService : IChatService public int ToastMaxVisible { get; set; } = 3; public PDChatButtonPosition ToastAnchor { get; set; } = PDChatButtonPosition.BottomRight; public PDChatVoiceEndpoints? VoiceEndpoints { get; set; } + public IReadOnlyList? Agents { get; set; } public IReadOnlyList Messages => Store; public bool SupportsConversations { get; init; } diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs index 45d527daa..220e58702 100644 --- a/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs @@ -1,5 +1,6 @@ using AwesomeAssertions; using Bunit; +using Microsoft.AspNetCore.Components.Web; using Microsoft.JSInterop; using PanoramicData.Blazor.Interfaces; using PanoramicData.Blazor.Models; @@ -7,14 +8,17 @@ namespace PanoramicData.Blazor.Test; /// -/// Voice Mode: offered only when the service supplies endpoints, off by default, and when on, a spoken question is -/// sent like a typed one and the first finished reply is read aloud. +/// Voice Mode: offered in the input area only when the service supplies endpoints, off by default. When on, words are +/// dictated into the text box, sent after a pause unless the user is editing, and the first finished reply is read aloud. /// public partial class PDChatTests { private const string VoiceModulePath = "./_content/PanoramicData.Blazor/js/pdchat-voice.js"; private static readonly PDChatVoiceEndpoints _voiceEndpoints = new("/voice/listen", "/voice/speak"); + /// Long enough that a send which should not happen would have. + private static readonly TimeSpan _settle = TimeSpan.FromMilliseconds(400); + /// Verifies that a service without voice endpoints offers no Voice Mode at all. [Fact] public void Without_voice_endpoints_there_is_no_Voice_Mode_control() @@ -24,21 +28,57 @@ public void Without_voice_endpoints_there_is_no_Voice_Mode_control() [Fact] public void Voice_Mode_is_offered_but_off_by_default() { - var component = RenderChat(new FakeChatService { VoiceEndpoints = _voiceEndpoints }); + var component = RenderChat(VoiceService()); var toggle = component.Find(".pdchat-voice-toggle"); toggle.TextContent.Should().Contain("Voice"); - toggle.GetAttribute("title").Should().Be("Turn Voice Mode on: ask out loud and hear the answer"); + toggle.GetAttribute("title").Should().Be("Voice: speak instead of typing"); toggle.GetAttribute("aria-pressed").Should().Be("false"); component.FindAll(".pdchat-voice-status").Should().BeEmpty(); component.Instance.VoiceState.Should().Be(PDChatVoiceState.Off); } + /// Verifies that the Voice control sits in the input area beside Send, not in the header. + [Fact] + public void The_Voice_control_is_in_the_input_area() + { + var component = RenderChat(VoiceService()); + + component.FindAll(".pdchat-header .pdchat-voice-toggle").Should().BeEmpty(); + component.FindAll(".chat-input-container .chat-input-accessories .pdchat-voice-toggle").Should().ContainSingle(); + } + + /// + /// Verifies that the microphone is not asked for until Voice is first turned on: rendering the chat, its input area + /// and the agent picker loads no voice module and starts nothing. + /// + [Fact] + public async Task The_microphone_is_not_requested_until_Voice_is_turned_on() + { + var module = SetUpVoiceModule(); + var service = VoiceService(); + service.Agents = [new("merlin", "Merlin"), new("alice", "Alice")]; + var component = RenderChat(service); + component.Find(".pdchat-voice-toggle"); + await component.Find(".pdchat-agent-picker select").ChangeAsync(new Microsoft.AspNetCore.Components.ChangeEventArgs { Value = "alice" }); + + VoiceModuleImports().Should().BeEmpty(); + module.Invocations.Should().BeEmpty(); + + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + + VoiceModuleImports().Should().ContainSingle(); + module.Invocations["start"].Should().ContainSingle(); + } + + private IEnumerable VoiceModuleImports() + => JSInterop.Invocations["import"].Where(call => VoiceModulePath.Equals(call.Arguments[0])); + /// Verifies that a chat whose input is not permitted offers no Voice Mode either. [Fact] public void Voice_Mode_is_not_offered_where_typing_is_not_permitted() { - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; + var service = VoiceService(); ((IChatService)service).IsInputPermitted = false; RenderChat(service).FindAll(".pdchat-voice-toggle").Should().BeEmpty(); @@ -49,43 +89,158 @@ public void Voice_Mode_is_not_offered_where_typing_is_not_permitted() public async Task Turning_Voice_Mode_on_starts_listening() { var module = SetUpVoiceModule(); - var component = RenderChat(new FakeChatService { VoiceEndpoints = _voiceEndpoints }); + var component = RenderChat(VoiceService()); await component.Find(".pdchat-voice-toggle").ClickAsync(new()); module.Invocations["start"].Should().ContainSingle().Which.Arguments[0].Should().Be("/voice/listen"); component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); component.Find(".pdchat-voice-toggle").GetAttribute("aria-pressed").Should().Be("true"); + component.Find(".pdchat-voice-toggle").GetAttribute("title").Should().Be("Voice: stop listening"); component.Find(".pdchat-voice-status").TextContent.Trim().Should().Be("Listening. Ask your question, then pause."); } - /// Verifies that a spoken question is sent as the user, as typed text is, and the microphone pauses. + /// Verifies that the auto-send delay defaults to one second and can be set on a service that does not implement it. + [Fact] + public void The_auto_send_delay_defaults_to_one_second() + { + IChatService service = new FakeChatService(); + + service.VoiceAutoSendDelay.Should().Be(TimeSpan.FromMilliseconds(1000)); + + service.VoiceAutoSendDelay = TimeSpan.FromMilliseconds(250); + service.VoiceAutoSendDelay.Should().Be(TimeSpan.FromMilliseconds(250)); + } + + /// Verifies that recognised words are dictated into the text box, and nothing is sent as they arrive. [Fact] - public async Task A_spoken_question_is_sent_like_a_typed_one() + public async Task Dictated_words_fill_the_text_box() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMinutes(10)); + var component = await RenderListeningAsync(service); + + await component.InvokeAsync(() => component.Instance.OnVoiceWord("Is")); + await component.InvokeAsync(() => component.Instance.OnVoiceWord("MS-26473")); + await component.InvokeAsync(() => component.Instance.OnVoiceWord("done?")); + + component.Find("textarea").GetAttribute("value").Should().Be("Is MS-26473 done?"); + service.Sent.Should().BeEmpty(); + } + + /// Verifies that the pause does not repeat words already dictated, and a host that reports only pauses still fills the box. + [Fact] + public async Task A_pause_adds_only_what_was_not_dictated() + { + SetUpVoiceModule(); + var component = await RenderListeningAsync(VoiceService(TimeSpan.FromMinutes(10))); + + await SayAsync(component, "Is", "it", "done?"); + await component.InvokeAsync(() => component.Instance.OnVoiceTurn("And the other one?")); + + component.Find("textarea").GetAttribute("value").Should().Be("Is it done? And the other one?"); + } + + /// Verifies that after a pause, with the text box not focused, the text is sent as the user once the delay has passed. + [Fact] + public async Task Dictation_is_sent_after_the_delay_when_the_text_box_is_not_focused() { var module = SetUpVoiceModule(); - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; - var component = RenderChat(service); - await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + var service = VoiceService(TimeSpan.FromMilliseconds(150)); + var component = await RenderListeningAsync(service); - await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Is MS-26473 done?")); + await SayAsync(component, "Is", "it", "done?"); + service.Sent.Should().BeEmpty("the delay has not passed"); - var sent = service.Sent.Should().ContainSingle().Subject; - sent.Message.Should().Be("Is MS-26473 done?"); + component.WaitForAssertion(() => service.Sent.Should().ContainSingle(), Patience); + var sent = service.Sent[0]; + sent.Message.Should().Be("Is it done?"); sent.Sender.Should().BeSameAs(_user); module.Invocations["pause"].Should().ContainSingle().Which.Arguments[0].Should().Be(true); + component.WaitForAssertion(() => component.Find("textarea").GetAttribute("value").Should().BeNullOrEmpty(), Patience); component.Instance.VoiceState.Should().Be(PDChatVoiceState.Thinking); } + /// Verifies that while the user is editing the text box, a pause never sends; the user presses Send. + [Fact] + public async Task Dictation_is_never_auto_sent_while_the_text_box_has_focus() + { + var module = SetUpVoiceModule(); + var service = VoiceService(); + var component = await RenderListeningAsync(service); + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + + await SayAsync(component, "Is", "it", "done?"); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + service.Sent.Should().BeEmpty(); + component.Find(".pdchat-voice-status").TextContent.Trim().Should().Be("Listening. You are editing, so press Send when ready."); + + await component.Find(".chat-input-container > button").ClickAsync(new MouseEventArgs()); + + service.Sent.Should().ContainSingle().Which.Message.Should().Be("Is it done?"); + module.Invocations["pause"].Should().ContainSingle().Which.Arguments[0].Should().Be(true); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Thinking); + } + + /// Verifies that focusing the text box while a send is pending cancels it. + [Fact] + public async Task Focusing_the_text_box_cancels_a_pending_send() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMilliseconds(200)); + var component = await RenderListeningAsync(service); + + await SayAsync(component, "Is", "it", "done?"); + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + await component.Find("textarea").BlurAsync(new FocusEventArgs()); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + service.Sent.Should().BeEmpty(); + } + + /// Verifies that speaking again before the delay has passed cancels the pending send until the next pause. + [Fact] + public async Task Speaking_again_cancels_a_pending_send() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMilliseconds(200)); + var component = await RenderListeningAsync(service); + + await SayAsync(component, "Is", "it"); + await component.InvokeAsync(() => component.Instance.OnVoiceWord("done?")); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + service.Sent.Should().BeEmpty(); + + await component.InvokeAsync(() => component.Instance.OnVoiceTurn("done?")); + + component.WaitForAssertion(() => service.Sent.Should().ContainSingle().Which.Message.Should().Be("Is it done?"), Patience); + } + + /// Verifies that turning Voice Mode off cancels a pending send. + [Fact] + public async Task Turning_Voice_Mode_off_cancels_a_pending_send() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMilliseconds(200)); + var component = await RenderListeningAsync(service); + + await SayAsync(component, "Is", "it", "done?"); + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + service.Sent.Should().BeEmpty(); + component.Find("textarea").GetAttribute("value").Should().Be("Is it done?", "dictation stays for the user to send or edit"); + } + /// Verifies that only the finished answer is read aloud, as plain text, and listening then resumes. [Fact] public async Task The_finished_answer_is_spoken_and_listening_resumes() { var module = SetUpVoiceModule(); - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; - var component = RenderChat(service); - await component.Find(".pdchat-voice-toggle").ClickAsync(new()); - await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Is it done?")); + var service = VoiceService(); + var component = await RenderListeningAsync(service); + await AskAsync(component, service, "Is it done?"); var typing = Message("Looking it up", MessageType.Typing); await component.InvokeAsync(() => service.Receive(typing)); @@ -111,10 +266,9 @@ public async Task The_finished_answer_is_spoken_and_listening_resumes() public async Task Only_the_first_reply_to_a_spoken_question_is_spoken() { var module = SetUpVoiceModule(); - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; - var component = RenderChat(service); - await component.Find(".pdchat-voice-toggle").ClickAsync(new()); - await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Is it done?")); + var service = VoiceService(); + var component = await RenderListeningAsync(service); + await AskAsync(component, service, "Is it done?"); await component.InvokeAsync(() => service.Receive(Message("Yes."))); await component.InvokeAsync(() => service.Receive(Message("An unrelated notification"))); @@ -127,7 +281,7 @@ public async Task Only_the_first_reply_to_a_spoken_question_is_spoken() public async Task With_Voice_Mode_off_nothing_is_spoken() { var module = SetUpVoiceModule(); - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; + var service = VoiceService(); var component = RenderChat(service); await component.InvokeAsync(() => service.Receive(Message("An answer"))); @@ -140,8 +294,7 @@ public async Task With_Voice_Mode_off_nothing_is_spoken() public async Task Turning_Voice_Mode_off_stops_it() { var module = SetUpVoiceModule(); - var component = RenderChat(new FakeChatService { VoiceEndpoints = _voiceEndpoints }); - await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + var component = await RenderListeningAsync(VoiceService()); await component.Find(".pdchat-voice-toggle").ClickAsync(new()); @@ -156,7 +309,7 @@ public async Task A_refused_microphone_is_explained() { var module = SetUpVoiceModule(); _ = module.SetupVoid("start", _ => true).SetException(new JSException("NotAllowedError")); - var component = RenderChat(new FakeChatService { VoiceEndpoints = _voiceEndpoints }); + var component = RenderChat(VoiceService()); await component.Find(".pdchat-voice-toggle").ClickAsync(new()); @@ -164,6 +317,65 @@ public async Task A_refused_microphone_is_explained() component.Find(".pdchat-voice-status").TextContent.Should().Contain("microphone could not be opened"); } + private static FakeChatService VoiceService(TimeSpan? autoSendDelay = null) + { + var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; + ((IChatService)service).VoiceAutoSendDelay = autoSendDelay ?? TimeSpan.Zero; + return service; + } + + private async Task> RenderListeningAsync(FakeChatService service) + { + var component = RenderChat(service); + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + return component; + } + + private static async Task SayAsync(IRenderedComponent component, params string[] words) + { + foreach (var word in words) + { + await component.InvokeAsync(() => component.Instance.OnVoiceWord(word)); + } + + await component.InvokeAsync(() => component.Instance.OnVoiceTurn(string.Join(' ', words))); + } + + private static async Task AskAsync(IRenderedComponent component, FakeChatService service, string question) + { + await SayAsync(component, question.Split(' ')); + component.WaitForAssertion(() => service.Sent.Should().ContainSingle(), Patience); + } + + /// Verifies that endpoints naming their own module have Voice Mode driven by it, not the standard one. + [Fact] + public async Task Simulated_Voice_Mode_uses_its_own_module() + { + var simulated = PDChatVoiceEndpoints.Simulated; + var module = JSInterop.SetupModule(simulated.ModulePath!); + module.Mode = JSRuntimeMode.Loose; + var service = VoiceService(); + service.VoiceEndpoints = simulated; + var component = RenderChat(service); + + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + + VoiceModuleImports().Should().BeEmpty(); + module.Invocations["start"].Should().ContainSingle(); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + } + + /// Verifies that the demo service shows Voice Mode and the agent picker with no speech service behind it. + [Fact] + public void The_demo_service_offers_simulated_Voice_Mode_and_agents() + { + using var service = new PanoramicData.Blazor.Services.DumbChatService(); + IChatService chat = service; + + chat.VoiceEndpoints.Should().Be(PDChatVoiceEndpoints.Simulated); + chat.Agents.Should().HaveCountGreaterThan(1); + } + private BunitJSModuleInterop SetUpVoiceModule() { var module = JSInterop.SetupModule(VoiceModulePath); diff --git a/PanoramicData.Blazor.Test/Components/PDMessagesTests.ChatInput.cs b/PanoramicData.Blazor.Test/Components/PDMessagesTests.ChatInput.cs new file mode 100644 index 000000000..98e180a49 --- /dev/null +++ b/PanoramicData.Blazor.Test/Components/PDMessagesTests.ChatInput.cs @@ -0,0 +1,108 @@ +using AwesomeAssertions; +using Bunit; +using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Components.Web; +using PanoramicData.Blazor.Interfaces; + +namespace PanoramicData.Blazor.Test; + +/// +/// as an : text appended from elsewhere, focus tracking, a virtual +/// Send, and host controls in the input row. +/// +public partial class PDMessagesTests +{ + private const string FocusIdentifier = "Blazor._internal.domWrapper.focus"; + + /// Verifies that appended text joins what was typed with a space and is pushed to the parent. + [Fact] + public async Task Appended_text_follows_what_was_typed() + { + var events = new List(); + var component = RenderLive(events); + await component.InvokeAsync(() => component.Find("textarea").InputAsync(new ChangeEventArgs { Value = "Is" })); + + await component.InvokeAsync(() => component.Instance.AppendAsync(" it done? ")); + await component.InvokeAsync(() => component.Instance.AppendAsync(" ")); + + component.Instance.Text.Should().Be("Is it done?"); + component.Find("textarea").GetAttribute("value").Should().Be("Is it done?"); + events.Should().Equal("input:Is it done?"); + } + + /// Verifies that the text box reports focus and blur, and loses focus when cleared. + [Fact] + public async Task Focus_is_tracked_and_reported() + { + var reported = new List(); + var component = Render(parameters => parameters + .Add(p => p.IsLive, true) + .Add(p => p.InputFocusChanged, isFocused => reported.Add(isFocused))); + + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + component.Instance.IsFocused.Should().BeTrue(); + + await component.Find("textarea").BlurAsync(new FocusEventArgs()); + component.Instance.IsFocused.Should().BeFalse(); + + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + await component.InvokeAsync(component.Instance.ClearInput); + component.Instance.IsFocused.Should().BeFalse("the text box was replaced"); + + reported.Should().Equal(true, false, true); + } + + /// Verifies that a virtual Send behaves as the button does, and sends nothing when empty. + [Fact] + public async Task SendAsync_is_the_same_as_pressing_send() + { + var events = new List(); + var component = RenderLive(events); + + await component.InvokeAsync(component.Instance.SendAsync); + events.Should().BeEmpty(); + + await component.InvokeAsync(() => component.Instance.AppendAsync("spoken")); + await component.InvokeAsync(component.Instance.SendAsync); + + events.Should().Equal("input:spoken", "input:spoken", "send"); + } + + /// Verifies that host controls are shown between the text box and Send, and not at all when absent. + [Fact] + public void Input_accessories_sit_beside_send() + { + RenderLive([]).FindAll(".chat-input-accessories").Should().BeEmpty(); + + var component = Render(parameters => parameters + .Add(p => p.IsLive, true) + .Add(p => p.InputAccessories, builder => builder.AddMarkupContent(0, ""))); + + var row = component.Find(".chat-input-container"); + row.Children.Select(child => child.LocalName).Should().Equal("textarea", "div", "button"); + row.QuerySelector(".chat-input-accessories .extra").Should().NotBeNull(); + } + + /// Verifies that the text box takes focus when shown, unless that is turned off. + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task The_text_box_takes_focus_only_when_auto_focus_is_on(bool isAutoFocused) + { + var attached = SignalOn("attachEnterHandler"); + var component = Render(parameters => parameters + .Add(p => p.IsLive, true) + .Add(p => p.IsInputAutoFocused, isAutoFocused)); + await attached.WaitForCountAsync(1); + + if (isAutoFocused) + { + component.WaitForAssertion(() => JSInterop.Invocations[FocusIdentifier].Should().ContainSingle(), TimeSpan.FromSeconds(10)); + } + else + { + await Task.Delay(100, Xunit.TestContext.Current.CancellationToken); + JSInterop.Invocations[FocusIdentifier].Should().BeEmpty(); + } + } +} diff --git a/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs b/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs index 6f5d3506c..70da9c6e5 100644 --- a/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs +++ b/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs @@ -46,6 +46,10 @@ internal sealed class ChatServiceDefaultState public PDChatButtonPosition ToastAnchor { get; set; } = PDChatButtonPosition.BottomRight; + public TimeSpan VoiceAutoSendDelay { get; set; } = TimeSpan.FromMilliseconds(1000); + + public string? SelectedAgentId { get; set; } + /// /// Gets or sets the handlers subscribed to the default . /// They are kept so that subscribing and unsubscribing behave as they would on any event, but a service that diff --git a/PanoramicData.Blazor/Interfaces/IChatInput.cs b/PanoramicData.Blazor/Interfaces/IChatInput.cs new file mode 100644 index 000000000..341914aa7 --- /dev/null +++ b/PanoramicData.Blazor/Interfaces/IChatInput.cs @@ -0,0 +1,24 @@ +namespace PanoramicData.Blazor.Interfaces; + +/// +/// The text box a chat message is composed in, whatever wrote the text: the keyboard, dictation, or both. +/// +/// +/// implements this, and 's Voice +/// control writes through it, so dictated words land where typed ones do and can be edited before sending. +/// +public interface IChatInput +{ + /// Gets the text currently in the box. + string Text { get; } + + /// Gets a value indicating whether the box has focus, meaning the user may be editing it. + bool IsFocused { get; } + + /// Appends text to the end of the box, separated from what is already there by a space. + /// The text to add. + Task AppendAsync(string text); + + /// Sends the box's text exactly as pressing Send would; does nothing when there is nothing to send. + Task SendAsync(); +} diff --git a/PanoramicData.Blazor/Interfaces/IChatService.cs b/PanoramicData.Blazor/Interfaces/IChatService.cs index e108f280a..cbaa41c20 100644 --- a/PanoramicData.Blazor/Interfaces/IChatService.cs +++ b/PanoramicData.Blazor/Interfaces/IChatService.cs @@ -94,6 +94,33 @@ bool IsInputPermitted /// PDChatVoiceEndpoints? VoiceEndpoints => null; + /// + /// Gets or sets how long after the speaker pauses Voice Mode sends the dictated text, when the user is not + /// editing it. Defaults to one second. + /// + TimeSpan VoiceAutoSendDelay + { + get => ChatServiceDefaultState.For(this).VoiceAutoSendDelay; + set => ChatServiceDefaultState.For(this).VoiceAutoSendDelay = value; + } + + /// + /// Gets the agents the user may choose to talk to, or null (the default) when the host offers no choice. + /// A picker is shown in the input area only when there are two or more. + /// + IReadOnlyList? Agents => null; + + /// + /// Gets or sets the of the agent the user is talking to, or null (the default) + /// for the host's default. The chat sets this when the user picks an agent; a host that needs to react implements + /// the setter. + /// + string? SelectedAgentId + { + get => ChatServiceDefaultState.For(this).SelectedAgentId; + set => ChatServiceDefaultState.For(this).SelectedAgentId = value; + } + /// /// Gets or sets whether the chat should auto-restore when new messages arrive. /// diff --git a/PanoramicData.Blazor/Models/PDChatAgentOption.cs b/PanoramicData.Blazor/Models/PDChatAgentOption.cs new file mode 100644 index 000000000..be0a2b7d0 --- /dev/null +++ b/PanoramicData.Blazor/Models/PDChatAgentOption.cs @@ -0,0 +1,11 @@ +namespace PanoramicData.Blazor.Models; + +/// +/// One of the agents a host offers in 's input area, for the user to choose +/// who they are talking to. +/// +/// The identifier the host recognises, stored in . +/// The name shown in the picker. +/// An optional description, shown as the option's tooltip. +/// An optional icon, shown beside the picker while this agent is selected. +public sealed record PDChatAgentOption(string Id, string Name, string? Description = null, string? IconUrl = null); diff --git a/PanoramicData.Blazor/Models/PDChatVoiceEndpoints.cs b/PanoramicData.Blazor/Models/PDChatVoiceEndpoints.cs index 3b1e36fcd..4e5bf24be 100644 --- a/PanoramicData.Blazor/Models/PDChatVoiceEndpoints.cs +++ b/PanoramicData.Blazor/Models/PDChatVoiceEndpoints.cs @@ -12,5 +12,16 @@ namespace PanoramicData.Blazor.Models; /// Receives one JSON message, {"text":...}, and answers with 24 kHz mono float32 audio as binary messages, /// then {"type":"done"}. /// +/// +/// An optional replacement for the browser module that drives the microphone and plays answers, exporting the same +/// start, pause, speak and stop functions; null uses the standard one. +/// /// Relative URLs are resolved against the page. Speech is processed wherever the host's endpoints send it. -public sealed record PDChatVoiceEndpoints(string ListenUrl, string SpeakUrl); +public sealed record PDChatVoiceEndpoints(string ListenUrl, string SpeakUrl, string? ModulePath = null) +{ + /// + /// Voice Mode with no speech service, for demos: it hears that the user is talking but makes the words up, and + /// answers in the browser's own voice. No audio leaves the browser. + /// + public static PDChatVoiceEndpoints Simulated { get; } = new("simulated", "simulated", "./_content/PanoramicData.Blazor/js/pdchat-voice-simulated.js"); +} diff --git a/PanoramicData.Blazor/PDChat.Agents.cs b/PanoramicData.Blazor/PDChat.Agents.cs new file mode 100644 index 000000000..bb40aeafb --- /dev/null +++ b/PanoramicData.Blazor/PDChat.Agents.cs @@ -0,0 +1,25 @@ +namespace PanoramicData.Blazor; + +/// +/// PDChat: the agent picker in the input area, offered when the chat service lists two or more +/// . +/// +public partial class PDChat +{ + private bool IsAgentPickerOffered => ChatService.Agents is { Count: > 1 }; + + // Null or unknown means the host's default, which the picker shows as its first agent. + private PDChatAgentOption? SelectedAgent => ChatService.Agents?.FirstOrDefault(agent => agent.Id == ChatService.SelectedAgentId) + ?? (ChatService.Agents is { Count: > 0 } agents ? agents[0] : null); + + private RenderFragment? InputAccessories => IsVoiceModeOffered || IsAgentPickerOffered ? InputAccessoriesContent : null; + + private void OnAgentSelected(ChangeEventArgs args) + { + var id = args.Value?.ToString(); + if (ChatService.Agents?.Any(agent => agent.Id == id) == true) + { + ChatService.SelectedAgentId = id; + } + } +} diff --git a/PanoramicData.Blazor/PDChat.Voice.cs b/PanoramicData.Blazor/PDChat.Voice.cs index b87374d49..106411db7 100644 --- a/PanoramicData.Blazor/PDChat.Voice.cs +++ b/PanoramicData.Blazor/PDChat.Voice.cs @@ -22,12 +22,14 @@ public enum PDChatVoiceState } /// -/// Voice Mode for : the user speaks a question, pauses, and hears the answer. Off by default, and +/// Voice Mode for : the user dictates into the text box, and hears the answer. Off by default, and /// offered only when the chat service supplies . /// /// -/// A spoken question goes through the same path as a typed one, so the question and the written answer are both in -/// the transcript. Only the first finished reply to a spoken question is read aloud. +/// Recognised words are appended to the text box, where they can be edited. When the speaker pauses and the box does +/// not have focus, the text is sent after as if Send had been pressed; +/// while the user is editing, they press Send themselves. The first finished reply to a message sent while Voice Mode +/// is listening is read aloud. /// public partial class PDChat { @@ -36,7 +38,8 @@ public partial class PDChat private IJSObjectReference? _voiceModule; private DotNetObjectReference? _voiceReference; private bool _isAwaitingSpokenAnswer; - private string _voiceHeard = string.Empty; + private bool _hasDictatedSinceTurn; + private CancellationTokenSource? _autoSendCancellation; /// Gets where Voice Mode is in a spoken exchange. public PDChatVoiceState VoiceState { get; private set; } = PDChatVoiceState.Off; @@ -44,22 +47,27 @@ public partial class PDChat /// Gets the last problem Voice Mode reported, shown in place of its status. public string? VoiceError { get; private set; } + // Dictation goes through the same text box as typing, via its IChatInput members. + private PDMessages? VoiceInput => MessagesComponent; + private bool IsVoiceModeOffered => ChatService.VoiceEndpoints is not null && ChatService.IsInputPermitted; private bool IsVoiceModeOn => VoiceState != PDChatVoiceState.Off; private string VoiceButtonTitle => IsVoiceModeOn - ? "Turn Voice Mode off" - : "Turn Voice Mode on: ask out loud and hear the answer"; + ? "Voice: stop listening" + : "Voice: speak instead of typing"; private string VoiceStatusCssClass => $"pdchat-voice-status pdchat-voice-{VoiceState.ToString().ToLowerInvariant()}" + (VoiceError is null ? string.Empty : " pdchat-voice-error"); private string VoiceStatusText => VoiceError ?? VoiceState switch { PDChatVoiceState.Starting => "Opening the microphone…", - PDChatVoiceState.Listening => _voiceHeard.Length > 0 ? _voiceHeard : "Listening. Ask your question, then pause.", + PDChatVoiceState.Listening => VoiceInput?.IsFocused == true + ? "Listening. You are editing, so press Send when ready." + : "Listening. Ask your question, then pause.", PDChatVoiceState.Thinking => "Thinking…", - PDChatVoiceState.Speaking => "Speaking. Turn Voice Mode off to stop.", + PDChatVoiceState.Speaking => "Speaking. Turn Voice off to stop.", _ => string.Empty, }; @@ -80,7 +88,7 @@ private async Task ToggleVoiceModeAsync() VoiceState = PDChatVoiceState.Starting; try { - _voiceModule ??= await JSRuntime.InvokeAsync("import", _voiceModulePath); + _voiceModule ??= await JSRuntime.InvokeAsync("import", endpoints.ModulePath ?? _voiceModulePath); _voiceReference ??= DotNetObjectReference.Create(this); await _voiceModule.InvokeVoidAsync("start", endpoints.ListenUrl, _voiceReference); VoiceState = PDChatVoiceState.Listening; @@ -94,44 +102,61 @@ private async Task ToggleVoiceModeAsync() private async Task StopVoiceModeAsync() { + CancelAutoSend(); VoiceState = PDChatVoiceState.Off; _isAwaitingSpokenAnswer = false; - _voiceHeard = string.Empty; + _hasDictatedSinceTurn = false; if (_voiceModule is not null) { await _voiceModule.InvokeVoidAsync("stop"); } } - /// Called by the voice module as words are recognised. + /// Called by the voice module as words are recognised; appends them to the text box. /// The word just heard. [JSInvokable] - public Task OnVoiceWord(string text) + public Task OnVoiceWord(string text) => InvokeAsync(async () => { - _voiceHeard = $"{_voiceHeard} {text}".Trim(); - return InvokeAsync(StateHasChanged); - } + if (VoiceState != PDChatVoiceState.Listening || VoiceInput is not { } input) + { + return; + } - /// Called by the voice module when the speaker has finished; sends what they said as a question. - /// The whole question. + // The speaker has carried on, so the pause that scheduled a send is over. + CancelAutoSend(); + _hasDictatedSinceTurn = true; + await input.AppendAsync(text); + }); + + /// + /// Called by the voice module when the speaker pauses. Unless the user is editing the text box, its text is sent + /// after . + /// + /// Everything said since the last pause. [JSInvokable] - public async Task OnVoiceTurn(string text) + public Task OnVoiceTurn(string text) => InvokeAsync(async () => { - if (VoiceState != PDChatVoiceState.Listening || string.IsNullOrWhiteSpace(text)) + if (VoiceState != PDChatVoiceState.Listening || VoiceInput is not { } input) { return; } - _voiceHeard = string.Empty; - _currentInput = text; - VoiceState = PDChatVoiceState.Thinking; - _isAwaitingSpokenAnswer = true; + CancelAutoSend(); - // Half duplex: while Merlin thinks and speaks, the microphone sends nothing, so it never hears itself. - await (_voiceModule?.InvokeVoidAsync("pause", true) ?? ValueTask.CompletedTask); - await SendCurrentMessageAsync(); - await InvokeAsync(StateHasChanged); - } + // A host that reports no words still has its pause recorded. + if (!_hasDictatedSinceTurn) + { + await input.AppendAsync(text); + } + + _hasDictatedSinceTurn = false; + if (!input.IsFocused && !string.IsNullOrWhiteSpace(input.Text)) + { + ScheduleAutoSend(); + } + + StateHasChanged(); + }); /// Called by the voice module when speech recognition reports a problem. /// What went wrong, for the user. @@ -146,12 +171,85 @@ public Task OnVoiceError(string text) [JSInvokable] public Task OnVoiceClosed() { + CancelAutoSend(); VoiceState = PDChatVoiceState.Off; _isAwaitingSpokenAnswer = false; VoiceError ??= "Voice Mode stopped: the connection closed."; return InvokeAsync(StateHasChanged); } + private Task OnInputFocusChangedAsync(bool isFocused) + { + // The user is editing, so whatever was dictated waits for them to press Send. + if (isFocused) + { + CancelAutoSend(); + } + + return Task.CompletedTask; + } + + private void ScheduleAutoSend() + { + var cancellation = new CancellationTokenSource(); + _autoSendCancellation = cancellation; + _ = AutoSendAfterDelayAsync(cancellation.Token); + } + + private void CancelAutoSend() + { + if (_autoSendCancellation is null) + { + return; + } + + _autoSendCancellation.Cancel(); + _autoSendCancellation.Dispose(); + _autoSendCancellation = null; + } + + private async Task AutoSendAfterDelayAsync(CancellationToken cancellationToken) + { + try + { + var delay = ChatService.VoiceAutoSendDelay; + await Task.Delay(delay < TimeSpan.Zero ? TimeSpan.Zero : delay, cancellationToken); + await InvokeAsync(async () => + { + if (cancellationToken.IsCancellationRequested || VoiceState != PDChatVoiceState.Listening + || VoiceInput is not { IsFocused: false } input) + { + return; + } + + await input.SendAsync(); + }); + } + catch (Exception ex) when (ex is JSDisconnectedException or OperationCanceledException or ObjectDisposedException) + { + // Cancelled by the speaker or the user, or the circuit is going away. + } + } + + /// + /// Marks a message about to be sent while listening as a spoken exchange: the microphone pauses and the reply is read aloud. + /// + private async Task BeginSpokenExchangeAsync() + { + if (VoiceState != PDChatVoiceState.Listening) + { + return; + } + + CancelAutoSend(); + _hasDictatedSinceTurn = false; + VoiceState = PDChatVoiceState.Thinking; + _isAwaitingSpokenAnswer = true; + + // Half duplex: while Merlin thinks and speaks, the microphone sends nothing, so it never hears itself. + await (_voiceModule?.InvokeVoidAsync("pause", true) ?? ValueTask.CompletedTask); + } + /// Reads the first finished reply to a spoken question aloud, then listens again. private async Task SpeakAnswerIfAwaitedAsync(ChatMessage message) { @@ -192,6 +290,7 @@ private static string ToSpeakableText(ChatMessage message) private async ValueTask DisposeVoiceAsync() { + CancelAutoSend(); if (_voiceModule is not null) { try diff --git a/PanoramicData.Blazor/PDChat.razor b/PanoramicData.Blazor/PDChat.razor index 0ab5fbde1..55244cee2 100644 --- a/PanoramicData.Blazor/PDChat.razor +++ b/PanoramicData.Blazor/PDChat.razor @@ -84,15 +84,6 @@ 🗑️ } - @if (IsVoiceModeOffered) - { - - } @@ -208,6 +199,9 @@ IsInputPermitted="@ChatService.IsInputPermitted" InputDisabledMessage="@ChatService.InputDisabledMessage" OnSendClicked="SendCurrentMessageAsync" + InputAccessories="InputAccessories" + InputFocusChanged="OnInputFocusChangedAsync" + IsInputAutoFocused="@(!IsVoiceModeOn)" UserIconSelector="UserIconSelector" UseFullWidthMessages="@ChatService.UseFullWidthMessages" MessageMetadataDisplayMode="@ChatService.MessageMetadataDisplayMode" @@ -255,6 +249,9 @@ IsInputPermitted="@ChatService.IsInputPermitted" InputDisabledMessage="@ChatService.InputDisabledMessage" OnSendClicked="SendCurrentMessageAsync" + InputAccessories="InputAccessories" + InputFocusChanged="OnInputFocusChangedAsync" + IsInputAutoFocused="@(!IsVoiceModeOn)" UserIconSelector="UserIconSelector" UseFullWidthMessages="@ChatService.UseFullWidthMessages" MessageMetadataDisplayMode="@ChatService.MessageMetadataDisplayMode" @@ -276,6 +273,9 @@ IsInputPermitted="@ChatService.IsInputPermitted" InputDisabledMessage="@ChatService.InputDisabledMessage" OnSendClicked="SendCurrentMessageAsync" + InputAccessories="InputAccessories" + InputFocusChanged="OnInputFocusChangedAsync" + IsInputAutoFocused="@(!IsVoiceModeOn)" UserIconSelector="UserIconSelector" UseFullWidthMessages="@ChatService.UseFullWidthMessages" MessageMetadataDisplayMode="@ChatService.MessageMetadataDisplayMode" @@ -289,3 +289,36 @@ } + +@code { + private RenderFragment InputAccessoriesContent => @ + @if (IsAgentPickerOffered) + { +
+ @if (SelectedAgent?.IconUrl is { } iconUrl) + { + + } + +
+ } + @if (IsVoiceModeOffered) + { + + } +
; +} \ No newline at end of file diff --git a/PanoramicData.Blazor/PDChat.razor.cs b/PanoramicData.Blazor/PDChat.razor.cs index 9af92c3d9..d4f623735 100644 --- a/PanoramicData.Blazor/PDChat.razor.cs +++ b/PanoramicData.Blazor/PDChat.razor.cs @@ -537,6 +537,9 @@ private async Task SendCurrentMessageAsync() Timestamp = DateTime.UtcNow }; + // Before sending, so a reply that arrives at once is still the one spoken. + await BeginSpokenExchangeAsync(); + // Fire it off ChatService.SendMessage(message); diff --git a/PanoramicData.Blazor/PDChat.razor.css b/PanoramicData.Blazor/PDChat.razor.css index 71610bde4..79c330574 100644 --- a/PanoramicData.Blazor/PDChat.razor.css +++ b/PanoramicData.Blazor/PDChat.razor.css @@ -543,18 +543,34 @@ body:has(.pdchat-container.dock-fullscreen) { outline-offset: 1px; } -/* Voice Mode: the switch carries a word as well as an icon, so it says what it does. */ +/* Input-area controls: the Voice switch carries a word as well as an icon, so it says what it does. */ .pdchat-voice-toggle { - font-size: 14px; - min-width: auto; white-space: nowrap; } .pdchat-voice-toggle.active { - background: rgba(255,255,255,0.25); - box-shadow: inset 0 0 0 1px rgba(255,255,255,0.6); + color: var(--pd-chat-bg, #fff); + background: #dc3545; + border-color: #dc3545; } +.pdchat-agent-picker { + display: flex; + align-items: center; + gap: 4px; + min-width: 0; +} + +.pdchat-agent-picker select { + min-width: 0; +} + +.pdchat-agent-icon { + width: 20px; + height: 20px; + flex-shrink: 0; + border-radius: 50%; +} .pdchat-voice-status { padding: 6px 12px; font-size: 13px; diff --git a/PanoramicData.Blazor/PDMessages.Interop.cs b/PanoramicData.Blazor/PDMessages.Interop.cs index ba61599da..21cf841c7 100644 --- a/PanoramicData.Blazor/PDMessages.Interop.cs +++ b/PanoramicData.Blazor/PDMessages.Interop.cs @@ -38,7 +38,10 @@ await JSRuntime.InvokeAsync( _dotNetRef ??= DotNetObjectReference.Create(this); await _module.InvokeVoidAsync("attachEnterHandler", InputRef, _dotNetRef); _enterHandlerAttached = true; - await InputRef.FocusAsync(); + if (IsInputAutoFocused) + { + await InputRef.FocusAsync(); + } } catch (Exception) { diff --git a/PanoramicData.Blazor/PDMessages.razor b/PanoramicData.Blazor/PDMessages.razor index ff112e3fa..030ca2291 100644 --- a/PanoramicData.Blazor/PDMessages.razor +++ b/PanoramicData.Blazor/PDMessages.razor @@ -25,9 +25,15 @@ @ref="InputRef" value="@_localInput" @oninput="OnInputChanged" + @onfocus="() => SetInputFocusedAsync(true)" + @onblur="() => SetInputFocusedAsync(false)" placeholder="Type a message..." rows="3" disabled="@(!IsLive)"> + @if (InputAccessories is not null) + { +
@InputAccessories
+ } } diff --git a/PanoramicData.Blazor/PDMessages.razor.cs b/PanoramicData.Blazor/PDMessages.razor.cs index 9dc6366f5..3450cb2bf 100644 --- a/PanoramicData.Blazor/PDMessages.razor.cs +++ b/PanoramicData.Blazor/PDMessages.razor.cs @@ -3,7 +3,7 @@ namespace PanoramicData.Blazor; /// /// A Blazor component that displays a list of chat messages with a send input area. /// -public partial class PDMessages +public partial class PDMessages : IChatInput { /// /// Gets or sets the injected JavaScript runtime. @@ -99,11 +99,51 @@ public partial class PDMessages /// internal ElementReference InputRef { get; set; } + /// + /// Gets or sets optional controls shown in the input row beside the Send button, such as a voice or agent control. + /// + [Parameter] public RenderFragment? InputAccessories { get; set; } + + /// + /// An event callback that is invoked when the text box gains (true) or loses (false) focus. + /// + [Parameter] public EventCallback InputFocusChanged { get; set; } + + /// + /// Gets or sets whether the text box takes focus when first shown and after each send. Defaults to true. + /// + [Parameter] public bool IsInputAutoFocused { get; set; } = true; + private string _localInput = string.Empty; private string _inputKey = Guid.NewGuid().ToString(); private bool CanSendLocal => IsInputPermitted && IsLive && !string.IsNullOrWhiteSpace(_localInput); + /// + public string Text => _localInput; + + /// + public bool IsFocused { get; private set; } + + /// + public async Task AppendAsync(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + var addition = text.Trim(); + _localInput = _localInput.Length == 0 || char.IsWhiteSpace(_localInput[^1]) + ? _localInput + addition + : $"{_localInput} {addition}"; + await CurrentInputChanged.InvokeAsync(_localInput); + StateHasChanged(); + } + + /// + public Task SendAsync() => OnSendClickedInternal(); + /// /// Clears the textarea. Called by the parent after a message is sent. /// @@ -111,10 +151,19 @@ public void ClearInput() { _localInput = string.Empty; _inputKey = Guid.NewGuid().ToString(); + + // The text box is replaced, and a removed element does not reliably report losing focus. + IsFocused = false; ResetEnterHandler(); StateHasChanged(); } + private async Task SetInputFocusedAsync(bool isFocused) + { + IsFocused = isFocused; + await InputFocusChanged.InvokeAsync(isFocused); + } + /// /// Called from JavaScript when Enter is pressed in the textarea. /// diff --git a/PanoramicData.Blazor/PDMessages.razor.css b/PanoramicData.Blazor/PDMessages.razor.css index 103468262..eb5d65e56 100644 --- a/PanoramicData.Blazor/PDMessages.razor.css +++ b/PanoramicData.Blazor/PDMessages.razor.css @@ -113,6 +113,17 @@ flex-shrink: 0; } +/* Host-supplied controls (voice, agent) stacked beside Send. */ +.chat-input-accessories { + display: flex; + flex-direction: column; + align-items: stretch; + gap: 4px; + flex-shrink: 0; + min-width: 0; + max-width: 40%; +} + /* Message shown in place of the input when input is not permitted (IsInputPermitted = false). Theme-aware via the --pd-chat-* variables. */ .chat-input-disabled { diff --git a/PanoramicData.Blazor/Services/DumbChatService.Voice.cs b/PanoramicData.Blazor/Services/DumbChatService.Voice.cs new file mode 100644 index 000000000..50116c496 --- /dev/null +++ b/PanoramicData.Blazor/Services/DumbChatService.Voice.cs @@ -0,0 +1,17 @@ +namespace PanoramicData.Blazor.Services; + +/// +/// The Voice Mode and agent picker of , so a demo shows both without a speech service. +/// +public partial class DumbChatService +{ + /// Gets or sets Voice Mode's endpoints; simulated by default, and null hides Voice Mode. + public PDChatVoiceEndpoints? VoiceEndpoints { get; set; } = PDChatVoiceEndpoints.Simulated; + + /// Gets the agents offered in the input area. + public IReadOnlyList? Agents { get; } = + [ + new("dumbbot", "DumbBot", "Answers by keyword, and not very well."), + new("parrot", "Parrot", "Exactly as helpful as DumbBot, with more feathers."), + ]; +} diff --git a/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js b/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js new file mode 100644 index 000000000..5557b86df --- /dev/null +++ b/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js @@ -0,0 +1,129 @@ +// PDChat Voice Mode, simulated: hears THAT you are talking, never WHAT, and makes the words up. +// Same exports as pdchat-voice.js. Audio never leaves the browser. + +const PHRASES = [ + "I know this is not what you are actually saying, but to use real voice recognition you need to hook in a streaming speech service.", + "blah blah blah blah blah blah", + "This is a simulated transcript, and none of your words were sent anywhere.", + "Pretend I just asked something really clever.", + "Supply VoiceEndpoints from your chat service to hear the real thing.", + "Testing, testing, one two three. Is this thing on?", +]; + +const LOUD = 0.02; +const WORD_MS = 220; +const PAUSE_MS = 700; + +let listening = null; +let phraseIndex = 0; +let wordIndex = 0; + +function nextWord() { + const words = PHRASES[phraseIndex].split(" "); + const word = words[wordIndex]; + wordIndex++; + if (wordIndex >= words.length) { + wordIndex = 0; + phraseIndex = (phraseIndex + 1) % PHRASES.length; + } + return word; +} + +function level(analyser, samples) { + analyser.getFloatTimeDomainData(samples); + let sum = 0; + for (let i = 0; i < samples.length; i++) { + sum += samples[i] * samples[i]; + } + return Math.sqrt(sum / samples.length); +} + +function tick(state, dotNet) { + if (state.paused) { + return; + } + const now = Date.now(); + if (level(state.analyser, state.samples) > LOUD) { + state.lastLoud = now; + if (now - state.lastWord >= WORD_MS) { + state.lastWord = now; + const word = nextWord(); + state.heard.push(word); + dotNet.invokeMethodAsync("OnVoiceWord", word); + } + } else if (state.heard.length > 0 && now - state.lastLoud >= PAUSE_MS) { + const text = state.heard.join(" "); + state.heard = []; + dotNet.invokeMethodAsync("OnVoiceTurn", text); + } +} + +/** Starts listening. Rejects when the microphone is refused or unavailable. */ +export async function start(listenUrl, dotNet) { + stop(); + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const context = new AudioContext(); + const analyser = context.createAnalyser(); + analyser.fftSize = 1024; + context.createMediaStreamSource(stream).connect(analyser); + const state = { + stream, + context, + analyser, + samples: new Float32Array(analyser.fftSize), + paused: false, + heard: [], + lastLoud: 0, + lastWord: 0, + }; + state.timer = setInterval(() => tick(state, dotNet), 50); + listening = state; +} + +/** While paused nothing is "heard", so the assistant does not hear itself. */ +export function pause(paused) { + if (listening) { + listening.paused = paused; + listening.heard = []; + } +} + +/** Stops listening and speaking, and releases the microphone. */ +export function stop() { + stopSpeaking(); + if (!listening) { + return; + } + const state = listening; + listening = null; + clearInterval(state.timer); + state.stream.getTracks().forEach((track) => track.stop()); + state.context.close(); +} + +/** Stops an answer part-way through. */ +export function stopSpeaking() { + if (globalThis.speechSynthesis) { + globalThis.speechSynthesis.cancel(); + } +} + +/** Speaks an answer with the browser's own voice, and calls OnVoiceSpoken when it ends. */ +export function speak(speakUrl, text, dotNet) { + stopSpeaking(); + let finished = false; + const finish = () => { + if (!finished) { + finished = true; + dotNet.invokeMethodAsync("OnVoiceSpoken"); + } + }; + if (!globalThis.speechSynthesis) { + setTimeout(finish, 60 * text.length); + return; + } + const utterance = new SpeechSynthesisUtterance(text); + utterance.onend = finish; + utterance.onerror = finish; + globalThis.speechSynthesis.speak(utterance); +}