From 9a158552079bdba10f6d59a39d7c4bc62dbde485 Mon Sep 17 00:00:00 2001 From: Kamil Tomaszek Date: Thu, 22 Oct 2026 17:06:45 -0700 Subject: [PATCH] fix(live): replay stored transcriptions as conversation history PiperOrigin-RevId: 1005163325 --- .../google/adk/flows/llmflows/Contents.java | 220 +++++-- .../adk/flows/llmflows/ContentsTest.java | 571 +++++++++++++++++- .../com/google/adk/runner/RunnerTest.java | 52 ++ 3 files changed, 792 insertions(+), 51 deletions(-) diff --git a/core/src/main/java/com/google/adk/flows/llmflows/Contents.java b/core/src/main/java/com/google/adk/flows/llmflows/Contents.java index e571b1bc6..536bb1698 100644 --- a/core/src/main/java/com/google/adk/flows/llmflows/Contents.java +++ b/core/src/main/java/com/google/adk/flows/llmflows/Contents.java @@ -35,6 +35,7 @@ import com.google.genai.types.FunctionCall; import com.google.genai.types.FunctionResponse; import com.google.genai.types.Part; +import com.google.genai.types.Transcription; import io.reactivex.rxjava3.core.Single; import java.util.ArrayList; import java.util.Collections; @@ -111,14 +112,54 @@ private ImmutableList getCurrentTurnContents( // Find the latest event that starts the current turn and process from there. for (int i = events.size() - 1; i >= 0; i--) { Event event = events.get(i); - if (event.author().equals(Role.USER) || isOtherAgentReply(agentName, event)) { + TranscriptionKind kind = transcriptionKind(event); + int start = -1; + if (kind != null) { + start = spokenTurnStart(currentBranch, events, agentName, i, kind); + } else if (shouldIncludeEventInContext(currentBranch, event) + && (event.author().equals(Role.USER) || isOtherAgentReply(agentName, event))) { + start = i; + } + if (start >= 0) { return getContents( - currentBranch, events.subList(i, events.size()), agentName, groupFunctionResponses); + currentBranch, events.subList(start, events.size()), agentName, groupFunctionResponses); } } return ImmutableList.of(); } + /** + * Returns where the spoken run ending at {@code end} starts, so the turn keeps the whole run + * rather than its last chunk. Returns -1 when the run does not start a turn: the agent's own + * speech, a chunk outside the context, or a blank run, which the merge drops. + */ + private int spokenTurnStart( + @Nullable String currentBranch, + List events, + String agentName, + int end, + TranscriptionKind kind) { + Event event = events.get(end); + boolean startsTurn = kind == TranscriptionKind.INPUT || isOtherAgentReply(agentName, event); + if (!startsTurn || !shouldIncludeEventInContext(currentBranch, event)) { + return -1; + } + int start = end; + boolean blank = kind.text(event).isBlank(); + for (int i = end - 1; i >= 0; i--) { + // Events left out of the context do not break the run. + if (!shouldIncludeEventInContext(currentBranch, events.get(i))) { + continue; + } + if (!continuesRun(kind, events.get(start), events.get(i))) { + break; + } + start = i; + blank &= kind.text(events.get(i)).isBlank(); + } + return blank ? -1 : start; + } + private ImmutableList getContents( @Nullable String currentBranch, List events, @@ -127,47 +168,31 @@ private ImmutableList getContents( List filteredEvents = new ArrayList<>(); boolean hasCompactEvent = false; - // Filter the events, leaving the contents and the function calls and responses from the current - // agent. for (Event event : events) { - if (event.actions().compaction().isPresent()) { - // Always include the compaction event for the later processCompactionEvent call. - // The compaction event is used to filter out normal events that are covered by the - // compaction event. - hasCompactEvent = true; + if (shouldIncludeEventInContext(currentBranch, event)) { + hasCompactEvent |= event.actions().compaction().isPresent(); filteredEvents.add(event); - continue; - } - - // Skip events without content, or generated neither by user nor by model or has empty text. - // E.g. events purely for mutating session states. - if (isEmptyContent(event)) { - continue; - } - if (!isEventBelongsToBranch(currentBranch, event)) { - continue; - } - if (isRequestConfirmationEvent(event)) { - continue; } + } - // TODO: Skip auth events. + if (hasCompactEvent) { + filteredEvents = processCompactionEvent(filteredEvents, agentName); + } + // Merge after compaction, which may end mid-run, and before other agents are narrated. + List resultEvents = new ArrayList<>(); + for (Event event : mergeTranscriptions(filteredEvents)) { if (isOtherAgentReply(agentName, event)) { Event foreignEvent = convertForeignEvent(event); if (foreignEvent != null) { - filteredEvents.add(foreignEvent); + resultEvents.add(foreignEvent); } } else { - filteredEvents.add(event); + resultEvents.add(event); } } - if (hasCompactEvent) { - filteredEvents = processCompactionEvent(filteredEvents); - } - - List resultEvents = rearrangeEventsForLatestFunctionResponse(filteredEvents); + resultEvents = rearrangeEventsForLatestFunctionResponse(resultEvents); resultEvents = rearrangeEventsForAsyncFunctionResponsesInHistory(resultEvents, groupFunctionResponses); @@ -177,11 +202,25 @@ private ImmutableList getContents( .collect(toImmutableList()); } + /** + * Whether {@code event} goes into the LLM context. Compaction events always do, so {@link + * #processCompactionEvent} can apply them. + */ + private boolean shouldIncludeEventInContext(@Nullable String currentBranch, Event event) { + if (event.actions().compaction().isPresent()) { + return true; + } + // TODO: Skip auth events. + return !isEmptyContent(event) + && isEventBelongsToBranch(currentBranch, event) + && !isRequestConfirmationEvent(event); + } + /** * Check if an event has missing or empty content. * - *

This can happen to the events that only changed session state. When both content and - * transcriptions are empty, the event will be considered as empty. The content is considered + *

This can happen to the events that only changed session state. When the content is empty and + * no transcription has text, the event will be considered as empty. The content is considered * empty if none of its parts contain text, inline data, file data, function call, function * response, server-side tool call, or server-side tool response. Parts with only thoughts are * also considered empty. @@ -190,15 +229,104 @@ private ImmutableList getContents( * @return {@code true} if the event is considered to have empty content, {@code false} otherwise. */ private boolean isEmptyContent(Event event) { - if (event.content().isEmpty()) { - return true; + return !hasVisibleContent(event) && transcriptionKind(event) == null; + } + + /** Whether {@code event} has a role and at least one part that is visible in the LLM context. */ + private boolean hasVisibleContent(Event event) { + return event + .content() + .filter(content -> !content.role().orElse("").isEmpty()) + .flatMap(Content::parts) + .map(parts -> parts.stream().anyMatch(part -> !isPartInvisible(part))) + .orElse(false); + } + + /** + * Merges each spoken run into one text event, so later invocations replay what was said: input as + * a user turn, output as a model turn. A blank run is dropped. + */ + private ImmutableList mergeTranscriptions(List events) { + ImmutableList.Builder merged = ImmutableList.builder(); + StringBuilder text = new StringBuilder(); + for (int i = 0; i < events.size(); i++) { + Event event = events.get(i); + TranscriptionKind kind = transcriptionKind(event); + if (kind == null) { + merged.add(event); + continue; + } + text.append(kind.text(event)); + if (i + 1 < events.size() && continuesRun(kind, event, events.get(i + 1))) { + continue; + } + // A blank run says nothing, while a space chunk inside a run still joins the words around it. + String runText = text.toString(); + if (!runText.isBlank()) { + merged.add(kind.toTextEvent(event, runText)); + } + text.setLength(0); + } + return merged.build(); + } + + /** + * Whether {@code other} belongs to the {@code kind} run that {@code event} is in. An output run + * is one agent's speech; an input run is the user's even when stored under different agents. + */ + private boolean continuesRun(TranscriptionKind kind, Event event, Event other) { + return transcriptionKind(other) == kind + && (kind == TranscriptionKind.INPUT || event.author().equals(other.author())); + } + + /** + * The kind of transcription {@code event} is merged as, or null if it has visible content or no + * transcription text. + */ + private @Nullable TranscriptionKind transcriptionKind(Event event) { + if (hasVisibleContent(event)) { + return null; + } + if (!TranscriptionKind.INPUT.text(event).isEmpty()) { + return TranscriptionKind.INPUT; + } + if (!TranscriptionKind.OUTPUT.text(event).isEmpty()) { + return TranscriptionKind.OUTPUT; + } + return null; + } + + /** Which side of a live conversation a transcription records. */ + private enum TranscriptionKind { + INPUT, + OUTPUT; + + String text(Event event) { + Optional transcription = + switch (this) { + case INPUT -> event.inputTranscription(); + case OUTPUT -> event.outputTranscription(); + }; + return transcription.flatMap(Transcription::text).orElse(""); + } + + /** The run's last {@code event}, carrying the run's {@code text} instead of a transcription. */ + Event toTextEvent(Event event, String text) { + return switch (this) { + // A live run stores the user's speech under the listening agent's name. + case INPUT -> + event.toBuilder() + .author(Role.USER) + .content(Content.builder().role(Role.USER).parts(Part.fromText(text)).build()) + .inputTranscription(null) + .build(); + case OUTPUT -> + event.toBuilder() + .content(Content.builder().role("model").parts(Part.fromText(text)).build()) + .outputTranscription(null) + .build(); + }; } - var content = event.content().get(); - return (content.role().isEmpty() - || content.role().get().isEmpty() - || content.parts().isEmpty() - || content.parts().get().isEmpty() - || content.parts().get().stream().allMatch(this::isPartInvisible)); } /** @@ -314,9 +442,10 @@ private boolean isPartInvisible(Part part) { * * * @param events the list of event to filter. + * @param agentName the current agent, which authors each summary. * @return a new list with compaction applied. */ - private List processCompactionEvent(List events) { + private List processCompactionEvent(List events, String agentName) { // Step 1: Split events into compaction events and regular events. List compactionEvents = new ArrayList<>(); List regularEvents = new ArrayList<>(); @@ -350,13 +479,13 @@ private List processCompactionEvent(List events) { e--; } else { // Otherwise the event is before the compaction, let's move to the next compaction event; - result.add(createCompactionEvent(compactionEvents.get(c))); + result.add(createCompactionEvent(compactionEvents.get(c), agentName)); c--; } } // Flush any remaining compactions. while (c >= 0) { - result.add(createCompactionEvent(compactionEvents.get(c))); + result.add(createCompactionEvent(compactionEvents.get(c), agentName)); c--; } // Flush any remaining regular events. @@ -390,11 +519,12 @@ private static List removeOverlappingCompactions(List events) { return Lists.reverse(result); } - private static Event createCompactionEvent(Event event) { + private static Event createCompactionEvent(Event event, String agentName) { EventCompaction compaction = event.actions().compaction().get(); return event.toBuilder() .timestamp(compaction.endTimestamp()) - .author("model") + // Authored by this agent, so convertForeignEvent leaves the summary alone. + .author(agentName.isEmpty() ? "model" : agentName) .content(compaction.compactedContent()) .build(); } diff --git a/core/src/test/java/com/google/adk/flows/llmflows/ContentsTest.java b/core/src/test/java/com/google/adk/flows/llmflows/ContentsTest.java index c133668a8..a574859d2 100644 --- a/core/src/test/java/com/google/adk/flows/llmflows/ContentsTest.java +++ b/core/src/test/java/com/google/adk/flows/llmflows/ContentsTest.java @@ -20,6 +20,7 @@ import static com.google.common.truth.Correspondence.transforming; import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; import static org.junit.Assert.assertThrows; import com.google.adk.agents.InvocationContext; @@ -41,6 +42,7 @@ import com.google.genai.types.Part; import com.google.genai.types.ToolCall; import com.google.genai.types.ToolResponse; +import com.google.genai.types.Transcription; import java.util.ArrayList; import java.util.ConcurrentModificationException; import java.util.Iterator; @@ -50,6 +52,7 @@ import java.util.Objects; import java.util.Optional; import java.util.stream.Stream; +import org.jspecify.annotations.Nullable; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -1372,6 +1375,522 @@ public void processRequest_noInvocationBranch_includesBranchedEvent() { otherAgentPreamblePart(), otherAgentPart("[agent_1] said:", "sibling output"))); } + // A live run stores what was said as transcription chunks; a later run replays them. + @Test + public void processRequest_transcriptionOnlyEvents_becomeUserAndModelText() { + ImmutableList events = + ImmutableList.of(heard("My name "), heard("is Ada."), spoken("Hi "), spoken("Ada!")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)) + .containsExactly("user: My name is Ada.", "model: Hi Ada!") + .inOrder(); + } + + @Test + public void processRequest_typedAndSpokenTurns_keepTheirOrder() { + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Hello."), + createAgentEvent("a1", "Hi, who is this?"), + heard("It is Ada."), + spoken("Nice to meet you, "), + spoken("Ada."), + createUserEvent("u2", "What is my name?")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)) + .containsExactly( + "user: Hello.", + "model: Hi, who is this?", + "user: It is Ada.", + "model: Nice to meet you, Ada.", + "user: What is my name?") + .inOrder(); + } + + @Test + public void processRequest_alternatingSpokenTurns_staySeparate() { + ImmutableList events = + ImmutableList.of(heard("a"), spoken("b"), heard("c"), spoken("d")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)) + .containsExactly("user: a", "model: b", "user: c", "model: d") + .inOrder(); + } + + @Test + public void processRequest_spokenRunsSplitByATypedTurn_stayApart() { + ImmutableList events = + ImmutableList.of(heard("a"), createUserEvent("u1", "typed"), heard("b")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: a", "user: typed", "user: b").inOrder(); + } + + // The event with content is kept as it is, so the run before it must not carry over past it. + @Test + public void processRequest_runBeforeAnEventWithContent_isFlushedFirst() { + Event typed = + createUserEvent("u1", "typed").toBuilder().inputTranscription(transcription("x")).build(); + ImmutableList events = ImmutableList.of(heard("a"), typed, heard("b")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: a", "user: typed", "user: b").inOrder(); + } + + @Test + public void processRequest_eventWithContentAndTranscription_keepsOnlyItsContent() { + Event typed = + createUserEvent("u1", "typed").toBuilder().inputTranscription(transcription("x")).build(); + + List contents = runContentsProcessor(ImmutableList.of(typed)); + + assertThat(turns(contents)).containsExactly("user: typed"); + } + + @Test + public void processRequest_emptyChunkInsideRun_doesNotSplitIt() { + Event noText = + Event.builder() + .author(AGENT) + .inputTranscription(Transcription.builder().finished(false).build()) + .build(); + ImmutableList events = ImmutableList.of(heard("My "), heard(""), noText, heard("name.")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: My name."); + } + + @Test + public void processRequest_emptyTranscription_isDropped() { + ImmutableList events = + ImmutableList.of(heard(""), spoken(""), createUserEvent("u1", "Hello.")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: Hello."); + } + + @Test + public void processRequest_spaceChunkBetweenWords_isKept() { + ImmutableList events = ImmutableList.of(heard("Hello"), heard(" "), heard("world.")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: Hello world."); + } + + @Test + public void processRequest_blankSpokenRun_isDropped() { + ImmutableList events = ImmutableList.of(heard("Hi."), spoken(" "), spoken(" ")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: Hi."); + } + + // Thoughts or an empty part list say nothing, so they must not hide what was said. + @Test + public void processRequest_emptyOrThoughtOnlyContentWithTranscription_isMergedAsTranscription() { + Event emptyContent = + heard("Hi.").toBuilder() + .content(Content.builder().role(USER).parts(ImmutableList.of()).build()) + .build(); + Event thoughtOnly = + spoken("Hello.").toBuilder() + .content( + Content.builder() + .role("model") + .parts(Part.builder().text("Greet them.").thought(true).build()) + .build()) + .build(); + + List contents = runContentsProcessor(ImmutableList.of(emptyContent, thoughtOnly)); + + assertThat(turns(contents)).containsExactly("user: Hi.", "model: Hello.").inOrder(); + } + + @Test + public void processRequest_otherAgentsSpokenReply_isPresentedAsContext() { + ImmutableList events = + ImmutableList.of( + heard("Is it sunny?"), spoken(OTHER_AGENT, "Yes, "), spoken(OTHER_AGENT, "sunny.")); + + List contents = runContentsProcessor(events); + + assertThat(contents) + .containsExactly( + Content.fromParts(Part.fromText("Is it sunny?")), + Content.fromParts( + otherAgentPreamblePart(), otherAgentPart("[other_agent] said:", "Yes, sunny."))) + .inOrder(); + } + + @Test + public void processRequest_spokenRepliesFromTwoAgents_stayAttributedToEach() { + ImmutableList events = + ImmutableList.of( + heard("Weather and time?"), + spoken(OTHER_AGENT, "Sunny."), + spoken("clock_agent", "Noon.")); + + List contents = runContentsProcessor(events); + + assertThat(contents) + .containsExactly( + Content.fromParts(Part.fromText("Weather and time?")), + Content.fromParts( + otherAgentPreamblePart(), otherAgentPart("[other_agent] said:", "Sunny.")), + Content.fromParts( + otherAgentPreamblePart(), otherAgentPart("[clock_agent] said:", "Noon."))) + .inOrder(); + } + + // A live run stores the user's speech under the listening agent's name; it is still the user's. + @Test + public void processRequest_speechHeardByAnotherAgent_isTheUsersTurn() { + ImmutableList events = ImmutableList.of(heard(OTHER_AGENT, "Is it sunny?")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: Is it sunny?"); + } + + @Test + public void processRequest_speechHeardByTwoAgents_staysOneTurn() { + ImmutableList events = + ImmutableList.of(heard(OTHER_AGENT, "Is it "), heard(AGENT, "sunny?")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: Is it sunny?"); + } + + @Test + public void processRequest_compactionEndingInsideASpokenRun_keepsOnlyTheUncoveredChunks() { + ImmutableList events = + ImmutableList.of( + heard("one ").toBuilder().timestamp(100).build(), + heard("two").toBuilder().timestamp(200).build(), + createCompactedEvent(100, 150, "Summary")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("model: Summary", "user: two").inOrder(); + } + + @Test + public void processRequest_includeContentsNone_keepsTheWholeSpokenRun() { + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Earlier question."), + createAgentEvent("a1", "Earlier answer."), + heard("My name "), + heard("is Ada.")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: My name is Ada."); + } + + @Test + public void processRequest_includeContentsNone_userAuthoredSpeech_keepsTheWholeSpokenRun() { + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Earlier question."), + heard(USER, "My name "), + heard(USER, "is Ada.")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: My name is Ada."); + } + + @Test + public void processRequest_includeContentsNone_keepsAnotherAgentsWholeSpokenRun() { + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Is it sunny?"), + spoken(OTHER_AGENT, "Yes, "), + spoken(OTHER_AGENT, "sunny.")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(contents) + .containsExactly( + Content.fromParts( + otherAgentPreamblePart(), otherAgentPart("[other_agent] said:", "Yes, sunny."))); + } + + @Test + public void processRequest_includeContentsNone_otherAgentsSpokenRunWithThoughts_startsTheTurn() { + Content thought = + Content.builder() + .role("model") + .parts(Part.builder().text("Look it up.").thought(true).build()) + .build(); + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Is it sunny?"), + spoken(OTHER_AGENT, "Yes, ").toBuilder().content(thought).build(), + spoken(OTHER_AGENT, "sunny.").toBuilder().content(thought).build()); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(contents) + .containsExactly( + Content.fromParts( + otherAgentPreamblePart(), otherAgentPart("[other_agent] said:", "Yes, sunny."))); + } + + // The agent's own speech answers the turn rather than starting one. + @Test + public void processRequest_includeContentsNone_ownSpokenReply_staysInTheUsersTurn() { + ImmutableList events = + ImmutableList.of(createUserEvent("u1", "Is it sunny?"), spoken("Yes, "), spoken("sunny.")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)) + .containsExactly("user: Is it sunny?", "model: Yes, sunny.") + .inOrder(); + } + + // Speech heard on a sibling branch is outside this agent's context, so it cannot start the turn. + @Test + public void processRequest_includeContentsNone_speechOnAnotherBranch_doesNotStartTheTurn() { + Event offBranch = heard("agent_1", "Hi.").toBuilder().branch("root.agent_1").build(); + ImmutableList events = + ImmutableList.of(createUserEvent("u1", "Is it sunny?"), offBranch); + + List contents = + runContentsProcessorOnBranch( + events, "agent_10", "root.agent_10", LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: Is it sunny?"); + } + + @Test + public void processRequest_includeContentsNone_replyOnAnotherBranch_doesNotStartTheTurn() { + Event offBranch = + createAgentEvent("agent_1", "o1", "Sibling reply.").toBuilder() + .branch("root.agent_1") + .build(); + ImmutableList events = + ImmutableList.of(createUserEvent("u1", "Is it sunny?"), offBranch); + + List contents = + runContentsProcessorOnBranch( + events, "agent_10", "root.agent_10", LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: Is it sunny?"); + } + + @Test + public void processRequest_includeContentsNone_trailingEmptyChunk_keepsTheTurn() { + ImmutableList events = ImmutableList.of(heard(USER, "Hello."), heard(USER, "")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: Hello."); + } + + @Test + public void processRequest_includeContentsNone_runWithAnExcludedChunk_staysWhole() { + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Earlier question."), heard("My "), heard(""), heard("name.")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: My name."); + } + + @Test + public void processRequest_includeContentsNone_blankSpokenRun_doesNotBlankTheTurn() { + ImmutableList events = + ImmutableList.of(createUserEvent("u1", "Is it sunny?"), spoken(OTHER_AGENT, " ")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: Is it sunny?"); + } + + @Test + public void processRequest_includeContentsNone_otherAgentsTextReplies_keepsOnlyTheLast() { + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Is it sunny?"), + createAgentEvent(OTHER_AGENT, "o1", "Checking."), + createAgentEvent(OTHER_AGENT, "o2", "Sunny.")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(contents) + .containsExactly( + Content.fromParts( + otherAgentPreamblePart(), otherAgentPart("[other_agent] said:", "Sunny."))); + } + + @Test + public void processRequest_includeContentsNone_otherAgentsTrailingEmptyChunk_keepsTheTurn() { + ImmutableList events = ImmutableList.of(heard("Hello."), heard(OTHER_AGENT, "")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: Hello."); + } + + @Test + public void processRequest_includeContentsNone_otherAgentsRunWithTrailingEmptyChunk_staysWhole() { + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Is it sunny?"), + spoken(OTHER_AGENT, "Yes, "), + spoken(OTHER_AGENT, "sunny."), + spoken(OTHER_AGENT, "")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(contents) + .containsExactly( + Content.fromParts( + otherAgentPreamblePart(), otherAgentPart("[other_agent] said:", "Yes, sunny."))); + } + + // A live run stores a turnComplete-only event after each turn, under the agent that spoke. + @Test + public void + processRequest_includeContentsNone_otherAgentsRunBeforeItsTurnComplete_startsTheTurn() { + ImmutableList events = + ImmutableList.of( + createUserEvent("u1", "Is it sunny?"), + spoken(OTHER_AGENT, "Yes."), + Event.builder().author(OTHER_AGENT).turnComplete(true).build()); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(contents) + .containsExactly( + Content.fromParts( + otherAgentPreamblePart(), otherAgentPart("[other_agent] said:", "Yes."))); + } + + // The confirmation response is left out of the context, so the turn starts at the user's message. + @Test + public void processRequest_includeContentsNone_trailingConfirmationResponse_keepsTheUsersTurn() { + Event confirmationResponse = + Event.builder() + .author(USER) + .content( + Content.fromParts( + Part.fromFunctionResponse( + Functions.REQUEST_CONFIRMATION_FUNCTION_CALL_NAME, + ImmutableMap.of("confirmed", true)))) + .build(); + ImmutableList events = + ImmutableList.of(createUserEvent("u1", "Delete it."), confirmationResponse); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: Delete it."); + } + + @Test + public void processRequest_includeContentsNone_spokenRunOpeningTheSession_staysWhole() { + ImmutableList events = ImmutableList.of(heard("My name "), heard("is Ada.")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: My name is Ada."); + } + + @Test + public void processRequest_includeContentsNone_earlierSpokenRunBeforeAReply_isLeftOut() { + ImmutableList events = + ImmutableList.of(heard("Earlier."), createAgentEvent("a1", "Answer."), heard("Now.")); + + List contents = + runContentsProcessorWithIncludeContents(events, LlmAgent.IncludeContents.NONE); + + assertThat(turns(contents)).containsExactly("user: Now."); + } + + @Test + public void processRequest_blankRunBeforeAnotherRun_addsNothingToIt() { + ImmutableList events = ImmutableList.of(spoken(" "), heard("Hi.")); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)).containsExactly("user: Hi."); + } + + // Pins today's behavior: an event carrying both transcriptions is merged as input, as ADK + // Python's merge does, and its output is lost. + @Test + public void processRequest_eventWithBothTranscriptions_isMergedAsInput() { + Event both = heard("Hi.").toBuilder().outputTranscription(transcription("Hello.")).build(); + + List contents = runContentsProcessor(ImmutableList.of(both)); + + assertThat(turns(contents)).containsExactly("user: Hi."); + } + + // Content with no role is not visible, so it does not hide a transcription. + @Test + public void processRequest_rolelessContentWithTranscription_isMergedAsTranscription() { + Event event = + spoken("said").toBuilder() + .content(Content.builder().parts(Part.fromText("raw")).build()) + .build(); + + List contents = runContentsProcessor(ImmutableList.of(event)); + + assertThat(turns(contents)).containsExactly("model: said"); + } + + // A live run stores a turnComplete-only event after each turn, which must not split a run. + @Test + public void processRequest_turnCompleteEventsBetweenChunks_doNotSplitARun() { + Event turnComplete = Event.builder().author(AGENT).turnComplete(true).build(); + ImmutableList events = + ImmutableList.of( + heard("My name "), + heard("is Ada."), + spoken("Hi "), + turnComplete, + spoken("Ada!"), + turnComplete); + + List contents = runContentsProcessor(events); + + assertThat(turns(contents)) + .containsExactly("user: My name is Ada.", "model: Hi Ada!") + .inOrder(); + } + private static Event createUserEvent(String id, String text) { return Event.builder() .id(id) @@ -1620,12 +2139,17 @@ private List runContentsProcessorGrouped(List events) { } private List runContentsProcessorOnBranch( - List events, String agentName, String invocationBranch) { - LlmAgent agent = - LlmAgent.builder() - .name(agentName) - .includeContents(LlmAgent.IncludeContents.DEFAULT) - .build(); + List events, String agentName, @Nullable String invocationBranch) { + return runContentsProcessorOnBranch( + events, agentName, invocationBranch, LlmAgent.IncludeContents.DEFAULT); + } + + private List runContentsProcessorOnBranch( + List events, + String agentName, + @Nullable String invocationBranch, + LlmAgent.IncludeContents includeContents) { + LlmAgent agent = LlmAgent.builder().name(agentName).includeContents(includeContents).build(); Session session = sessionService.createSession("test-app", "test-user", null, "test-session").blockingGet(); session.addEvents(events); @@ -1696,4 +2220,39 @@ private Event createCompactedEvent(long startTimestamp, long endTimestamp, Strin .build()) .build(); } + + /** A chunk of the user's speech, stored under the listening agent's name as a live run does. */ + private static Event heard(String text) { + return heard(AGENT, text); + } + + private static Event heard(String author, String text) { + return Event.builder().author(author).inputTranscription(transcription(text)).build(); + } + + /** A chunk of an agent's speech. */ + private static Event spoken(String text) { + return spoken(AGENT, text); + } + + private static Event spoken(String author, String text) { + return Event.builder().author(author).outputTranscription(transcription(text)).build(); + } + + private static Transcription transcription(String text) { + return Transcription.builder().text(text).build(); + } + + /** Each content as "role: text", joining the text of its parts. */ + private static ImmutableList turns(List contents) { + return contents.stream() + .map( + content -> + content.role().orElse("") + + ": " + + content.parts().orElse(ImmutableList.of()).stream() + .map(part -> part.text().orElse("")) + .collect(joining(" | "))) + .collect(toImmutableList()); + } } diff --git a/core/src/test/java/com/google/adk/runner/RunnerTest.java b/core/src/test/java/com/google/adk/runner/RunnerTest.java index c92f67ff3..2b9951121 100644 --- a/core/src/test/java/com/google/adk/runner/RunnerTest.java +++ b/core/src/test/java/com/google/adk/runner/RunnerTest.java @@ -101,6 +101,7 @@ import com.google.genai.types.Part; import com.google.genai.types.PartialArg; import com.google.genai.types.Schema; +import com.google.genai.types.Transcription; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; import io.opentelemetry.context.ContextKey; @@ -2122,6 +2123,57 @@ public void runLive_success() throws Exception { assertThat(simplifyEvents(testSubscriber.values())).containsExactly("test agent: from llm"); } + // Each runLive opens a new connection and sends it the session's stored history. + @Test + public void runLive_newConnection_sendsStoredTranscriptionsAsHistory() throws Exception { + TestLlm liveLlm = + createTestLlm( + heardResponse("My name "), + heardResponse("is Ada."), + spokenResponse("Nice to meet you, "), + spokenResponse("Ada.")); + Runner liveRunner = + Runner.builder() + .app( + App.builder() + .name("test") + .rootAgent(createTestAgentBuilder(liveLlm).build()) + .build()) + .build(); + Session liveSession = liveRunner.sessionService().createSession("test", "user").blockingGet(); + + runLiveToCompletion(liveRunner, liveSession.sessionKey()); + runLiveToCompletion(liveRunner, liveSession.sessionKey()); + + LlmRequest secondConnection = liveLlm.getRequests().get(1); + assertThat(secondConnection.contents()) + .containsExactly( + Content.builder().role("user").parts(Part.fromText("My name is Ada.")).build(), + Content.builder().role("model").parts(Part.fromText("Nice to meet you, Ada.")).build()) + .inOrder(); + } + + private static void runLiveToCompletion(Runner runner, SessionKey sessionKey) throws Exception { + LiveRequestQueue liveRequestQueue = new LiveRequestQueue(); + TestSubscriber testSubscriber = + runner.runLive(sessionKey, liveRequestQueue, RunConfig.builder().build()).test(); + liveRequestQueue.close(); + testSubscriber.await(); + testSubscriber.assertComplete(); + } + + private static LlmResponse heardResponse(String text) { + return LlmResponse.builder() + .inputTranscription(Transcription.builder().text(text).build()) + .build(); + } + + private static LlmResponse spokenResponse(String text) { + return LlmResponse.builder() + .outputTranscription(Transcription.builder().text(text).build()) + .build(); + } + @Test public void runLive_asyncSessionService_persistsEvents() throws Exception { BaseSessionService asyncSessionService =