Skip to content

fix(live): replay stored transcriptions as conversation history #1670

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Oct 23, 2026
+792 −51
Merged

fix(live): replay stored transcriptions as conversation history #1670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 175 additions & 45 deletions core/src/main/java/com/google/adk/flows/llmflows/Contents.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import com.google.genai.types.FunctionCall;
import com.google.genai.types.FunctionResponse;
import com.google.genai.types.Part;
import com.google.genai.types.Transcription;
import io.reactivex.rxjava3.core.Single;
import java.util.ArrayList;
import java.util.Collections;
Expand Down Expand Up @@ -111,14 +112,54 @@ private ImmutableList<Content> getCurrentTurnContents(
// Find the latest event that starts the current turn and process from there.
for (int i = events.size() - 1; i >= 0; i--) {
Event event = events.get(i);
if (event.author().equals(Role.USER) || isOtherAgentReply(agentName, event)) {
TranscriptionKind kind = transcriptionKind(event);
int start = -1;
if (kind != null) {
start = spokenTurnStart(currentBranch, events, agentName, i, kind);
} else if (shouldIncludeEventInContext(currentBranch, event)
&& (event.author().equals(Role.USER) || isOtherAgentReply(agentName, event))) {
start = i;
}
if (start >= 0) {
return getContents(
currentBranch, events.subList(i, events.size()), agentName, groupFunctionResponses);
currentBranch, events.subList(start, events.size()), agentName, groupFunctionResponses);
}
}
return ImmutableList.of();
}

/**
* Returns where the spoken run ending at {@code end} starts, so the turn keeps the whole run
* rather than its last chunk. Returns -1 when the run does not start a turn: the agent's own
* speech, a chunk outside the context, or a blank run, which the merge drops.
*/
private int spokenTurnStart(
@Nullable String currentBranch,
List<Event> events,
String agentName,
int end,
TranscriptionKind kind) {
Event event = events.get(end);
boolean startsTurn = kind == TranscriptionKind.INPUT || isOtherAgentReply(agentName, event);
if (!startsTurn || !shouldIncludeEventInContext(currentBranch, event)) {
return -1;
}
int start = end;
boolean blank = kind.text(event).isBlank();
for (int i = end - 1; i >= 0; i--) {
// Events left out of the context do not break the run.
if (!shouldIncludeEventInContext(currentBranch, events.get(i))) {
continue;
}
if (!continuesRun(kind, events.get(start), events.get(i))) {
break;
}
start = i;
blank &= kind.text(events.get(i)).isBlank();
}
return blank ? -1 : start;
}

private ImmutableList<Content> getContents(
@Nullable String currentBranch,
List<Event> events,
Expand All @@ -127,47 +168,31 @@ private ImmutableList<Content> getContents(
List<Event> filteredEvents = new ArrayList<>();
boolean hasCompactEvent = false;

// Filter the events, leaving the contents and the function calls and responses from the current
// agent.
for (Event event : events) {
if (event.actions().compaction().isPresent()) {
// Always include the compaction event for the later processCompactionEvent call.
// The compaction event is used to filter out normal events that are covered by the
// compaction event.
hasCompactEvent = true;
if (shouldIncludeEventInContext(currentBranch, event)) {
hasCompactEvent |= event.actions().compaction().isPresent();
filteredEvents.add(event);
continue;
}

// Skip events without content, or generated neither by user nor by model or has empty text.
// E.g. events purely for mutating session states.
if (isEmptyContent(event)) {
continue;
}
if (!isEventBelongsToBranch(currentBranch, event)) {
continue;
}
if (isRequestConfirmationEvent(event)) {
continue;
}
}

// TODO: Skip auth events.
if (hasCompactEvent) {
filteredEvents = processCompactionEvent(filteredEvents, agentName);
}

// Merge after compaction, which may end mid-run, and before other agents are narrated.
List<Event> resultEvents = new ArrayList<>();
for (Event event : mergeTranscriptions(filteredEvents)) {
if (isOtherAgentReply(agentName, event)) {
Event foreignEvent = convertForeignEvent(event);
if (foreignEvent != null) {
filteredEvents.add(foreignEvent);
resultEvents.add(foreignEvent);
}
} else {
filteredEvents.add(event);
resultEvents.add(event);
}
}

if (hasCompactEvent) {
filteredEvents = processCompactionEvent(filteredEvents);
}

List<Event> resultEvents = rearrangeEventsForLatestFunctionResponse(filteredEvents);
resultEvents = rearrangeEventsForLatestFunctionResponse(resultEvents);
resultEvents =
rearrangeEventsForAsyncFunctionResponsesInHistory(resultEvents, groupFunctionResponses);

Expand All @@ -177,11 +202,25 @@ private ImmutableList<Content> getContents(
.collect(toImmutableList());
}

/**
* Whether {@code event} goes into the LLM context. Compaction events always do, so {@link
* #processCompactionEvent} can apply them.
*/
private boolean shouldIncludeEventInContext(@Nullable String currentBranch, Event event) {
if (event.actions().compaction().isPresent()) {
return true;
}
// TODO: Skip auth events.
return !isEmptyContent(event)
&& isEventBelongsToBranch(currentBranch, event)
&& !isRequestConfirmationEvent(event);
}

/**
* Check if an event has missing or empty content.
*
* <p>This can happen to the events that only changed session state. When both content and
* transcriptions are empty, the event will be considered as empty. The content is considered
* <p>This can happen to the events that only changed session state. When the content is empty and
* no transcription has text, the event will be considered as empty. The content is considered
* empty if none of its parts contain text, inline data, file data, function call, function
* response, server-side tool call, or server-side tool response. Parts with only thoughts are
* also considered empty.
Expand All @@ -190,15 +229,104 @@ private ImmutableList<Content> getContents(
* @return {@code true} if the event is considered to have empty content, {@code false} otherwise.
*/
private boolean isEmptyContent(Event event) {
if (event.content().isEmpty()) {
return true;
return !hasVisibleContent(event) && transcriptionKind(event) == null;
}

/** Whether {@code event} has a role and at least one part that is visible in the LLM context. */
private boolean hasVisibleContent(Event event) {
return event
.content()
.filter(content -> !content.role().orElse("").isEmpty())
.flatMap(Content::parts)
.map(parts -> parts.stream().anyMatch(part -> !isPartInvisible(part)))
.orElse(false);
}

/**
* Merges each spoken run into one text event, so later invocations replay what was said: input as
* a user turn, output as a model turn. A blank run is dropped.
*/
private ImmutableList<Event> mergeTranscriptions(List<Event> events) {
ImmutableList.Builder<Event> merged = ImmutableList.builder();
StringBuilder text = new StringBuilder();
for (int i = 0; i < events.size(); i++) {
Event event = events.get(i);
TranscriptionKind kind = transcriptionKind(event);
if (kind == null) {
merged.add(event);
continue;
}
text.append(kind.text(event));
if (i + 1 < events.size() && continuesRun(kind, event, events.get(i + 1))) {
continue;
}
// A blank run says nothing, while a space chunk inside a run still joins the words around it.
String runText = text.toString();
if (!runText.isBlank()) {
merged.add(kind.toTextEvent(event, runText));
}
text.setLength(0);
}
return merged.build();
}

/**
* Whether {@code other} belongs to the {@code kind} run that {@code event} is in. An output run
* is one agent's speech; an input run is the user's even when stored under different agents.
*/
private boolean continuesRun(TranscriptionKind kind, Event event, Event other) {
return transcriptionKind(other) == kind
&& (kind == TranscriptionKind.INPUT || event.author().equals(other.author()));
}

/**
* The kind of transcription {@code event} is merged as, or null if it has visible content or no
* transcription text.
*/
private @Nullable TranscriptionKind transcriptionKind(Event event) {
if (hasVisibleContent(event)) {
return null;
}
if (!TranscriptionKind.INPUT.text(event).isEmpty()) {
return TranscriptionKind.INPUT;
}
if (!TranscriptionKind.OUTPUT.text(event).isEmpty()) {
return TranscriptionKind.OUTPUT;
}
return null;
}

/** Which side of a live conversation a transcription records. */
private enum TranscriptionKind {
INPUT,
OUTPUT;

String text(Event event) {
Optional<Transcription> transcription =
switch (this) {
case INPUT -> event.inputTranscription();
case OUTPUT -> event.outputTranscription();
};
return transcription.flatMap(Transcription::text).orElse("");
}

/** The run's last {@code event}, carrying the run's {@code text} instead of a transcription. */
Event toTextEvent(Event event, String text) {
return switch (this) {
// A live run stores the user's speech under the listening agent's name.
case INPUT ->
event.toBuilder()
.author(Role.USER)
.content(Content.builder().role(Role.USER).parts(Part.fromText(text)).build())
.inputTranscription(null)
.build();
case OUTPUT ->
event.toBuilder()
.content(Content.builder().role("model").parts(Part.fromText(text)).build())
.outputTranscription(null)
.build();
};
}
var content = event.content().get();
return (content.role().isEmpty()
|| content.role().get().isEmpty()
|| content.parts().isEmpty()
|| content.parts().get().isEmpty()
|| content.parts().get().stream().allMatch(this::isPartInvisible));
}

/**
Expand Down Expand Up @@ -314,9 +442,10 @@ private boolean isPartInvisible(Part part) {
* </pre>
*
* @param events the list of event to filter.
* @param agentName the current agent, which authors each summary.
* @return a new list with compaction applied.
*/
private List<Event> processCompactionEvent(List<Event> events) {
private List<Event> processCompactionEvent(List<Event> events, String agentName) {
// Step 1: Split events into compaction events and regular events.
List<Event> compactionEvents = new ArrayList<>();
List<Event> regularEvents = new ArrayList<>();
Expand Down Expand Up @@ -350,13 +479,13 @@ private List<Event> processCompactionEvent(List<Event> events) {
e--;
} else {
// Otherwise the event is before the compaction, let's move to the next compaction event;
result.add(createCompactionEvent(compactionEvents.get(c)));
result.add(createCompactionEvent(compactionEvents.get(c), agentName));
c--;
}
}
// Flush any remaining compactions.
while (c >= 0) {
result.add(createCompactionEvent(compactionEvents.get(c)));
result.add(createCompactionEvent(compactionEvents.get(c), agentName));
c--;
}
// Flush any remaining regular events.
Expand Down Expand Up @@ -390,11 +519,12 @@ private static List<Event> removeOverlappingCompactions(List<Event> events) {
return Lists.reverse(result);
}

private static Event createCompactionEvent(Event event) {
private static Event createCompactionEvent(Event event, String agentName) {
EventCompaction compaction = event.actions().compaction().get();
return event.toBuilder()
.timestamp(compaction.endTimestamp())
.author("model")
// Authored by this agent, so convertForeignEvent leaves the summary alone.
.author(agentName.isEmpty() ? "model" : agentName)
.content(compaction.compactedContent())
.build();
}
Expand Down
Loading
Loading