diff --git a/cmd/lk/simulate_ci.go b/cmd/lk/simulate_ci.go index 2bae9f0b..0de3880c 100644 --- a/cmd/lk/simulate_ci.go +++ b/cmd/lk/simulate_ci.go @@ -20,12 +20,10 @@ import ( "io" "os" "os/signal" - "strings" "sync/atomic" "time" "github.com/livekit/protocol/livekit" - agent "github.com/livekit/protocol/livekit/agent" ) type toggleWriter struct { @@ -173,7 +171,19 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { // --- Results --- - report.Results(run, agent) + if !out.Interactive() { + report.Results(run, agent) + } else { + // A terminal is watching; we just couldn't open the TUI (e.g. stdin + // isn't a TTY). Keep it to counts and pointers, the per-scenario + // transcripts go to a report file like the TUI's. + dashboardURL := simulationDashboardURL(config.pc.ProjectId, runID) + if path := newRunReporter().Finish(run, agent, brokenAgent, dashboardURL); path != "" { + out.Statusf("Run report: %s", path) + } + total, _, passed, failedN := simulationJobCounts(run) + fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN) + } if brokenAgent && agent != nil { writeBrokenAgentNote(out.WarnWriter(), agent) @@ -183,13 +193,10 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { out.Statusf("Dashboard: %s", url) } + // the returned error is printed by main and reports the failure; the + // counts line / full dump above already carries the detail _, _, _, failed := simulationJobCounts(run) if failed > 0 || run.Status == livekit.SimulationRun_STATUS_FAILED { - if failed > 0 { - out.Resultf("::error::%d simulation(s) failed\n", failed) - } else { - out.Resultf("::error::Simulation run failed: %s\n", run.Error) - } if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 { return fmt.Errorf("simulation failed: %s", run.Error) } @@ -198,166 +205,3 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { return nil } - -// writeRunResults writes the per-job results and the run summary, with GitHub -// group markers (a useful delimiter outside GitHub too). -func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) { - if run == nil { - return - } - summary := decodeRunSummary(run) - - if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 { - fmt.Fprintf(w, "✗ Simulation failed: %s\n", run.Error) - return - } - - for i, job := range run.Jobs { - icon := "⏺" - switch job.Status { - case livekit.SimulationRun_Job_STATUS_COMPLETED: - icon = "✓" - case livekit.SimulationRun_Job_STATUS_FAILED: - icon = "✗" - } - - label := job.Label - if label == "" { - label = fmt.Sprintf("Job %d", i+1) - } - - fmt.Fprintf(w, "::group::%s %s (%s)\n", icon, label, job.Id) - - if job.Instructions != "" { - fmt.Fprintln(w, "Instructions:") - for line := range strings.SplitSeq(job.Instructions, "\n") { - fmt.Fprintf(w, " %s\n", line) - } - } - - if job.AgentExpectations != "" { - fmt.Fprintln(w, "Expected:") - for line := range strings.SplitSeq(job.AgentExpectations, "\n") { - fmt.Fprintf(w, " %s\n", line) - } - } - - if job.Error != "" { - if job.Status == livekit.SimulationRun_Job_STATUS_COMPLETED { - fmt.Fprintf(w, "Result: %s\n", job.Error) - } else { - fmt.Fprintf(w, "Error: %s\n", job.Error) - } - } - - if summary != nil && summary.ChatHistory != nil { - writeChatHistory(w, summary.ChatHistory[job.Id]) - } - - if ap != nil && job.RoomName != "" { - logs := ap.RecentRoomLogs(0, job.RoomName) - if len(logs) > 0 { - fmt.Fprintln(w, "Logs:") - for _, line := range logs { - fmt.Fprintf(w, " %s\n", ansiEscapeRe.ReplaceAllString(line, "")) - } - } - } - - fmt.Fprintln(w, "::endgroup::") - - if job.Status == livekit.SimulationRun_Job_STATUS_FAILED { - firstLine, _, _ := strings.Cut(job.Error, "\n") - fmt.Fprintf(w, "::error::Job %d failed: %s\n", i+1, firstLine) - } - } - - if summary != nil { - writeRunSummary(w, run, summary) - } else { - msg := "The summary for this run is not available" - if run.Error != "" { - msg = run.Error - } - fmt.Fprintln(w) - fmt.Fprintln(w, "⚠ "+msg) - } -} - -func writeRunSummary(w io.Writer, run *livekit.SimulationRun, summary *livekit.SimulationRunSummary) { - total, _, passed, failed := simulationJobCounts(run) - - fmt.Fprintln(w) - fmt.Fprintln(w, "::group::Summary") - fmt.Fprintf(w, "%d total, %d passed, %d failed\n", total, passed, failed) - - if summary.GoingWell != "" { - fmt.Fprintln(w) - fmt.Fprintln(w, "Going well:") - for line := range strings.SplitSeq(summary.GoingWell, "\n") { - fmt.Fprintf(w, " %s\n", line) - } - } - - if summary.ToImprove != "" { - fmt.Fprintln(w) - fmt.Fprintln(w, "To improve:") - for line := range strings.SplitSeq(summary.ToImprove, "\n") { - fmt.Fprintf(w, " %s\n", line) - } - } - - if len(summary.Issues) > 0 { - fmt.Fprintln(w) - fmt.Fprintln(w, "Issues:") - for i, issue := range summary.Issues { - fmt.Fprintf(w, " %d. %s\n", i+1, issue.Description) - if issue.Suggestion != "" { - fmt.Fprintf(w, " Suggestion: %s\n", issue.Suggestion) - } - } - } - - fmt.Fprintln(w, "::endgroup::") -} - -func writeChatHistory(w io.Writer, chatCtx *agent.ChatContext) { - if chatCtx == nil || len(chatCtx.Items) == 0 { - return - } - fmt.Fprintln(w, "Transcript:") - for _, item := range chatCtx.Items { - switch v := item.Item.(type) { - case *agent.ChatContext_ChatItem_Message: - msg := v.Message - text := chatMessageText(msg) - if text == "" { - continue - } - switch msg.Role { - case agent.ChatRole_USER: - fmt.Fprintf(w, " ● You\n") - case agent.ChatRole_ASSISTANT: - fmt.Fprintf(w, " ● Agent\n") - default: - fmt.Fprintf(w, " ● %s\n", msg.Role) - } - for tl := range strings.SplitSeq(text, "\n") { - fmt.Fprintf(w, " %s\n", tl) - } - case *agent.ChatContext_ChatItem_FunctionCall: - fc := v.FunctionCall - fmt.Fprintf(w, " [call] %s(%s)\n", fc.Name, fc.Arguments) - case *agent.ChatContext_ChatItem_FunctionCallOutput: - fco := v.FunctionCallOutput - label := "output" - if fco.IsError { - label = "error" - } - fmt.Fprintf(w, " [%s] %s -> %s\n", label, fco.Name, fco.Output) - case *agent.ChatContext_ChatItem_AgentHandoff: - h := v.AgentHandoff - fmt.Fprintf(w, " [handoff] -> %s\n", h.NewAgentId) - } - } -} diff --git a/cmd/lk/simulate_report.go b/cmd/lk/simulate_report.go index 5fa78d06..db196777 100644 --- a/cmd/lk/simulate_report.go +++ b/cmd/lk/simulate_report.go @@ -22,6 +22,7 @@ import ( "time" "github.com/livekit/protocol/livekit" + agent "github.com/livekit/protocol/livekit/agent" ) // simLog is the single source of the plain-text simulation output, shared by @@ -144,6 +145,169 @@ func (a asciiWriter) Write(p []byte) (int, error) { return len(p), err } +// writeRunResults writes the per-job results and the run summary, with GitHub +// group markers (a useful delimiter outside GitHub too). +func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) { + if run == nil { + return + } + summary := decodeRunSummary(run) + + if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 { + fmt.Fprintf(w, "✗ Simulation failed: %s\n", run.Error) + return + } + + for i, job := range run.Jobs { + icon := "⏺" + switch job.Status { + case livekit.SimulationRun_Job_STATUS_COMPLETED: + icon = "✓" + case livekit.SimulationRun_Job_STATUS_FAILED: + icon = "✗" + } + + label := job.Label + if label == "" { + label = fmt.Sprintf("Job %d", i+1) + } + + fmt.Fprintf(w, "::group::%s %s (%s)\n", icon, label, job.Id) + + if job.Instructions != "" { + fmt.Fprintln(w, "Instructions:") + for line := range strings.SplitSeq(job.Instructions, "\n") { + fmt.Fprintf(w, " %s\n", line) + } + } + + if job.AgentExpectations != "" { + fmt.Fprintln(w, "Expected:") + for line := range strings.SplitSeq(job.AgentExpectations, "\n") { + fmt.Fprintf(w, " %s\n", line) + } + } + + if job.Error != "" { + if job.Status == livekit.SimulationRun_Job_STATUS_COMPLETED { + fmt.Fprintf(w, "Result: %s\n", job.Error) + } else { + fmt.Fprintf(w, "Error: %s\n", job.Error) + } + } + + if summary != nil && summary.ChatHistory != nil { + writeChatHistory(w, summary.ChatHistory[job.Id]) + } + + if ap != nil && job.RoomName != "" { + logs := ap.RecentRoomLogs(0, job.RoomName) + if len(logs) > 0 { + fmt.Fprintln(w, "Logs:") + for _, line := range logs { + fmt.Fprintf(w, " %s\n", ansiEscapeRe.ReplaceAllString(line, "")) + } + } + } + + fmt.Fprintln(w, "::endgroup::") + + if job.Status == livekit.SimulationRun_Job_STATUS_FAILED { + firstLine, _, _ := strings.Cut(job.Error, "\n") + fmt.Fprintf(w, "::error::Job %d failed: %s\n", i+1, firstLine) + } + } + + if summary != nil { + writeRunSummary(w, run, summary) + } else { + msg := "The summary for this run is not available" + if run.Error != "" { + msg = run.Error + } + fmt.Fprintln(w) + fmt.Fprintln(w, "⚠ "+msg) + } +} + +func writeRunSummary(w io.Writer, run *livekit.SimulationRun, summary *livekit.SimulationRunSummary) { + total, _, passed, failed := simulationJobCounts(run) + + fmt.Fprintln(w) + fmt.Fprintln(w, "::group::Summary") + fmt.Fprintf(w, "%d total, %d passed, %d failed\n", total, passed, failed) + + if summary.GoingWell != "" { + fmt.Fprintln(w) + fmt.Fprintln(w, "Going well:") + for line := range strings.SplitSeq(summary.GoingWell, "\n") { + fmt.Fprintf(w, " %s\n", line) + } + } + + if summary.ToImprove != "" { + fmt.Fprintln(w) + fmt.Fprintln(w, "To improve:") + for line := range strings.SplitSeq(summary.ToImprove, "\n") { + fmt.Fprintf(w, " %s\n", line) + } + } + + if len(summary.Issues) > 0 { + fmt.Fprintln(w) + fmt.Fprintln(w, "Issues:") + for i, issue := range summary.Issues { + fmt.Fprintf(w, " %d. %s\n", i+1, issue.Description) + if issue.Suggestion != "" { + fmt.Fprintf(w, " Suggestion: %s\n", issue.Suggestion) + } + } + } + + fmt.Fprintln(w, "::endgroup::") +} + +func writeChatHistory(w io.Writer, chatCtx *agent.ChatContext) { + if chatCtx == nil || len(chatCtx.Items) == 0 { + return + } + fmt.Fprintln(w, "Transcript:") + for _, item := range chatCtx.Items { + switch v := item.Item.(type) { + case *agent.ChatContext_ChatItem_Message: + msg := v.Message + text := chatMessageText(msg) + if text == "" { + continue + } + switch msg.Role { + case agent.ChatRole_USER: + fmt.Fprintf(w, " ● You\n") + case agent.ChatRole_ASSISTANT: + fmt.Fprintf(w, " ● Agent\n") + default: + fmt.Fprintf(w, " ● %s\n", msg.Role) + } + for tl := range strings.SplitSeq(text, "\n") { + fmt.Fprintf(w, " %s\n", tl) + } + case *agent.ChatContext_ChatItem_FunctionCall: + fc := v.FunctionCall + fmt.Fprintf(w, " [call] %s(%s)\n", fc.Name, fc.Arguments) + case *agent.ChatContext_ChatItem_FunctionCallOutput: + fco := v.FunctionCallOutput + label := "output" + if fco.IsError { + label = "error" + } + fmt.Fprintf(w, " [%s] %s -> %s\n", label, fco.Name, fco.Output) + case *agent.ChatContext_ChatItem_AgentHandoff: + h := v.AgentHandoff + fmt.Fprintf(w, " [handoff] -> %s\n", h.NewAgentId) + } + } +} + // runReporter writes the simLog to a temp file so TUI runs leave the same // record the non-TUI mode prints. type runReporter struct {