feat(simulate): save generated scenarios, generate from sessions, default to ./scenarios.yaml

autocomplete/fish_autocomplete

@@ -222,17 +222,19 @@ complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subco
 complete -x -c lk -n '__fish_seen_subcommand_from agent a; and not __fish_seen_subcommand_from init create dockerfile config deploy promote status update restart rollback logs tail delete destroy versions list secrets update-secrets private-link start dev console daemon simulate help h' -a 'simulate' -d 'Run agent simulations against LiveKit Cloud'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l num-simulations -s n -r -d 'Number of scenarios to generate'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l concurrency -r -d 'Max simulations running in parallel (default: server-side limit)'
-complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l scenarios -r -d 'Path to a scenarios `FILE` (yaml). If omitted, scenarios are generated from the agent\'s source'
+complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l scenarios -r -d 'Path to a scenarios `FILE` (yaml). Defaults to ./scenarios.yaml when it exists; otherwise scenarios are generated from the agent\'s source'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l yes -s y -d 'Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l view -r -d 'Open a pre-existing simulation'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l export -r -d 'Print the run with run `ID` and its exact per-job chat contexts as JSON. Nothing is run or polled: the run must already be finished'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l agent-name -r -d 'Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or "" to target the project\'s default agent (the one that auto-joins every room). Requires --scenarios.'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l help -s h -d 'show help'
-complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from audio' -a 'audio' -d 'Simulate speech-to-speech interactions using the agent\'s full audio pipeline'
+complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from audio generate' -a 'audio' -d 'Simulate speech-to-speech interactions using the agent\'s full audio pipeline'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from audio' -f -l background-noise -d 'Mix ambient noise into the simulated user\'s audio'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from audio' -f -l low-quality-microphone -d 'Publish the simulated user\'s audio as a low-quality microphone would capture it'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from audio' -f -l packet-loss -d 'Drop packets from the simulated user\'s audio track'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from audio' -f -l help -s h -d 'show help'
+complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from audio generate' -a 'generate' -d 'Turn recent agent sessions into scenarios and save them. Nothing is run'
+complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from generate' -f -l help -s h -d 'show help'
 complete -x -c lk -n '__fish_seen_subcommand_from agent a; and not __fish_seen_subcommand_from init create dockerfile config deploy promote status update restart rollback logs tail delete destroy versions list secrets update-secrets private-link start dev console daemon simulate help h' -a 'help' -d 'Shows a list of commands or help for one command'
 complete -c lk -n '__fish_seen_subcommand_from analytics' -f -l experimental -d 'Enable experimental features'
 complete -c lk -n '__fish_seen_subcommand_from analytics' -f -l help -s h -d 'show help'

cmd/lk/analytics.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	authutil "github.com/livekit/livekit-cli/v2/pkg/auth"
+	"github.com/livekit/livekit-cli/v2/pkg/config"
 	"github.com/livekit/livekit-cli/v2/pkg/util"
 	"github.com/livekit/protocol/auth"
 	"github.com/urfave/cli/v3"
@@ -316,21 +317,27 @@ func callAnalyticsAPI(ctx context.Context, cmd *cli.Command, sessionID string, q
 		return nil, err
 	}
 
-	projectID, err := resolveAnalyticsProjectID()
-	if err != nil {
+	if _, err := resolveAnalyticsProjectID(); err != nil {
 		return nil, err
 	}
 
-	token, err := createAnalyticsAccessToken(project.APIKey, project.APISecret)
+	path := "sessions"
+	if sessionID != "" {
+		path += "/" + url.PathEscape(sessionID)
+	}
+	return analyticsGET(ctx, project, path, query)
+}
+
+// analyticsGET fetches /api/project/{project_id}/{path} from the cloud API with
+// a token minted from the project's key; pc.ProjectId must be set.
+func analyticsGET(ctx context.Context, pc *config.ProjectConfig, path string, query url.Values) ([]byte, error) {
+	token, err := createAnalyticsAccessToken(pc.APIKey, pc.APISecret)
 	if err != nil {
 		return nil, err
 	}
 
 	baseURL := strings.TrimSuffix(serverURL, "/")
-	endpoint := fmt.Sprintf("%s/api/project/%s/sessions", baseURL, url.PathEscape(projectID))
-	if sessionID != "" {
-		endpoint += "/" + url.PathEscape(sessionID)
-	}
+	endpoint := fmt.Sprintf("%s/api/project/%s/%s", baseURL, url.PathEscape(pc.ProjectId), path)
 
 	reqURL, err := url.Parse(endpoint)
 	if err != nil {

cmd/lk/simulate.go

@@ -73,7 +73,7 @@ var simulateCommand = &cli.Command{
 	Action: func(ctx context.Context, cmd *cli.Command) error {
 		return runSimulate(ctx, cmd, livekit.SimulationMode_SIMULATION_MODE_TEXT)
 	},
-	Commands: []*cli.Command{simulateAudioCommand},
+	Commands: []*cli.Command{simulateAudioCommand, simulateGenerateCommand},
 	Flags: []cli.Flag{
 		&cli.IntFlag{
 			Name:    "num-simulations",
@@ -86,7 +86,7 @@ var simulateCommand = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:  "scenarios",
-			Usage: "Path to a scenarios `FILE` (yaml). If omitted, scenarios are generated from the agent's source",
+			Usage: "Path to a scenarios `FILE` (yaml). Defaults to ./scenarios.yaml when it exists; otherwise scenarios are generated from the agent's source",
 		},
 		&cli.BoolFlag{
 			Name:    "yes",
@@ -324,9 +324,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 	runID := cmd.String("view")
 	liveAgentName := cmd.String("agent-name")
 
-	// never auto-discovered: an explicit --scenarios file is the source of
-	// truth, otherwise scenarios are generated from the agent's source
-	scenariosPath := cmd.String("scenarios")
+	scenariosPath := scenariosPathOrDefault(cmd)
 
 	var (
 		agentName   string
@@ -342,7 +340,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 	if cmd.IsSet("agent-name") {
 		// nothing is spawned, so there's no source to generate scenarios from.
 		if scenariosPath == "" {
-			return fmt.Errorf("--agent-name requires --scenarios (no source to generate scenarios from when running against a live agent)")
+			return fmt.Errorf("--agent-name requires a scenarios file (--scenarios or ./%s): nothing is spawned, so there is no source to generate scenarios from", defaultScenariosFile)
 		}
 		liveAgent = true
 		agentName = liveAgentName

cmd/lk/simulate_generate.go

@@ -0,0 +1,204 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"charm.land/huh/v2"
+	"github.com/urfave/cli/v3"
+
+	"github.com/livekit/livekit-cli/v2/pkg/config"
+	"github.com/livekit/livekit-cli/v2/pkg/util"
+	"github.com/livekit/protocol/livekit"
+	lksdk "github.com/livekit/server-sdk-go/v2"
+)
+
+// defaultScenariosFile is where `simulate` looks when --scenarios is omitted,
+// and where derived scenarios are saved.
+const defaultScenariosFile = "scenarios.yaml"
+
+const recentSessionsLimit = 20
+
+var simulateGenerateCommand = &cli.Command{
+	Name:            "generate",
+	Usage:           "Turn recent agent sessions into scenarios and save them. Nothing is run",
+	ArgsUsage:       "[SESSION_ID...]",
+	Description:     "Without SESSION_IDs, pick from the project's recent sessions. Scenarios are appended to the --scenarios file (default scenarios.yaml).",
+	HideHelpCommand: true,
+	Action: func(ctx context.Context, cmd *cli.Command) error {
+		pc := simulateProjectConfig
+		group, err := deriveScenarios(ctx, pc, cmd.Args().Slice())
+		if err != nil {
+			return err
+		}
+		path := cmd.String("scenarios")
+		if path == "" {
+			path = defaultScenariosFile
+		}
+		return saveScenarioGroup(path, group)
+	},
+}
+
+// scenariosPathOrDefault resolves --scenarios, falling back to scenarios.yaml
+// in the working directory when it exists; "" means no file.
+func scenariosPathOrDefault(cmd *cli.Command) string {
+	if path := cmd.String("scenarios"); path != "" {
+		return path
+	}
+	if _, err := os.Stat(defaultScenariosFile); err == nil {
+		return defaultScenariosFile
+	}
+	return ""
+}
+
+// deriveScenarios has the cloud derive one scenario per recorded session.
+// Without IDs the user picks from the project's recent sessions, which needs a
+// terminal. A session the cloud can't derive from is skipped with a warning.
+func deriveScenarios(ctx context.Context, pc *config.ProjectConfig, sessionIDs []string) (*livekit.ScenarioGroup, error) {
+	if len(sessionIDs) == 0 {
+		if !isInteractive() {
+			return nil, errors.New("pass one or more session IDs (from the dashboard's Sessions page) to generate scenarios non-interactively")
+		}
+		sessions, err := listRecentSessions(ctx, pc)
+		if err != nil {
+			return nil, err
+		}
+		if len(sessions) == 0 {
+			return nil, errors.New("no finished sessions in this project yet; talk to your agent first, then re-run")
+		}
+		sessionIDs, err = pickSessions(sessions)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	client := lksdk.NewAgentSimulationClient(serverURL, pc.APIKey, pc.APISecret)
+	group := &livekit.ScenarioGroup{Name: scenarioGroupName()}
+	for _, id := range sessionIDs {
+		var scenario *livekit.Scenario
+		err := out.Await("Deriving a scenario from session "+id, ctx, func(ctx context.Context) error {
+			resp, err := client.CreateScenarioFromSession(ctx, &livekit.Scenario_CreateFromSession_Request{
+				ProjectId: pc.ProjectId,
+				RoomId:    id,
+			})
+			if err != nil {
+				return err
+			}
+			scenario = resp.GetScenario()
+			return nil
+		})
+		if err != nil {
+			if ctx.Err() != nil {
+				return nil, ctx.Err()
+			}
+			out.Warnf("Warning: skipping session %s: %v", id, err)
+			continue
+		}
+		group.Scenarios = append(group.Scenarios, scenario)
+	}
+	if len(group.Scenarios) == 0 {
+		return nil, errors.New("no scenarios could be derived from the selected sessions")
+	}
+
+	preview, err := scenarioGroupToYAML(group)
+	if err != nil {
+		return nil, err
+	}
+	out.Result(string(preview))
+	return group, nil
+}
+
+// scenarioGroupName names a new group after the project directory, which is
+// what the dashboard's runs list shows.
+func scenarioGroupName() string {
+	wd, err := os.Getwd()
+	if err != nil {
+		return ""
+	}
+	return filepath.Base(wd)
+}
+
+// listRecentSessions returns the project's newest finished sessions; a live
+// session has no chat history to derive from yet.
+func listRecentSessions(ctx context.Context, pc *config.ProjectConfig) ([]*analyticsSession, error) {
+	query := url.Values{}
+	query.Set("limit", fmt.Sprint(recentSessionsLimit))
+	query.Set("status", "closed")
+	body, err := analyticsGET(ctx, pc, "sessions", query)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list recent sessions: %w", err)
+	}
+	var res analyticsListResponse
+	if err := json.Unmarshal(body, &res); err != nil {
+		return nil, fmt.Errorf("failed to parse sessions response: %w", err)
+	}
+	return res.Sessions, nil
+}
+
+func pickSessions(sessions []*analyticsSession) ([]string, error) {
+	var options []huh.Option[string]
+	for _, s := range sessions {
+		label := fmt.Sprintf("%s  %s  %d participants", emptyDash(s.RoomName), emptyDash(s.CreatedAt), s.NumParticipants)
+		options = append(options, huh.NewOption(label, s.SessionID))
+	}
+	var picked []string
+	err := huh.NewForm(huh.NewGroup(huh.NewMultiSelect[string]().
+		Title("Which sessions should become scenarios?").
+		Description("Each session is turned into one scenario: what the user did, and what the agent is expected to do.").
+		Options(options...).
+		Height(len(options) + 2).
+		Value(&picked))).
+		WithTheme(util.FormTheme()).
+		Run()
+	if err != nil {
+		return nil, err
+	}
+	if len(picked) == 0 {
+		return nil, errors.New("no sessions selected")
+	}
+	return picked, nil
+}
+
+// saveScenarioGroup appends the group's scenarios to the scenarios file at
+// path, creating it when missing. An existing file keeps its name.
+func saveScenarioGroup(path string, group *livekit.ScenarioGroup) error {
+	merged := group
+	if existing, err := loadScenarioGroup(path); err == nil {
+		existing.Scenarios = append(existing.Scenarios, group.Scenarios...)
+		merged = existing
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	data, err := scenarioGroupToYAML(merged)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		return err
+	}
+	noun := "scenarios"
+	if len(group.Scenarios) == 1 {
+		noun = "scenario"
+	}
+	out.Statusf("Saved %d %s to %s", len(group.Scenarios), noun, path)
+	return nil
+}

cmd/lk/simulate_generate_test.go

@@ -0,0 +1,67 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/livekit/livekit-cli/v2/pkg/util"
+	"github.com/livekit/protocol/livekit"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSaveScenarioGroup(t *testing.T) {
+	out = util.NewPrinter(io.Discard, io.Discard, true)
+	path := filepath.Join(t.TempDir(), "scenarios.yaml")
+
+	first := &livekit.ScenarioGroup{Name: "frontdesk", Scenarios: []*livekit.Scenario{
+		{Label: "book a table", Instructions: "Ask for a table", AgentExpectations: "Confirms the party size"},
+	}}
+	require.NoError(t, saveScenarioGroup(path, first))
+
+	got, err := loadScenarioGroup(path)
+	require.NoError(t, err)
+	require.Equal(t, "frontdesk", got.Name)
+	require.Len(t, got.Scenarios, 1)
+
+	// a second save appends and keeps the file's own name
+	second := &livekit.ScenarioGroup{Name: "other", Scenarios: []*livekit.Scenario{
+		{Label: "cancel", Instructions: "Cancel the booking", AgentExpectations: "Confirms cancellation"},
+	}}
+	require.NoError(t, saveScenarioGroup(path, second))
+
+	got, err = loadScenarioGroup(path)
+	require.NoError(t, err)
+	require.Equal(t, "frontdesk", got.Name)
+	require.Len(t, got.Scenarios, 2)
+	require.Equal(t, "cancel", got.Scenarios[1].Label)
+}
+
+func TestSaveScenarioGroupRejectsUnparseableFile(t *testing.T) {
+	out = util.NewPrinter(io.Discard, io.Discard, true)
+	path := filepath.Join(t.TempDir(), "scenarios.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("scenarios: [\n"), 0o644))
+
+	err := saveScenarioGroup(path, &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Label: "x"}}})
+	require.Error(t, err)
+
+	// the broken file is left untouched
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Equal(t, "scenarios: [\n", string(data))
+}