diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index bc1eac4a..60d6d318 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -36,6 +36,7 @@ import ( "github.com/livekit/livekit-cli/v2/pkg/config" "github.com/livekit/livekit-cli/v2/pkg/util" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils" lksdk "github.com/livekit/server-sdk-go/v2" "github.com/livekit/server-sdk-go/v2/pkg/cloudagents" "google.golang.org/protobuf/proto" @@ -91,7 +92,7 @@ var simulateCommand = &cli.Command{ &cli.BoolFlag{ Name: "yes", Aliases: []string{"y"}, - Usage: "Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)", + Usage: "Skip confirmation prompts: source upload, and adding missing ids to the --scenarios file (required for non-interactive runs)", }, &cli.StringFlag{ Name: "view", @@ -164,14 +165,21 @@ func writeGeneratedScenariosTemp(run *livekit.SimulationRun) (string, error) { // scenarioGroupToYAML renders a ScenarioGroup as a scenarios.yaml document, the // inverse of loadScenarioGroup. func scenarioGroupToYAML(group *livekit.ScenarioGroup) ([]byte, error) { - f := scenariosFile{Name: group.GetName()} + f := scenariosFile{ID: group.GetId(), Name: group.GetName()} + if f.ID == "" { + f.ID = utils.NewGuid(utils.ScenarioGroupPrefix) + } for _, s := range group.GetScenarios() { ys := yamlScenario{ + ID: s.GetId(), Label: s.GetLabel(), Instructions: s.GetInstructions(), AgentExpectations: s.GetAgentExpectations(), Tags: s.GetTags(), } + if ys.ID == "" { + ys.ID = utils.NewGuid(utils.ScenarioPrefix) + } if s.GetUserdata() != "" { var ud map[string]any if err := json.Unmarshal([]byte(s.GetUserdata()), &ud); err != nil { @@ -187,11 +195,13 @@ func scenarioGroupToYAML(group *livekit.ScenarioGroup) ([]byte, error) { // scenariosFile mirrors a scenarios.yaml; `userdata` is a nested mapping here // and JSON-encoded into the proto's string field. type scenariosFile struct { + ID string `yaml:"id"` Name string `yaml:"name"` Scenarios []yamlScenario `yaml:"scenarios"` } type yamlScenario struct { + ID string `yaml:"id"` Label string `yaml:"label"` Instructions string `yaml:"instructions"` AgentExpectations string `yaml:"agent_expectations"` @@ -255,8 +265,19 @@ func loadScenarioGroup(path string) (*livekit.ScenarioGroup, error) { return nil, fmt.Errorf("failed to parse scenarios file: %w", err) } - group := &livekit.ScenarioGroup{Name: f.Name} + if f.ID == "" { + return nil, fmt.Errorf("scenarios file has no id") + } + group := &livekit.ScenarioGroup{Id: f.ID, Name: f.Name} + seen := map[string]bool{} for _, s := range f.Scenarios { + if s.ID == "" { + return nil, fmt.Errorf("scenario %q has no id", s.Label) + } + if seen[s.ID] { + return nil, fmt.Errorf("duplicate scenario id %q", s.ID) + } + seen[s.ID] = true var userdata string if len(s.Userdata) > 0 { b, err := json.Marshal(s.Userdata) @@ -266,6 +287,7 @@ func loadScenarioGroup(path string) (*livekit.ScenarioGroup, error) { userdata = string(b) } group.Scenarios = append(group.Scenarios, &livekit.Scenario{ + Id: s.ID, Label: s.Label, Instructions: s.Instructions, AgentExpectations: s.AgentExpectations, @@ -377,6 +399,9 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S var scenarioGroup *livekit.ScenarioGroup if scenariosPath != "" { + if err := ensureScenarioIDs(cmd, scenariosPath); err != nil { + return err + } scenarioGroup, err = loadScenarioGroup(scenariosPath) if err != nil { return err diff --git a/cmd/lk/simulate_scenario_ids.go b/cmd/lk/simulate_scenario_ids.go new file mode 100644 index 00000000..647b94e8 --- /dev/null +++ b/cmd/lk/simulate_scenario_ids.go @@ -0,0 +1,141 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "os" + + "charm.land/huh/v2" + "github.com/urfave/cli/v3" + "gopkg.in/yaml.v3" + + "github.com/livekit/livekit-cli/v2/pkg/util" + "github.com/livekit/protocol/utils" +) + +// ensureScenarioIDs refuses to run a scenarios file whose group or scenarios +// lack an `id`, offering to insert generated ones in place first. Without a +// stable id nothing correlates a scenario across runs once its text changes. +func ensureScenarioIDs(cmd *cli.Command, path string) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read scenarios file: %w", err) + } + fixed, added, err := insertScenarioIDs(data) + if err != nil { + return fmt.Errorf("failed to parse scenarios file: %w", err) + } + if added == 0 { + return nil + } + if !cmd.Bool("yes") { + if !isInteractive() { + return fmt.Errorf("%s is missing %d scenario id(s); re-run with --yes to have them added to the file", path, added) + } + confirmed := false + err := huh.NewForm(huh.NewGroup(huh.NewConfirm(). + Title("Add scenario IDs?"). + Description(fmt.Sprintf( + "%d entries in %s have no `id`. IDs let LiveKit track a scenario\n"+ + "across runs even as its text changes, so every scenario needs one.", + added, util.Accented(path), + )). + Affirmative("Add IDs"). + Negative("Cancel"). + Value(&confirmed))). + WithTheme(util.FormTheme()). + Run() + if err != nil { + return err + } + if !confirmed { + return fmt.Errorf("aborted: scenarios must have ids to run") + } + } + if err := os.WriteFile(path, fixed, 0o644); err != nil { + return fmt.Errorf("failed to write scenarios file: %w", err) + } + fmt.Fprintf(os.Stderr, "Added %d id(s) to %s\n", added, path) + return nil +} + +// insertScenarioIDs adds a generated `id` as the first key of the document +// and of every scenario that lacks one. Editing the node tree rather than +// re-marshalling keeps the user's comments and key order intact. Returns nil +// output when nothing was added. +func insertScenarioIDs(data []byte) ([]byte, int, error) { + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, 0, err + } + if len(doc.Content) == 0 || doc.Content[0].Kind != yaml.MappingNode { + return nil, 0, nil + } + root := doc.Content[0] + added := 0 + if addMissingID(root, utils.ScenarioGroupPrefix) { + added++ + } + if scenarios := mappingValue(root, "scenarios"); scenarios != nil && scenarios.Kind == yaml.SequenceNode { + for _, s := range scenarios.Content { + if s.Kind == yaml.MappingNode && addMissingID(s, utils.ScenarioPrefix) { + added++ + } + } + } + if added == 0 { + return nil, 0, nil + } + var out bytes.Buffer + enc := yaml.NewEncoder(&out) + enc.SetIndent(detectIndent(data)) + if err := enc.Encode(&doc); err != nil { + return nil, 0, err + } + return out.Bytes(), added, nil +} + +// detectIndent is the leading-space width of the first indented line, so the +// rewrite keeps the file's own indentation. +func detectIndent(data []byte) int { + for _, line := range bytes.Split(data, []byte("\n")) { + if n := len(line) - len(bytes.TrimLeft(line, " ")); n > 0 && n < len(line) { + return n + } + } + return 2 +} + +func addMissingID(m *yaml.Node, prefix string) bool { + if mappingValue(m, "id") != nil { + return false + } + m.Content = append([]*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "id"}, + {Kind: yaml.ScalarNode, Value: utils.NewGuid(prefix)}, + }, m.Content...) + return true +} + +func mappingValue(m *yaml.Node, key string) *yaml.Node { + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i+1] + } + } + return nil +} diff --git a/cmd/lk/simulate_scenario_ids_test.go b/cmd/lk/simulate_scenario_ids_test.go new file mode 100644 index 00000000..fa6c1b5c --- /dev/null +++ b/cmd/lk/simulate_scenario_ids_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/livekit/protocol/livekit" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func writeScenariosFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "scenarios.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + return path +} + +func TestInsertScenarioIDsPreservesFileAndAddsMissing(t *testing.T) { + in := `# suite for the drive-thru agent +name: drive-thru +scenarios: + - label: order a burger # happy path + instructions: order a burger + agent_expectations: confirms the order + - id: keep-me + label: already has id + instructions: say hi +` + out, added, err := insertScenarioIDs([]byte(in)) + require.NoError(t, err) + require.Equal(t, 2, added) // group id + first scenario + + var doc yaml.Node + require.NoError(t, yaml.Unmarshal(out, &doc)) + root := doc.Content[0] + require.Equal(t, "id", root.Content[0].Value, "group id is inserted as the first key") + require.True(t, strings.HasPrefix(root.Content[1].Value, "SCNG_")) + require.Equal(t, "name", root.Content[2].Value) + + scenarios := root.Content[5] + require.Equal(t, "id", scenarios.Content[0].Content[0].Value, "scenario id is inserted as the first key") + require.True(t, strings.HasPrefix(scenarios.Content[0].Content[1].Value, "SCN_")) + require.Equal(t, "keep-me", scenarios.Content[1].Content[1].Value, "existing ids are untouched") + + s := string(out) + require.Contains(t, s, "# suite for the drive-thru agent") + require.Contains(t, s, " - id: ", "two-space indentation is kept") + require.Contains(t, s, "label: order a burger # happy path") + + // a fully identified file is left alone + out2, added2, err := insertScenarioIDs(out) + require.NoError(t, err) + require.Equal(t, 0, added2) + require.Nil(t, out2) +} + +func TestLoadScenarioGroupRequiresUniqueIDs(t *testing.T) { + _, err := loadScenarioGroup(writeScenariosFile(t, ` +id: g1 +name: n +scenarios: + - label: first + instructions: a + - id: s2 + label: second + instructions: b +`)) + require.ErrorContains(t, err, `"first"`) + require.ErrorContains(t, err, "id") + + _, err = loadScenarioGroup(writeScenariosFile(t, ` +id: g1 +name: n +scenarios: + - id: dup + label: first + instructions: a + - id: dup + label: second + instructions: b +`)) + require.ErrorContains(t, err, `duplicate scenario id "dup"`) + + _, err = loadScenarioGroup(writeScenariosFile(t, ` +name: n +scenarios: + - id: s1 + label: first + instructions: a +`)) + require.ErrorContains(t, err, "scenarios file has no id") + + group, err := loadScenarioGroup(writeScenariosFile(t, ` +id: g1 +name: n +scenarios: + - id: s1 + label: first + instructions: a +`)) + require.NoError(t, err) + require.Equal(t, "g1", group.GetId()) + require.Equal(t, "s1", group.GetScenarios()[0].GetId()) +} + +func TestScenarioGroupToYAMLFillsIDs(t *testing.T) { + group := &livekit.ScenarioGroup{ + Name: "generated", + Scenarios: []*livekit.Scenario{ + {Label: "a", Instructions: "x"}, + {Id: "fixed", Label: "b", Instructions: "y"}, + }, + } + out, err := scenarioGroupToYAML(group) + require.NoError(t, err) + + path := writeScenariosFile(t, string(out)) + loaded, err := loadScenarioGroup(path) + require.NoError(t, err) + require.True(t, strings.HasPrefix(loaded.GetId(), "SCNG_")) + require.True(t, strings.HasPrefix(loaded.GetScenarios()[0].GetId(), "SCN_")) + require.Equal(t, "fixed", loaded.GetScenarios()[1].GetId()) +} diff --git a/go.mod b/go.mod index f6c34cf6..f6c96a84 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.51.1-0.20260908073808-6cde54c87840 + github.com/livekit/protocol v1.51.1-0.20260909121420-e586831b17e5 github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 @@ -233,6 +233,7 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.56.0 // indirect golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect + golang.org/x/mod v0.39.0 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index 55c46e11..24de6957 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,10 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.51.1-0.20260908073808-6cde54c87840 h1:I9hIEcud1mJzaN0uycftHJwdwBpD0A7OaheEMbJuNm0= -github.com/livekit/protocol v1.51.1-0.20260908073808-6cde54c87840/go.mod h1:zxowkRnQlJ2VMn6ZyinXMDi985wcKXuWNeXmEERqFAs= +github.com/livekit/protocol v1.51.1-0.20260908185743-ed98e87d30d1 h1:0ANcI9XgK5dHP0QALg/YVw2sDnZYHCHZqSZs/NMqXjA= +github.com/livekit/protocol v1.51.1-0.20260908185743-ed98e87d30d1/go.mod h1:zxowkRnQlJ2VMn6ZyinXMDi985wcKXuWNeXmEERqFAs= +github.com/livekit/protocol v1.51.1-0.20260909121420-e586831b17e5 h1:2vijMbplOTFsMky19mcbWFd9D8ql1xcGXDEf/rt+s9A= +github.com/livekit/protocol v1.51.1-0.20260909121420-e586831b17e5/go.mod h1:zxowkRnQlJ2VMn6ZyinXMDi985wcKXuWNeXmEERqFAs= github.com/livekit/psrpc v0.7.6 h1:YG07lUMTtf+eaYI2goT9zcVZ0kGJNWN1K6ETNFtv1HQ= github.com/livekit/psrpc v0.7.6/go.mod h1:DMw15RO7x5XmcgfwzWJYk2In605kx+wu1QRVbPfzf8M= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0=