From 77a0cbcc568778391400d6a60a8aa4988db8da7b Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Wed, 9 Sep 2026 08:16:26 -0400 Subject: [PATCH] feat(simulate): require stable ids in scenarios.yaml, offer to add them Labels, instructions, and expectations all change over a scenario's life, so nothing correlates runs of the same scenario across time. Each scenario and the file itself now carry an id that is sent with the run. Ids are generated with protocol's shared guid utility, prefixed SCN_ (scenario) and SCNG_ (scenario group). When a file is missing ids, the CLI offers to insert them in place, preserving comments, key order, and indentation. --- cmd/lk/simulate.go | 31 +++++- cmd/lk/simulate_scenario_ids.go | 141 +++++++++++++++++++++++++++ cmd/lk/simulate_scenario_ids_test.go | 140 ++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 6 +- 5 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 cmd/lk/simulate_scenario_ids.go create mode 100644 cmd/lk/simulate_scenario_ids_test.go diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index bc1eac4a..60d6d318 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -36,6 +36,7 @@ import ( "github.com/livekit/livekit-cli/v2/pkg/config" "github.com/livekit/livekit-cli/v2/pkg/util" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils" lksdk "github.com/livekit/server-sdk-go/v2" "github.com/livekit/server-sdk-go/v2/pkg/cloudagents" "google.golang.org/protobuf/proto" @@ -91,7 +92,7 @@ var simulateCommand = &cli.Command{ &cli.BoolFlag{ Name: "yes", Aliases: []string{"y"}, - Usage: "Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)", + Usage: "Skip confirmation prompts: source upload, and adding missing ids to the --scenarios file (required for non-interactive runs)", }, &cli.StringFlag{ Name: "view", @@ -164,14 +165,21 @@ func writeGeneratedScenariosTemp(run *livekit.SimulationRun) (string, error) { // scenarioGroupToYAML renders a ScenarioGroup as a scenarios.yaml document, the // inverse of loadScenarioGroup. func scenarioGroupToYAML(group *livekit.ScenarioGroup) ([]byte, error) { - f := scenariosFile{Name: group.GetName()} + f := scenariosFile{ID: group.GetId(), Name: group.GetName()} + if f.ID == "" { + f.ID = utils.NewGuid(utils.ScenarioGroupPrefix) + } for _, s := range group.GetScenarios() { ys := yamlScenario{ + ID: s.GetId(), Label: s.GetLabel(), Instructions: s.GetInstructions(), AgentExpectations: s.GetAgentExpectations(), Tags: s.GetTags(), } + if ys.ID == "" { + ys.ID = utils.NewGuid(utils.ScenarioPrefix) + } if s.GetUserdata() != "" { var ud map[string]any if err := json.Unmarshal([]byte(s.GetUserdata()), &ud); err != nil { @@ -187,11 +195,13 @@ func scenarioGroupToYAML(group *livekit.ScenarioGroup) ([]byte, error) { // scenariosFile mirrors a scenarios.yaml; `userdata` is a nested mapping here // and JSON-encoded into the proto's string field. type scenariosFile struct { + ID string `yaml:"id"` Name string `yaml:"name"` Scenarios []yamlScenario `yaml:"scenarios"` } type yamlScenario struct { + ID string `yaml:"id"` Label string `yaml:"label"` Instructions string `yaml:"instructions"` AgentExpectations string `yaml:"agent_expectations"` @@ -255,8 +265,19 @@ func loadScenarioGroup(path string) (*livekit.ScenarioGroup, error) { return nil, fmt.Errorf("failed to parse scenarios file: %w", err) } - group := &livekit.ScenarioGroup{Name: f.Name} + if f.ID == "" { + return nil, fmt.Errorf("scenarios file has no id") + } + group := &livekit.ScenarioGroup{Id: f.ID, Name: f.Name} + seen := map[string]bool{} for _, s := range f.Scenarios { + if s.ID == "" { + return nil, fmt.Errorf("scenario %q has no id", s.Label) + } + if seen[s.ID] { + return nil, fmt.Errorf("duplicate scenario id %q", s.ID) + } + seen[s.ID] = true var userdata string if len(s.Userdata) > 0 { b, err := json.Marshal(s.Userdata) @@ -266,6 +287,7 @@ func loadScenarioGroup(path string) (*livekit.ScenarioGroup, error) { userdata = string(b) } group.Scenarios = append(group.Scenarios, &livekit.Scenario{ + Id: s.ID, Label: s.Label, Instructions: s.Instructions, AgentExpectations: s.AgentExpectations, @@ -377,6 +399,9 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S var scenarioGroup *livekit.ScenarioGroup if scenariosPath != "" { + if err := ensureScenarioIDs(cmd, scenariosPath); err != nil { + return err + } scenarioGroup, err = loadScenarioGroup(scenariosPath) if err != nil { return err diff --git a/cmd/lk/simulate_scenario_ids.go b/cmd/lk/simulate_scenario_ids.go new file mode 100644 index 00000000..647b94e8 --- /dev/null +++ b/cmd/lk/simulate_scenario_ids.go @@ -0,0 +1,141 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "os" + + "charm.land/huh/v2" + "github.com/urfave/cli/v3" + "gopkg.in/yaml.v3" + + "github.com/livekit/livekit-cli/v2/pkg/util" + "github.com/livekit/protocol/utils" +) + +// ensureScenarioIDs refuses to run a scenarios file whose group or scenarios +// lack an `id`, offering to insert generated ones in place first. Without a +// stable id nothing correlates a scenario across runs once its text changes. +func ensureScenarioIDs(cmd *cli.Command, path string) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read scenarios file: %w", err) + } + fixed, added, err := insertScenarioIDs(data) + if err != nil { + return fmt.Errorf("failed to parse scenarios file: %w", err) + } + if added == 0 { + return nil + } + if !cmd.Bool("yes") { + if !isInteractive() { + return fmt.Errorf("%s is missing %d scenario id(s); re-run with --yes to have them added to the file", path, added) + } + confirmed := false + err := huh.NewForm(huh.NewGroup(huh.NewConfirm(). + Title("Add scenario IDs?"). + Description(fmt.Sprintf( + "%d entries in %s have no `id`. IDs let LiveKit track a scenario\n"+ + "across runs even as its text changes, so every scenario needs one.", + added, util.Accented(path), + )). + Affirmative("Add IDs"). + Negative("Cancel"). + Value(&confirmed))). + WithTheme(util.FormTheme()). + Run() + if err != nil { + return err + } + if !confirmed { + return fmt.Errorf("aborted: scenarios must have ids to run") + } + } + if err := os.WriteFile(path, fixed, 0o644); err != nil { + return fmt.Errorf("failed to write scenarios file: %w", err) + } + fmt.Fprintf(os.Stderr, "Added %d id(s) to %s\n", added, path) + return nil +} + +// insertScenarioIDs adds a generated `id` as the first key of the document +// and of every scenario that lacks one. Editing the node tree rather than +// re-marshalling keeps the user's comments and key order intact. Returns nil +// output when nothing was added. +func insertScenarioIDs(data []byte) ([]byte, int, error) { + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, 0, err + } + if len(doc.Content) == 0 || doc.Content[0].Kind != yaml.MappingNode { + return nil, 0, nil + } + root := doc.Content[0] + added := 0 + if addMissingID(root, utils.ScenarioGroupPrefix) { + added++ + } + if scenarios := mappingValue(root, "scenarios"); scenarios != nil && scenarios.Kind == yaml.SequenceNode { + for _, s := range scenarios.Content { + if s.Kind == yaml.MappingNode && addMissingID(s, utils.ScenarioPrefix) { + added++ + } + } + } + if added == 0 { + return nil, 0, nil + } + var out bytes.Buffer + enc := yaml.NewEncoder(&out) + enc.SetIndent(detectIndent(data)) + if err := enc.Encode(&doc); err != nil { + return nil, 0, err + } + return out.Bytes(), added, nil +} + +// detectIndent is the leading-space width of the first indented line, so the +// rewrite keeps the file's own indentation. +func detectIndent(data []byte) int { + for _, line := range bytes.Split(data, []byte("\n")) { + if n := len(line) - len(bytes.TrimLeft(line, " ")); n > 0 && n < len(line) { + return n + } + } + return 2 +} + +func addMissingID(m *yaml.Node, prefix string) bool { + if mappingValue(m, "id") != nil { + return false + } + m.Content = append([]*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "id"}, + {Kind: yaml.ScalarNode, Value: utils.NewGuid(prefix)}, + }, m.Content...) + return true +} + +func mappingValue(m *yaml.Node, key string) *yaml.Node { + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i+1] + } + } + return nil +} diff --git a/cmd/lk/simulate_scenario_ids_test.go b/cmd/lk/simulate_scenario_ids_test.go new file mode 100644 index 00000000..fa6c1b5c --- /dev/null +++ b/cmd/lk/simulate_scenario_ids_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/livekit/protocol/livekit" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func writeScenariosFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "scenarios.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + return path +} + +func TestInsertScenarioIDsPreservesFileAndAddsMissing(t *testing.T) { + in := `# suite for the drive-thru agent +name: drive-thru +scenarios: + - label: order a burger # happy path + instructions: order a burger + agent_expectations: confirms the order + - id: keep-me + label: already has id + instructions: say hi +` + out, added, err := insertScenarioIDs([]byte(in)) + require.NoError(t, err) + require.Equal(t, 2, added) // group id + first scenario + + var doc yaml.Node + require.NoError(t, yaml.Unmarshal(out, &doc)) + root := doc.Content[0] + require.Equal(t, "id", root.Content[0].Value, "group id is inserted as the first key") + require.True(t, strings.HasPrefix(root.Content[1].Value, "SCNG_")) + require.Equal(t, "name", root.Content[2].Value) + + scenarios := root.Content[5] + require.Equal(t, "id", scenarios.Content[0].Content[0].Value, "scenario id is inserted as the first key") + require.True(t, strings.HasPrefix(scenarios.Content[0].Content[1].Value, "SCN_")) + require.Equal(t, "keep-me", scenarios.Content[1].Content[1].Value, "existing ids are untouched") + + s := string(out) + require.Contains(t, s, "# suite for the drive-thru agent") + require.Contains(t, s, " - id: ", "two-space indentation is kept") + require.Contains(t, s, "label: order a burger # happy path") + + // a fully identified file is left alone + out2, added2, err := insertScenarioIDs(out) + require.NoError(t, err) + require.Equal(t, 0, added2) + require.Nil(t, out2) +} + +func TestLoadScenarioGroupRequiresUniqueIDs(t *testing.T) { + _, err := loadScenarioGroup(writeScenariosFile(t, ` +id: g1 +name: n +scenarios: + - label: first + instructions: a + - id: s2 + label: second + instructions: b +`)) + require.ErrorContains(t, err, `"first"`) + require.ErrorContains(t, err, "id") + + _, err = loadScenarioGroup(writeScenariosFile(t, ` +id: g1 +name: n +scenarios: + - id: dup + label: first + instructions: a + - id: dup + label: second + instructions: b +`)) + require.ErrorContains(t, err, `duplicate scenario id "dup"`) + + _, err = loadScenarioGroup(writeScenariosFile(t, ` +name: n +scenarios: + - id: s1 + label: first + instructions: a +`)) + require.ErrorContains(t, err, "scenarios file has no id") + + group, err := loadScenarioGroup(writeScenariosFile(t, ` +id: g1 +name: n +scenarios: + - id: s1 + label: first + instructions: a +`)) + require.NoError(t, err) + require.Equal(t, "g1", group.GetId()) + require.Equal(t, "s1", group.GetScenarios()[0].GetId()) +} + +func TestScenarioGroupToYAMLFillsIDs(t *testing.T) { + group := &livekit.ScenarioGroup{ + Name: "generated", + Scenarios: []*livekit.Scenario{ + {Label: "a", Instructions: "x"}, + {Id: "fixed", Label: "b", Instructions: "y"}, + }, + } + out, err := scenarioGroupToYAML(group) + require.NoError(t, err) + + path := writeScenariosFile(t, string(out)) + loaded, err := loadScenarioGroup(path) + require.NoError(t, err) + require.True(t, strings.HasPrefix(loaded.GetId(), "SCNG_")) + require.True(t, strings.HasPrefix(loaded.GetScenarios()[0].GetId(), "SCN_")) + require.Equal(t, "fixed", loaded.GetScenarios()[1].GetId()) +} diff --git a/go.mod b/go.mod index f6c34cf6..f6c96a84 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.51.1-0.20260908073808-6cde54c87840 + github.com/livekit/protocol v1.51.1-0.20260909121420-e586831b17e5 github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 @@ -233,6 +233,7 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.56.0 // indirect golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect + golang.org/x/mod v0.39.0 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index 55c46e11..24de6957 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,10 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.51.1-0.20260908073808-6cde54c87840 h1:I9hIEcud1mJzaN0uycftHJwdwBpD0A7OaheEMbJuNm0= -github.com/livekit/protocol v1.51.1-0.20260908073808-6cde54c87840/go.mod h1:zxowkRnQlJ2VMn6ZyinXMDi985wcKXuWNeXmEERqFAs= +github.com/livekit/protocol v1.51.1-0.20260908185743-ed98e87d30d1 h1:0ANcI9XgK5dHP0QALg/YVw2sDnZYHCHZqSZs/NMqXjA= +github.com/livekit/protocol v1.51.1-0.20260908185743-ed98e87d30d1/go.mod h1:zxowkRnQlJ2VMn6ZyinXMDi985wcKXuWNeXmEERqFAs= +github.com/livekit/protocol v1.51.1-0.20260909121420-e586831b17e5 h1:2vijMbplOTFsMky19mcbWFd9D8ql1xcGXDEf/rt+s9A= +github.com/livekit/protocol v1.51.1-0.20260909121420-e586831b17e5/go.mod h1:zxowkRnQlJ2VMn6ZyinXMDi985wcKXuWNeXmEERqFAs= github.com/livekit/psrpc v0.7.6 h1:YG07lUMTtf+eaYI2goT9zcVZ0kGJNWN1K6ETNFtv1HQ= github.com/livekit/psrpc v0.7.6/go.mod h1:DMw15RO7x5XmcgfwzWJYk2In605kx+wu1QRVbPfzf8M= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0=