feat(simulate): require stable ids in scenarios.yaml, offer to add them

cmd/lk/simulate.go

@@ -36,6 +36,7 @@ import (
 	"github.com/livekit/livekit-cli/v2/pkg/config"
 	"github.com/livekit/livekit-cli/v2/pkg/util"
 	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/utils"
 	lksdk "github.com/livekit/server-sdk-go/v2"
 	"github.com/livekit/server-sdk-go/v2/pkg/cloudagents"
 	"google.golang.org/protobuf/proto"
@@ -91,7 +92,7 @@ var simulateCommand = &cli.Command{
 		&cli.BoolFlag{
 			Name:    "yes",
 			Aliases: []string{"y"},
-			Usage:   "Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)",
+			Usage:   "Skip confirmation prompts: source upload, and adding missing ids to the --scenarios file (required for non-interactive runs)",
 		},
 		&cli.StringFlag{
 			Name:  "view",
@@ -164,14 +165,21 @@ func writeGeneratedScenariosTemp(run *livekit.SimulationRun) (string, error) {
 // scenarioGroupToYAML renders a ScenarioGroup as a scenarios.yaml document, the
 // inverse of loadScenarioGroup.
 func scenarioGroupToYAML(group *livekit.ScenarioGroup) ([]byte, error) {
-	f := scenariosFile{Name: group.GetName()}
+	f := scenariosFile{ID: group.GetId(), Name: group.GetName()}
+	if f.ID == "" {
+		f.ID = utils.NewGuid(utils.ScenarioGroupPrefix)
+	}
 	for _, s := range group.GetScenarios() {
 		ys := yamlScenario{
+			ID:                s.GetId(),
 			Label:             s.GetLabel(),
 			Instructions:      s.GetInstructions(),
 			AgentExpectations: s.GetAgentExpectations(),
 			Tags:              s.GetTags(),
 		}
+		if ys.ID == "" {
+			ys.ID = utils.NewGuid(utils.ScenarioPrefix)
+		}
 		if s.GetUserdata() != "" {
 			var ud map[string]any
 			if err := json.Unmarshal([]byte(s.GetUserdata()), &ud); err != nil {
@@ -187,11 +195,13 @@ func scenarioGroupToYAML(group *livekit.ScenarioGroup) ([]byte, error) {
 // scenariosFile mirrors a scenarios.yaml; `userdata` is a nested mapping here
 // and JSON-encoded into the proto's string field.
 type scenariosFile struct {
+	ID        string         `yaml:"id"`
 	Name      string         `yaml:"name"`
 	Scenarios []yamlScenario `yaml:"scenarios"`
 }
 
 type yamlScenario struct {
+	ID                string            `yaml:"id"`
 	Label             string            `yaml:"label"`
 	Instructions      string            `yaml:"instructions"`
 	AgentExpectations string            `yaml:"agent_expectations"`
@@ -255,8 +265,19 @@ func loadScenarioGroup(path string) (*livekit.ScenarioGroup, error) {
 		return nil, fmt.Errorf("failed to parse scenarios file: %w", err)
 	}
 
-	group := &livekit.ScenarioGroup{Name: f.Name}
+	if f.ID == "" {
+		return nil, fmt.Errorf("scenarios file has no id")
+	}
+	group := &livekit.ScenarioGroup{Id: f.ID, Name: f.Name}
+	seen := map[string]bool{}
 	for _, s := range f.Scenarios {
+		if s.ID == "" {
+			return nil, fmt.Errorf("scenario %q has no id", s.Label)
+		}
+		if seen[s.ID] {
+			return nil, fmt.Errorf("duplicate scenario id %q", s.ID)
+		}
+		seen[s.ID] = true
 		var userdata string
 		if len(s.Userdata) > 0 {
 			b, err := json.Marshal(s.Userdata)
@@ -266,6 +287,7 @@ func loadScenarioGroup(path string) (*livekit.ScenarioGroup, error) {
 			userdata = string(b)
 		}
 		group.Scenarios = append(group.Scenarios, &livekit.Scenario{
+			Id:                s.ID,
 			Label:             s.Label,
 			Instructions:      s.Instructions,
 			AgentExpectations: s.AgentExpectations,
@@ -377,6 +399,9 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 
 	var scenarioGroup *livekit.ScenarioGroup
 	if scenariosPath != "" {
+		if err := ensureScenarioIDs(cmd, scenariosPath); err != nil {
+			return err
+		}
 		scenarioGroup, err = loadScenarioGroup(scenariosPath)
 		if err != nil {
 			return err

cmd/lk/simulate_scenario_ids.go

@@ -0,0 +1,141 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"charm.land/huh/v2"
+	"github.com/urfave/cli/v3"
+	"gopkg.in/yaml.v3"
+
+	"github.com/livekit/livekit-cli/v2/pkg/util"
+	"github.com/livekit/protocol/utils"
+)
+
+// ensureScenarioIDs refuses to run a scenarios file whose group or scenarios
+// lack an `id`, offering to insert generated ones in place first. Without a
+// stable id nothing correlates a scenario across runs once its text changes.
+func ensureScenarioIDs(cmd *cli.Command, path string) error {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read scenarios file: %w", err)
+	}
+	fixed, added, err := insertScenarioIDs(data)
+	if err != nil {
+		return fmt.Errorf("failed to parse scenarios file: %w", err)
+	}
+	if added == 0 {
+		return nil
+	}
+	if !cmd.Bool("yes") {
+		if !isInteractive() {
+			return fmt.Errorf("%s is missing %d scenario id(s); re-run with --yes to have them added to the file", path, added)
+		}
+		confirmed := false
+		err := huh.NewForm(huh.NewGroup(huh.NewConfirm().
+			Title("Add scenario IDs?").
+			Description(fmt.Sprintf(
+				"%d entries in %s have no `id`. IDs let LiveKit track a scenario\n"+
+					"across runs even as its text changes, so every scenario needs one.",
+				added, util.Accented(path),
+			)).
+			Affirmative("Add IDs").
+			Negative("Cancel").
+			Value(&confirmed))).
+			WithTheme(util.FormTheme()).
+			Run()
+		if err != nil {
+			return err
+		}
+		if !confirmed {
+			return fmt.Errorf("aborted: scenarios must have ids to run")
+		}
+	}
+	if err := os.WriteFile(path, fixed, 0o644); err != nil {
+		return fmt.Errorf("failed to write scenarios file: %w", err)
+	}
+	fmt.Fprintf(os.Stderr, "Added %d id(s) to %s\n", added, path)
+	return nil
+}
+
+// insertScenarioIDs adds a generated `id` as the first key of the document
+// and of every scenario that lacks one. Editing the node tree rather than
+// re-marshalling keeps the user's comments and key order intact. Returns nil
+// output when nothing was added.
+func insertScenarioIDs(data []byte) ([]byte, int, error) {
+	var doc yaml.Node
+	if err := yaml.Unmarshal(data, &doc); err != nil {
+		return nil, 0, err
+	}
+	if len(doc.Content) == 0 || doc.Content[0].Kind != yaml.MappingNode {
+		return nil, 0, nil
+	}
+	root := doc.Content[0]
+	added := 0
+	if addMissingID(root, utils.ScenarioGroupPrefix) {
+		added++
+	}
+	if scenarios := mappingValue(root, "scenarios"); scenarios != nil && scenarios.Kind == yaml.SequenceNode {
+		for _, s := range scenarios.Content {
+			if s.Kind == yaml.MappingNode && addMissingID(s, utils.ScenarioPrefix) {
+				added++
+			}
+		}
+	}
+	if added == 0 {
+		return nil, 0, nil
+	}
+	var out bytes.Buffer
+	enc := yaml.NewEncoder(&out)
+	enc.SetIndent(detectIndent(data))
+	if err := enc.Encode(&doc); err != nil {
+		return nil, 0, err
+	}
+	return out.Bytes(), added, nil
+}
+
+// detectIndent is the leading-space width of the first indented line, so the
+// rewrite keeps the file's own indentation.
+func detectIndent(data []byte) int {
+	for _, line := range bytes.Split(data, []byte("\n")) {
+		if n := len(line) - len(bytes.TrimLeft(line, " ")); n > 0 && n < len(line) {
+			return n
+		}
+	}
+	return 2
+}
+
+func addMissingID(m *yaml.Node, prefix string) bool {
+	if mappingValue(m, "id") != nil {
+		return false
+	}
+	m.Content = append([]*yaml.Node{
+		{Kind: yaml.ScalarNode, Value: "id"},
+		{Kind: yaml.ScalarNode, Value: utils.NewGuid(prefix)},
+	}, m.Content...)
+	return true
+}
+
+func mappingValue(m *yaml.Node, key string) *yaml.Node {
+	for i := 0; i+1 < len(m.Content); i += 2 {
+		if m.Content[i].Value == key {
+			return m.Content[i+1]
+		}
+	}
+	return nil
+}

cmd/lk/simulate_scenario_ids_test.go

@@ -0,0 +1,140 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func writeScenariosFile(t *testing.T, content string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "scenarios.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
+	return path
+}
+
+func TestInsertScenarioIDsPreservesFileAndAddsMissing(t *testing.T) {
+	in := `# suite for the drive-thru agent
+name: drive-thru
+scenarios:
+  - label: order a burger # happy path
+    instructions: order a burger
+    agent_expectations: confirms the order
+  - id: keep-me
+    label: already has id
+    instructions: say hi
+`
+	out, added, err := insertScenarioIDs([]byte(in))
+	require.NoError(t, err)
+	require.Equal(t, 2, added) // group id + first scenario
+
+	var doc yaml.Node
+	require.NoError(t, yaml.Unmarshal(out, &doc))
+	root := doc.Content[0]
+	require.Equal(t, "id", root.Content[0].Value, "group id is inserted as the first key")
+	require.True(t, strings.HasPrefix(root.Content[1].Value, "SCNG_"))
+	require.Equal(t, "name", root.Content[2].Value)
+
+	scenarios := root.Content[5]
+	require.Equal(t, "id", scenarios.Content[0].Content[0].Value, "scenario id is inserted as the first key")
+	require.True(t, strings.HasPrefix(scenarios.Content[0].Content[1].Value, "SCN_"))
+	require.Equal(t, "keep-me", scenarios.Content[1].Content[1].Value, "existing ids are untouched")
+
+	s := string(out)
+	require.Contains(t, s, "# suite for the drive-thru agent")
+	require.Contains(t, s, "  - id: ", "two-space indentation is kept")
+	require.Contains(t, s, "label: order a burger # happy path")
+
+	// a fully identified file is left alone
+	out2, added2, err := insertScenarioIDs(out)
+	require.NoError(t, err)
+	require.Equal(t, 0, added2)
+	require.Nil(t, out2)
+}
+
+func TestLoadScenarioGroupRequiresUniqueIDs(t *testing.T) {
+	_, err := loadScenarioGroup(writeScenariosFile(t, `
+id: g1
+name: n
+scenarios:
+  - label: first
+    instructions: a
+  - id: s2
+    label: second
+    instructions: b
+`))
+	require.ErrorContains(t, err, `"first"`)
+	require.ErrorContains(t, err, "id")
+
+	_, err = loadScenarioGroup(writeScenariosFile(t, `
+id: g1
+name: n
+scenarios:
+  - id: dup
+    label: first
+    instructions: a
+  - id: dup
+    label: second
+    instructions: b
+`))
+	require.ErrorContains(t, err, `duplicate scenario id "dup"`)
+
+	_, err = loadScenarioGroup(writeScenariosFile(t, `
+name: n
+scenarios:
+  - id: s1
+    label: first
+    instructions: a
+`))
+	require.ErrorContains(t, err, "scenarios file has no id")
+
+	group, err := loadScenarioGroup(writeScenariosFile(t, `
+id: g1
+name: n
+scenarios:
+  - id: s1
+    label: first
+    instructions: a
+`))
+	require.NoError(t, err)
+	require.Equal(t, "g1", group.GetId())
+	require.Equal(t, "s1", group.GetScenarios()[0].GetId())
+}
+
+func TestScenarioGroupToYAMLFillsIDs(t *testing.T) {
+	group := &livekit.ScenarioGroup{
+		Name: "generated",
+		Scenarios: []*livekit.Scenario{
+			{Label: "a", Instructions: "x"},
+			{Id: "fixed", Label: "b", Instructions: "y"},
+		},
+	}
+	out, err := scenarioGroupToYAML(group)
+	require.NoError(t, err)
+
+	path := writeScenariosFile(t, string(out))
+	loaded, err := loadScenarioGroup(path)
+	require.NoError(t, err)
+	require.True(t, strings.HasPrefix(loaded.GetId(), "SCNG_"))
+	require.True(t, strings.HasPrefix(loaded.GetScenarios()[0].GetId(), "SCN_"))
+	require.Equal(t, "fixed", loaded.GetScenarios()[1].GetId())
+}

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/google/go-querystring v1.2.0
 	github.com/joho/godotenv v1.5.1
 	github.com/klauspost/compress v1.20.0
-	github.com/livekit/protocol v1.51.1-0.20260908073808-6cde54c87840
+	github.com/livekit/protocol v1.51.1-0.20260909121420-e586831b17e5
 	github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795
 	github.com/mattn/go-isatty v0.0.22
 	github.com/moby/moby/client v0.4.1
@@ -233,6 +233,7 @@ require (
 	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	golang.org/x/crypto v0.56.0 // indirect
 	golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect
+	golang.org/x/mod v0.39.0 // indirect
 	golang.org/x/net v0.58.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect