diff --git a/pkg/executor/flow_runner.go b/pkg/executor/flow_runner.go index 5b5df9c5..8ba59a35 100644 --- a/pkg/executor/flow_runner.go +++ b/pkg/executor/flow_runner.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strconv" "strings" "time" @@ -39,6 +40,9 @@ type FlowRunner struct { conditionSkipped bool // Sub-command tracking for compound steps (runFlow, repeat, retry) subCommands []report.Command + // live is the top-level compound step that runs now, whose steps go to + // the report as they start and end (liveSubs); nil between them. + live *liveSteps // nestedArtifactSeq numbers failure artifacts captured for nested steps // (runFlow / repeat / retry), which have no top-level command index. nestedArtifactSeq int @@ -684,7 +688,12 @@ func (fr *FlowRunner) executeStep(idx int, step flow.Step) (report.Status, strin if err := fr.script.ExpandStep(step); err != nil { result = expressionFailed(err) } else { + switch step.(type) { + case *flow.RepeatStep, *flow.RetryStep, *flow.RunFlowStep: + fr.live = &liveSteps{cmd: idx} + } result = fr.runStep(idx, step, &artifacts) + fr.live = nil } result = fr.maskTypedResult(step, result) @@ -1772,6 +1781,17 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { // inside a runFlow showed the password itself. describe, desc := fr.stepNames(step) + // The step as it starts, in the report while it runs (liveSubs). + running := report.Command{ + ID: fmt.Sprintf("sub-%d", len(fr.subCommands)), + Index: len(fr.subCommands), + Type: string(step.Type()), + Label: step.Label(), + YAML: describe, + Status: report.StatusRunning, + StartTime: &start, + } + // For nested compound steps, we need to track their sub-commands separately var nestedSubCommands, parentSubCommands []report.Command isCompoundStep := false @@ -1782,6 +1802,8 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { parentSubCommands = fr.subCommands fr.subCommands = nil } + fr.live.begin(isCompoundStep, parentSubCommands, running) + fr.liveSubs() // Expand once, before the handler, as executeStep does: a ${...} that // throws fails the step and nothing runs. @@ -1828,6 +1850,7 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { nestedSubCommands = fr.subCommands fr.subCommands = parentSubCommands } + fr.live.end(isCompoundStep) // Add to parent's sub-commands for report status := report.StatusPassed @@ -1877,10 +1900,70 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { } fr.subCommands = append(fr.subCommands, cmd) + fr.liveSubs() return result } +// liveSteps is a top-level runFlow, repeat or retry while it runs: the +// compound steps running inside it, outermost first, each with the list it +// goes into once it ends, and the step running in the innermost one. +type liveSteps struct { + cmd int + frames []liveFrame + step *report.Command +} + +type liveFrame struct { + parent []report.Command + cmd report.Command +} + +// begin records a nested step that starts: a compound one with the list it +// goes into (its parent's), another as the step running. +func (l *liveSteps) begin(compound bool, parent []report.Command, c report.Command) { + if l == nil { + return + } + if compound { + l.frames = append(l.frames, liveFrame{parent: parent, cmd: c}) + } else { + l.step = &c + } +} + +// end forgets the nested step begin recorded last, which ended. +func (l *liveSteps) end(compound bool) { + if l == nil { + return + } + if compound { + l.frames = l.frames[:len(l.frames)-1] + } else { + l.step = nil + } +} + +// liveSubs writes the steps of the top-level compound step that runs now to +// the report as far as they got: those that ended and those running, so a +// runFlow's steps can be followed while it runs and not only once it ended. +func (fr *FlowRunner) liveSubs() { + l := fr.live + if l == nil || fr.flowWriter == nil { + return + } + list := slices.Clone(fr.subCommands) + if l.step != nil { + list = append(list, *l.step) + } + for i := len(l.frames) - 1; i >= 0; i-- { + c := l.frames[i].cmd + c.SubCommands = list + list = append(slices.Clone(l.frames[i].parent), c) + } + fr.flowWriter.CommandSubs(l.cmd, list) +} + // executeSubFlow executes a sub-flow without separate report tracking. func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult { // Save current flow dir diff --git a/pkg/executor/nested_report_test.go b/pkg/executor/nested_report_test.go index 6031c260..0c2feafb 100644 --- a/pkg/executor/nested_report_test.go +++ b/pkg/executor/nested_report_test.go @@ -3,8 +3,10 @@ package executor import ( "context" "path/filepath" + "strings" "testing" + "github.com/devicelab-dev/maestro-runner/pkg/core" "github.com/devicelab-dev/maestro-runner/pkg/flow" "github.com/devicelab-dev/maestro-runner/pkg/report" ) @@ -104,3 +106,75 @@ func TestConsoleLogsAreReportedPerStep(t *testing.T) { t.Errorf("runFlow logs %q, tapOn logs %q: want none", cmds[1].Logs, cmds[2].Logs) } } + +// A runFlow's steps are in the report while they run, each as it starts and +// as it ends, the nested runFlow's too: they used to arrive with its end. +func TestRunFlowStepsAreReportedAsTheyRun(t *testing.T) { + dir := t.TempDir() + var shape func(cs []report.Command) string + shape = func(cs []report.Command) string { + var out []string + for _, c := range cs { + s := c.Type + ":" + string(c.Status) + if len(c.SubCommands) > 0 { + s += "[" + shape(c.SubCommands) + "]" + } + out = append(out, s) + } + return strings.Join(out, ",") + } + seen := map[string]string{} // a tap's id → its runFlow's steps in the report as it ran + driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult { + if tap, ok := step.(*flow.TapOnStep); ok { + d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json")) + if err != nil { + t.Errorf("read the report while %s runs: %v", tap.Selector.ID, err) + } else { + seen[tap.Selector.ID] = shape(d.Commands[0].SubCommands) + } + } + return &core.CommandResult{Success: true} + }} + tap := func(id string) *flow.TapOnStep { + return &flow.TapOnStep{BaseStep: flow.BaseStep{StepType: flow.StepTapOn}, Selector: flow.Selector{ID: id}} + } + runner := New(driver, RunnerConfig{ + OutputDir: dir, + Artifacts: ArtifactNever, + Device: report.Device{ID: "test", Platform: "android"}, + }) + f := flow.Flow{ + SourcePath: "test.yaml", + Config: flow.Config{Name: "live steps"}, + Steps: []flow.Step{ + &flow.RunFlowStep{ + BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, + Steps: []flow.Step{ + tap("a"), + &flow.RunFlowStep{BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, Steps: []flow.Step{tap("b")}}, + tap("c"), + }, + }, + }, + } + if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil { + t.Fatalf("Run() error = %v", err) + } + want := map[string]string{ + "a": "tapOn:running", + "b": "tapOn:passed,runFlow:running[tapOn:running]", + "c": "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:running", + } + for id, w := range want { + if seen[id] != w { + t.Errorf("while %s ran the report had %q, want %q", id, seen[id], w) + } + } + d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json")) + if err != nil { + t.Fatal(err) + } + if got, w := shape(d.Commands[0].SubCommands), "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:passed"; got != w { + t.Errorf("at the end the report has %q, want %q", got, w) + } +} diff --git a/pkg/report/flow.go b/pkg/report/flow.go index 3b3790f4..5c702d79 100644 --- a/pkg/report/flow.go +++ b/pkg/report/flow.go @@ -91,6 +91,19 @@ func (w *FlowWriter) CommandEndWithSubs(cmdIndex int, status Status, element *El w.updateIndexProgress() } +// CommandSubs writes a running command's sub-commands so far: the steps of a +// runFlow, repeat or retry that ended, and the ones running now, so a reader +// of the report sees them before the command ends. CommandEndWithSubs writes +// the final list. +func (w *FlowWriter) CommandSubs(cmdIndex int, subCommands []Command) { + if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) { + return + } + w.flow.Commands[cmdIndex].SubCommands = subCommands + w.flush() + w.updateIndexProgress() +} + // CommandYAML replaces the command's description, written when the report // was built, with the one known once it ran. func (w *FlowWriter) CommandYAML(cmdIndex int, yaml string) {