Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions pkg/executor/flow_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -39,6 +40,9 @@ type FlowRunner struct {
conditionSkipped bool
// Sub-command tracking for compound steps (runFlow, repeat, retry)
subCommands []report.Command
// live is the top-level compound step that runs now, whose steps go to
// the report as they start and end (liveSubs); nil between them.
live *liveSteps
// nestedArtifactSeq numbers failure artifacts captured for nested steps
// (runFlow / repeat / retry), which have no top-level command index.
nestedArtifactSeq int
Expand Down Expand Up @@ -684,7 +688,12 @@ func (fr *FlowRunner) executeStep(idx int, step flow.Step) (report.Status, strin
if err := fr.script.ExpandStep(step); err != nil {
result = expressionFailed(err)
} else {
switch step.(type) {
case *flow.RepeatStep, *flow.RetryStep, *flow.RunFlowStep:
fr.live = &liveSteps{cmd: idx}
}
result = fr.runStep(idx, step, &artifacts)
fr.live = nil
}
result = fr.maskTypedResult(step, result)

Expand Down Expand Up @@ -1772,6 +1781,17 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
// inside a runFlow showed the password itself.
describe, desc := fr.stepNames(step)

// The step as it starts, in the report while it runs (liveSubs).
running := report.Command{
ID: fmt.Sprintf("sub-%d", len(fr.subCommands)),
Index: len(fr.subCommands),
Type: string(step.Type()),
Label: step.Label(),
YAML: describe,
Status: report.StatusRunning,
StartTime: &start,
}

// For nested compound steps, we need to track their sub-commands separately
var nestedSubCommands, parentSubCommands []report.Command
isCompoundStep := false
Expand All @@ -1782,6 +1802,8 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
parentSubCommands = fr.subCommands
fr.subCommands = nil
}
fr.live.begin(isCompoundStep, parentSubCommands, running)
fr.liveSubs()

// Expand once, before the handler, as executeStep does: a ${...} that
// throws fails the step and nothing runs.
Expand Down Expand Up @@ -1828,6 +1850,7 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
nestedSubCommands = fr.subCommands
fr.subCommands = parentSubCommands
}
fr.live.end(isCompoundStep)

// Add to parent's sub-commands for report
status := report.StatusPassed
Expand Down Expand Up @@ -1877,10 +1900,70 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult {
}

fr.subCommands = append(fr.subCommands, cmd)
fr.liveSubs()

return result
}

// liveSteps is a top-level runFlow, repeat or retry while it runs: the
// compound steps running inside it, outermost first, each with the list it
// goes into once it ends, and the step running in the innermost one.
type liveSteps struct {
cmd int
frames []liveFrame
step *report.Command
}

type liveFrame struct {
parent []report.Command
cmd report.Command
}

// begin records a nested step that starts: a compound one with the list it
// goes into (its parent's), another as the step running.
func (l *liveSteps) begin(compound bool, parent []report.Command, c report.Command) {
if l == nil {
return
}
if compound {
l.frames = append(l.frames, liveFrame{parent: parent, cmd: c})
} else {
l.step = &c
}
}

// end forgets the nested step begin recorded last, which ended.
func (l *liveSteps) end(compound bool) {
if l == nil {
return
}
if compound {
l.frames = l.frames[:len(l.frames)-1]
} else {
l.step = nil
}
}

// liveSubs writes the steps of the top-level compound step that runs now to
// the report as far as they got: those that ended and those running, so a
// runFlow's steps can be followed while it runs and not only once it ended.
func (fr *FlowRunner) liveSubs() {
l := fr.live
if l == nil || fr.flowWriter == nil {
return
}
list := slices.Clone(fr.subCommands)
if l.step != nil {
list = append(list, *l.step)
}
for i := len(l.frames) - 1; i >= 0; i-- {
c := l.frames[i].cmd
c.SubCommands = list
list = append(slices.Clone(l.frames[i].parent), c)
}
fr.flowWriter.CommandSubs(l.cmd, list)
}

// executeSubFlow executes a sub-flow without separate report tracking.
func (fr *FlowRunner) executeSubFlow(subFlow flow.Flow) *core.CommandResult {
// Save current flow dir
Expand Down
74 changes: 74 additions & 0 deletions pkg/executor/nested_report_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package executor
import (
"context"
"path/filepath"
"strings"
"testing"

"github.com/devicelab-dev/maestro-runner/pkg/core"
"github.com/devicelab-dev/maestro-runner/pkg/flow"
"github.com/devicelab-dev/maestro-runner/pkg/report"
)
Expand Down Expand Up @@ -104,3 +106,75 @@ func TestConsoleLogsAreReportedPerStep(t *testing.T) {
t.Errorf("runFlow logs %q, tapOn logs %q: want none", cmds[1].Logs, cmds[2].Logs)
}
}

// A runFlow's steps are in the report while they run, each as it starts and
// as it ends, the nested runFlow's too: they used to arrive with its end.
func TestRunFlowStepsAreReportedAsTheyRun(t *testing.T) {
dir := t.TempDir()
var shape func(cs []report.Command) string
shape = func(cs []report.Command) string {
var out []string
for _, c := range cs {
s := c.Type + ":" + string(c.Status)
if len(c.SubCommands) > 0 {
s += "[" + shape(c.SubCommands) + "]"
}
out = append(out, s)
}
return strings.Join(out, ",")
}
seen := map[string]string{} // a tap's id → its runFlow's steps in the report as it ran
driver := &mockDriver{executeFunc: func(step flow.Step) *core.CommandResult {
if tap, ok := step.(*flow.TapOnStep); ok {
d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json"))
if err != nil {
t.Errorf("read the report while %s runs: %v", tap.Selector.ID, err)
} else {
seen[tap.Selector.ID] = shape(d.Commands[0].SubCommands)
}
}
return &core.CommandResult{Success: true}
}}
tap := func(id string) *flow.TapOnStep {
return &flow.TapOnStep{BaseStep: flow.BaseStep{StepType: flow.StepTapOn}, Selector: flow.Selector{ID: id}}
}
runner := New(driver, RunnerConfig{
OutputDir: dir,
Artifacts: ArtifactNever,
Device: report.Device{ID: "test", Platform: "android"},
})
f := flow.Flow{
SourcePath: "test.yaml",
Config: flow.Config{Name: "live steps"},
Steps: []flow.Step{
&flow.RunFlowStep{
BaseStep: flow.BaseStep{StepType: flow.StepRunFlow},
Steps: []flow.Step{
tap("a"),
&flow.RunFlowStep{BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, Steps: []flow.Step{tap("b")}},
tap("c"),
},
},
},
}
if _, err := runner.Run(context.Background(), []flow.Flow{f}); err != nil {
t.Fatalf("Run() error = %v", err)
}
want := map[string]string{
"a": "tapOn:running",
"b": "tapOn:passed,runFlow:running[tapOn:running]",
"c": "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:running",
}
for id, w := range want {
if seen[id] != w {
t.Errorf("while %s ran the report had %q, want %q", id, seen[id], w)
}
}
d, err := report.ReadFlowDetail(filepath.Join(dir, "flows", "flow-000.json"))
if err != nil {
t.Fatal(err)
}
if got, w := shape(d.Commands[0].SubCommands), "tapOn:passed,runFlow:passed[tapOn:passed],tapOn:passed"; got != w {
t.Errorf("at the end the report has %q, want %q", got, w)
}
}
13 changes: 13 additions & 0 deletions pkg/report/flow.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,19 @@ func (w *FlowWriter) CommandEndWithSubs(cmdIndex int, status Status, element *El
w.updateIndexProgress()
}

// CommandSubs writes a running command's sub-commands so far: the steps of a
// runFlow, repeat or retry that ended, and the ones running now, so a reader
// of the report sees them before the command ends. CommandEndWithSubs writes
// the final list.
func (w *FlowWriter) CommandSubs(cmdIndex int, subCommands []Command) {
if cmdIndex < 0 || cmdIndex >= len(w.flow.Commands) {
return
}
w.flow.Commands[cmdIndex].SubCommands = subCommands
w.flush()
w.updateIndexProgress()
}

// CommandYAML replaces the command's description, written when the report
// was built, with the one known once it ran.
func (w *FlowWriter) CommandYAML(cmdIndex int, yaml string) {
Expand Down
Loading