Support async and remote workloads

resources/shared/benchmark.mjs

@@ -1,5 +1,5 @@
 /* eslint-disable no-case-declarations */
-import { StepRunner } from "./step-runner.mjs";
+import { StepRunner, AsyncStepRunner } from "./step-runner.mjs";
 import { Params } from "./params.mjs";
 
 /**
@@ -20,15 +20,31 @@ export class BenchmarkStep {
     }
 }
 
+export class AsyncBenchmarkStep extends BenchmarkStep {
+    async runAndRecord(params, suite, test, callback) {
+        const testRunner = new AsyncStepRunner(null, null, params, suite, test, callback);
+        const result = await testRunner.runTest();
+        return result;
+    }
+}
+
+export const BENCHMARK_SUITE_TYPE = Object.freeze({
+    __proto__: null,
+    sync: "sync",
+    async: "async",
+});
+
 /**
  * BenchmarkSuite
  *
  * A single test suite that contains one or more test steps.
  */
 export class BenchmarkSuite {
-    constructor(name, steps) {
+    constructor(name, steps, type = BENCHMARK_SUITE_TYPE.sync) {
         this.name = name;
         this.steps = steps;
+        this.type = type;
+        console.assert(this.type in BENCHMARK_SUITE_TYPE, "Invalid Type", this.type);
     }
 
     record(_step, syncTime, asyncTime) {
@@ -54,6 +70,7 @@ export class BenchmarkSuite {
 
         for (const step of this.steps) {
             const result = await step.runAndRecordStep(params, this, step, this.record);
+            console.assert(result, "Missing test return value", step);
             measuredValues.tests[step.name] = result;
             measuredValues.total += result.total;
             onProgress?.(step.name);
@@ -66,11 +83,31 @@ export class BenchmarkSuite {
             type: "suite-tests-complete",
             status: "success",
             result: measuredValues,
-            suitename: this.name,
+            suiteName: this.name,
         };
     }
 }
 
+export class AsyncBenchmarkSuite extends BenchmarkSuite {
+    constructor(name, steps) {
+        super(name, steps, BENCHMARK_SUITE_TYPE.async);
+    }
+}
+
+export const MESSAGE_TYPE = Object.freeze({
+    __proto__: null,
+    appReady: "app-ready",
+    suiteStart: "suite-start",
+    stepComplete: "step-complete",
+    suiteComplete: "suite-complete",
+});
+
+export const MESSAGE_STATUS = Object.freeze({
+    __proto__: null,
+    success: "success",
+    error: "error",
+});
+
 /** **********************************************************************
  * BenchmarkConnector
  *
@@ -95,31 +132,42 @@ export class BenchmarkConnector {
     }
 
     async onMessage(event) {
-        if (event.data.id !== this.appId || event.data.key !== "benchmark-connector")
+        const message = event.data;
+        if (message.appId !== this.appId || message.key !== "benchmark-connector") {
+            console.warning("Invalid message", message);
             return;
+        }
 
-        switch (event.data.type) {
-            case "benchmark-suite":
+        switch (message.type) {
+            case MESSAGE_TYPE.suiteStart:
                 const params = new Params(new URLSearchParams(window.location.search));
-                const suite = this.suites[event.data.name];
+                const { name } = message.payload;
+                const suite = this.suites[name];
                 if (!suite)
-                    console.error(`Suite with the name of "${event.data.name}" not found!`);
-                const { result } = await suite.runAndRecordSuite(params, (test) => this.sendMessage({ type: "step-complete", status: "success", appId: this.appId, name: this.name, test }));
-                this.sendMessage({ type: "suite-complete", status: "success", appId: this.appId, result });
+                    console.error(`Suite with the name of "${name}" not found!`);
+                const onProgress = (step) => this._sendMessage(MESSAGE_TYPE.stepComplete, { name: this.name, step });
+                const { result } = await suite.runAndRecordSuite(params, onProgress);
+                this._sendMessage(MESSAGE_TYPE.suiteComplete, { result });
                 this.disconnect();
                 break;
             default:
-                console.error(`Message data type not supported: ${event.data.type}`);
+                console.error(`Message data type not supported: ${message.type}`);
         }
     }
 
-    sendMessage(message) {
+    _sendMessage(type, payload, status = MESSAGE_STATUS.success) {
+        const message = {
+            appId: this.appId,
+            type: type,
+            payload: payload,
+            status: status,
+        };
         window.top.postMessage(message, "*");
     }
 
     connect() {
         window.addEventListener("message", this.onMessage);
-        this.sendMessage({ type: "app-ready", status: "success", appId: this.appId });
+        this._sendMessage(MESSAGE_TYPE.appReady, { appId: this.appId });
     }
 
     disconnect() {

resources/suite-runner.mjs

@@ -1,4 +1,5 @@
 import { STEP_RUNNER_LOOKUP } from "./shared/step-runner.mjs";
+import { MESSAGE_TYPE } from "./shared/benchmark.mjs";
 import { WarmupSuite } from "./benchmark-runner.mjs";
 
 function delay(ms) {
@@ -172,19 +173,17 @@ export class RemoteSuiteRunner extends SuiteRunner {
         const suiteName = this.suite.name;
         const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
         const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
-
         performance.mark(suitePrepareStartLabel);
 
         // Wait for the app-ready message from the workload.
-        const appReadyPromise = this._subscribeOnce("app-ready");
+        const appReadyPromise = this._subscribeOnce(MESSAGE_TYPE.appReady);
         await this._loadFrame(this.suite);
+        // The suite will send out the app-ready message.
         const response = await appReadyPromise;
+        this._initializeAppId(response.appId);
         await this.suite.prepare?.(this.page);
-        // Capture appId to pass along with messages.
-        this.appId = response?.appId;
 
         performance.mark(suitePrepareEndLabel);
-
         const entry = performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
         this.#prepareTime = entry.duration;
     }
@@ -193,26 +192,37 @@ export class RemoteSuiteRunner extends SuiteRunner {
         await delay(this.params.waitAfterSetup);
 
         // Ask workload to run its own tests.
-        this.frame.contentWindow.postMessage({ id: this.appId, key: "benchmark-connector", type: "benchmark-suite", name: this.suite.config?.name || "default" }, "*");
+        this._sendMessage(MESSAGE_TYPE.suiteStart, { name: this.suite.config?.name || "default" });
         // Capture metrics from the completed tests.
-        const response = await this._subscribeOnce("suite-complete");
+        const { result } = await this._subscribeOnce(MESSAGE_TYPE.suiteComplete);
 
         await delay(this.params.waitAfterSuite);
 
         this.suiteResults.tests = {
             ...this.suiteResults.tests,
-            ...response.result.tests,
+            ...result.tests,
         };
 
         this.suiteResults.prepare = this.#prepareTime;
-        this.suiteResults.total = response.result.total;
+        this.suiteResults.total = result.total;
 
         this._validateSuiteResults();
         await this._updateClient();
     }
 
+    _sendMessage(type, payload) {
+        const message = {
+            appId: this.appId,
+            key: "benchmark-connector",
+            type: type,
+            payload: payload,
+        };
+        this.frame.contentWindow.postMessage(message, "*");
+    }
+
     _handlePostMessage(event) {
-        const callback = this.postMessageCallbacks.get(event.data.type);
+        const message = event.data;
+        const callback = this.postMessageCallbacks.get(message.type);
         if (callback)
             callback(event);
     }
@@ -234,11 +244,20 @@ export class RemoteSuiteRunner extends SuiteRunner {
     _subscribeOnce(type) {
         return new Promise((resolve) => {
             this._startSubscription(type, (e) => {
+                const message = e.data;
+                if (type !== MESSAGE_TYPE.appReady && message.appId !== this.appId)
+                    throw new Error(`Got message for invalid app: ${message.appId} instead of ${this.appId}`);
                 this._stopSubscription(type);
-                resolve(e.data);
+                resolve(message.payload);
             });
         });
     }
+
+    _initializeAppId(appId) {
+        console.assert(!this.appId, `Cannot receive ${MESSAGE_TYPE.appReady} twice`);
+        console.assert(appId, `${MESSAGE_TYPE.appReady} sent no appId`);
+        this.appId = appId;
+    }
 }
 
 export const SUITE_RUNNER_LOOKUP = {

suites-experimental/javascript-wc-indexeddb/dist/resources.txt

@@ -17,8 +17,8 @@ src/index.mjs
 src/speedometer-utils/benchmark.mjs
 src/speedometer-utils/helpers.mjs
 src/speedometer-utils/params.mjs
-src/speedometer-utils/test-invoker.mjs
-src/speedometer-utils/test-runner.mjs
+src/speedometer-utils/step-runner.mjs
+src/speedometer-utils/step-scheduler.mjs
 src/speedometer-utils/todomvc-utils.mjs
 src/speedometer-utils/translations.mjs
 src/storage/base-storage-manager.js

suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/benchmark.mjs

@@ -1,5 +1,5 @@
 /* eslint-disable no-case-declarations */
-import { TestRunner, AsyncTestRunner } from "./test-runner.mjs";
+import { StepRunner, AsyncStepRunner } from "./step-runner.mjs";
 import { Params } from "./params.mjs";
 
 /**
@@ -15,10 +15,10 @@ export class BenchmarkStep {
     }
 
     async runAndRecord(params, suite, test, callback) {
-        const TestRunnerClass = params.useAsyncSteps ? AsyncTestRunner : TestRunner;
+        const StepRunnerClass = params.useAsyncSteps ? AsyncStepRunner : StepRunner;
         const type = params.useAsyncSteps ? "async" : "sync";
-        const testRunner = new TestRunnerClass(null, null, params, suite, test, callback, type);
-        const result = await testRunner.runTest();
+        const stepRunner = new StepRunnerClass(null, null, params, suite, test, callback, type);
+        const result = await stepRunner.runStep();
         return result;
     }
 }
@@ -75,6 +75,20 @@ export class BenchmarkSuite {
     }
 }
 
+export const MESSAGE_TYPE = Object.freeze({
+    __proto__: null,
+    appReady: "app-ready",
+    suiteStart: "suite-start",
+    stepComplete: "step-complete",
+    suiteComplete: "suite-complete",
+});
+
+export const MESSAGE_STATUS = Object.freeze({
+    __proto__: null,
+    success: "success",
+    error: "error",
+});
+
 /** **********************************************************************
  * BenchmarkConnector
  *
@@ -99,31 +113,43 @@ export class BenchmarkConnector {
     }
 
     async onMessage(event) {
-        if (event.data.id !== this.appId || event.data.key !== "benchmark-connector")
+        const message = event.data;
+        if (message.appId !== this.appId || message.key !== "benchmark-connector") {
+            console.warn("Invalid message", message);
             return;
+        }
 
-        switch (event.data.type) {
-            case "benchmark-suite":
+        switch (message.type) {
+            case MESSAGE_TYPE.suiteStart:
                 const params = new Params(new URLSearchParams(window.location.search));
-                const suite = this.suites[event.data.name];
+                const { name } = message.payload;
+                const suite = this.suites[name];
                 if (!suite)
-                    console.error(`Suite with the name of "${event.data.name}" not found!`);
-                const { result } = await suite.runAndRecord(params, (test) => this.sendMessage({ type: "step-complete", status: "success", appId: this.appId, name: this.name, test }));
-                this.sendMessage({ type: "suite-complete", status: "success", appId: this.appId, result });
+                    console.error(`Suite with the name of "${name}" not found!`);
+                const onProgress = (step) => this._sendMessage(MESSAGE_TYPE.stepComplete, { name: this.name, step });
+                const { result } = await suite.runAndRecord(params, onProgress);
+                this._sendMessage(MESSAGE_TYPE.suiteComplete, { result });
                 this.disconnect();
                 break;
             default:
-                console.error(`Message data type not supported: ${event.data.type}`);
+                console.error(`Message data type not supported: ${message.type}`);
         }
     }
 
-    sendMessage(message) {
+    _sendMessage(type, payload, status = MESSAGE_STATUS.success) {
+        const message = {
+            appId: this.appId,
+            key: "benchmark-connector",
+            type: type,
+            payload: payload,
+            status: status,
+        };
         window.top.postMessage(message, "*");
     }
 
     connect() {
         window.addEventListener("message", this.onMessage);
-        this.sendMessage({ type: "app-ready", status: "success", appId: this.appId });
+        this._sendMessage(MESSAGE_TYPE.appReady, { appId: this.appId });
     }
 
     disconnect() {