Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## Unreleased

### Added
- The rspack bundler now logs when test compilation starts and how long it took. #TINYMCE-14795

### Changed
- Updated various dependencies, including webdriverio, rspack, webpack-dev-server and eslint. #TINYMCE-14572
- Changed the default bundler to rspack. #TINYMCE-14649
- Batched test results are now posted with the next test start instead of waiting for their own request. #TINYMCE-14795

### Fixed
- A rejected promise or exception in a `before` or `after` hook aborted the entire run instead of being reported as a failure. #TINYMCE-14759
- An error arriving after the test that caused it had finished was silently discarded, it is now reported against the most recent test. #TINYMCE-14759
- A test that timed out no longer hides the error from work that was still in flight, it is reported as `<test> (error after test)`. #TINYMCE-14759
- A promise rejected without a reason was treated as a pass. #TINYMCE-14759
- An `after` or `afterEach` hook failure replaced the error of the test that had already failed. #TINYMCE-14759
- The mouse position reset code was only running on page load due to the reduced network traffic in v15.0.0. #TINYMCE-14795
- The mouse position reset logic now works on Firefox and Safari. #TINYMCE-14795

## 16.2.0 - 2026-03-25

Expand Down
6 changes: 5 additions & 1 deletion modules/runner/src/main/ts/api/Main.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Failure, Global } from '@ephox/bedrock-common';
import * as Globals from '../core/Globals';
import { MouseWatch } from '../core/MouseWatch';
import * as TestLoader from '../core/TestLoader';
import { UrlParams } from '../core/UrlParams';
import { makeSessionId } from '../core/Utils';
Expand All @@ -14,11 +15,14 @@ declare const $: JQueryStatic;
// Setup the globals
Globals.setup();

// Watch for mouse effects globally, before tests load
const mouse = MouseWatch();

const setupAndRun = (loadError?: Error) => {
const params = UrlParams.parse(window.location.search, makeSessionId);
const ui = Ui($('body'));
const callbacks = Callbacks();
const reporter = Reporter(params, callbacks, ui);
const reporter = Reporter(params, callbacks, ui, mouse);

const runner = Runner(Globals.rootSuite(), params, callbacks, reporter, ui);
runner.init().then((data) => {
Expand Down
32 changes: 32 additions & 0 deletions modules/runner/src/main/ts/core/MouseWatch.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
export interface MouseWatch {
readonly hasMoved: () => boolean;
readonly clear: () => void;
}

type Open = XMLHttpRequest['open'];

const isMouseEffect = (url: string): boolean => url.replace(/[?#].*$/, '').endsWith('/mouse');

/*
* Mouse effects are ajax calls the test code makes directly to the server, so the runner is
* never told when the driver has moved the real mouse pointer. Watching XMLHttpRequest is the
* only way to know that the pointer needs to be put back before the next test runs.
*/
export const MouseWatch = (): MouseWatch => {
let moved = false;
const open: Open = XMLHttpRequest.prototype.open;

XMLHttpRequest.prototype.open = function (this: XMLHttpRequest, method: string, url: string | URL, ...rest: unknown[]): void {
if (isMouseEffect(String(url))) {
moved = true;
}
(open as (...args: unknown[]) => void).apply(this, [ method, url, ...rest ]);
};

return {
hasMoved: () => moved,
clear: () => {
moved = false;
}
};
};
6 changes: 4 additions & 2 deletions modules/runner/src/main/ts/reporter/Callbacks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export interface Callbacks {
readonly loadHarness: () => Promise<HarnessResponse>
readonly sendKeepAlive: (session: string) => Promise<void>;
readonly sendInit: (session: string) => Promise<void>;
readonly sendTestStart: (session: string, number: number, totalTests: number, file: string, name: string) => Promise<void>;
readonly sendTestStart: (session: string, number: number, totalTests: number, file: string, name: string, results: TestReport[]) => Promise<void>;
readonly sendTestResults: (session: string, results: TestReport[]) => Promise<void>;
readonly sendDone: (session: string, error?: string) => Promise<void>;
}
Expand Down Expand Up @@ -82,13 +82,15 @@ export const Callbacks = (): Callbacks => {
});
};

const sendTestStart = (session: string, number: number, totalTests: number, file: string, name: string): Promise<void> => {
// any results batched up so far ride along, so the server gets them without a second request
const sendTestStart = (session: string, number: number, totalTests: number, file: string, name: string, results: TestReport[]): Promise<void> => {
return sendJson('/tests/start', {
number,
totalTests,
session,
file,
name,
results,
});
};

Expand Down
59 changes: 43 additions & 16 deletions modules/runner/src/main/ts/reporter/Reporter.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import { LoggedError, Reporter as ErrorReporter } from '@ephox/bedrock-common';
import { Callbacks, TestReport } from './Callbacks';
import { MouseWatch } from '../core/MouseWatch';
import { UrlParams } from '../core/UrlParams';
import { formatElapsedTime, mapStackTrace, setStack } from '../core/Utils';

type LoggedError = LoggedError.LoggedError;

export interface TestReporter {
readonly start: () => void;
readonly start: () => Promise<void>;
readonly retry: () => void;
readonly pass: () => void;
readonly skip: (reason: string) => void;
Expand Down Expand Up @@ -48,7 +49,7 @@ const mapError = (e: LoggedError) => mapStackTrace(e.stack).then((mappedStack) =
return e;
});

export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi): Reporter => {
export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi, mouse: MouseWatch): Reporter => {
const initial = Date.now();
let timeOfLastReport = initial;
let currentCount = params.offset || 0;
Expand All @@ -63,19 +64,24 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi
// A global list of requests that were sent to the server, we must wait for these before sending `/done` or it may confuse the HUD
const requestsInFlight: Promise<void>[] = [];

const forceReportResults = (): void => {
const takeResults = (): TestReport[] => {
// Assume the results will be posted once they are returned
timeOfLastReport = Date.now();
// splice deletes the values specified and returns them
return testResults.splice(0, testResults.length);
};

const reportResults = (): void => {
if (testResults.length > 0) {
requestsInFlight.push(callbacks.sendTestResults(params.session, testResults));
testResults.length = 0;
requestsInFlight.push(callbacks.sendTestResults(params.session, takeResults()));
}
};

const reportResult = (result: TestReport): void => {
testResults.push(result);
if (Date.now() - timeOfLastReport > 30 * 1000) {
// ping the server with results every 30 seconds or so, as a form of keep-alive
forceReportResults();
timeOfLastReport = Date.now();
reportResults();
}
};

Expand Down Expand Up @@ -116,18 +122,39 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi
let started = false;
const testUi = ui.test();

const start = (): void => {
const sendStart = (): Promise<void> => {
if (currentCount === 1) {
// we need to send test start once to establish the session
requestsInFlight.push(callbacks.sendTestStart(params.session, currentCount, totalNumTests, file, name, takeResults()));
return Promise.resolve();
} else if (mouse.hasMoved()) {
// Send a new test start so the server resets the mouse - and wait for it
const pending = takeResults();
return callbacks.sendTestStart(params.session, currentCount, totalNumTests, file, name, pending).then(() => {
mouse.clear();
}, (e) => {
// Assume failures are temporary. `mouse.clear()` hasn't run so the next test will try again. requeue the results.
testResults.unshift(...pending);
console.error('Failed to reset the mouse position', e);
});
} else {
return Promise.resolve();
}
};

const start = (): Promise<void> => {
if (!started) {
started = true;
starttime = Date.now();
currentCount++;

testUi.start(file, name);

if (currentCount === 1) {
// we need to send test start once to establish the session
requestsInFlight.push(callbacks.sendTestStart(params.session, currentCount, totalNumTests, file, name));
}
// remove mouse reset time from test records
return sendStart().then(() => {
starttime = Date.now();
});
} else {
return Promise.resolve();
}
};

Expand All @@ -154,7 +181,7 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi
if (params.retry > 0) {
// a test that was under reload/retry status has now passed.
// this needs to be reported immediately, otherwise we might bump up against server timeouts.
forceReportResults();
reportResults();
}
}
};
Expand Down Expand Up @@ -207,13 +234,13 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi
reportFailure(file, name, elapsed(Date.now()), e, (err) => {
ui.error(err);
// nothing is waiting on this result and the page may be about to reload, so send it now
forceReportResults();
reportResults();
});
}
};

const waitForResults = async (): Promise<void> => {
forceReportResults();
reportResults();
if (requestsInFlight.length > 0) {
const currentRequests = requestsInFlight.slice(0);
requestsInFlight.length = 0;
Expand Down
4 changes: 2 additions & 2 deletions modules/runner/src/main/ts/runner/TestRun.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ export const runTest = (test: Test, state: RunState, actions: RunActions, report
if (!state.auto) {
console.log(`Starting test ${state.testCount} of ${state.totalTests}: ${test.fullTitle()} (${test.file})`);
}
testReport.start();
return runTestWithRetry(test, state, testReport, 0)
return testReport.start()
.then(() => runTestWithRetry(test, state, testReport, 0))
.then(() => pass(testReport), (e: LoggedError | InternalError) => {
if (e instanceof SkipError) {
return skip(testReport);
Expand Down
61 changes: 61 additions & 0 deletions modules/runner/src/test/ts/core/MouseWatchTest.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import { assert } from 'chai';
import { afterEach, beforeEach, describe, it } from 'mocha';
import { MouseWatch } from '../../../main/ts/core/MouseWatch';

const global = globalThis as any;

// The runner watches XMLHttpRequest to spot mouse effects, so tests need one to watch.
// A new class per test keeps each watcher's patch off the next test's prototype.
const fakeXhr = () => class {
public opened: string[] = [];

public open(_method: string, url: string): void {
this.opened.push(url);
}
};

describe('MouseWatch', () => {
let original: any;

beforeEach(() => {
original = global.XMLHttpRequest;
global.XMLHttpRequest = fakeXhr();
});

afterEach(() => {
global.XMLHttpRequest = original;
});

const openUrl = (url: string) => {
const xhr = new global.XMLHttpRequest();
xhr.open('post', url);
return xhr;
};

it('should not report the mouse as moved before any effects are sent', () => {
const mouse = MouseWatch();
openUrl('/tests/start');
assert.isFalse(mouse.hasMoved());
});

it('should report the mouse as moved once a mouse effect is sent', () => {
const mouse = MouseWatch();
openUrl('/mouse');
assert.isTrue(mouse.hasMoved());
mouse.clear();
assert.isFalse(mouse.hasMoved());
});

it('should not treat urls that merely contain "mouse" as mouse effects', () => {
const mouse = MouseWatch();
openUrl('/project/src/test/ts/browser/MouseTest.js');
openUrl('/mouseover');
assert.isFalse(mouse.hasMoved());
});

it('should still send the original request', () => {
MouseWatch();
const xhr = openUrl('/mouse?session=1');
assert.deepEqual(xhr.opened, [ '/mouse?session=1' ]);
});
});
Loading
Loading