diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d00876a..eeffa7d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added +- The rspack bundler now logs when test compilation starts and how long it took. #TINYMCE-14795 + ### Changed - Updated various dependencies, including webdriverio, rspack, webpack-dev-server and eslint. #TINYMCE-14572 - Changed the default bundler to rspack. #TINYMCE-14649 +- Batched test results are now posted with the next test start instead of waiting for their own request. #TINYMCE-14795 ### Fixed - A rejected promise or exception in a `before` or `after` hook aborted the entire run instead of being reported as a failure. #TINYMCE-14759 @@ -16,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A test that timed out no longer hides the error from work that was still in flight, it is reported as ` (error after test)`. #TINYMCE-14759 - A promise rejected without a reason was treated as a pass. #TINYMCE-14759 - An `after` or `afterEach` hook failure replaced the error of the test that had already failed. #TINYMCE-14759 +- The mouse position reset code was only running on page load due to the reduced network traffic in v15.0.0. #TINYMCE-14795 +- The mouse position reset logic now works on Firefox and Safari. #TINYMCE-14795 ## 16.2.0 - 2026-03-25 diff --git a/modules/runner/src/main/ts/api/Main.ts b/modules/runner/src/main/ts/api/Main.ts index 0d47a570..2cf06363 100644 --- a/modules/runner/src/main/ts/api/Main.ts +++ b/modules/runner/src/main/ts/api/Main.ts @@ -1,5 +1,6 @@ import { Failure, Global } from '@ephox/bedrock-common'; import * as Globals from '../core/Globals'; +import { MouseWatch } from '../core/MouseWatch'; import * as TestLoader from '../core/TestLoader'; import { UrlParams } from '../core/UrlParams'; import { makeSessionId } from '../core/Utils'; @@ -14,11 +15,14 @@ declare const $: JQueryStatic; // Setup the globals Globals.setup(); +// Watch for mouse effects globally, before tests load +const mouse = MouseWatch(); + const setupAndRun = (loadError?: Error) => { const params = UrlParams.parse(window.location.search, makeSessionId); const ui = Ui($('body')); const callbacks = Callbacks(); - const reporter = Reporter(params, callbacks, ui); + const reporter = Reporter(params, callbacks, ui, mouse); const runner = Runner(Globals.rootSuite(), params, callbacks, reporter, ui); runner.init().then((data) => { diff --git a/modules/runner/src/main/ts/core/MouseWatch.ts b/modules/runner/src/main/ts/core/MouseWatch.ts new file mode 100644 index 00000000..c23ce613 --- /dev/null +++ b/modules/runner/src/main/ts/core/MouseWatch.ts @@ -0,0 +1,32 @@ +export interface MouseWatch { + readonly hasMoved: () => boolean; + readonly clear: () => void; +} + +type Open = XMLHttpRequest['open']; + +const isMouseEffect = (url: string): boolean => url.replace(/[?#].*$/, '').endsWith('/mouse'); + +/* + * Mouse effects are ajax calls the test code makes directly to the server, so the runner is + * never told when the driver has moved the real mouse pointer. Watching XMLHttpRequest is the + * only way to know that the pointer needs to be put back before the next test runs. + */ +export const MouseWatch = (): MouseWatch => { + let moved = false; + const open: Open = XMLHttpRequest.prototype.open; + + XMLHttpRequest.prototype.open = function (this: XMLHttpRequest, method: string, url: string | URL, ...rest: unknown[]): void { + if (isMouseEffect(String(url))) { + moved = true; + } + (open as (...args: unknown[]) => void).apply(this, [ method, url, ...rest ]); + }; + + return { + hasMoved: () => moved, + clear: () => { + moved = false; + } + }; +}; diff --git a/modules/runner/src/main/ts/reporter/Callbacks.ts b/modules/runner/src/main/ts/reporter/Callbacks.ts index 78bd4c4c..607c7d00 100644 --- a/modules/runner/src/main/ts/reporter/Callbacks.ts +++ b/modules/runner/src/main/ts/reporter/Callbacks.ts @@ -20,7 +20,7 @@ export interface Callbacks { readonly loadHarness: () => Promise readonly sendKeepAlive: (session: string) => Promise; readonly sendInit: (session: string) => Promise; - readonly sendTestStart: (session: string, number: number, totalTests: number, file: string, name: string) => Promise; + readonly sendTestStart: (session: string, number: number, totalTests: number, file: string, name: string, results: TestReport[]) => Promise; readonly sendTestResults: (session: string, results: TestReport[]) => Promise; readonly sendDone: (session: string, error?: string) => Promise; } @@ -82,13 +82,15 @@ export const Callbacks = (): Callbacks => { }); }; - const sendTestStart = (session: string, number: number, totalTests: number, file: string, name: string): Promise => { + // any results batched up so far ride along, so the server gets them without a second request + const sendTestStart = (session: string, number: number, totalTests: number, file: string, name: string, results: TestReport[]): Promise => { return sendJson('/tests/start', { number, totalTests, session, file, name, + results, }); }; diff --git a/modules/runner/src/main/ts/reporter/Reporter.ts b/modules/runner/src/main/ts/reporter/Reporter.ts index 5ad1b2be..c4bec087 100644 --- a/modules/runner/src/main/ts/reporter/Reporter.ts +++ b/modules/runner/src/main/ts/reporter/Reporter.ts @@ -1,12 +1,13 @@ import { LoggedError, Reporter as ErrorReporter } from '@ephox/bedrock-common'; import { Callbacks, TestReport } from './Callbacks'; +import { MouseWatch } from '../core/MouseWatch'; import { UrlParams } from '../core/UrlParams'; import { formatElapsedTime, mapStackTrace, setStack } from '../core/Utils'; type LoggedError = LoggedError.LoggedError; export interface TestReporter { - readonly start: () => void; + readonly start: () => Promise; readonly retry: () => void; readonly pass: () => void; readonly skip: (reason: string) => void; @@ -48,7 +49,7 @@ const mapError = (e: LoggedError) => mapStackTrace(e.stack).then((mappedStack) = return e; }); -export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi): Reporter => { +export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi, mouse: MouseWatch): Reporter => { const initial = Date.now(); let timeOfLastReport = initial; let currentCount = params.offset || 0; @@ -63,10 +64,16 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi // A global list of requests that were sent to the server, we must wait for these before sending `/done` or it may confuse the HUD const requestsInFlight: Promise[] = []; - const forceReportResults = (): void => { + const takeResults = (): TestReport[] => { + // Assume the results will be posted once they are returned + timeOfLastReport = Date.now(); + // splice deletes the values specified and returns them + return testResults.splice(0, testResults.length); + }; + + const reportResults = (): void => { if (testResults.length > 0) { - requestsInFlight.push(callbacks.sendTestResults(params.session, testResults)); - testResults.length = 0; + requestsInFlight.push(callbacks.sendTestResults(params.session, takeResults())); } }; @@ -74,8 +81,7 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi testResults.push(result); if (Date.now() - timeOfLastReport > 30 * 1000) { // ping the server with results every 30 seconds or so, as a form of keep-alive - forceReportResults(); - timeOfLastReport = Date.now(); + reportResults(); } }; @@ -116,18 +122,39 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi let started = false; const testUi = ui.test(); - const start = (): void => { + const sendStart = (): Promise => { + if (currentCount === 1) { + // we need to send test start once to establish the session + requestsInFlight.push(callbacks.sendTestStart(params.session, currentCount, totalNumTests, file, name, takeResults())); + return Promise.resolve(); + } else if (mouse.hasMoved()) { + // Send a new test start so the server resets the mouse - and wait for it + const pending = takeResults(); + return callbacks.sendTestStart(params.session, currentCount, totalNumTests, file, name, pending).then(() => { + mouse.clear(); + }, (e) => { + // Assume failures are temporary. `mouse.clear()` hasn't run so the next test will try again. requeue the results. + testResults.unshift(...pending); + console.error('Failed to reset the mouse position', e); + }); + } else { + return Promise.resolve(); + } + }; + + const start = (): Promise => { if (!started) { started = true; - starttime = Date.now(); currentCount++; testUi.start(file, name); - if (currentCount === 1) { - // we need to send test start once to establish the session - requestsInFlight.push(callbacks.sendTestStart(params.session, currentCount, totalNumTests, file, name)); - } + // remove mouse reset time from test records + return sendStart().then(() => { + starttime = Date.now(); + }); + } else { + return Promise.resolve(); } }; @@ -154,7 +181,7 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi if (params.retry > 0) { // a test that was under reload/retry status has now passed. // this needs to be reported immediately, otherwise we might bump up against server timeouts. - forceReportResults(); + reportResults(); } } }; @@ -207,13 +234,13 @@ export const Reporter = (params: UrlParams, callbacks: Callbacks, ui: ReporterUi reportFailure(file, name, elapsed(Date.now()), e, (err) => { ui.error(err); // nothing is waiting on this result and the page may be about to reload, so send it now - forceReportResults(); + reportResults(); }); } }; const waitForResults = async (): Promise => { - forceReportResults(); + reportResults(); if (requestsInFlight.length > 0) { const currentRequests = requestsInFlight.slice(0); requestsInFlight.length = 0; diff --git a/modules/runner/src/main/ts/runner/TestRun.ts b/modules/runner/src/main/ts/runner/TestRun.ts index 9d4f7551..374cb128 100644 --- a/modules/runner/src/main/ts/runner/TestRun.ts +++ b/modules/runner/src/main/ts/runner/TestRun.ts @@ -97,8 +97,8 @@ export const runTest = (test: Test, state: RunState, actions: RunActions, report if (!state.auto) { console.log(`Starting test ${state.testCount} of ${state.totalTests}: ${test.fullTitle()} (${test.file})`); } - testReport.start(); - return runTestWithRetry(test, state, testReport, 0) + return testReport.start() + .then(() => runTestWithRetry(test, state, testReport, 0)) .then(() => pass(testReport), (e: LoggedError | InternalError) => { if (e instanceof SkipError) { return skip(testReport); diff --git a/modules/runner/src/test/ts/core/MouseWatchTest.ts b/modules/runner/src/test/ts/core/MouseWatchTest.ts new file mode 100644 index 00000000..939204b2 --- /dev/null +++ b/modules/runner/src/test/ts/core/MouseWatchTest.ts @@ -0,0 +1,61 @@ +import { assert } from 'chai'; +import { afterEach, beforeEach, describe, it } from 'mocha'; +import { MouseWatch } from '../../../main/ts/core/MouseWatch'; + +const global = globalThis as any; + +// The runner watches XMLHttpRequest to spot mouse effects, so tests need one to watch. +// A new class per test keeps each watcher's patch off the next test's prototype. +const fakeXhr = () => class { + public opened: string[] = []; + + public open(_method: string, url: string): void { + this.opened.push(url); + } +}; + +describe('MouseWatch', () => { + let original: any; + + beforeEach(() => { + original = global.XMLHttpRequest; + global.XMLHttpRequest = fakeXhr(); + }); + + afterEach(() => { + global.XMLHttpRequest = original; + }); + + const openUrl = (url: string) => { + const xhr = new global.XMLHttpRequest(); + xhr.open('post', url); + return xhr; + }; + + it('should not report the mouse as moved before any effects are sent', () => { + const mouse = MouseWatch(); + openUrl('/tests/start'); + assert.isFalse(mouse.hasMoved()); + }); + + it('should report the mouse as moved once a mouse effect is sent', () => { + const mouse = MouseWatch(); + openUrl('/mouse'); + assert.isTrue(mouse.hasMoved()); + mouse.clear(); + assert.isFalse(mouse.hasMoved()); + }); + + it('should not treat urls that merely contain "mouse" as mouse effects', () => { + const mouse = MouseWatch(); + openUrl('/project/src/test/ts/browser/MouseTest.js'); + openUrl('/mouseover'); + assert.isFalse(mouse.hasMoved()); + }); + + it('should still send the original request', () => { + MouseWatch(); + const xhr = openUrl('/mouse?session=1'); + assert.deepEqual(xhr.opened, [ '/mouse?session=1' ]); + }); +}); diff --git a/modules/runner/src/test/ts/reporter/ReporterTest.ts b/modules/runner/src/test/ts/reporter/ReporterTest.ts index 7bba475e..6e6a054f 100644 --- a/modules/runner/src/test/ts/reporter/ReporterTest.ts +++ b/modules/runner/src/test/ts/reporter/ReporterTest.ts @@ -2,8 +2,9 @@ import { Failure, LoggedError } from '@ephox/bedrock-common'; import { assert } from 'chai'; import * as fc from 'fast-check'; import { beforeEach, describe, it } from 'mocha'; +import { MouseWatch } from '../../../main/ts/core/MouseWatch'; import { UrlParams } from '../../../main/ts/core/UrlParams'; -import { Callbacks, TestErrorData } from '../../../main/ts/reporter/Callbacks'; +import { Callbacks, TestErrorData, TestReport } from '../../../main/ts/reporter/Callbacks'; import { Reporter } from '../../../main/ts/reporter/Reporter'; import { noop, wait } from '../TestUtils'; @@ -13,6 +14,7 @@ interface StartTestData { readonly totalTests: number; readonly file: string; readonly name: string; + readonly results: TestReport[]; } interface EndTestData { @@ -48,12 +50,23 @@ const ui = { describe('Reporter.test', () => { let reporter: Reporter, startTestData: StartTestData[], endTestData: EndTestData[]; let doneCalled: boolean, doneError: string | undefined, offset: number; + let mouseMoved: boolean, startFails: boolean; + const mouse: MouseWatch = { + hasMoved: () => mouseMoved, + clear: () => { + mouseMoved = false; + } + }; const callbacks: Callbacks = { loadHarness: () => Promise.resolve({ retries: 0, chunk: 100, stopOnFailure: true, mode: 'manual', timeout: 10000 }), sendKeepAlive: () => Promise.resolve(), sendInit: () => Promise.resolve(), - sendTestStart: (session, currentCount, totalTests, file, name) => { - startTestData.push({ session, currentCount, totalTests, file, name }); + sendTestStart: (session, currentCount, totalTests, file, name, results) => { + if (startFails) { + return Promise.reject(new Error('start failed')); + } + startTestData.push({ session, currentCount, totalTests, file, name, results }); + results.forEach(r => endTestData.push({ session, ...r })); return Promise.resolve(); }, sendTestResults: (session, results) => { @@ -69,7 +82,9 @@ describe('Reporter.test', () => { const reset = (newOffset: number = Math.floor(Math.random() * 1000)) => { offset = newOffset; - reporter = Reporter({ ...params, offset }, callbacks, ui); + mouseMoved = false; + startFails = false; + reporter = Reporter({ ...params, offset }, callbacks, ui, mouse); startTestData = []; endTestData = []; doneCalled = false; @@ -90,7 +105,8 @@ describe('Reporter.test', () => { session: sessionId, totalTests: testCount, file: fileName + 'Test.ts', - name: testName + name: testName, + results: [] }, 'Checking start test data contents'); assert.equal(endTestData.length, 0, 'Checking there is no end test data'); @@ -106,6 +122,54 @@ describe('Reporter.test', () => { })); }); + it('should send batched results along with a test start', () => { + reset(5); + const first = reporter.test('SomeTest.ts', 'first', 10); + return first.start().then(() => { + first.pass(); + mouseMoved = true; + return reporter.test('SomeTest.ts', 'second', 10).start(); + }).then(() => { + assert.equal(startTestData.length, 1, 'Checking start test data was sent'); + assert.deepEqual(startTestData[0].results.map((r) => r.name), [ 'first' ], 'Checking the batch rode along'); + assert.equal(endTestData.length, 1, 'Checking the result was recorded'); + return reporter.waitForResults().then(() => { + assert.equal(endTestData.length, 1, 'Checking the result was not also sent separately'); + }); + }); + }); + + it('should keep batched results when a test start fails', () => { + reset(5); + const first = reporter.test('SomeTest.ts', 'first', 10); + return first.start().then(() => { + first.pass(); + mouseMoved = true; + startFails = true; + return reporter.test('SomeTest.ts', 'second', 10).start(); + }).then(() => { + assert.equal(startTestData.length, 0, 'Checking the start was not recorded'); + assert.isTrue(mouseMoved, 'Checking the mouse watch was not cleared'); + return reporter.waitForResults().then(() => { + assert.deepEqual(endTestData.map((r) => r.name), [ 'first' ], 'Checking the result was sent later instead of lost'); + }); + }); + }); + + it('should only send a test start mid-run when the mouse has moved', () => { + reset(5); + const first = reporter.test('SomeTest.ts', 'first', 10); + return first.start().then(() => { + assert.equal(startTestData.length, 0, 'Checking no start test data was sent'); + mouseMoved = true; + return reporter.test('SomeTest.ts', 'second', 10).start(); + }).then(() => { + assert.equal(startTestData.length, 1, 'Checking start test data was sent'); + assert.equal(startTestData[0].currentCount, 7, 'Checking the test number'); + assert.isFalse(mouseMoved, 'Checking the mouse watch was cleared'); + }); + }); + it('should report the session id, file, name, passed state and time on a skipped test', () => { return fc.assert(fc.asyncProperty(fc.string(), fc.string(), fc.string(), fc.integer({ max: offset }), (fileName, testName, skippedMessage, testCount) => { reset(); diff --git a/modules/sample/src/test/ts/client/pass/MouseResetTest.ts b/modules/sample/src/test/ts/client/pass/MouseResetTest.ts new file mode 100644 index 00000000..7560b740 --- /dev/null +++ b/modules/sample/src/test/ts/client/pass/MouseResetTest.ts @@ -0,0 +1,45 @@ +import { after, Assert, before, describe, it } from '@ephox/bedrock-client'; + +import { sendMouse } from '../../utils/Utils'; + +describe('Mouse reset', () => { + let target: HTMLDivElement; + let hovered = false; + let movedAway = false; + + // We can't depend on Waiter here because it's + const waitFor = async (label: string, predicate: () => boolean): Promise => { + for (let i = 0; i < 50 && !predicate(); i++) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + Assert.eq(label, true, predicate()); + }; + + before(() => { + target = document.createElement('div'); + target.id = 'mouse-reset-target'; + target.style.cssText = 'position: fixed; left: 150px; top: 150px; width: 100px; height: 100px; z-index: 1000;'; + target.addEventListener('mouseover', () => { + hovered = true; + }); + target.addEventListener('mouseout', () => { + movedAway = true; + }); + document.body.appendChild(target); + }); + + after(() => { + document.body.removeChild(target); + }); + + // A click is used to park the pointer rather than a move, because Safari reports success for a + // move without the page ever seeing it + it('should move the real mouse over an element', async () => { + await sendMouse('#mouse-reset-target', 'click'); + await waitFor('Target should have been hovered', () => hovered); + }); + + it('should have moved the mouse off the element before this test', async () => { + await waitFor('Mouse should have left the target between tests', () => movedAway); + }); +}); diff --git a/modules/server/src/main/ts/bedrock/compiler/Rspack.ts b/modules/server/src/main/ts/bedrock/compiler/Rspack.ts index 8cc90507..4d394ebe 100644 --- a/modules/server/src/main/ts/bedrock/compiler/Rspack.ts +++ b/modules/server/src/main/ts/bedrock/compiler/Rspack.ts @@ -124,6 +124,8 @@ const getWebPackConfigTs = (tsConfigFile: string, scratchFile: string, dest: str const compileTests = (compileInfo: RspackCompileInfo, exitOnCompileError: boolean, srcFiles: string[], polyfills: string[]): Promise => { return new Promise((resolve) => { + console.log(`Compiling ${srcFiles.length} tests...`); + const start = Date.now(); mkdirp.sync(path.dirname(compileInfo.scratchFile)); fs.writeFileSync(compileInfo.scratchFile, Imports.generateImports(true, compileInfo.scratchFile, srcFiles, polyfills)); @@ -145,6 +147,7 @@ const compileTests = (compileInfo: RspackCompileInfo, exitOnCompileError: boolea } } + console.log(`Compiled ${srcFiles.length} tests in ${Date.now() - start}ms`); resolve(compileInfo.dest); }); }); diff --git a/modules/server/src/main/ts/bedrock/server/Apis.ts b/modules/server/src/main/ts/bedrock/server/Apis.ts index dc303045..02ab25d9 100644 --- a/modules/server/src/main/ts/bedrock/server/Apis.ts +++ b/modules/server/src/main/ts/bedrock/server/Apis.ts @@ -25,6 +25,7 @@ interface StartData { readonly file: string; readonly number: number; readonly totalTests: number; + readonly results: Controller.TestResult[]; } export interface ResultsData { @@ -47,6 +48,7 @@ const maxInvalidAttempts = 300; export const create = (master: DriverMaster | null, pMaybeDriver: Promise>, projectdir: string, basedir: string, stickyFirstSession: boolean, overallTimeout: number, testfiles: string[], loglevel: 'simple' | 'advanced', resetMousePosition: boolean): Apis => { let pageHasLoaded = false; let needsMousePositionReset = true; + let mousePositionResetSupported = true; // On IE, the webdriver seems to load the page before it's ready to start // responding to commands. If the testing page itself tries to interact with @@ -113,17 +115,26 @@ export const create = (master: DriverMaster | null, pMaybeDriver: Promise Promise.reject('Resetting mouse position not supported without webdriver running. Use bedrock-auto to get this feature.'), (driver) => waitForDriverReady(maxInvalidAttempts, async () => { const shouldResetMousePos = force || needsMousePositionReset; - // TODO re-enable resetting the mouse on other browsers when mouseMove gets fixed on Firefox/IE - const browserName = (driver.capabilities as Record).browserName; - if (shouldResetMousePos && (browserName === 'chrome' || browserName === 'msedge')) { - // Reset the mouse position to the top left of the window - await driver.performActions([{ - type: 'pointer', - id: 'finger1', - parameters: { pointerType: 'mouse' }, - actions: [{ type: 'pointerMove', duration: 0, x: 0, y: 0 }] - }]); - needsMousePositionReset = false; + if (shouldResetMousePos && mousePositionResetSupported) { + try { + // Park the mouse in the top left corner, out of the way of the browser. + // Use two actions to account for drivers thinking this is a no-op; it still only + // sends one driver request to the browser. + await driver.performActions([{ + type: 'pointer', + id: 'finger1', + parameters: { pointerType: 'mouse' }, + actions: [ + { type: 'pointerMove', duration: 0, x: 1, y: 1 }, + { type: 'pointerMove', duration: 0, x: 0, y: 0 } + ] + }]); + needsMousePositionReset = false; + } catch (e) { + // Log the failure instead of crashing the whole test run + mousePositionResetSupported = false; + console.error('Unable to reset the mouse position, continuing without it: ', e); + } } }) ); @@ -153,6 +164,7 @@ export const create = (master: DriverMaster | null, pMaybeDriver: Promise resetMousePositionAction(true)), Routes.effect('POST', '/tests/start', (data: StartData) => { + c.recordTestResults(data.session, data.results); c.recordTestStart(data.session, data.name, data.file, data.number, data.totalTests); return resetMousePositionAction(); }),