diff --git a/packages/cli/e2e/__tests__/deploy.spec.ts b/packages/cli/e2e/__tests__/deploy.spec.ts index 5fe40d24..f047526c 100644 --- a/packages/cli/e2e/__tests__/deploy.spec.ts +++ b/packages/cli/e2e/__tests__/deploy.spec.ts @@ -483,6 +483,57 @@ describe('deploy', { timeout: 45_000 }, () => { }) }) + describe('preview-playwright-project', () => { + let fixt: FixtureSandbox + + beforeAll(async () => { + fixt = await FixtureSandbox.create({ + source: path.join(__dirname, 'fixtures', 'preview-playwright-project'), + template: 'playwright', + }) + }, 180_000) + + afterAll(async () => { + await fixt?.destroy() + }) + + // A deployed suite first: the preview renders a construct diff only for + // an updated resource. Both runs bundle the Playwright project, which is + // what takes the time; the enclosing suite's budget is smaller than one + // deploy's own. + it('Should render a renamed Playwright check suite as a construct diff', async () => { + await runDeploy(fixt, ['--force'], { + env: { + PROJECT_LOGICAL_ID: projectLogicalId, + CHECKLY_E2E_CLI_VERSION: '4.8.0', + }, + }) + const { stdout } = await runDeploy(fixt, ['--preview'], { + env: { + PROJECT_LOGICAL_ID: projectLogicalId, + SUITE_NAME: 'Renamed suite', + CHECKLY_E2E_CLI_VERSION: '4.8.0', + }, + }) + expect(stdout).toMatch(tableRows([['~', 'PlaywrightCheck', 'suite']])) + expect(stdout).not.toContain('could not render this resource') + // The construct diff needs the API's preview endpoint; against an API + // without it the CLI prints the overview only and says so. + if (stdout.includes('for the deploy preview endpoint')) { + return + } + expect(stdout).toMatch(/^\s*-\s+name: 'Suite',$/m) + expect(stdout).toMatch(/^\s*\+\s+name: 'Renamed suite',$/m) + // Context lines on both sides: the deployed side unfolds the config + // path and the projects from the stored test command exactly as the + // local side does, and spells the engine the same way. The rename is + // the only change the diff shows. + expect(stdout).toContain('playwrightConfigPath: \'playwright.config.ts\'') + expect(stdout).toContain('engine: Engine.node(\'22\')') + expect(stdout.match(/^\s*[-+]\s+[A-Za-z]+: /gm)).toHaveLength(2) + }, 300_000) + }) + describe('snapshot-project', () => { let fixt: FixtureSandbox diff --git a/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/checkly.config.ts b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/checkly.config.ts new file mode 100644 index 00000000..e0ff3445 --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/checkly.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from 'checkly' + +// A project with one Playwright check suite, for the deploy preview's +// rendering of a suite as a construct. No lockfile is committed: the test +// sandbox copies its template's lockfile in, which the suite's validation +// requires. +export default defineConfig({ + projectName: 'Preview Playwright Project', + logicalId: process.env.PROJECT_LOGICAL_ID!, + repoUrl: 'https://github.com/checkly/checkly-cli', + checks: { + checkMatch: '**/*.check.ts', + }, + cli: { + runLocation: 'us-east-1', + }, +}) diff --git a/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/package.json b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/package.json new file mode 100644 index 00000000..439f6b6c --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/package.json @@ -0,0 +1,8 @@ +{ + "name": "preview-playwright-project", + "version": "1.0.0", + "private": true, + "dependencies": { + "@playwright/test": "^1.59.1" + } +} diff --git a/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/playwright.config.ts b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/playwright.config.ts new file mode 100644 index 00000000..96e2af5d --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/playwright.config.ts @@ -0,0 +1,11 @@ +import { defineConfig, devices } from '@playwright/test' + +export default defineConfig({ + testDir: './tests', + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}) diff --git a/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/suite.check.ts b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/suite.check.ts new file mode 100644 index 00000000..ebd0b4d8 --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/suite.check.ts @@ -0,0 +1,10 @@ +import { Engine, PlaywrightCheck } from 'checkly/constructs' + +new PlaywrightCheck('suite', { + name: process.env.SUITE_NAME ?? 'Suite', + playwrightConfigPath: './playwright.config.ts', + pwProjects: 'chromium', + engine: Engine.node('22'), + frequency: 10, + locations: ['us-east-1'], +}) diff --git a/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/tests/homepage.test.ts b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/tests/homepage.test.ts new file mode 100644 index 00000000..f353a943 --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/preview-playwright-project/tests/homepage.test.ts @@ -0,0 +1,6 @@ +import { expect, test } from '@playwright/test' + +test('homepage loads', async ({ page }) => { + await page.goto('https://www.checklyhq.com') + await expect(page).toHaveTitle(/Checkly/) +}) diff --git a/packages/cli/src/commands/import/plan.ts b/packages/cli/src/commands/import/plan.ts index cd811d61..b5bcdeaf 100644 --- a/packages/cli/src/commands/import/plan.ts +++ b/packages/cli/src/commands/import/plan.ts @@ -18,6 +18,7 @@ import { ChecklyConfig, ConfigNotFoundError, loadChecklyConfig } from '../../ser import { ImportPlan, ProjectNotFoundError, ImportPlanFilter, ImportPlanOptions, ResourceSync, ImportPlanFriend, FriendResourceSync, NoImportableResourcesFoundError } from '../../rest/projects.js' import { cased, Comment, docComment, Program } from '../../sourcegen/index.js' import { ConstructCodegen, sortResources } from '../../constructs/construct-codegen.js' +import { PREVIEW_ONLY_CHECK_TYPES } from '../../constructs/check-codegen.js' import { Context } from '../../constructs/internal/codegen/index.js' import { isSnippet, @@ -1280,6 +1281,16 @@ ${chalk.cyan('For safety, resources are not deletable until the plan has been co } try { + // A codegen that exists for the deploy preview only must not + // write a construct here: a suite that reaches a plan (an API + // without the rule that leaves them out) is reported as not + // importable instead. + const previewOnly = resource.type === 'check' + ? PREVIEW_ONLY_CHECK_TYPES.get(resource.payload?.checkType) + : undefined + if (previewOnly !== undefined) { + throw new Error(previewOnly) + } codegen.gencode(resource.logicalId, resource as any, context) } catch (cause) { if (!(cause instanceof Error)) { diff --git a/packages/cli/src/constructs/__tests__/generated-code-compiles.spec.ts b/packages/cli/src/constructs/__tests__/generated-code-compiles.spec.ts index 9165747a..6c2f967b 100644 --- a/packages/cli/src/constructs/__tests__/generated-code-compiles.spec.ts +++ b/packages/cli/src/constructs/__tests__/generated-code-compiles.spec.ts @@ -10,6 +10,7 @@ import { AgenticCheckCodegen, AgenticCheckResource } from '../agentic-check-code import { ApiCheckCodegen, ApiCheckResource } from '../api-check-codegen.js' import { CheckGroupCodegen, CheckGroupResource } from '../check-group-codegen.js' import { IncidentioAlertChannelCodegen, IncidentioAlertChannelResource } from '../incidentio-alert-channel-codegen.js' +import { PlaywrightCheckCodegen, PlaywrightCheckResource } from '../playwright-check-codegen.js' import { Context, MASKED_VALUE } from '../internal/codegen/index.js' import { Session } from '../session.js' import { Program } from '../../sourcegen/index.js' @@ -82,6 +83,31 @@ describe('generated code compiles', () => { locations: [], shouldFail: false, } + // A Playwright suite's props are unfolded from its test command; the + // engine is spelled as the construct's own `Engine` for the engines it + // offers and as a plain object otherwise. A command that cannot be + // unfolded leaves the required `playwrightConfigPath` out and is not + // expected to compile, so it is not part of this program. + const suite: PlaywrightCheckResource = { + id: 'suite', + checkType: 'PLAYWRIGHT', + name: 'Suite', + testCommand: 'npx playwright test --config playwright.config.ts --project chromium --grep \'@smoke|@checkout\'', + installCommand: 'npm ci', + engine: 'node', + engineVersion: '22', + locations: ['us-east-1'], + tags: ['e2e'], + frequency: 10, + } + const suiteOnUnknownEngine: PlaywrightCheckResource = { + id: 'suite-deno', + checkType: 'PLAYWRIGHT', + name: 'Suite on Deno', + testCommand: 'deno task e2e --config playwright.config.ts', + engine: 'deno', + engineVersion: '2', + } const group: CheckGroupResource = { id: 7, name: 'Group', @@ -116,12 +142,14 @@ describe('generated code compiles', () => { new ApiCheckCodegen(program).gencode('api', apiCheck, context) new ApiCheckCodegen(program).gencode('sub-minute', subMinute, context) new AgenticCheckCodegen(program).gencode('agentic', agentic, context) + new PlaywrightCheckCodegen(program).gencode('suite', suite, context) + new PlaywrightCheckCodegen(program).gencode('suite-deno', suiteOnUnknownEngine, context) groupCodegen.gencode('group', group, context) channelCodegen.gencode('incidents', channel, context) await program.realize() const generated = program.paths.filter(file => file.endsWith('.ts')) - expect(generated.length).toBeGreaterThanOrEqual(5) + expect(generated.length).toBeGreaterThanOrEqual(7) // TypeScript reports file names with forward slashes on every platform. const generatedNames = new Set(generated.map(file => file.replaceAll('\\', '/'))) diff --git a/packages/cli/src/constructs/__tests__/playwright-check-codegen.spec.ts b/packages/cli/src/constructs/__tests__/playwright-check-codegen.spec.ts new file mode 100644 index 00000000..cb2c98fe --- /dev/null +++ b/packages/cli/src/constructs/__tests__/playwright-check-codegen.spec.ts @@ -0,0 +1,238 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' + +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' + +import { PlaywrightCheckCodegen, PlaywrightCheckResource, unfoldTestCommand } from '../playwright-check-codegen.js' +import { PREVIEW_ONLY_CHECK_TYPES } from '../check-codegen.js' +import { CheckGroupCodegen, CheckGroupResource } from '../check-group-codegen.js' +import { Context } from '../internal/codegen/context.js' +import { Session } from '../session.js' +import { Program } from '../../sourcegen/index.js' + +/** + * The construct generated for a Playwright check suite: the test command + * unfolded into the props it was built from, the suite's own props, the + * shared check props, and never the props a suite does not have. + */ + +const DEFAULT_TEST_COMMAND = Session.packageManager.execCommand(['playwright', 'test']).unsafeDisplayCommand + +let rootDirectory: string + +beforeAll(async () => { + rootDirectory = await mkdtemp(path.join(tmpdir(), 'playwright-check-codegen-')) +}) + +afterAll(async () => { + await rm(rootDirectory, { recursive: true, force: true }) +}) + +afterEach(() => { + Session.reset() +}) + +async function render ( + resource: PlaywrightCheckResource, + prepare?: (program: Program, context: Context) => void, +): Promise { + const program = new Program({ + rootDirectory: await mkdtemp(path.join(rootDirectory, 'render-')), + constructFileSuffix: '.check', + specFileSuffix: '.spec', + language: 'typescript', + }) + const context = new Context() + prepare?.(program, context) + new PlaywrightCheckCodegen(program).gencode(resource.id, resource, context) + await program.realize() + const constructFile = program.paths.find(file => file.includes('playwright-check-suites') && file.endsWith('.check.ts')) + if (constructFile === undefined) { + throw new Error('PlaywrightCheckCodegen did not register a construct file') + } + return readFile(constructFile, 'utf8') +} + +/** A suite as the API describes it, bundle keys included. */ +const suite = (overrides: Partial = {}): PlaywrightCheckResource => ({ + id: 'suite-uuid', + checkType: 'PLAYWRIGHT', + name: 'Checkout flows', + activated: true, + muted: false, + locations: ['us-east-1', 'eu-west-1'], + tags: ['e2e'], + frequency: 10, + testCommand: `${DEFAULT_TEST_COMMAND} --config playwright.config.ts`, + installCommand: null, + engine: null, + engineVersion: null, + ...({ + codeBundlePath: 'bundles/suite.tar.gz', + cacheHash: 'abc123', + playwrightVersion: '1.59.1', + browsers: ['chromium'], + workingDir: '.', + doubleCheck: false, + } as Partial), + ...overrides, +}) + +describe('unfoldTestCommand', () => { + it('splits the CLI-built command into the user command and the props', () => { + expect(unfoldTestCommand('npx playwright test --config playwright.config.ts')).toEqual({ + command: 'npx playwright test', + playwrightConfigPath: 'playwright.config.ts', + }) + expect(unfoldTestCommand( + 'pnpm exec playwright test --config \'e2e/pw config.ts\' --project chromium \'Mobile Safari\' --grep \'@smoke|@checkout\'', + )).toEqual({ + command: 'pnpm exec playwright test', + playwrightConfigPath: 'e2e/pw config.ts', + pwProjects: ['chromium', 'Mobile Safari'], + pwTags: ['@smoke', '@checkout'], + }) + }) + + it('never reads a bare flag-like word as a value', () => { + // A project named `--config` cannot be told from a user's trailing flag, + // so the trailing flag reading wins: the user's part ends with it. + expect(unfoldTestCommand('npx playwright test --config user.ts --project --config real.ts')).toEqual({ + command: 'npx playwright test --config user.ts --project', + playwrightConfigPath: 'real.ts', + }) + }) + + it('drops a backslash-newline as a line continuation', () => { + expect(unfoldTestCommand('npx playwright test \\\n --config a.ts')).toEqual({ + command: 'npx playwright test', + playwrightConfigPath: 'a.ts', + }) + }) + + it('reads the quoting forms shellQuote produces and the usual hand-written ones', () => { + expect(unfoldTestCommand('npx playwright test --config \'it\'"\'"\'s.config.ts\'').playwrightConfigPath).toBe('it\'s.config.ts') + expect(unfoldTestCommand('npx playwright test --config "a \\"b\\".ts" --project a\\ b')).toEqual({ + command: 'npx playwright test', + playwrightConfigPath: 'a "b".ts', + pwProjects: ['a b'], + }) + // A backslash in double quotes escapes only what the shell would expand. + expect(unfoldTestCommand('npx playwright test --config "e2e\\config.ts"').playwrightConfigPath).toBe('e2e\\config.ts') + }) + + it('keeps the user command as written and takes the last --config word as the boundary', () => { + expect(unfoldTestCommand('FOO="x y" npx playwright test --config a.ts --config b.ts')).toEqual({ + command: 'FOO="x y" npx playwright test --config a.ts', + playwrightConfigPath: 'b.ts', + }) + // A tag pattern containing the text is a word, not a boundary. + expect(unfoldTestCommand('npx playwright test --config a.ts --grep \'--config\'')).toEqual({ + command: 'npx playwright test', + playwrightConfigPath: 'a.ts', + pwTags: ['--config'], + }) + }) + + it('returns a command it cannot unfold whole', () => { + for (const command of [ + 'npx playwright test', + 'npx playwright test --config', + '--config a.ts', + 'npx playwright test --config a.ts --workers 2', + 'npx playwright test --config a.ts --project', + 'npx playwright test --config a.ts --grep a --grep b', + 'npx playwright test --config a.ts --project a --project b', + 'npx playwright test --config \'unterminated', + ]) { + expect(unfoldTestCommand(command), command).toEqual({ command }) + } + }) +}) + +describe('PlaywrightCheckCodegen', () => { + it('is a preview-only codegen, which checkly import refuses', () => { + expect(PREVIEW_ONLY_CHECK_TYPES.get('PLAYWRIGHT')).toMatch(/cannot be imported/) + }) + + it('describes the suite', () => { + const codegen = new PlaywrightCheckCodegen(new Program({ + rootDirectory: '.', + constructFileSuffix: '.check', + specFileSuffix: '.spec', + language: 'typescript', + })) + expect(codegen.describe(suite())).toBe('Playwright Check Suite: Checkout flows') + }) + + it('renders the suite props from the test command and the shared check props', async () => { + const source = await render(suite({ + testCommand: `${DEFAULT_TEST_COMMAND} --config 'e2e/pw config.ts' --project chromium firefox --grep '@smoke|@checkout'`, + installCommand: 'pnpm install --frozen-lockfile', + engine: 'node', + engineVersion: '22', + groupId: 7, + runtimeId: '2025.04', + environmentVariables: [{ key: 'BASE_URL', value: 'https://example.com', locked: false }], + }), (program, context) => { + const group: CheckGroupResource = { id: 7, name: 'Checkout', concurrency: 1, useGlobalAlertSettings: true, alertSettings: {} } + new CheckGroupCodegen(program).prepare('checkout', group, context) + }) + expect(source).toContain('import { Engine, Frequency, PlaywrightCheck } from \'checkly/constructs\'') + expect(source).toContain('new PlaywrightCheck(\'suite-uuid\', {') + expect(source).toContain('name: \'Checkout flows\'') + expect(source).toContain('playwrightConfigPath: \'e2e/pw config.ts\'') + expect(source).toContain('pwProjects: [\n \'chromium\',\n \'firefox\',\n ]') + expect(source).toContain('pwTags: [\n \'@smoke\',\n \'@checkout\',\n ]') + expect(source).toContain('installCommand: \'pnpm install --frozen-lockfile\'') + expect(source).toContain('engine: Engine.node(\'22\')') + expect(source).toContain('locations: [\n \'us-east-1\',\n \'eu-west-1\',\n ]') + expect(source).toContain('tags: [\n \'e2e\',\n ]') + expect(source).toContain('frequency: Frequency.EVERY_10M') + expect(source).toContain('group: checkoutGroup') + expect(source).toContain('runtimeId: \'2025.04\'') + expect(source).toContain('key: \'BASE_URL\'') + // The package manager's own command is what the construct fills in. + expect(source).not.toContain('testCommand') + }) + + it('never prints the props a suite does not have, nor the bundle', async () => { + const source = await render(suite({ + ...({ + retryStrategy: { type: 'LINEAR', baseBackoffSeconds: 60, maxRetries: 2, maxDurationSeconds: 600, sameRegion: true }, + doubleCheck: true, + aiAutoRepairEnabled: true, + } as Partial), + })) + for (const prop of [ + 'retryStrategy', 'RetryStrategyBuilder', 'doubleCheck', 'aiAutoRepairEnabled', + 'codeBundlePath', 'codeBundleSha256', 'cacheHash', 'playwrightVersion', 'browsers', 'workingDir', + ]) { + expect(source, prop).not.toContain(prop) + } + }) + + it('prints a custom test command as the user wrote it', async () => { + const source = await render(suite({ testCommand: 'yarn e2e --reporter=line --config playwright.config.ts' })) + expect(source).toContain('testCommand: \'yarn e2e --reporter=line\'') + expect(source).toContain('playwrightConfigPath: \'playwright.config.ts\'') + }) + + it('prints a command it cannot unfold whole, without a config path', async () => { + const source = await render(suite({ testCommand: 'npx playwright test --workers 2' })) + expect(source).toContain('testCommand: \'npx playwright test --workers 2\'') + expect(source).not.toContain('playwrightConfigPath') + }) + + it('leaves out an engine without a version', async () => { + const source = await render(suite({ engine: 'node', engineVersion: null })) + expect(source).not.toContain('engine') + }) + + it('prints an engine the construct does not offer as a plain object', async () => { + const source = await render(suite({ engine: 'deno', engineVersion: '2' })) + expect(source).toContain('engine: {\n name: \'deno\',\n version: \'2\',\n }') + expect(source).not.toContain('Engine') + }) +}) diff --git a/packages/cli/src/constructs/__tests__/render-construct.spec.ts b/packages/cli/src/constructs/__tests__/render-construct.spec.ts index 59b27995..15b75ff8 100644 --- a/packages/cli/src/constructs/__tests__/render-construct.spec.ts +++ b/packages/cli/src/constructs/__tests__/render-construct.spec.ts @@ -161,8 +161,8 @@ describe('renderConstruct()', () => { expect(() => renderConstruct(codegen, 'suite', { type: 'check' as const, logicalId: 'suite', - payload: { id: 'suite', checkType: 'PLAYWRIGHT', name: 'Suite' }, - })).toThrow(/unsupported check type 'PLAYWRIGHT'/) + payload: { id: 'suite', checkType: 'NOPE', name: 'Suite' }, + })).toThrow(/unsupported check type 'NOPE'/) }) }) diff --git a/packages/cli/src/constructs/check-codegen.ts b/packages/cli/src/constructs/check-codegen.ts index fe540666..817eddae 100644 --- a/packages/cli/src/constructs/check-codegen.ts +++ b/packages/cli/src/constructs/check-codegen.ts @@ -10,6 +10,7 @@ import { FrequencyResource, valueForFrequency } from './frequency-codegen.js' import { HeartbeatMonitorCodegen, HeartbeatMonitorResource } from './heartbeat-monitor-codegen.js' import { valueForKeyValuePair } from './key-value-pair-codegen.js' import { MultiStepCheckCodegen, MultiStepCheckResource } from './multi-step-check-codegen.js' +import { PlaywrightCheckCodegen, PlaywrightCheckResource } from './playwright-check-codegen.js' import { RetryStrategyResource, valueForRetryStrategy } from './retry-strategy-codegen.js' import { TcpMonitorCodegen, TcpMonitorResource } from './tcp-monitor-codegen.js' import { UrlMonitorCodegen, UrlMonitorResource } from './url-monitor-codegen.js' @@ -390,6 +391,18 @@ export function buildRuntimeCheckProps ( } } +/** + * Check types whose codegen exists for the deploy preview only, with the + * reason `checkly import` gives when it refuses one. A check of such a type + * is defined by an uploaded code bundle, which an import plan cannot hand + * back as source, so the import refuses the resource instead of generating + * a construct without its files (the API leaves such checks out of import + * plans as well). + */ +export const PREVIEW_ONLY_CHECK_TYPES: ReadonlyMap = new Map([ + ['PLAYWRIGHT', 'Playwright check suites cannot be imported: their code bundle cannot be unpacked as source.'], +]) + export class CheckCodegen extends Codegen { agenticCheckCodegen: AgenticCheckCodegen apiCheckCodegen: ApiCheckCodegen @@ -397,6 +410,7 @@ export class CheckCodegen extends Codegen { checkGroupCodegen: CheckGroupCodegen heartbeatMonitorCodegen: HeartbeatMonitorCodegen multiStepCheckCodegen: MultiStepCheckCodegen + playwrightCheckCodegen: PlaywrightCheckCodegen tcpMonitorCodegen: TcpMonitorCodegen urlMonitorCodegen: UrlMonitorCodegen dnsMonitorCodegen: DnsMonitorCodegen @@ -413,6 +427,7 @@ export class CheckCodegen extends Codegen { this.checkGroupCodegen = new CheckGroupCodegen(program) this.heartbeatMonitorCodegen = new HeartbeatMonitorCodegen(program) this.multiStepCheckCodegen = new MultiStepCheckCodegen(program) + this.playwrightCheckCodegen = new PlaywrightCheckCodegen(program) this.tcpMonitorCodegen = new TcpMonitorCodegen(program) this.urlMonitorCodegen = new UrlMonitorCodegen(program) this.dnsMonitorCodegen = new DnsMonitorCodegen(program) @@ -436,6 +451,8 @@ export class CheckCodegen extends Codegen { return this.tcpMonitorCodegen.describe(resource as TcpMonitorResource) case 'MULTI_STEP': return this.multiStepCheckCodegen.describe(resource as MultiStepCheckResource) + case 'PLAYWRIGHT': + return this.playwrightCheckCodegen.describe(resource as PlaywrightCheckResource) case 'HEARTBEAT': return this.heartbeatMonitorCodegen.describe(resource as HeartbeatMonitorResource) case 'URL': @@ -474,6 +491,9 @@ export class CheckCodegen extends Codegen { case 'MULTI_STEP': this.multiStepCheckCodegen.gencode(logicalId, resource as MultiStepCheckResource, context) return + case 'PLAYWRIGHT': + this.playwrightCheckCodegen.gencode(logicalId, resource as PlaywrightCheckResource, context) + return case 'HEARTBEAT': this.heartbeatMonitorCodegen.gencode(logicalId, resource as HeartbeatMonitorResource, context) return diff --git a/packages/cli/src/constructs/internal/codegen/render.ts b/packages/cli/src/constructs/internal/codegen/render.ts index 8a00ae40..baa8728c 100644 --- a/packages/cli/src/constructs/internal/codegen/render.ts +++ b/packages/cli/src/constructs/internal/codegen/render.ts @@ -67,7 +67,7 @@ export interface RenderConstructOptions { * * Errors from the codegens themselves are deliberately left as they are, the * way `commands/import/plan.ts` takes them: a resource type they do not cover - * (a Playwright check suite) throws a plain `Error`, and a script they cannot + * throws a plain `Error`, and a script they cannot * parse throws `UnsupportedScriptError`. **So a caller rendering for a reader * catches `Error`, not only `ConstructRenderError`**, and treats any of them * as "show the coarser listing for this resource". diff --git a/packages/cli/src/constructs/playwright-check-codegen.ts b/packages/cli/src/constructs/playwright-check-codegen.ts new file mode 100644 index 00000000..2c0ce61a --- /dev/null +++ b/packages/cli/src/constructs/playwright-check-codegen.ts @@ -0,0 +1,289 @@ +import { Codegen, Context } from './internal/codegen/index.js' +import { expr, GeneratedFile, ident, unknown, Value } from '../sourcegen/index.js' +import { buildRuntimeCheckProps, RuntimeCheckResource } from './check-codegen.js' +import { PlaywrightCheck } from './playwright-check.js' + +/** + * Generates a `PlaywrightCheck` construct from a Playwright check suite's + * API representation. + * + * The deploy preview is this codegen's consumer: it renders the deployed and + * the local version of an updated suite as constructs and prints the + * difference. `checkly import` never writes a suite: the API leaves them out + * of import plans, because a suite is defined by its uploaded code bundle, + * which an import plan cannot hand back as source, and the import command + * refuses one on its own too (`PREVIEW_ONLY_CHECK_TYPES` in + * `check-codegen.ts`). The code generated here is displayed, not written to + * a project. + * + * What the API holds differs from what the construct takes. The construct's + * `playwrightConfigPath`, `pwProjects` and `pwTags` are not stored: the CLI + * folds them into `testCommand` on deploy (`PlaywrightCheck.buildTestCommand` + * appends `--config `, `--project …` and `--grep |` to + * the user's command, or to the package manager's `playwright test`). This + * codegen unfolds that command again, so the construct reads as the user + * wrote it, and prints `testCommand` only when the user's part differs from + * the package manager's default (the project's current one: a suite deployed + * under another package manager shows its old default as a `testCommand` + * the new deploy removes). A command that is not in the CLI's own + * format is printed whole and `playwrightConfigPath` is left out: the + * construct requires the prop, but a made-up path would show a change that + * did not happen. Such a construct would not type-check; the preview only + * displays it. + * + * Deliberately not printed: the code bundle (`codeBundlePath`, + * `codeBundleSha256`), the dependency cache hash, the Playwright version, the + * browsers and the working directory. They describe the bundled project + * rather than the construct's props; the API reports the bundle, the version + * and the cache as changes with a cause, which the preview prints as notes + * beside the construct. A `browsers` or `workingDir` change has no cause and + * is not listed beside a construct diff; both derive from the bundled + * Playwright project, whose change the code bundle note announces. + * + * A suite has no retry strategy and no double check (Playwright retries + * tests itself), and no automatic check repair, so none of those is printed + * whatever the API sends. + */ + +export interface PlaywrightCheckResource extends RuntimeCheckResource { + checkType: 'PLAYWRIGHT' + /** The full command the runner executes; see {@link unfoldTestCommand}. */ + testCommand?: string | null + installCommand?: string | null + /** The JavaScript engine's name (`node`, `bun`) and version. */ + engine?: string | null + engineVersion?: string | null +} + +/** A test command split into the construct props it was built from. */ +export interface UnfoldedTestCommand { + /** + * The command without the flags the CLI appends; the whole command when it + * could not be unfolded. + */ + command: string + playwrightConfigPath?: string + pwProjects?: string[] + pwTags?: string[] +} + +interface Word { + text: string + start: number + /** Whether any part of the word was quoted or escaped; a flag never is. */ + quoted: boolean +} + +/** The characters a backslash escapes inside double quotes in a POSIX shell. */ +const DOUBLE_QUOTE_ESCAPABLE = new Set(['$', '`', '"', '\\']) + +/** + * Splits a command into words the way a POSIX shell would: whitespace + * separates, single quotes take everything literally, double quotes let a + * backslash escape the few characters the shell would expand, a backslash + * outside quotes escapes the next character, and a backslash before a + * newline continues the line. This covers what + * `shellQuote` produces (bare words and single-quoted words with `'"'"'` for + * an embedded quote) and the usual hand-written forms. An unterminated quote + * makes the command unreadable. + */ +function shellWords (command: string): Word[] | undefined { + const words: Word[] = [] + let current: Word | undefined + let quote: '\'' | '"' | undefined + for (let index = 0; index < command.length; index++) { + const char = command[index] + if (quote === undefined) { + if (char === ' ' || char === '\t' || char === '\n') { + if (current !== undefined) { + words.push(current) + current = undefined + } + continue + } + if (char === '\\' && command[index + 1] === '\n') { + index++ + continue + } + current ??= { text: '', start: index, quoted: false } + if (char === '\'' || char === '"') { + quote = char + current.quoted = true + } else if (char === '\\' && index + 1 < command.length) { + current.text += command[++index] + current.quoted = true + } else { + current.text += char + } + continue + } + if (char === quote) { + quote = undefined + } else if (quote === '"' && char === '\\' && command[index + 1] === '\n') { + index++ + } else if (quote === '"' && char === '\\' && DOUBLE_QUOTE_ESCAPABLE.has(command[index + 1] ?? '')) { + current!.text += command[++index] + } else { + current!.text += char + } + } + if (quote !== undefined) { + return undefined + } + if (current !== undefined) { + words.push(current) + } + return words +} + +const isFlag = (word: Word): boolean => !word.quoted && word.text.startsWith('--') + +/** + * Reads the part the CLI appended, starting at the word after `--config`: + * the config path, then `--project …` and `--grep |`, each at + * most once. A value never looks like a flag: `shellQuote` leaves a bare + * `--foo` project name as it is, and reading such a tail could put the + * user's own trailing flag together with the CLI's config path, so the + * command is left whole instead. Anything else means the CLI did not build + * this tail. + */ +function readAppendedFlags (words: Word[]): Omit | undefined { + const [config, ...rest] = words + if (config === undefined) { + return undefined + } + let pwProjects: string[] | undefined + let pwTags: string[] | undefined + for (let index = 0; index < rest.length;) { + const flag = rest[index++].text + const values: string[] = [] + while (index < rest.length && !isFlag(rest[index])) { + values.push(rest[index++].text) + } + if (flag === '--project' && pwProjects === undefined && values.length > 0) { + pwProjects = values + } else if (flag === '--grep' && pwTags === undefined && values.length === 1) { + pwTags = values[0].split('|') + } else { + return undefined + } + } + return { playwrightConfigPath: config.text, pwProjects, pwTags } +} + +/** + * Splits a stored test command back into the user's command and the + * construct props the CLI folded into it. A bare `--config` word marks where + * the CLI's part may start; the earliest one whose tail reads as the CLI's + * format wins, so a user's own command keeps whatever it had before it + * (spacing and quoting included) and a quoted value that merely spells a + * flag is never mistaken for the boundary. A command with no such boundary + * was not built by the CLI and is returned whole. + * + * The `--grep` pattern joins the tags with `|`, so a single tag that itself + * contains `|` cannot be told from two tags and comes back as two. + */ +export function unfoldTestCommand (testCommand: string): UnfoldedTestCommand { + const words = shellWords(testCommand) ?? [] + for (let index = 1; index < words.length; index++) { + if (!isFlag(words[index]) || words[index].text !== '--config') { + continue + } + const appended = readAppendedFlags(words.slice(index + 1)) + if (appended !== undefined) { + // Whitespace and a line continuation before the boundary belong to + // neither side. + const command = testCommand.slice(0, words[index].start).replace(/(\s|\\\n)+$/, '') + return { command, ...appended } + } + } + return { command: testCommand } +} + +const construct = 'PlaywrightCheck' + +export class PlaywrightCheckCodegen extends Codegen { + describe (resource: PlaywrightCheckResource): string { + return `Playwright Check Suite: ${resource.name}` + } + + gencode (logicalId: string, resource: PlaywrightCheckResource, context: Context): void { + const filePath = context.filePath('resources/playwright-check-suites', resource.name, { + tags: resource.tags, + isolate: true, + unique: true, + }) + + const file = this.program.generatedConstructFile(filePath.fullPath) + + file.namedImport(construct, 'checkly/constructs') + + file.section(expr(ident(construct), builder => { + builder.new(builder => { + builder.string(logicalId) + builder.object(builder => { + const { command, playwrightConfigPath, pwProjects, pwTags }: Partial = + resource.testCommand != null ? unfoldTestCommand(resource.testCommand) : {} + + if (playwrightConfigPath !== undefined) { + builder.string('playwrightConfigPath', playwrightConfigPath) + } + + if (pwProjects !== undefined) { + builder.array('pwProjects', builder => { + for (const project of pwProjects) { + builder.string(project) + } + }) + } + + if (pwTags !== undefined) { + builder.array('pwTags', builder => { + for (const tag of pwTags) { + builder.string(tag) + } + }) + } + + // The construct fills in the package manager's `playwright test` + // when no command is given, so that command is not spelled out. + if (command !== undefined && command !== PlaywrightCheck.defaultTestCommand()) { + builder.string('testCommand', command) + } + + if (resource.installCommand != null) { + builder.string('installCommand', resource.installCommand) + } + + // An engine without a version cannot be spelled as the construct's + // `Engine` (which always carries one) and is left out. + if (resource.engine != null && resource.engineVersion != null) { + builder.value('engine', valueForEngine(file, resource.engine, resource.engineVersion)) + } + + buildRuntimeCheckProps(this.program, file, builder, resource, context, { + omit: ['retryStrategy'], + }) + }) + }) + })) + } +} + +/** + * `Engine.node(version)` or `Engine.bun(version)` for the engines the + * construct offers; any other name as a plain object of the same shape, + * which the prop's type accepts, so an unfamiliar engine still shows. + */ +function valueForEngine (file: GeneratedFile, name: string, version: string): Value { + if (name === 'node' || name === 'bun') { + file.namedImport('Engine', 'checkly/constructs') + return expr(ident('Engine'), builder => { + builder.member(ident(name)) + builder.call(builder => { + builder.string(version) + }) + }) + } + return unknown({ name, version }) +} diff --git a/packages/cli/src/constructs/playwright-check.ts b/packages/cli/src/constructs/playwright-check.ts index 71d87902..4ef6b623 100644 --- a/packages/cli/src/constructs/playwright-check.ts +++ b/packages/cli/src/constructs/playwright-check.ts @@ -497,7 +497,7 @@ export class PlaywrightCheck extends RuntimeCheck { bundler.registerFiles(...files) const testCommand = PlaywrightCheck.buildTestCommand( - this.testCommand ?? this.#defaultTestCommand(), + this.testCommand ?? PlaywrightCheck.defaultTestCommand(), relativePlaywrightConfigPath, this.pwProjects, this.pwTags, @@ -516,7 +516,8 @@ export class PlaywrightCheck extends RuntimeCheck { }) } - #defaultTestCommand (): string { + /** The test command a suite runs when none is given: the package manager's `playwright test`. */ + static defaultTestCommand (): string { return Session.packageManager.execCommand(['playwright', 'test']).unsafeDisplayCommand } diff --git a/packages/cli/src/services/deploy-diff/__tests__/render.spec.ts b/packages/cli/src/services/deploy-diff/__tests__/render.spec.ts index 63cee0ec..05314567 100644 --- a/packages/cli/src/services/deploy-diff/__tests__/render.spec.ts +++ b/packages/cli/src/services/deploy-diff/__tests__/render.spec.ts @@ -823,14 +823,14 @@ describe('renderResourceDiff', () => { it('falls back to a listing when the codegen throws, and never throws itself', () => { const { local } = scenario() const check = local.find(resource => resource.logicalId === 'api') as ResourceSync - check.payload = { ...check.payload, checkType: 'PLAYWRIGHT' } + check.payload = { ...check.payload, checkType: 'NOPE' } const lines = render( { type: 'check', logicalId: 'api', action: 'UPDATE', changes: [{ path: '/name', origin: 'code', before: 'API', after: 'Suite' }], - before: deployed({ checkType: 'PLAYWRIGHT' }), + before: deployed({ checkType: 'NOPE' }), redactions: [], }, local, @@ -1031,3 +1031,108 @@ describe('renderResourceDiff', () => { }) }) }) + +describe('renderResourceDiff for a Playwright check suite', () => { + /** The suite's deploy payload: the construct's props folded into the test command, plus the bundle keys. */ + const localSuite = (name: string): ResourceSync[] => [{ + type: 'check', + logicalId: 'suite', + member: true, + payload: { + checkType: 'PLAYWRIGHT', + name, + activated: true, + muted: false, + shouldFail: false, + locations: ['us-east-1'], + tags: [], + frequency: 10, + groupId: null, + alertSettings: {}, + useGlobalAlertSettings: true, + runParallel: false, + doubleCheck: false, + testCommand: 'npx playwright test --config playwright.config.ts --project chromium', + installCommand: null, + engine: 'node', + engineVersion: '22', + codeBundlePath: 'bundles/suite.tar.gz', + codeBundleSha256: 'c'.repeat(64), + cacheHash: 'abc123', + playwrightVersion: '1.59.1', + browsers: ['chromium'], + workingDir: '.', + }, + }] + /** The suite as Checkly has it: the import format the API projects for a PLAYWRIGHT row. */ + const deployedSuite = (name: string): Record => ({ + id: 'suite-uuid', + checkType: 'PLAYWRIGHT', + name, + activated: true, + muted: false, + locations: ['us-east-1'], + tags: [], + frequency: 10, + testCommand: 'npx playwright test --config playwright.config.ts --project chromium', + installCommand: null, + cacheHash: 'abc123', + playwrightVersion: '1.59.1', + browsers: ['chromium'], + workingDir: '.', + // Strings on both sides, as the API projects a suite's engine. + engine: 'node', + engineVersion: '22', + alertChannelSubscriptions: [], + privateLocationAssignments: [], + }) + const renderSuite = (changes: DiffEntry['changes'], name = 'Suite') => { + const local = localSuite(name) + const entry: DiffEntry = { + type: 'check', + logicalId: 'suite', + physicalId: 'suite-uuid', + action: 'UPDATE', + changes, + before: deployedSuite('Suite'), + redactions: [], + } + return plain(renderResourceDiff({ + entry, + local: local[0], + localResources: local, + diff: [entry], + project, + ids: physicalIdsFromPlan([entry], local), + pruneRelations: false, + })) + } + + it('renders a renamed suite as a PlaywrightCheck construct diff', () => { + const lines = renderSuite([{ path: '/name', origin: 'code', before: 'Suite', after: 'Renamed' }], 'Renamed') + const text = lines.join('\n') + expect(text).toContain('- name: \'Suite\'') + expect(text).toContain('+ name: \'Renamed\'') + expect(text).toContain(' playwrightConfigPath: \'playwright.config.ts\'') + expect(text).toContain(' \'chromium\'') + // The engine is a string pair on both sides and never a changed line. + expect(text).not.toMatch(/^[-+] +engine/m) + expect(text).not.toContain('retryStrategy') + expect(text).not.toContain('could not render') + expect(lines.filter(line => /^[-+](?![-+]{2} )/.test(line))).toHaveLength(2) + }) + + it('prints a new code bundle as a line when nothing else changed', () => { + expect(renderSuite([{ path: '/codeBundle', origin: 'code', cause: 'code bundle' }])).toEqual(['changed: code bundle']) + }) + + it('prints a new code bundle as a note beside the construct diff', () => { + const lines = renderSuite([ + { path: '/name', origin: 'code', before: 'Suite', after: 'Renamed' }, + { path: '/codeBundle', origin: 'code', cause: 'code bundle' }, + { path: '/playwrightVersion', origin: 'code', cause: 'playwright version', before: '1.59.1', after: '1.60.0' }, + ], 'Renamed') + expect(lines.join('\n')).toContain('+ name: \'Renamed\'') + expect(lines.slice(-2)).toEqual(['/codeBundle: changed (code bundle)', '/playwrightVersion: changed (playwright version)']) + }) +}) diff --git a/packages/cli/src/services/deploy-diff/render.ts b/packages/cli/src/services/deploy-diff/render.ts index 5b9c50c8..4a197d25 100644 --- a/packages/cli/src/services/deploy-diff/render.ts +++ b/packages/cli/src/services/deploy-diff/render.ts @@ -238,7 +238,7 @@ export function renderResourceDiff (input: RenderResourceInput): RenderedLine[] ) } catch (cause) { // A payload this CLI cannot shape like an import resource, a codegen that - // does not cover the type (a Playwright check suite), a script it cannot + // does not cover the type, a script it cannot // parse, a construct it refuses: the listing says what changed even when // the rendering cannot. const reason = cause instanceof UnshapeableError