diff --git a/package.json b/package.json index 3b07a2d00257..1c7e4156421c 100644 --- a/package.json +++ b/package.json @@ -144,16 +144,16 @@ "oxfmt --write --no-error-on-unmatched-pattern" ] }, - "packageManager": "pnpm@11.22.0", + "packageManager": "pnpm@12.8.1", "devEngines": { "packageManager": { "name": "pnpm", - "version": "11.22.0", + "version": "12.8.1", "onFail": "error" } }, "volta": { "node": "24.18.0", - "pnpm": "11.22.0" + "pnpm": "12.8.1" } } diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts index 4183c70f9841..22c6b6a9bc83 100644 --- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts +++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts @@ -703,7 +703,11 @@ export async function handleCloudflareChallenge( // check if we ended up on the CF challenge page const isChallenge = async () => { - return options.isChallengeCallback!(page).catch(() => false); + return options.isChallengeCallback!(page).catch(async () => { + // the challenge page navigates itself, which destroys the context of an in-flight check + await page.waitForLoadState('domcontentloaded').catch(() => {}); + return options.isChallengeCallback!(page).catch(() => false); + }); }; if (!(await isChallenge())) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8b664436d0ed..3b0dd53e141e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,195 +6,154 @@ importers: .: configDependencies: {} packageManagerDependencies: - '@pnpm/exe': - specifier: 11.22.0 - version: 11.22.0 pnpm: - specifier: 11.22.0 - version: 11.22.0 + specifier: 12.8.1 + version: 12.8.1 packages: - '@pnpm/exe@11.22.0': - resolution: {integrity: sha512-B1SGeKm+v9pX9YkzMmrnO2FbgBd8TDwzZ3jSj6J6ThxdyGvI4TsOfMeHARW4Wb25visPpmMDfIXrU76EZdJM4g==} - hasBin: true - - '@pnpm/linux-arm64@11.22.0': - resolution: {integrity: sha512-xzzn3jYG9QaiFZPaHcWM3yX4Fm9UGz5E42mzpbvUEvXYf5O9fwNolenOhGLpRl2qS5u35fjZSvDZbWlOwHMcug==} + '@pnpm/exe.android-arm64@12.8.1': + resolution: {integrity: sha512-siDxcidfz5eM6L2tJfik+i2bbUEw8UC7MZ3Yl2HYdRmqbdhHGAOQACtOOu6QVW4Jul4/w5wVKm+wM8ACtgh1Pg==} cpu: [arm64] - os: [linux] + os: [android] - '@pnpm/linux-x64@11.22.0': - resolution: {integrity: sha512-isvaPctGinbsM2hsTtRsMarN8Sr5QhXDTNmn8Xv9Lp1PjincCvH2RBbhpo+xYIOxzgls1dtQSdgoORUbfRVALQ==} + '@pnpm/exe.android-x64@12.8.1': + resolution: {integrity: sha512-0F/uqPRc3aN4vLHsa0f+6fTtmsigFQ9/yxEU43fionF//pNHEd5e97b3w/nKIgqYMCUQCJphmBRMA9Q8/wlo9Q==} cpu: [x64] - os: [linux] + os: [android] - '@pnpm/linuxstatic-arm64@11.22.0': - resolution: {integrity: sha512-i4J+AQWW0T3JBdXaLsvxu8ZmMG1HLS6JZQESdOR9uBv8Zumb4yg8GYT83eWRLacr6ngVdhEBiC3W4eOG64MFbA==} + '@pnpm/exe.darwin-arm64@12.8.1': + resolution: {integrity: sha512-/rwavvMQJsl2RIxOHBnDF+4Gk+fhQFAY4M3PQZoKskJOEnaJcygG9p1xby6wcQcbS4d9dIubv09CQhufTgfmpw==} cpu: [arm64] - os: [linux] - libc: [musl] + os: [darwin] - '@pnpm/linuxstatic-x64@11.22.0': - resolution: {integrity: sha512-QYzk8jhSuSbVthW/OxEOhU3f3zhjpw4HqgedrlwbVNb7btCWn5del2Hb5PsYYka2PbmKq5EtF5IzYN8Yp1CfxQ==} + '@pnpm/exe.darwin-x64@12.8.1': + resolution: {integrity: sha512-htYt2gN7zqJKLhC99Ft6EUiO7myjZAZXfmGCTJUL1v+o7vE9hZdE1+fAkk4Oj1ZHNRti6b2woOxEkRkAusi8OQ==} cpu: [x64] - os: [linux] - libc: [musl] - - '@pnpm/macos-arm64@11.22.0': - resolution: {integrity: sha512-Io8Axk5kutPgMuAfOg1QGj3J0/KpLvH5iiPcmp7up8D4q7BNWT02ndQo3RyGWqrlkf5nwspM41GFpWTShrZ4Aw==} - cpu: [arm64] os: [darwin] - '@pnpm/win-arm64@11.22.0': - resolution: {integrity: sha512-QgaRuKGQKov7xW2utPCgDT83fn/PU5cD6HFgib48oTslz7wv26E89d4jMCxL4GRmEL2YCfkYuj5ITj1oVWg5WQ==} - cpu: [arm64] - os: [win32] - - '@pnpm/win-x64@11.22.0': - resolution: {integrity: sha512-iWYsiSwpgxqur+TwsnSoPdOQaEfd4ygB84mb5tJUOgim6PHr1xhKJBndaQzH+At/WkLxLdYN+K8dYc4M7naezg==} + '@pnpm/exe.freebsd-x64@12.8.1': + resolution: {integrity: sha512-PB5BULNgytxvExoOXUUeseS/DcpPENs+cg3iA6xHHeaLT4ctEGKSGXEbuj7USeqvo8JBpAx039hivnbzGqDwnA==} cpu: [x64] - os: [win32] + os: [freebsd] - '@reflink/reflink-darwin-arm64@0.1.19': - resolution: {integrity: sha512-ruy44Lpepdk1FqDz38vExBY/PVUsjxZA+chd9wozjUH9JjuDT/HEaQYA6wYN9mf041l0yLVar6BCZuWABJvHSA==} - engines: {node: '>= 10'} + '@pnpm/exe.linux-arm64-musl@12.8.1': + resolution: {integrity: sha512-kKeyC/ArjrgdciQRBrOL0j9HtOI4gDRgoyP+eJ1CQKyadpKlf/DhDDyspXe07R3B9KagTVLUF2nlt6Dr5quFIg==} cpu: [arm64] - os: [darwin] - - '@reflink/reflink-darwin-x64@0.1.19': - resolution: {integrity: sha512-By85MSWrMZa+c26TcnAy8SDk0sTUkYlNnwknSchkhHpGXOtjNDUOxJE9oByBnGbeuIE1PiQsxDG3Ud+IVV9yuA==} - engines: {node: '>= 10'} - cpu: [x64] - os: [darwin] + os: [linux] + libc: [musl] - '@reflink/reflink-linux-arm64-gnu@0.1.19': - resolution: {integrity: sha512-7P+er8+rP9iNeN+bfmccM4hTAaLP6PQJPKWSA4iSk2bNvo6KU6RyPgYeHxXmzNKzPVRcypZQTpFgstHam6maVg==} - engines: {node: '>= 10'} + '@pnpm/exe.linux-arm64@12.8.1': + resolution: {integrity: sha512-q4s9a9X2O3N9aeUFooW24orNrxvu20+jyVUFR5RanPCqOKKPBrgs2iywMkBBx1aXkkzdWT54/mfz8Be8sJlWEw==} cpu: [arm64] os: [linux] libc: [glibc] - '@reflink/reflink-linux-arm64-musl@0.1.19': - resolution: {integrity: sha512-37iO/Dp6m5DDaC2sf3zPtx/hl9FV3Xze4xoYidrxxS9bgP3S8ALroxRK6xBG/1TtfXKTvolvp+IjrUU6ujIGmA==} - engines: {node: '>= 10'} - cpu: [arm64] + '@pnpm/exe.linux-ppc64@12.8.1': + resolution: {integrity: sha512-F7XSjKJthwCh/L6abZiMpAfo2Cm61SZIT3e4dfgiIjESndlWaXCnfLKngSgZ/SEMsdmHT6dud3+kWIB3tw0/xQ==} + cpu: [ppc64] os: [linux] - libc: [musl] + libc: [glibc] - '@reflink/reflink-linux-x64-gnu@0.1.19': - resolution: {integrity: sha512-jbI8jvuYCaA3MVUdu8vLoLAFqC+iNMpiSuLbxlAgg7x3K5bsS8nOpTRnkLF7vISJ+rVR8W+7ThXlXlUQ93ulkw==} - engines: {node: '>= 10'} - cpu: [x64] + '@pnpm/exe.linux-riscv64@12.8.1': + resolution: {integrity: sha512-z1ecpIK/ZY08Unk69KWWE9867oaKUytGUjrHnBoGFAAhsK+q+0cfvBH+f5fWobJ7FX97leMlLO+gkJolddM45Q==} + cpu: [riscv64] os: [linux] libc: [glibc] - '@reflink/reflink-linux-x64-musl@0.1.19': - resolution: {integrity: sha512-e9FBWDe+lv7QKAwtKOt6A2W/fyy/aEEfr0g6j/hWzvQcrzHCsz07BNQYlNOjTfeytrtLU7k449H1PI95jA4OjQ==} - engines: {node: '>= 10'} + '@pnpm/exe.linux-s390x@12.8.1': + resolution: {integrity: sha512-KU6tnrBfu4uKXTq6N5qJwSm8FSrs9Wc8b5AUEsHK9bRXwNdsZ1/lRxG1++X29SKRW9OSNuDSjxASv5mAvGo1gA==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@pnpm/exe.linux-x64-musl@12.8.1': + resolution: {integrity: sha512-eUE8BWJkDr2tJb/yrk0yVnszlmNqFarVxUlqiZ9BAkct28Nq/5g2H0tvMDmPA7fAQW3STNj+omJPPRVT+4s5sQ==} cpu: [x64] os: [linux] libc: [musl] - '@reflink/reflink-win32-arm64-msvc@0.1.19': - resolution: {integrity: sha512-09PxnVIQcd+UOn4WAW73WU6PXL7DwGS6wPlkMhMg2zlHHG65F3vHepOw06HFCq+N42qkaNAc8AKIabWvtk6cIQ==} - engines: {node: '>= 10'} + '@pnpm/exe.linux-x64@12.8.1': + resolution: {integrity: sha512-8bDZ0lZlCdi2rvnFul7EYgcC7zKeWSe76y8JV2oXobaS8p9i9d6PSf6LgLtTM0fI7R9Oh4eLCbveXyNDMmvZ+g==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@pnpm/exe.win32-arm64@12.8.1': + resolution: {integrity: sha512-MsT0dlrRxnCRCHCaosNbhWpEiAFnFuMEu3KuDJDU41BXgTwiINxU8goloUSUDejWZUqKaUqNC/qUTihl9nFtqg==} cpu: [arm64] os: [win32] - '@reflink/reflink-win32-x64-msvc@0.1.19': - resolution: {integrity: sha512-E//yT4ni2SyhwP8JRjVGWr3cbnhWDiPLgnQ66qqaanjjnMiu3O/2tjCPQXlcGc/DEYofpDc9fvhv6tALQsMV9w==} - engines: {node: '>= 10'} + '@pnpm/exe.win32-x64@12.8.1': + resolution: {integrity: sha512-SWtPe0PsbTTk//gJND10tMdOfCYjrkq5+qV49hzVhY7xz2iJ339Rc+xwASlcvbwTfrvH39YNqpKYibD+CRGLwA==} cpu: [x64] os: [win32] - '@reflink/reflink@0.1.19': - resolution: {integrity: sha512-DmCG8GzysnCZ15bres3N5AHCmwBwYgp0As6xjhQ47rAUTUXxJiK+lLUxaGsX3hd/30qUpVElh05PbGuxRPgJwA==} - engines: {node: '>= 10'} - - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - pnpm@11.22.0: - resolution: {integrity: sha512-H/hwxMYTPf2I+yr8Rt0T1H8JyXlLQ4xv20fKmMrzvBY4HuC+k6CRuOOCTPAfiJ9G19niCRD7C+GrD7W6qA3WIQ==} - engines: {node: '>=22.13'} + pnpm@12.8.1: + resolution: {integrity: sha512-9kupB1B/XOr+BsjTjmBS0BeURFgOwSed3Vv8EctIqoomRLZlmOB+dh2oSHKp/FfV+QK4f6SdAkGY1VhhKqu+RQ==} + engines: {node: '>=18.*'} hasBin: true snapshots: - '@pnpm/exe@11.22.0': - dependencies: - '@reflink/reflink': 0.1.19 - detect-libc: 2.1.2 - optionalDependencies: - '@pnpm/linux-arm64': 11.22.0 - '@pnpm/linux-x64': 11.22.0 - '@pnpm/linuxstatic-arm64': 11.22.0 - '@pnpm/linuxstatic-x64': 11.22.0 - '@pnpm/macos-arm64': 11.22.0 - '@pnpm/win-arm64': 11.22.0 - '@pnpm/win-x64': 11.22.0 - - '@pnpm/linux-arm64@11.22.0': - optional: true - - '@pnpm/linux-x64@11.22.0': + '@pnpm/exe.android-arm64@12.8.1': optional: true - '@pnpm/linuxstatic-arm64@11.22.0': + '@pnpm/exe.android-x64@12.8.1': optional: true - '@pnpm/linuxstatic-x64@11.22.0': + '@pnpm/exe.darwin-arm64@12.8.1': optional: true - '@pnpm/macos-arm64@11.22.0': + '@pnpm/exe.darwin-x64@12.8.1': optional: true - '@pnpm/win-arm64@11.22.0': + '@pnpm/exe.freebsd-x64@12.8.1': optional: true - '@pnpm/win-x64@11.22.0': + '@pnpm/exe.linux-arm64-musl@12.8.1': optional: true - '@reflink/reflink-darwin-arm64@0.1.19': + '@pnpm/exe.linux-arm64@12.8.1': optional: true - '@reflink/reflink-darwin-x64@0.1.19': + '@pnpm/exe.linux-ppc64@12.8.1': optional: true - '@reflink/reflink-linux-arm64-gnu@0.1.19': + '@pnpm/exe.linux-riscv64@12.8.1': optional: true - '@reflink/reflink-linux-arm64-musl@0.1.19': + '@pnpm/exe.linux-s390x@12.8.1': optional: true - '@reflink/reflink-linux-x64-gnu@0.1.19': + '@pnpm/exe.linux-x64-musl@12.8.1': optional: true - '@reflink/reflink-linux-x64-musl@0.1.19': + '@pnpm/exe.linux-x64@12.8.1': optional: true - '@reflink/reflink-win32-arm64-msvc@0.1.19': + '@pnpm/exe.win32-arm64@12.8.1': optional: true - '@reflink/reflink-win32-x64-msvc@0.1.19': + '@pnpm/exe.win32-x64@12.8.1': optional: true - '@reflink/reflink@0.1.19': + pnpm@12.8.1: optionalDependencies: - '@reflink/reflink-darwin-arm64': 0.1.19 - '@reflink/reflink-darwin-x64': 0.1.19 - '@reflink/reflink-linux-arm64-gnu': 0.1.19 - '@reflink/reflink-linux-arm64-musl': 0.1.19 - '@reflink/reflink-linux-x64-gnu': 0.1.19 - '@reflink/reflink-linux-x64-musl': 0.1.19 - '@reflink/reflink-win32-arm64-msvc': 0.1.19 - '@reflink/reflink-win32-x64-msvc': 0.1.19 - - detect-libc@2.1.2: {} - - pnpm@11.22.0: {} + '@pnpm/exe.android-arm64': 12.8.1 + '@pnpm/exe.android-x64': 12.8.1 + '@pnpm/exe.darwin-arm64': 12.8.1 + '@pnpm/exe.darwin-x64': 12.8.1 + '@pnpm/exe.freebsd-x64': 12.8.1 + '@pnpm/exe.linux-arm64': 12.8.1 + '@pnpm/exe.linux-arm64-musl': 12.8.1 + '@pnpm/exe.linux-ppc64': 12.8.1 + '@pnpm/exe.linux-riscv64': 12.8.1 + '@pnpm/exe.linux-s390x': 12.8.1 + '@pnpm/exe.linux-x64': 12.8.1 + '@pnpm/exe.linux-x64-musl': 12.8.1 + '@pnpm/exe.win32-arm64': 12.8.1 + '@pnpm/exe.win32-x64': 12.8.1 --- lockfileVersion: '9.0' diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts index 3964d3b1e7e2..074e38a55ced 100644 --- a/test/core/playwright_utils.test.ts +++ b/test/core/playwright_utils.test.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { MemoryStorageBackend, serviceLocator } from '@crawlee/core'; import { KeyValueStore, launchPlaywright, playwrightUtils, Request } from '@crawlee/playwright'; +import { sleep } from '@crawlee/utils'; import type { Browser, Page } from 'playwright'; import { chromium } from 'playwright'; import { runExampleComServer } from '../shared/_helper.js'; @@ -495,6 +496,41 @@ describe('playwrightUtils', () => { ); }); + test('detects the challenge when the challenge page navigates itself during the check', async () => { + const url = 'https://challenge.test/'; + await page.route(`${url}**`, async (route) => { + if (!route.request().url().includes('__cf_chl_tk')) { + // keeps the check busy until the navigation below commits and destroys its execution context + await route.fulfill({ + contentType: 'text/html', + body: `${currentChallengeBody}`, + }); + return; + } + + await sleep(200); + await route.fulfill({ + contentType: 'text/html', + body: `${currentChallengeBody}`, + }); + }); + + const selfNavigation = page.waitForRequest((request) => request.url().includes('__cf_chl_tk')); + await page.goto(url); + await selfNavigation; + + await expect(handleCloudflareChallenge(page, url, fastOptions())).rejects.toThrow(/Blocked by Cloudflare/); + await page.unrouteAll(); + }); + test('resolves without detection on a regular page', async () => { await page.setContent('

Welcome

'); await expect( diff --git a/website/versioned_docs/version-3.17/yarn.lock b/website/versioned_docs/version-3.17/yarn.lock index bcd982b16581..1a4ceb637581 100644 --- a/website/versioned_docs/version-3.17/yarn.lock +++ b/website/versioned_docs/version-3.17/yarn.lock @@ -36,12 +36,12 @@ __metadata: linkType: hard "brace-expansion@npm:^1.1.7": - version: 1.1.18 - resolution: "brace-expansion@npm:1.1.18" + version: 1.1.21 + resolution: "brace-expansion@npm:1.1.21" dependencies: balanced-match: "npm:^1.0.0" concat-map: "npm:0.0.1" - checksum: 10c0/3432c18a9e2ebf94162d4effb62198bd0adea06a9f332b2c0188df5d5e30b1e51ea3c848b6608e47d0b857ebe1ea5b3888ed3326dd3c4f6f9645c94153cf9c14 + checksum: 10c0/8f0a68a720f9cb28c2ecaea5cdaf15fafd686d6b6e4e722a81c9d4bd2f68c1fd0619731cac05875d842221eaf4d97d564d329a12eb3c4c22137502fb3c65be8b languageName: node linkType: hard