From f90b51c9841eb31124e1cd89a1dffd250d842d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Wed, 30 Sep 2026 09:59:31 +0200 Subject: [PATCH 1/3] fix(playwright): keep detecting a Cloudflare challenge that navigates itself (#4170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloudflare's challenge page navigates on its own (e.g. to `?__cf_chl_tk=…` and back). When `handleCloudflareChallenge` runs its detection check during that navigation, `page.evaluate` fails with "Execution context was destroyed", and the `.catch(() => false)` around it read that as "not a challenge". So the hook either returned early and the request handler ran on the challenge page, or it left the click loop without clicking and then failed the request. Both happen in the `camoufox-cloudflare` E2E runs on the platform. A failed check now waits for the next document to load and checks once more before giving up. The new test reproduces the bug deterministically in Chromium with a challenge page that navigates itself while the check is still running. --- .../src/internals/utils/playwright-utils.ts | 6 +++- test/core/playwright_utils.test.ts | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts index 4183c70f9841..22c6b6a9bc83 100644 --- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts +++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts @@ -703,7 +703,11 @@ export async function handleCloudflareChallenge( // check if we ended up on the CF challenge page const isChallenge = async () => { - return options.isChallengeCallback!(page).catch(() => false); + return options.isChallengeCallback!(page).catch(async () => { + // the challenge page navigates itself, which destroys the context of an in-flight check + await page.waitForLoadState('domcontentloaded').catch(() => {}); + return options.isChallengeCallback!(page).catch(() => false); + }); }; if (!(await isChallenge())) { diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts index 3964d3b1e7e2..074e38a55ced 100644 --- a/test/core/playwright_utils.test.ts +++ b/test/core/playwright_utils.test.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { MemoryStorageBackend, serviceLocator } from '@crawlee/core'; import { KeyValueStore, launchPlaywright, playwrightUtils, Request } from '@crawlee/playwright'; +import { sleep } from '@crawlee/utils'; import type { Browser, Page } from 'playwright'; import { chromium } from 'playwright'; import { runExampleComServer } from '../shared/_helper.js'; @@ -495,6 +496,41 @@ describe('playwrightUtils', () => { ); }); + test('detects the challenge when the challenge page navigates itself during the check', async () => { + const url = 'https://challenge.test/'; + await page.route(`${url}**`, async (route) => { + if (!route.request().url().includes('__cf_chl_tk')) { + // keeps the check busy until the navigation below commits and destroys its execution context + await route.fulfill({ + contentType: 'text/html', + body: `${currentChallengeBody}`, + }); + return; + } + + await sleep(200); + await route.fulfill({ + contentType: 'text/html', + body: `${currentChallengeBody}`, + }); + }); + + const selfNavigation = page.waitForRequest((request) => request.url().includes('__cf_chl_tk')); + await page.goto(url); + await selfNavigation; + + await expect(handleCloudflareChallenge(page, url, fastOptions())).rejects.toThrow(/Blocked by Cloudflare/); + await page.unrouteAll(); + }); + test('resolves without detection on a regular page', async () => { await page.setContent('

Welcome

'); await expect( From 626d2c95188b66289f6f7faa8f5b5df4f17d3e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Wed, 30 Sep 2026 11:59:45 +0200 Subject: [PATCH 2/3] chore: bump pnpm to 12 (#4169) Bumps pnpm from 11.22.0 to 12.6.0 (`packageManager`, `devEngines` and the `volta` hint), the same move as apify/apify-client-js#1088 and apify/apify-sdk-js#752. apify-cli is already on 12. pnpm 12 fails on unrecognized `pnpm-workspace.yaml` settings, and it accepts everything we have there. Releases go through lerna, so the native `pnpm version`/`pnpm publish` changes don't apply. `lerna version` still works with pnpm as its `npmClient`. The lockfile only changes its header document, which pins pnpm itself. The dependency graph is unchanged. --- package.json | 6 +- pnpm-lock.yaml | 199 ++++++++++++++++++++----------------------------- 2 files changed, 82 insertions(+), 123 deletions(-) diff --git a/package.json b/package.json index 3b07a2d00257..1c7e4156421c 100644 --- a/package.json +++ b/package.json @@ -144,16 +144,16 @@ "oxfmt --write --no-error-on-unmatched-pattern" ] }, - "packageManager": "pnpm@11.22.0", + "packageManager": "pnpm@12.8.1", "devEngines": { "packageManager": { "name": "pnpm", - "version": "11.22.0", + "version": "12.8.1", "onFail": "error" } }, "volta": { "node": "24.18.0", - "pnpm": "11.22.0" + "pnpm": "12.8.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8b664436d0ed..3b0dd53e141e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,195 +6,154 @@ importers: .: configDependencies: {} packageManagerDependencies: - '@pnpm/exe': - specifier: 11.22.0 - version: 11.22.0 pnpm: - specifier: 11.22.0 - version: 11.22.0 + specifier: 12.8.1 + version: 12.8.1 packages: - '@pnpm/exe@11.22.0': - resolution: {integrity: sha512-B1SGeKm+v9pX9YkzMmrnO2FbgBd8TDwzZ3jSj6J6ThxdyGvI4TsOfMeHARW4Wb25visPpmMDfIXrU76EZdJM4g==} - hasBin: true - - '@pnpm/linux-arm64@11.22.0': - resolution: {integrity: sha512-xzzn3jYG9QaiFZPaHcWM3yX4Fm9UGz5E42mzpbvUEvXYf5O9fwNolenOhGLpRl2qS5u35fjZSvDZbWlOwHMcug==} + '@pnpm/exe.android-arm64@12.8.1': + resolution: {integrity: sha512-siDxcidfz5eM6L2tJfik+i2bbUEw8UC7MZ3Yl2HYdRmqbdhHGAOQACtOOu6QVW4Jul4/w5wVKm+wM8ACtgh1Pg==} cpu: [arm64] - os: [linux] + os: [android] - '@pnpm/linux-x64@11.22.0': - resolution: {integrity: sha512-isvaPctGinbsM2hsTtRsMarN8Sr5QhXDTNmn8Xv9Lp1PjincCvH2RBbhpo+xYIOxzgls1dtQSdgoORUbfRVALQ==} + '@pnpm/exe.android-x64@12.8.1': + resolution: {integrity: sha512-0F/uqPRc3aN4vLHsa0f+6fTtmsigFQ9/yxEU43fionF//pNHEd5e97b3w/nKIgqYMCUQCJphmBRMA9Q8/wlo9Q==} cpu: [x64] - os: [linux] + os: [android] - '@pnpm/linuxstatic-arm64@11.22.0': - resolution: {integrity: sha512-i4J+AQWW0T3JBdXaLsvxu8ZmMG1HLS6JZQESdOR9uBv8Zumb4yg8GYT83eWRLacr6ngVdhEBiC3W4eOG64MFbA==} + '@pnpm/exe.darwin-arm64@12.8.1': + resolution: {integrity: sha512-/rwavvMQJsl2RIxOHBnDF+4Gk+fhQFAY4M3PQZoKskJOEnaJcygG9p1xby6wcQcbS4d9dIubv09CQhufTgfmpw==} cpu: [arm64] - os: [linux] - libc: [musl] + os: [darwin] - '@pnpm/linuxstatic-x64@11.22.0': - resolution: {integrity: sha512-QYzk8jhSuSbVthW/OxEOhU3f3zhjpw4HqgedrlwbVNb7btCWn5del2Hb5PsYYka2PbmKq5EtF5IzYN8Yp1CfxQ==} + '@pnpm/exe.darwin-x64@12.8.1': + resolution: {integrity: sha512-htYt2gN7zqJKLhC99Ft6EUiO7myjZAZXfmGCTJUL1v+o7vE9hZdE1+fAkk4Oj1ZHNRti6b2woOxEkRkAusi8OQ==} cpu: [x64] - os: [linux] - libc: [musl] - - '@pnpm/macos-arm64@11.22.0': - resolution: {integrity: sha512-Io8Axk5kutPgMuAfOg1QGj3J0/KpLvH5iiPcmp7up8D4q7BNWT02ndQo3RyGWqrlkf5nwspM41GFpWTShrZ4Aw==} - cpu: [arm64] os: [darwin] - '@pnpm/win-arm64@11.22.0': - resolution: {integrity: sha512-QgaRuKGQKov7xW2utPCgDT83fn/PU5cD6HFgib48oTslz7wv26E89d4jMCxL4GRmEL2YCfkYuj5ITj1oVWg5WQ==} - cpu: [arm64] - os: [win32] - - '@pnpm/win-x64@11.22.0': - resolution: {integrity: sha512-iWYsiSwpgxqur+TwsnSoPdOQaEfd4ygB84mb5tJUOgim6PHr1xhKJBndaQzH+At/WkLxLdYN+K8dYc4M7naezg==} + '@pnpm/exe.freebsd-x64@12.8.1': + resolution: {integrity: sha512-PB5BULNgytxvExoOXUUeseS/DcpPENs+cg3iA6xHHeaLT4ctEGKSGXEbuj7USeqvo8JBpAx039hivnbzGqDwnA==} cpu: [x64] - os: [win32] + os: [freebsd] - '@reflink/reflink-darwin-arm64@0.1.19': - resolution: {integrity: sha512-ruy44Lpepdk1FqDz38vExBY/PVUsjxZA+chd9wozjUH9JjuDT/HEaQYA6wYN9mf041l0yLVar6BCZuWABJvHSA==} - engines: {node: '>= 10'} + '@pnpm/exe.linux-arm64-musl@12.8.1': + resolution: {integrity: sha512-kKeyC/ArjrgdciQRBrOL0j9HtOI4gDRgoyP+eJ1CQKyadpKlf/DhDDyspXe07R3B9KagTVLUF2nlt6Dr5quFIg==} cpu: [arm64] - os: [darwin] - - '@reflink/reflink-darwin-x64@0.1.19': - resolution: {integrity: sha512-By85MSWrMZa+c26TcnAy8SDk0sTUkYlNnwknSchkhHpGXOtjNDUOxJE9oByBnGbeuIE1PiQsxDG3Ud+IVV9yuA==} - engines: {node: '>= 10'} - cpu: [x64] - os: [darwin] + os: [linux] + libc: [musl] - '@reflink/reflink-linux-arm64-gnu@0.1.19': - resolution: {integrity: sha512-7P+er8+rP9iNeN+bfmccM4hTAaLP6PQJPKWSA4iSk2bNvo6KU6RyPgYeHxXmzNKzPVRcypZQTpFgstHam6maVg==} - engines: {node: '>= 10'} + '@pnpm/exe.linux-arm64@12.8.1': + resolution: {integrity: sha512-q4s9a9X2O3N9aeUFooW24orNrxvu20+jyVUFR5RanPCqOKKPBrgs2iywMkBBx1aXkkzdWT54/mfz8Be8sJlWEw==} cpu: [arm64] os: [linux] libc: [glibc] - '@reflink/reflink-linux-arm64-musl@0.1.19': - resolution: {integrity: sha512-37iO/Dp6m5DDaC2sf3zPtx/hl9FV3Xze4xoYidrxxS9bgP3S8ALroxRK6xBG/1TtfXKTvolvp+IjrUU6ujIGmA==} - engines: {node: '>= 10'} - cpu: [arm64] + '@pnpm/exe.linux-ppc64@12.8.1': + resolution: {integrity: sha512-F7XSjKJthwCh/L6abZiMpAfo2Cm61SZIT3e4dfgiIjESndlWaXCnfLKngSgZ/SEMsdmHT6dud3+kWIB3tw0/xQ==} + cpu: [ppc64] os: [linux] - libc: [musl] + libc: [glibc] - '@reflink/reflink-linux-x64-gnu@0.1.19': - resolution: {integrity: sha512-jbI8jvuYCaA3MVUdu8vLoLAFqC+iNMpiSuLbxlAgg7x3K5bsS8nOpTRnkLF7vISJ+rVR8W+7ThXlXlUQ93ulkw==} - engines: {node: '>= 10'} - cpu: [x64] + '@pnpm/exe.linux-riscv64@12.8.1': + resolution: {integrity: sha512-z1ecpIK/ZY08Unk69KWWE9867oaKUytGUjrHnBoGFAAhsK+q+0cfvBH+f5fWobJ7FX97leMlLO+gkJolddM45Q==} + cpu: [riscv64] os: [linux] libc: [glibc] - '@reflink/reflink-linux-x64-musl@0.1.19': - resolution: {integrity: sha512-e9FBWDe+lv7QKAwtKOt6A2W/fyy/aEEfr0g6j/hWzvQcrzHCsz07BNQYlNOjTfeytrtLU7k449H1PI95jA4OjQ==} - engines: {node: '>= 10'} + '@pnpm/exe.linux-s390x@12.8.1': + resolution: {integrity: sha512-KU6tnrBfu4uKXTq6N5qJwSm8FSrs9Wc8b5AUEsHK9bRXwNdsZ1/lRxG1++X29SKRW9OSNuDSjxASv5mAvGo1gA==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@pnpm/exe.linux-x64-musl@12.8.1': + resolution: {integrity: sha512-eUE8BWJkDr2tJb/yrk0yVnszlmNqFarVxUlqiZ9BAkct28Nq/5g2H0tvMDmPA7fAQW3STNj+omJPPRVT+4s5sQ==} cpu: [x64] os: [linux] libc: [musl] - '@reflink/reflink-win32-arm64-msvc@0.1.19': - resolution: {integrity: sha512-09PxnVIQcd+UOn4WAW73WU6PXL7DwGS6wPlkMhMg2zlHHG65F3vHepOw06HFCq+N42qkaNAc8AKIabWvtk6cIQ==} - engines: {node: '>= 10'} + '@pnpm/exe.linux-x64@12.8.1': + resolution: {integrity: sha512-8bDZ0lZlCdi2rvnFul7EYgcC7zKeWSe76y8JV2oXobaS8p9i9d6PSf6LgLtTM0fI7R9Oh4eLCbveXyNDMmvZ+g==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@pnpm/exe.win32-arm64@12.8.1': + resolution: {integrity: sha512-MsT0dlrRxnCRCHCaosNbhWpEiAFnFuMEu3KuDJDU41BXgTwiINxU8goloUSUDejWZUqKaUqNC/qUTihl9nFtqg==} cpu: [arm64] os: [win32] - '@reflink/reflink-win32-x64-msvc@0.1.19': - resolution: {integrity: sha512-E//yT4ni2SyhwP8JRjVGWr3cbnhWDiPLgnQ66qqaanjjnMiu3O/2tjCPQXlcGc/DEYofpDc9fvhv6tALQsMV9w==} - engines: {node: '>= 10'} + '@pnpm/exe.win32-x64@12.8.1': + resolution: {integrity: sha512-SWtPe0PsbTTk//gJND10tMdOfCYjrkq5+qV49hzVhY7xz2iJ339Rc+xwASlcvbwTfrvH39YNqpKYibD+CRGLwA==} cpu: [x64] os: [win32] - '@reflink/reflink@0.1.19': - resolution: {integrity: sha512-DmCG8GzysnCZ15bres3N5AHCmwBwYgp0As6xjhQ47rAUTUXxJiK+lLUxaGsX3hd/30qUpVElh05PbGuxRPgJwA==} - engines: {node: '>= 10'} - - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - pnpm@11.22.0: - resolution: {integrity: sha512-H/hwxMYTPf2I+yr8Rt0T1H8JyXlLQ4xv20fKmMrzvBY4HuC+k6CRuOOCTPAfiJ9G19niCRD7C+GrD7W6qA3WIQ==} - engines: {node: '>=22.13'} + pnpm@12.8.1: + resolution: {integrity: sha512-9kupB1B/XOr+BsjTjmBS0BeURFgOwSed3Vv8EctIqoomRLZlmOB+dh2oSHKp/FfV+QK4f6SdAkGY1VhhKqu+RQ==} + engines: {node: '>=18.*'} hasBin: true snapshots: - '@pnpm/exe@11.22.0': - dependencies: - '@reflink/reflink': 0.1.19 - detect-libc: 2.1.2 - optionalDependencies: - '@pnpm/linux-arm64': 11.22.0 - '@pnpm/linux-x64': 11.22.0 - '@pnpm/linuxstatic-arm64': 11.22.0 - '@pnpm/linuxstatic-x64': 11.22.0 - '@pnpm/macos-arm64': 11.22.0 - '@pnpm/win-arm64': 11.22.0 - '@pnpm/win-x64': 11.22.0 - - '@pnpm/linux-arm64@11.22.0': - optional: true - - '@pnpm/linux-x64@11.22.0': + '@pnpm/exe.android-arm64@12.8.1': optional: true - '@pnpm/linuxstatic-arm64@11.22.0': + '@pnpm/exe.android-x64@12.8.1': optional: true - '@pnpm/linuxstatic-x64@11.22.0': + '@pnpm/exe.darwin-arm64@12.8.1': optional: true - '@pnpm/macos-arm64@11.22.0': + '@pnpm/exe.darwin-x64@12.8.1': optional: true - '@pnpm/win-arm64@11.22.0': + '@pnpm/exe.freebsd-x64@12.8.1': optional: true - '@pnpm/win-x64@11.22.0': + '@pnpm/exe.linux-arm64-musl@12.8.1': optional: true - '@reflink/reflink-darwin-arm64@0.1.19': + '@pnpm/exe.linux-arm64@12.8.1': optional: true - '@reflink/reflink-darwin-x64@0.1.19': + '@pnpm/exe.linux-ppc64@12.8.1': optional: true - '@reflink/reflink-linux-arm64-gnu@0.1.19': + '@pnpm/exe.linux-riscv64@12.8.1': optional: true - '@reflink/reflink-linux-arm64-musl@0.1.19': + '@pnpm/exe.linux-s390x@12.8.1': optional: true - '@reflink/reflink-linux-x64-gnu@0.1.19': + '@pnpm/exe.linux-x64-musl@12.8.1': optional: true - '@reflink/reflink-linux-x64-musl@0.1.19': + '@pnpm/exe.linux-x64@12.8.1': optional: true - '@reflink/reflink-win32-arm64-msvc@0.1.19': + '@pnpm/exe.win32-arm64@12.8.1': optional: true - '@reflink/reflink-win32-x64-msvc@0.1.19': + '@pnpm/exe.win32-x64@12.8.1': optional: true - '@reflink/reflink@0.1.19': + pnpm@12.8.1: optionalDependencies: - '@reflink/reflink-darwin-arm64': 0.1.19 - '@reflink/reflink-darwin-x64': 0.1.19 - '@reflink/reflink-linux-arm64-gnu': 0.1.19 - '@reflink/reflink-linux-arm64-musl': 0.1.19 - '@reflink/reflink-linux-x64-gnu': 0.1.19 - '@reflink/reflink-linux-x64-musl': 0.1.19 - '@reflink/reflink-win32-arm64-msvc': 0.1.19 - '@reflink/reflink-win32-x64-msvc': 0.1.19 - - detect-libc@2.1.2: {} - - pnpm@11.22.0: {} + '@pnpm/exe.android-arm64': 12.8.1 + '@pnpm/exe.android-x64': 12.8.1 + '@pnpm/exe.darwin-arm64': 12.8.1 + '@pnpm/exe.darwin-x64': 12.8.1 + '@pnpm/exe.freebsd-x64': 12.8.1 + '@pnpm/exe.linux-arm64': 12.8.1 + '@pnpm/exe.linux-arm64-musl': 12.8.1 + '@pnpm/exe.linux-ppc64': 12.8.1 + '@pnpm/exe.linux-riscv64': 12.8.1 + '@pnpm/exe.linux-s390x': 12.8.1 + '@pnpm/exe.linux-x64': 12.8.1 + '@pnpm/exe.linux-x64-musl': 12.8.1 + '@pnpm/exe.win32-arm64': 12.8.1 + '@pnpm/exe.win32-x64': 12.8.1 --- lockfileVersion: '9.0' From c0f3bf4d6ea154d3a64164b0149e96d18de1c8ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:12:14 +0200 Subject: [PATCH 3/3] chore(deps): bump brace-expansion from 1.1.18 to 1.1.21 in /website/versioned_docs/version-3.17 (#4174) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 1.1.18 to 1.1.21.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=brace-expansion&package-manager=npm_and_yarn&previous-version=1.1.18&new-version=1.1.21)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apify/crawlee/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/versioned_docs/version-3.17/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/versioned_docs/version-3.17/yarn.lock b/website/versioned_docs/version-3.17/yarn.lock index bcd982b16581..1a4ceb637581 100644 --- a/website/versioned_docs/version-3.17/yarn.lock +++ b/website/versioned_docs/version-3.17/yarn.lock @@ -36,12 +36,12 @@ __metadata: linkType: hard "brace-expansion@npm:^1.1.7": - version: 1.1.18 - resolution: "brace-expansion@npm:1.1.18" + version: 1.1.21 + resolution: "brace-expansion@npm:1.1.21" dependencies: balanced-match: "npm:^1.0.0" concat-map: "npm:0.0.1" - checksum: 10c0/3432c18a9e2ebf94162d4effb62198bd0adea06a9f332b2c0188df5d5e30b1e51ea3c848b6608e47d0b857ebe1ea5b3888ed3326dd3c4f6f9645c94153cf9c14 + checksum: 10c0/8f0a68a720f9cb28c2ecaea5cdaf15fafd686d6b6e4e722a81c9d4bd2f68c1fd0619731cac05875d842221eaf4d97d564d329a12eb3c4c22137502fb3c65be8b languageName: node linkType: hard