diff --git a/docs/03_concepts/13_http_clients.mdx b/docs/03_concepts/13_http_clients.mdx index b7e5f44532..fc7cf955a5 100644 --- a/docs/03_concepts/13_http_clients.mdx +++ b/docs/03_concepts/13_http_clients.mdx @@ -103,7 +103,7 @@ Crawlee provides an abstract base class, `HttpCli To create a custom HTTP client, inherit from `HttpClient` and implement two methods: -- `_fetch` sends a single request, through the proxy from `proxy_info` if there is one, and returns the response without reading its body. It doesn't follow redirects or keep cookies of its own. Its `timeout` bounds both receiving the response and reading the body, and without one the default timeout of the underlying library applies. +- `_fetch` sends a single request, through the proxy from `proxy_info` if there is one, and returns the response without reading its body. It doesn't follow redirects or keep cookies of its own. Its `timeout` bounds both receiving the response and reading the body, and without one the default timeout of the underlying library applies. With `ignore_tls_errors`, it doesn't verify the TLS certificate of the server. - `cleanup` closes the connections and other resources the client holds. The response is a subclass of the abstract `HttpResponse` class. Its body is read on `read` or `read_stream`, and `aclose` releases the connection when the body isn't needed. The base class provides the async context manager, which closes the response on exit. diff --git a/docs/03_concepts/code_examples/http_clients/parsel_impit_example.py b/docs/03_concepts/code_examples/http_clients/parsel_impit_example.py index fe2a263b49..3fc7cf1eaf 100644 --- a/docs/03_concepts/code_examples/http_clients/parsel_impit_example.py +++ b/docs/03_concepts/code_examples/http_clients/parsel_impit_example.py @@ -9,7 +9,6 @@ async def main() -> None: # Optional additional keyword arguments for `impit.AsyncClient`. http3=True, browser='firefox', - verify=True, ) crawler = ParselCrawler( diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index c20d72a3a8..9593ac7e7c 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -411,6 +411,20 @@ crawler = ParselCrawler(save_response_cookies=False) With `BasicCrawler` and the HTTP crawlers, `context.send_request` always saves the response cookies to the session, unless you pass other cookies in its `cookies` argument. In v1, `persist_cookies_per_session=False` applied to `context.send_request` too. With `PlaywrightCrawler` and its default HTTP client, `context.send_request` stores the response cookies in the browser context, so `save_response_cookies=False` keeps them out of the session as well. +### `ignore_tls_errors` replaces `verify` + +`ImpitHttpClient` and `HttpxHttpClient` take `ignore_tls_errors` instead of `verify`, and `CurlImpersonateHttpClient` takes it too. TLS certificates are still verified by default. `HttpxHttpClient` no longer takes a CA bundle or an `SSLContext`. + +```python +# v1 +http_client = ImpitHttpClient(verify=False) + +# v2 +http_client = ImpitHttpClient(ignore_tls_errors=True) +``` + +To ignore TLS errors for a single call, pass `ignore_tls_errors=True` to `send_request` of the client or to `context.send_request`. + ### Some client arguments are ignored The following keyword arguments of the underlying libraries are now ignored with a warning: @@ -418,6 +432,7 @@ The following keyword arguments of the underlying libraries are now ignored with - `cookies` of every client and `cookie_jar` of `ImpitHttpClient`, since cookies are kept in sessions. To send other cookies with a single call, pass `cookies` to `send_request`. - `auth` and `params` of `HttpxHttpClient` and `CurlImpersonateHttpClient`, which would reach every request of a redirect chain. Pass the credentials in the `Authorization` header and the query in the URL instead. - `max_redirects` of every client and `allow_redirects` of `CurlImpersonateHttpClient`, since `send_request` follows the redirects. +- `verify` of every client. For what replaces it, see [`ignore_tls_errors` replaces `verify`](#ignore_tls_errors-replaces-verify). ### Custom clients implement `_fetch` @@ -439,7 +454,9 @@ class MyHttpClient(HttpClient): # v2 class MyHttpClient(HttpClient): - async def _fetch(self, url, *, method, headers, payload, proxy_info, timeout): ... + async def _fetch( + self, url, *, method, headers, payload, proxy_info, timeout, ignore_tls_errors + ): ... async def cleanup(self): ... ``` diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 3456d939e5..93d9026314 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -583,6 +583,7 @@ def __call__( headers: HttpHeaders | dict[str, str] | None = None, cookies: SessionCookies | None = None, timeout: timedelta | None = None, + ignore_tls_errors: bool = False, ) -> Coroutine[None, None, HttpResponse]: """Call send request function. @@ -596,6 +597,7 @@ def __call__( request handler unless `save_response_cookies` is disabled. timeout: Maximum time allowed to receive the response, including all redirects, and to read its body. Without it, the default timeout of the HTTP client applies. + ignore_tls_errors: Whether to ignore TLS certificate errors for this request. Returns: The HTTP response received from the server. diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index 3ecfc75e13..5de250c911 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -287,6 +287,9 @@ async def _create_browser_context( password=proxy_info.password, ) + if proxy_info.ignore_tls_errors: + filtered_options['ignore_https_errors'] = True + if self._fingerprint_generator: return await AsyncNewContext( browser=self._browser, diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 5865b5d369..9204cbb4dd 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1399,6 +1399,7 @@ async def send_request( headers: HttpHeaders | dict[str, str] | None = None, cookies: SessionCookies | None = None, timeout: timedelta | None = None, + ignore_tls_errors: bool = False, ) -> HttpResponse: response = await self._http_client.send_request( url=url, @@ -1409,6 +1410,7 @@ async def send_request( cookies=cookies, proxy_info=proxy_info, timeout=timeout, + ignore_tls_errors=ignore_tls_errors, ) # A response whose body the handler never read would hold its connection open. register_deferred_cleanup(response.aclose) diff --git a/src/crawlee/crawlers/_playwright/_playwright_http_client.py b/src/crawlee/crawlers/_playwright/_playwright_http_client.py index cf7b577fa1..3827353980 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_http_client.py +++ b/src/crawlee/crawlers/_playwright/_playwright_http_client.py @@ -61,6 +61,7 @@ async def send_request( cookies: SessionCookies | None = None, proxy_info: ProxyInfo | None = None, timeout: timedelta | None = None, + ignore_tls_errors: bool = False, ) -> HttpResponse: # TODO: Use `session` to restore all the fingerprint headers according to the `BrowserContext`, after resolved # https://github.com/apify/crawlee-python/issues/1055 @@ -88,6 +89,7 @@ async def send_request( cookies=cookies, proxy_info=proxy_info, timeout=timeout, + ignore_tls_errors=ignore_tls_errors, ) finally: _caller_cookies_var.reset(token) @@ -102,6 +104,7 @@ async def _fetch( payload: HttpPayload | None, proxy_info: ProxyInfo | None, timeout: timedelta | None, + ignore_tls_errors: bool, ) -> HttpResponse: # `proxy_info` are not used because `APIRequestContext` inherits the proxy from `BrowserContext` browser_context = _browser_page_context_var.get() @@ -121,6 +124,8 @@ async def _fetch( headers=fetch_headers or None, data=payload, timeout=timeout.total_seconds() * 1000 if timeout is not None else None, + # Without the flag, the TLS setting of the browser context applies. + ignore_https_errors=True if ignore_tls_errors else None, # Redirects are followed hop by hop by `send_request`. max_redirects=0, ) diff --git a/src/crawlee/http_clients/_base.py b/src/crawlee/http_clients/_base.py index a49ec9f703..110c2baf7f 100644 --- a/src/crawlee/http_clients/_base.py +++ b/src/crawlee/http_clients/_base.py @@ -226,6 +226,7 @@ async def send_request( cookies: SessionCookies | None = None, proxy_info: ProxyInfo | None = None, timeout: timedelta | None = None, + ignore_tls_errors: bool = False, ) -> HttpResponse: """Send an HTTP request via the client, following up to 10 redirects unless `follow_redirects` is disabled. @@ -247,6 +248,8 @@ async def send_request( proxy_info: The information about the proxy to be used. timeout: Maximum time allowed to receive the response, including all redirects, and to read its body. Without it, the default timeout of the HTTP client applies. + ignore_tls_errors: Whether to ignore TLS certificate errors for this request. TLS errors are also ignored + when the client or `proxy_info` sets `ignore_tls_errors`. Raises: ProxyError: Raised if a proxy-related error occurs. @@ -264,6 +267,9 @@ async def send_request( if cookies is None: cookies = session.cookies if session is not None else SessionCookies() + if proxy_info is not None and proxy_info.ignore_tls_errors: + ignore_tls_errors = True + # `encoded=True` keeps the path and the query byte for byte, including a safe `%2F` in query. current_url = _with_ascii_host(URL(url, encoded=True)) deadline = monotonic() + timeout.total_seconds() if timeout is not None else None @@ -282,6 +288,7 @@ async def send_request( payload=payload, proxy_info=proxy_info, timeout=timedelta(seconds=remaining) if remaining is not None else None, + ignore_tls_errors=ignore_tls_errors, ) try: @@ -327,6 +334,7 @@ async def _fetch( payload: HttpPayload | None, proxy_info: ProxyInfo | None, timeout: timedelta | None, + ignore_tls_errors: bool, ) -> HttpResponse: """Send a single HTTP request and return the response without reading its body. @@ -341,6 +349,8 @@ async def _fetch( proxy_info: The information about the proxy to be used. timeout: Maximum time allowed to receive the response and read its body. Without it, the default timeout of the client library applies. `send_request` also cancels the call when the time is up. + ignore_tls_errors: Whether to ignore TLS certificate errors, as requested by the caller of `send_request` + or by `proxy_info`. A client that ignores them on its own does so even when this is `False`. Raises: ProxyError: Raised if a proxy-related error occurs. diff --git a/src/crawlee/http_clients/_curl_impersonate.py b/src/crawlee/http_clients/_curl_impersonate.py index 0deccb8e12..b03102768b 100644 --- a/src/crawlee/http_clients/_curl_impersonate.py +++ b/src/crawlee/http_clients/_curl_impersonate.py @@ -170,25 +170,28 @@ class CurlImpersonateHttpClient(HttpClient): def __init__( self, *, + ignore_tls_errors: bool = False, follow_redirects: bool = True, **async_session_kwargs: Any, ) -> None: """Initialize a new instance. Args: + ignore_tls_errors: Whether to ignore TLS certificate errors for every request. follow_redirects: Whether to follow HTTP redirects. async_session_kwargs: Additional keyword arguments for `curl_cffi.requests.AsyncSession`. Some of them are ignored with a warning. Cookies are kept in sessions, so `cookies` is ignored. The `auth` and `params` arguments would reach every request of a redirect chain, so pass credentials in the `Authorization` header and the query in the URL instead. Redirects are followed by `send_request`, so `allow_redirects` - and `max_redirects` are ignored. + and `max_redirects` are ignored. Use `ignore_tls_errors` instead of `verify`. """ super().__init__( follow_redirects=follow_redirects, headers=async_session_kwargs.pop('headers', None), ) + self._ignore_tls_errors = ignore_tls_errors - for ignored_kwarg in ('cookies', 'auth', 'params', 'allow_redirects', 'max_redirects'): + for ignored_kwarg in ('cookies', 'auth', 'params', 'allow_redirects', 'max_redirects', 'verify'): if async_session_kwargs.pop(ignored_kwarg, None) is not None: warnings.warn( f'The `{ignored_kwarg}` argument of `CurlImpersonateHttpClient` is ignored.', @@ -198,7 +201,7 @@ def __init__( self._async_session_kwargs = async_session_kwargs - self._client_by_proxy_url = dict[str | None, AsyncSession]() + self._clients = dict[str | None, AsyncSession]() @override async def _fetch( @@ -210,6 +213,7 @@ async def _fetch( payload: HttpPayload | None, proxy_info: ProxyInfo | None, timeout: timedelta | None, + ignore_tls_errors: bool, ) -> _CurlImpersonateResponse: client = self._get_client(proxy_info.url if proxy_info else None) deadline = asyncio.get_running_loop().time() + timeout.total_seconds() if timeout is not None else None @@ -228,6 +232,7 @@ async def _fetch( data=payload, # Redirects are followed hop by hop by `send_request`. allow_redirects=False, + verify=not (ignore_tls_errors or self._ignore_tls_errors), stream=True, **timeout_kwargs, ) @@ -248,7 +253,7 @@ def _get_client(self, proxy_url: str | None) -> AsyncSession: Store the new session for future use. """ # Check if a session for the given proxy URL has already been created. - if proxy_url not in self._client_by_proxy_url: + if proxy_url not in self._clients: # Prepare a default kwargs for the new session. A provided proxy URL and a chrome for impersonation # are set as default options. kwargs: dict[str, Any] = { @@ -260,9 +265,9 @@ def _get_client(self, proxy_url: str | None) -> AsyncSession: kwargs.update(self._async_session_kwargs) # Create and store the new session with the specified kwargs. - self._client_by_proxy_url[proxy_url] = _AsyncSession(**kwargs) + self._clients[proxy_url] = _AsyncSession(**kwargs) - return self._client_by_proxy_url[proxy_url] + return self._clients[proxy_url] def _convert_method(self, method: HttpMethod) -> CurlHttpMethod: """Convert from Crawlee HTTP method to curl-cffi HTTP method. @@ -314,6 +319,6 @@ def _is_proxy_error(error: CurlRequestError) -> bool: return False async def cleanup(self) -> None: - for client in self._client_by_proxy_url.values(): + for client in self._clients.values(): await client.close() - self._client_by_proxy_url.clear() + self._clients.clear() diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index 234e539d90..dfe99322a1 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -22,7 +22,6 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator from datetime import timedelta - from ssl import SSLContext from crawlee._types import HttpMethod, HttpPayload from crawlee.proxy_configuration import ProxyInfo @@ -120,7 +119,7 @@ def __init__( *, http1: bool = True, http2: bool = True, - verify: str | bool | SSLContext = True, + ignore_tls_errors: bool = False, header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR, follow_redirects: bool = True, **async_client_kwargs: Any, @@ -130,16 +129,17 @@ def __init__( Args: http1: Whether to enable HTTP/1.1 support. http2: Whether to enable HTTP/2 support. - verify: SSL certificates used to verify the identity of requested hosts. + ignore_tls_errors: Whether to ignore TLS certificate errors for every request. header_generator: Header generator instance to use for generating browser-like headers. follow_redirects: Whether to follow HTTP redirects. async_client_kwargs: Additional keyword arguments for `httpx2.AsyncClient`. Some of them are ignored with a warning. The `mounts` and `transport` arguments would bypass the proxy of a `ProxyInfo`. Cookies are kept in sessions, so `cookies` is ignored. The `auth` and `params` arguments would reach every request of a redirect chain, so pass credentials in the `Authorization` header and the query in the URL instead. - Redirects are followed by `send_request`, so `max_redirects` is ignored. The `proxy` argument covers - only the requests made without a `ProxyInfo`, a `ProxyConfiguration` takes precedence over it. The - `limits` argument applies per proxy, because every proxy gets a connection pool of its own. + Redirects are followed by `send_request`, so `max_redirects` is ignored. Use `ignore_tls_errors` + instead of `verify`. The `proxy` argument covers only the requests made without a `ProxyInfo`, a + `ProxyConfiguration` takes precedence over it. The `limits` argument applies per proxy and TLS setting, + because each combination gets a connection pool of its own. """ super().__init__( follow_redirects=follow_redirects, @@ -161,7 +161,7 @@ def __init__( # covers the requests that carry no `ProxyInfo` of their own. self._proxy = async_client_kwargs.pop('proxy', None) - for ignored_kwarg in ('mounts', 'transport', 'cookies', 'auth', 'params', 'max_redirects'): + for ignored_kwarg in ('mounts', 'transport', 'cookies', 'auth', 'params', 'max_redirects', 'verify'): if async_client_kwargs.pop(ignored_kwarg, None) is not None: warnings.warn( f'The `{ignored_kwarg}` argument of `HttpxHttpClient` is ignored.', @@ -171,10 +171,13 @@ def __init__( self._async_client_kwargs = async_client_kwargs self._header_generator = header_generator + self._ignore_tls_errors = ignore_tls_errors - self._ssl_context = httpx2.create_ssl_context(verify=verify) + self._ssl_context = httpx2.create_ssl_context() + self._unverified_ssl_context = httpx2.create_ssl_context(verify=False) - self._client_by_proxy_url = dict[str | None, httpx2.AsyncClient]() + # The certificates are verified per transport, so the clients are kept per proxy URL and TLS setting. + self._clients = dict[tuple[str | None, bool], httpx2.AsyncClient]() @override async def send_request( @@ -188,6 +191,7 @@ async def send_request( cookies: SessionCookies | None = None, proxy_info: ProxyInfo | None = None, timeout: timedelta | None = None, + ignore_tls_errors: bool = False, ) -> HttpResponse: # The headers are generated once per call, so that every request of a redirect chain carries the same browser # profile. @@ -203,6 +207,7 @@ async def send_request( cookies=cookies, proxy_info=proxy_info, timeout=timeout, + ignore_tls_errors=ignore_tls_errors, ) @override @@ -215,8 +220,12 @@ async def _fetch( payload: HttpPayload | None, proxy_info: ProxyInfo | None, timeout: timedelta | None, + ignore_tls_errors: bool, ) -> _HttpxResponse: - client = self._get_client(proxy_info.url if proxy_info else None) + client = self._get_client( + proxy_info.url if proxy_info else None, + ignore_tls_errors=ignore_tls_errors or self._ignore_tls_errors, + ) deadline = asyncio.get_running_loop().time() + timeout.total_seconds() if timeout is not None else None http_request = client.build_request( @@ -238,18 +247,21 @@ async def _fetch( return _HttpxResponse(response, deadline=deadline) - def _get_client(self, proxy_url: str | None) -> httpx2.AsyncClient: - """Retrieve or create an HTTP client for the given proxy URL. + def _get_client(self, proxy_url: str | None, *, ignore_tls_errors: bool) -> httpx2.AsyncClient: + """Retrieve or create an HTTP client for the given proxy URL and TLS setting. - If a client for the specified proxy URL does not exist, create and store a new one. + If such a client does not exist, create and store a new one. """ - if proxy_url not in self._client_by_proxy_url: + key = (proxy_url, ignore_tls_errors) + if key not in self._clients: + ssl_context = self._unverified_ssl_context if ignore_tls_errors else self._ssl_context + # A client built with `proxy=` mounts its own transport and never calls the one given to `transport=`, # so the proxy goes on the transport. transport = httpx2.AsyncHTTPTransport( http1=self._http1, http2=self._http2, - verify=self._ssl_context, + verify=ssl_context, proxy=proxy_url or self._proxy, # Above the `httpx2` default of 20 kept-alive connections every request pays a TCP and TLS handshake. limits=self._async_client_kwargs.get( @@ -270,7 +282,7 @@ def _get_client(self, proxy_url: str | None) -> httpx2.AsyncClient: kwargs.update( { 'transport': transport, - 'verify': self._ssl_context, + 'verify': ssl_context, # Redirects are followed hop by hop by `send_request`, which also handles the cookies. 'follow_redirects': False, # With both cookie protocols off, the jar of the client never stores a cookie. @@ -279,9 +291,9 @@ def _get_client(self, proxy_url: str | None) -> httpx2.AsyncClient: ) client = httpx2.AsyncClient(**kwargs) - self._client_by_proxy_url[proxy_url] = client + self._clients[key] = client - return self._client_by_proxy_url[proxy_url] + return self._clients[key] def _combine_headers(self, explicit_headers: HttpHeaders) -> HttpHeaders: """Merge generated headers with the headers of the client and explicit headers for an HTTP request. @@ -314,6 +326,6 @@ def _is_proxy_error(error: httpx2.TransportError) -> bool: return False async def cleanup(self) -> None: - for client in self._client_by_proxy_url.values(): + for client in self._clients.values(): await client.aclose() - self._client_by_proxy_url.clear() + self._clients.clear() diff --git a/src/crawlee/http_clients/_impit.py b/src/crawlee/http_clients/_impit.py index 7e3312ada9..343d4a21c6 100644 --- a/src/crawlee/http_clients/_impit.py +++ b/src/crawlee/http_clients/_impit.py @@ -114,7 +114,7 @@ def __init__( self, *, http3: bool = False, - verify: bool = True, + ignore_tls_errors: bool = False, browser: Browser | None = 'firefox', follow_redirects: bool = True, **async_client_kwargs: Any, @@ -123,22 +123,22 @@ def __init__( Args: http3: Whether to enable HTTP/3 support. - verify: SSL certificates used to verify the identity of requested hosts. + ignore_tls_errors: Whether to ignore TLS certificate errors for every request. browser: Browser to impersonate. follow_redirects: Whether to follow HTTP redirects. async_client_kwargs: Additional keyword arguments for `impit.AsyncClient`. Some of them are ignored with a warning. Cookies are kept in sessions, so `cookies` and `cookie_jar` are ignored. Redirects are followed - by `send_request`, so `max_redirects` is ignored. + by `send_request`, so `max_redirects` is ignored. Use `ignore_tls_errors` instead of `verify`. """ super().__init__( follow_redirects=follow_redirects, headers=async_client_kwargs.pop('headers', None), ) self._http3 = http3 - self._verify = verify + self._ignore_tls_errors = ignore_tls_errors self._browser = browser - for ignored_kwarg in ('cookies', 'cookie_jar', 'max_redirects'): + for ignored_kwarg in ('cookies', 'cookie_jar', 'max_redirects', 'verify'): if async_client_kwargs.pop(ignored_kwarg, None) is not None: warnings.warn( f'The `{ignored_kwarg}` argument of `ImpitHttpClient` is ignored.', @@ -148,7 +148,8 @@ def __init__( self._async_client_kwargs = async_client_kwargs - self._client_by_proxy_url = dict[str | None, AsyncClient]() + # `impit` verifies the certificates per client, so the clients are kept per proxy URL and TLS setting. + self._clients = dict[tuple[str | None, bool], AsyncClient]() @override async def _fetch( @@ -160,8 +161,12 @@ async def _fetch( payload: HttpPayload | None, proxy_info: ProxyInfo | None, timeout: timedelta | None, + ignore_tls_errors: bool, ) -> _ImpitResponse: - client = self._get_client(proxy_info.url if proxy_info else None) + client = self._get_client( + proxy_info.url if proxy_info else None, + ignore_tls_errors=ignore_tls_errors or self._ignore_tls_errors, + ) deadline = asyncio.get_running_loop().time() + timeout.total_seconds() if timeout is not None else None try: @@ -183,19 +188,20 @@ async def _fetch( return _ImpitResponse(response, deadline=deadline) - def _get_client(self, proxy_url: str | None) -> AsyncClient: - """Retrieve or create an HTTP client for the given proxy URL. + def _get_client(self, proxy_url: str | None, *, ignore_tls_errors: bool) -> AsyncClient: + """Retrieve or create an HTTP client for the given proxy URL and TLS setting. - If a client for the specified proxy URL does not exist, create and store a new one. + If such a client does not exist, create and store a new one. """ - if client := self._client_by_proxy_url.get(proxy_url): + key = (proxy_url, ignore_tls_errors) + if client := self._clients.get(key): return client # Prepare a default kwargs for the new client. kwargs: dict[str, Any] = { 'proxy': proxy_url, 'http3': self._http3, - 'verify': self._verify, + 'verify': not ignore_tls_errors, 'browser': self._browser, } @@ -207,7 +213,7 @@ def _get_client(self, proxy_url: str | None) -> AsyncClient: client = AsyncClient(**kwargs) - self._client_by_proxy_url[proxy_url] = client + self._clients[key] = client return client @@ -228,4 +234,4 @@ def _is_proxy_error(error: HTTPError) -> bool: @override async def cleanup(self) -> None: """Clean up resources used by the HTTP client.""" - self._client_by_proxy_url.clear() + self._clients.clear() diff --git a/src/crawlee/proxy_configuration.py b/src/crawlee/proxy_configuration.py index 130ebc71e9..ae909fa291 100644 --- a/src/crawlee/proxy_configuration.py +++ b/src/crawlee/proxy_configuration.py @@ -44,6 +44,10 @@ class ProxyInfo: """The identifier of the used proxy session, if used. Using the same session ID guarantees getting the same proxy URL.""" + ignore_tls_errors: bool = False + """Whether the proxy intercepts HTTPS traffic, so that `HttpClient.send_request` and the browser contexts of + `PlaywrightBrowserController` ignore TLS certificate errors for the requests sent through it.""" + @docs_group('Configuration') class ProxyConfiguration: diff --git a/tests/integration/http_clients/test_curl_impersonate.py b/tests/integration/http_clients/test_curl_impersonate.py index 6a04a82b8d..a3d6cba35f 100644 --- a/tests/integration/http_clients/test_curl_impersonate.py +++ b/tests/integration/http_clients/test_curl_impersonate.py @@ -29,7 +29,7 @@ async def test_internal_session_cookie_jar_stays_empty(server_url: URL) -> None: async with await client.send_request(str(url), session=session): pass - curl_session = client._client_by_proxy_url[None] + curl_session = client._clients[None] assert list(curl_session._cookies.jar) == [] # The cookies are stored by `send_request`, so they still reach the crawlee session. @@ -69,6 +69,7 @@ async def test_sub_millisecond_timeout_not_rounded_to_zero(monkeypatch: pytest.M payload=None, proxy_info=None, timeout=timedelta(microseconds=100), + ignore_tls_errors=False, ) assert curl_session.request.call_args.kwargs['timeout'] == 0.001 @@ -87,11 +88,45 @@ async def test_no_timeout_uses_session_default(monkeypatch: pytest.MonkeyPatch) payload=None, proxy_info=None, timeout=None, + ignore_tls_errors=False, ) assert 'timeout' not in curl_session.request.call_args.kwargs +@pytest.mark.parametrize( + ('client_flag', 'request_flag', 'expected_verify'), + [ + pytest.param(False, False, True, id='verified'), + pytest.param(True, False, False, id='client'), + pytest.param(False, True, False, id='request'), + ], +) +async def test_ignore_tls_errors_turns_off_verification( + monkeypatch: pytest.MonkeyPatch, + *, + client_flag: bool, + request_flag: bool, + expected_verify: bool, +) -> None: + """Test that the `curl_cffi` request skips verification when the client or the request ignores TLS errors.""" + curl_session = mock_curl_session() + client = CurlImpersonateHttpClient(ignore_tls_errors=client_flag) + monkeypatch.setattr(client, '_get_client', Mock(return_value=curl_session)) + + await client._fetch( + 'https://example.com/', + method='GET', + headers=HttpHeaders(), + payload=None, + proxy_info=None, + timeout=None, + ignore_tls_errors=request_flag, + ) + + assert curl_session.request.call_args.kwargs['verify'] is expected_verify + + async def test_timed_out_request_closes_connection() -> None: """Test that a request that runs out of time before its response arrives closes the connection.""" client_closed = asyncio.Event() diff --git a/tests/integration/http_clients/test_http_clients.py b/tests/integration/http_clients/test_http_clients.py index 17c1c4847f..f9c687f4d3 100644 --- a/tests/integration/http_clients/test_http_clients.py +++ b/tests/integration/http_clients/test_http_clients.py @@ -408,7 +408,7 @@ async def test_sessions_share_one_client(http_client: HttpClient, server_url: UR async with await http_client.send_request(str(server_url / 'cookies'), session=Session()): pass - assert len(http_client._client_by_proxy_url) == 1 # ty: ignore[unresolved-attribute] + assert len(http_client._clients) == 1 # ty: ignore[unresolved-attribute] async def test_cookies_isolated_per_session(http_client: HttpClient, server_url: URL) -> None: diff --git a/tests/integration/http_clients/test_httpx.py b/tests/integration/http_clients/test_httpx.py index 4468d8ab9f..ec6781eac4 100644 --- a/tests/integration/http_clients/test_httpx.py +++ b/tests/integration/http_clients/test_httpx.py @@ -156,7 +156,7 @@ async def test_own_cookie_jar_stays_empty(server_url: URL) -> None: async with await client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=session): pass - assert list(client._get_client(None).cookies.jar) == [] + assert list(client._get_client(None, ignore_tls_errors=False).cookies.jar) == [] assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'a'} diff --git a/tests/integration/http_clients/test_impit.py b/tests/integration/http_clients/test_impit.py index 5740e724e4..f491b9a802 100644 --- a/tests/integration/http_clients/test_impit.py +++ b/tests/integration/http_clients/test_impit.py @@ -1,16 +1,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING from unittest.mock import AsyncMock, Mock +import pytest from impit import USE_CLIENT_DEFAULT, AsyncClient, Response from crawlee import HttpHeaders from crawlee.http_clients import ImpitHttpClient -if TYPE_CHECKING: - import pytest - async def test_no_timeout_uses_client_default(monkeypatch: pytest.MonkeyPatch) -> None: """Test that a request without a timeout gets the default timeout of the `impit` client, not none at all.""" @@ -26,6 +23,41 @@ async def test_no_timeout_uses_client_default(monkeypatch: pytest.MonkeyPatch) - payload=None, proxy_info=None, timeout=None, + ignore_tls_errors=False, ) assert impit_client.request.call_args.kwargs['timeout'] is USE_CLIENT_DEFAULT + + +@pytest.mark.parametrize( + ('client_flag', 'request_flag', 'expected_verify'), + [ + pytest.param(False, False, True, id='verified'), + pytest.param(True, False, False, id='client'), + pytest.param(False, True, False, id='request'), + ], +) +async def test_ignore_tls_errors_turns_off_verification( + monkeypatch: pytest.MonkeyPatch, + *, + client_flag: bool, + request_flag: bool, + expected_verify: bool, +) -> None: + """Test that the `impit` client skips verification when the client or the request ignores TLS errors.""" + async_client_class = Mock() + async_client_class.return_value.request = AsyncMock(return_value=Mock(spec=Response)) + monkeypatch.setattr('crawlee.http_clients._impit.AsyncClient', async_client_class) + client = ImpitHttpClient(ignore_tls_errors=client_flag) + + await client._fetch( + 'https://example.com/', + method='GET', + headers=HttpHeaders(), + payload=None, + proxy_info=None, + timeout=None, + ignore_tls_errors=request_flag, + ) + + assert async_client_class.call_args.kwargs['verify'] is expected_verify diff --git a/tests/unit/browsers/test_playwright_browser_controller.py b/tests/unit/browsers/test_playwright_browser_controller.py index f734ac8536..ed407f2310 100644 --- a/tests/unit/browsers/test_playwright_browser_controller.py +++ b/tests/unit/browsers/test_playwright_browser_controller.py @@ -2,11 +2,13 @@ import asyncio from typing import Any -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock -from playwright.async_api import BrowserContext, Page +import pytest +from playwright.async_api import Browser, BrowserContext, Page from crawlee.browsers import PlaywrightBrowserController, PlaywrightPersistentBrowser +from crawlee.proxy_configuration import ProxyInfo async def test_memory_leak_on_concurrent_context_creation() -> None: @@ -38,3 +40,44 @@ async def delayed_launch_persistent_context(*args: Any, **kwargs: Any) -> Any: await asyncio.gather(controller.new_page(), controller.new_page()) assert mocked_context_launcher.call_count == 1 + + +@pytest.mark.parametrize( + 'use_incognito_pages', + [ + pytest.param(True, id='incognito'), + pytest.param(False, id='persistent'), + ], +) +@pytest.mark.parametrize( + ('ignore_tls_errors', 'context_options', 'expected'), + [ + pytest.param(True, None, True, id='intercepting_proxy'), + pytest.param(False, None, None, id='regular_proxy'), + pytest.param(True, {'ignore_https_errors': False}, True, id='overrides_context_option'), + ], +) +async def test_proxy_ignore_tls_errors_reaches_context( + *, + ignore_tls_errors: bool, + context_options: dict[str, Any] | None, + expected: bool | None, + use_incognito_pages: bool, +) -> None: + """Test that a proxy with `ignore_tls_errors` makes the browser context ignore TLS certificate errors.""" + browser_context = AsyncMock(spec=BrowserContext) + browser_context.new_page.return_value = Mock(spec=Page) + browser = AsyncMock(spec=Browser, contexts=[]) + browser.new_context.return_value = browser_context + controller = PlaywrightBrowserController(browser, use_incognito_pages=use_incognito_pages, header_generator=None) + proxy_info = ProxyInfo( + url='http://proxy.example.com:8080', + scheme='http', + hostname='proxy.example.com', + port=8080, + ignore_tls_errors=ignore_tls_errors, + ) + + await controller.new_page(browser_new_context_options=context_options, proxy_info=proxy_info) + + assert browser.new_context.call_args.kwargs.get('ignore_https_errors') is expected diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 093a7215c2..7339bd4b5e 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -634,6 +634,7 @@ async def handler(context: BasicCrawlingContext) -> None: headers={'Content-Type': 'text/plain'}, cookies=cookies, timeout=timeout, + ignore_tls_errors=True, ) await crawler.run(['https://a.placeholder.com']) @@ -648,6 +649,7 @@ async def handler(context: BasicCrawlingContext) -> None: cookies=cookies, proxy_info=None, timeout=timeout, + ignore_tls_errors=True, ) diff --git a/tests/unit/crawlers/_playwright/test_playwright_http_client.py b/tests/unit/crawlers/_playwright/test_playwright_http_client.py index b9a1aa37e3..c730bcc6d3 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_http_client.py +++ b/tests/unit/crawlers/_playwright/test_playwright_http_client.py @@ -48,6 +48,7 @@ async def test_send_request_converts_timeout_to_milliseconds() -> None: headers=None, data=None, timeout=pytest.approx(12_000, abs=100), + ignore_https_errors=None, max_redirects=0, ) from_playwright_response.assert_awaited_once_with(playwright_response, protocol='') @@ -90,6 +91,33 @@ async def test_send_request_sends_cookies_argument_instead_of_context_cookies() assert page.request.fetch.call_args.kwargs['headers'] == {'cookie': 'from_argument=1'} +@pytest.mark.parametrize( + ('ignore_tls_errors', 'expected_ignore_https_errors'), + [ + pytest.param(False, None, id='browser_context_setting'), + pytest.param(True, True, id='ignored'), + ], +) +async def test_ignore_tls_errors_reaches_browser_request( + *, + ignore_tls_errors: bool, + expected_ignore_https_errors: bool | None, +) -> None: + """Test that `ignore_tls_errors` turns on `ignore_https_errors`, and the browser context decides without it.""" + page = mock_page() + client = PlaywrightHttpClient() + + with patch.object( + PlaywrightHttpResponse, + 'from_playwright_response', + new=AsyncMock(return_value=mock_response(200, 'https://example.com/')), + ): + async with browser_page_context(page): + await client.send_request('https://example.com/', ignore_tls_errors=ignore_tls_errors) + + assert page.request.fetch.call_args.kwargs['ignore_https_errors'] is expected_ignore_https_errors + + async def test_cookies_argument_keeps_context_cookies_out() -> None: """Test that the empty `Cookie` header keeping the browser context cookies out is sent only with `cookies`.""" page = mock_page() diff --git a/tests/unit/http_clients/test_http_clients.py b/tests/unit/http_clients/test_http_clients.py index b6e2378e0a..99eae39be0 100644 --- a/tests/unit/http_clients/test_http_clients.py +++ b/tests/unit/http_clients/test_http_clients.py @@ -15,6 +15,7 @@ from crawlee.http_clients._base import _is_cross_origin, _redirect_method from crawlee.http_clients._httpx import _HttpxResponse from crawlee.http_clients._impit import _ImpitResponse +from crawlee.proxy_configuration import ProxyInfo from crawlee.sessions import Session if TYPE_CHECKING: @@ -63,12 +64,15 @@ async def test_cookies_of_non_ascii_host_sent_back(monkeypatch: pytest.MonkeyPat [ pytest.param(ImpitHttpClient, 'cookies', id='impit_cookies'), pytest.param(ImpitHttpClient, 'cookie_jar', id='impit_cookie_jar'), + pytest.param(ImpitHttpClient, 'verify', id='impit_verify'), pytest.param(HttpxHttpClient, 'cookies', id='httpx_cookies'), pytest.param(HttpxHttpClient, 'auth', id='httpx_auth'), pytest.param(HttpxHttpClient, 'params', id='httpx_params'), + pytest.param(HttpxHttpClient, 'verify', id='httpx_verify'), pytest.param(CurlImpersonateHttpClient, 'cookies', id='curl_cookies'), pytest.param(CurlImpersonateHttpClient, 'auth', id='curl_auth'), pytest.param(CurlImpersonateHttpClient, 'params', id='curl_params'), + pytest.param(CurlImpersonateHttpClient, 'verify', id='curl_verify'), ], ) def test_ignored_argument_warns(client_class: type[HttpClient], kwarg: str) -> None: @@ -170,3 +174,66 @@ def test_see_other_keeps_head() -> None: def test_is_cross_origin(url: str, other: str, *, expected: bool) -> None: """Test that a redirect leaves the origin when the scheme, host or port changes.""" assert _is_cross_origin(URL(url, encoded=True), URL(other, encoded=True)) is expected + + +@pytest.mark.parametrize( + 'client_class', + [ + pytest.param(ImpitHttpClient, id='impit'), + pytest.param(HttpxHttpClient, id='httpx'), + ], +) +async def test_clients_cached_per_tls_setting(client_class: type[ImpitHttpClient | HttpxHttpClient]) -> None: + """Test that the requests ignoring TLS errors get a client of their own, while the others share one.""" + client = client_class() + + async with client: + verified = client._get_client(None, ignore_tls_errors=False) + unverified = client._get_client(None, ignore_tls_errors=True) + + assert verified is not unverified + assert client._get_client(None, ignore_tls_errors=False) is verified + assert client._get_client(None, ignore_tls_errors=True) is unverified + + +@pytest.mark.parametrize( + 'client_class', + [ + pytest.param(ImpitHttpClient, id='impit'), + pytest.param(HttpxHttpClient, id='httpx'), + pytest.param(CurlImpersonateHttpClient, id='curl'), + ], +) +@pytest.mark.parametrize( + ('request_flag', 'proxy_flag', 'expected_flag'), + [ + pytest.param(False, False, False, id='verified'), + pytest.param(True, False, True, id='request'), + pytest.param(False, True, True, id='proxy'), + ], +) +async def test_ignore_tls_errors_reaches_fetch( + monkeypatch: pytest.MonkeyPatch, + client_class: type[HttpClient], + *, + request_flag: bool, + proxy_flag: bool, + expected_flag: bool, +) -> None: + """Test that `send_request` asks `_fetch` to ignore TLS errors when the call or its proxy sets the flag.""" + url = 'https://example.com/' + response = Mock(spec=HttpResponse, status_code=200, headers={}, url=url, headers_set_cookie=[]) + fetch = AsyncMock(return_value=response) + client = client_class() + monkeypatch.setattr(client, '_fetch', fetch) + proxy_info = ProxyInfo( + url='http://proxy.com:8000', + scheme='http', + hostname='proxy.com', + port=8000, + ignore_tls_errors=proxy_flag, + ) + + await client.send_request(url, proxy_info=proxy_info, ignore_tls_errors=request_flag) + + assert fetch.call_args.kwargs['ignore_tls_errors'] is expected_flag diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index c817e18ac5..994c9ccc3b 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -1,11 +1,14 @@ from __future__ import annotations import logging +import ssl from typing import Any +from unittest.mock import AsyncMock, Mock import httpx2 import pytest +from crawlee import HttpHeaders from crawlee.http_clients import HttpxHttpClient @@ -22,8 +25,43 @@ async def test_transport_kwargs_do_not_reach_the_client(client_kwargs: dict[str, client = HttpxHttpClient(**client_kwargs) async with client: - assert client._get_client(None)._mounts == {} - assert client._get_client(None)._transport is not client_kwargs.get('transport') + assert client._get_client(None, ignore_tls_errors=False)._mounts == {} + assert client._get_client(None, ignore_tls_errors=False)._transport is not client_kwargs.get('transport') + + +@pytest.mark.parametrize( + ('client_flag', 'request_flag', 'expected_verify_mode'), + [ + pytest.param(False, False, ssl.CERT_REQUIRED, id='verified'), + pytest.param(True, False, ssl.CERT_NONE, id='client'), + pytest.param(False, True, ssl.CERT_NONE, id='request'), + ], +) +async def test_ignore_tls_errors_turns_off_verification( + monkeypatch: pytest.MonkeyPatch, + *, + client_flag: bool, + request_flag: bool, + expected_verify_mode: ssl.VerifyMode, +) -> None: + """Test that the `httpx2` transport skips verification when the client or the request ignores TLS errors.""" + transport_class = Mock() + transport_class.return_value.handle_async_request = AsyncMock(return_value=httpx2.Response(200)) + monkeypatch.setattr(httpx2, 'AsyncHTTPTransport', transport_class) + + client = HttpxHttpClient(ignore_tls_errors=client_flag) + + await client._fetch( + 'https://example.com/', + method='GET', + headers=HttpHeaders(), + payload=None, + proxy_info=None, + timeout=None, + ignore_tls_errors=request_flag, + ) + + assert transport_class.call_args.kwargs['verify'].verify_mode == expected_verify_mode def test_silences_httpx_request_logging(monkeypatch: pytest.MonkeyPatch) -> None: