feat!: Add ignore_tls_errors to HTTP clients, crawlers and ProxyInfo

docs/03_concepts/13_http_clients.mdx

@@ -103,7 +103,7 @@ Crawlee provides an abstract base class, <ApiLink to="class/HttpClient">`HttpCli
 
 To create a custom HTTP client, inherit from <ApiLink to="class/HttpClient">`HttpClient`</ApiLink> and implement two methods:
 
-- `_fetch` sends a single request, through the proxy from `proxy_info` if there is one, and returns the response without reading its body. It doesn't follow redirects or keep cookies of its own. Its `timeout` bounds both receiving the response and reading the body, and without one the default timeout of the underlying library applies.
+- `_fetch` sends a single request, through the proxy from `proxy_info` if there is one, and returns the response without reading its body. It doesn't follow redirects or keep cookies of its own. Its `timeout` bounds both receiving the response and reading the body, and without one the default timeout of the underlying library applies. With `ignore_tls_errors`, it doesn't verify the TLS certificate of the server.
 - `cleanup` closes the connections and other resources the client holds.
 
 The response is a subclass of the abstract <ApiLink to="class/HttpResponse">`HttpResponse`</ApiLink> class. Its body is read on `read` or `read_stream`, and `aclose` releases the connection when the body isn't needed. The base class provides the async context manager, which closes the response on exit.

docs/03_concepts/code_examples/http_clients/parsel_impit_example.py

@@ -9,7 +9,6 @@ async def main() -> None:
         # Optional additional keyword arguments for `impit.AsyncClient`.
         http3=True,
         browser='firefox',
-        verify=True,
     )
 
     crawler = ParselCrawler(

docs/06_upgrading/upgrading_to_v2.md

@@ -411,13 +411,28 @@ crawler = ParselCrawler(save_response_cookies=False)
 
 With `BasicCrawler` and the HTTP crawlers, `context.send_request` always saves the response cookies to the session, unless you pass other cookies in its `cookies` argument. In v1, `persist_cookies_per_session=False` applied to `context.send_request` too. With `PlaywrightCrawler` and its default HTTP client, `context.send_request` stores the response cookies in the browser context, so `save_response_cookies=False` keeps them out of the session as well.
 
+### `ignore_tls_errors` replaces `verify`
+
+`ImpitHttpClient` and `HttpxHttpClient` take `ignore_tls_errors` instead of `verify`, and `CurlImpersonateHttpClient` takes it too. TLS certificates are still verified by default. `HttpxHttpClient` no longer takes a CA bundle or an `SSLContext`.
+
+```python
+# v1
+http_client = ImpitHttpClient(verify=False)
+
+# v2
+http_client = ImpitHttpClient(ignore_tls_errors=True)
+```
+
+To ignore TLS errors for a single call, pass `ignore_tls_errors=True` to `send_request` of the client or to `context.send_request`.
+
 ### Some client arguments are ignored
 
 The following keyword arguments of the underlying libraries are now ignored with a warning:
 
 - `cookies` of every client and `cookie_jar` of `ImpitHttpClient`, since cookies are kept in sessions. To send other cookies with a single call, pass `cookies` to `send_request`.
 - `auth` and `params` of `HttpxHttpClient` and `CurlImpersonateHttpClient`, which would reach every request of a redirect chain. Pass the credentials in the `Authorization` header and the query in the URL instead.
 - `max_redirects` of every client and `allow_redirects` of `CurlImpersonateHttpClient`, since `send_request` follows the redirects.
+- `verify` of every client. For what replaces it, see [`ignore_tls_errors` replaces `verify`](#ignore_tls_errors-replaces-verify).
 
 ### Custom clients implement `_fetch`
 
@@ -439,7 +454,9 @@ class MyHttpClient(HttpClient):
 
 # v2
 class MyHttpClient(HttpClient):
-    async def _fetch(self, url, *, method, headers, payload, proxy_info, timeout): ...
+    async def _fetch(
+        self, url, *, method, headers, payload, proxy_info, timeout, ignore_tls_errors
+    ): ...
 
     async def cleanup(self): ...
 ```

src/crawlee/_types.py

@@ -583,6 +583,7 @@ def __call__(
         headers: HttpHeaders | dict[str, str] | None = None,
         cookies: SessionCookies | None = None,
         timeout: timedelta | None = None,
+        ignore_tls_errors: bool = False,
     ) -> Coroutine[None, None, HttpResponse]:
         """Call send request function.
 
@@ -596,6 +597,7 @@ def __call__(
                 request handler unless `save_response_cookies` is disabled.
             timeout: Maximum time allowed to receive the response, including all redirects, and to read its body.
                 Without it, the default timeout of the HTTP client applies.
+            ignore_tls_errors: Whether to ignore TLS certificate errors for this request.
 
         Returns:
             The HTTP response received from the server.

src/crawlee/browsers/_playwright_browser_controller.py

@@ -287,6 +287,9 @@ async def _create_browser_context(
                 password=proxy_info.password,
             )
 
+            if proxy_info.ignore_tls_errors:
+                filtered_options['ignore_https_errors'] = True
+
         if self._fingerprint_generator:
             return await AsyncNewContext(
                 browser=self._browser,

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -1399,6 +1399,7 @@ async def send_request(
             headers: HttpHeaders | dict[str, str] | None = None,
             cookies: SessionCookies | None = None,
             timeout: timedelta | None = None,
+            ignore_tls_errors: bool = False,
         ) -> HttpResponse:
             response = await self._http_client.send_request(
                 url=url,
@@ -1409,6 +1410,7 @@ async def send_request(
                 cookies=cookies,
                 proxy_info=proxy_info,
                 timeout=timeout,
+                ignore_tls_errors=ignore_tls_errors,
             )
             # A response whose body the handler never read would hold its connection open.
             register_deferred_cleanup(response.aclose)

src/crawlee/crawlers/_playwright/_playwright_http_client.py

@@ -61,6 +61,7 @@ async def send_request(
         cookies: SessionCookies | None = None,
         proxy_info: ProxyInfo | None = None,
         timeout: timedelta | None = None,
+        ignore_tls_errors: bool = False,
     ) -> HttpResponse:
         # TODO: Use `session` to restore all the fingerprint headers according to the `BrowserContext`, after resolved
         # https://github.com/apify/crawlee-python/issues/1055
@@ -88,6 +89,7 @@ async def send_request(
                 cookies=cookies,
                 proxy_info=proxy_info,
                 timeout=timeout,
+                ignore_tls_errors=ignore_tls_errors,
             )
         finally:
             _caller_cookies_var.reset(token)
@@ -102,6 +104,7 @@ async def _fetch(
         payload: HttpPayload | None,
         proxy_info: ProxyInfo | None,
         timeout: timedelta | None,
+        ignore_tls_errors: bool,
     ) -> HttpResponse:
         # `proxy_info` are not used because `APIRequestContext` inherits the proxy from `BrowserContext`
         browser_context = _browser_page_context_var.get()
@@ -121,6 +124,8 @@ async def _fetch(
             headers=fetch_headers or None,
             data=payload,
             timeout=timeout.total_seconds() * 1000 if timeout is not None else None,
+            # Without the flag, the TLS setting of the browser context applies.
+            ignore_https_errors=True if ignore_tls_errors else None,
             # Redirects are followed hop by hop by `send_request`.
             max_redirects=0,
         )

src/crawlee/http_clients/_base.py

@@ -226,6 +226,7 @@ async def send_request(
         cookies: SessionCookies | None = None,
         proxy_info: ProxyInfo | None = None,
         timeout: timedelta | None = None,
+        ignore_tls_errors: bool = False,
     ) -> HttpResponse:
         """Send an HTTP request via the client, following up to 10 redirects unless `follow_redirects` is disabled.
 
@@ -247,6 +248,8 @@ async def send_request(
             proxy_info: The information about the proxy to be used.
             timeout: Maximum time allowed to receive the response, including all redirects, and to read its body.
                 Without it, the default timeout of the HTTP client applies.
+            ignore_tls_errors: Whether to ignore TLS certificate errors for this request. TLS errors are also ignored
+                when the client or `proxy_info` sets `ignore_tls_errors`.
 
         Raises:
             ProxyError: Raised if a proxy-related error occurs.
@@ -264,6 +267,9 @@ async def send_request(
         if cookies is None:
             cookies = session.cookies if session is not None else SessionCookies()
 
+        if proxy_info is not None and proxy_info.ignore_tls_errors:
+            ignore_tls_errors = True
+
         # `encoded=True` keeps the path and the query byte for byte, including a safe `%2F` in query.
         current_url = _with_ascii_host(URL(url, encoded=True))
         deadline = monotonic() + timeout.total_seconds() if timeout is not None else None
@@ -282,6 +288,7 @@ async def send_request(
                     payload=payload,
                     proxy_info=proxy_info,
                     timeout=timedelta(seconds=remaining) if remaining is not None else None,
+                    ignore_tls_errors=ignore_tls_errors,
                 )
 
             try:
@@ -327,6 +334,7 @@ async def _fetch(
         payload: HttpPayload | None,
         proxy_info: ProxyInfo | None,
         timeout: timedelta | None,
+        ignore_tls_errors: bool,
     ) -> HttpResponse:
         """Send a single HTTP request and return the response without reading its body.
 
@@ -341,6 +349,8 @@ async def _fetch(
             proxy_info: The information about the proxy to be used.
             timeout: Maximum time allowed to receive the response and read its body. Without it, the default timeout
                 of the client library applies. `send_request` also cancels the call when the time is up.
+            ignore_tls_errors: Whether to ignore TLS certificate errors, as requested by the caller of `send_request`
+                or by `proxy_info`. A client that ignores them on its own does so even when this is `False`.
 
         Raises:
             ProxyError: Raised if a proxy-related error occurs.

src/crawlee/http_clients/_curl_impersonate.py

@@ -170,25 +170,28 @@ class CurlImpersonateHttpClient(HttpClient):
     def __init__(
         self,
         *,
+        ignore_tls_errors: bool = False,
         follow_redirects: bool = True,
         **async_session_kwargs: Any,
     ) -> None:
         """Initialize a new instance.
 
         Args:
+            ignore_tls_errors: Whether to ignore TLS certificate errors for every request.
             follow_redirects: Whether to follow HTTP redirects.
             async_session_kwargs: Additional keyword arguments for `curl_cffi.requests.AsyncSession`. Some of them are
                 ignored with a warning. Cookies are kept in sessions, so `cookies` is ignored. The `auth` and `params`
                 arguments would reach every request of a redirect chain, so pass credentials in the `Authorization`
                 header and the query in the URL instead. Redirects are followed by `send_request`, so `allow_redirects`
-                and `max_redirects` are ignored.
+                and `max_redirects` are ignored. Use `ignore_tls_errors` instead of `verify`.
         """
         super().__init__(
             follow_redirects=follow_redirects,
             headers=async_session_kwargs.pop('headers', None),
         )
+        self._ignore_tls_errors = ignore_tls_errors
 
-        for ignored_kwarg in ('cookies', 'auth', 'params', 'allow_redirects', 'max_redirects'):
+        for ignored_kwarg in ('cookies', 'auth', 'params', 'allow_redirects', 'max_redirects', 'verify'):
             if async_session_kwargs.pop(ignored_kwarg, None) is not None:
                 warnings.warn(
                     f'The `{ignored_kwarg}` argument of `CurlImpersonateHttpClient` is ignored.',
@@ -198,7 +201,7 @@ def __init__(
 
         self._async_session_kwargs = async_session_kwargs
 
-        self._client_by_proxy_url = dict[str | None, AsyncSession]()
+        self._clients = dict[str | None, AsyncSession]()
 
     @override
     async def _fetch(
@@ -210,6 +213,7 @@ async def _fetch(
         payload: HttpPayload | None,
         proxy_info: ProxyInfo | None,
         timeout: timedelta | None,
+        ignore_tls_errors: bool,
     ) -> _CurlImpersonateResponse:
         client = self._get_client(proxy_info.url if proxy_info else None)
         deadline = asyncio.get_running_loop().time() + timeout.total_seconds() if timeout is not None else None
@@ -228,6 +232,7 @@ async def _fetch(
                 data=payload,
                 # Redirects are followed hop by hop by `send_request`.
                 allow_redirects=False,
+                verify=not (ignore_tls_errors or self._ignore_tls_errors),
                 stream=True,
                 **timeout_kwargs,
             )
@@ -248,7 +253,7 @@ def _get_client(self, proxy_url: str | None) -> AsyncSession:
         Store the new session for future use.
         """
         # Check if a session for the given proxy URL has already been created.
-        if proxy_url not in self._client_by_proxy_url:
+        if proxy_url not in self._clients:
             # Prepare a default kwargs for the new session. A provided proxy URL and a chrome for impersonation
             # are set as default options.
             kwargs: dict[str, Any] = {
@@ -260,9 +265,9 @@ def _get_client(self, proxy_url: str | None) -> AsyncSession:
             kwargs.update(self._async_session_kwargs)
 
             # Create and store the new session with the specified kwargs.
-            self._client_by_proxy_url[proxy_url] = _AsyncSession(**kwargs)
+            self._clients[proxy_url] = _AsyncSession(**kwargs)
 
-        return self._client_by_proxy_url[proxy_url]
+        return self._clients[proxy_url]
 
     def _convert_method(self, method: HttpMethod) -> CurlHttpMethod:
         """Convert from Crawlee HTTP method to curl-cffi HTTP method.
@@ -314,6 +319,6 @@ def _is_proxy_error(error: CurlRequestError) -> bool:
         return False
 
     async def cleanup(self) -> None:
-        for client in self._client_by_proxy_url.values():
+        for client in self._clients.values():
             await client.close()
-        self._client_by_proxy_url.clear()
+        self._clients.clear()