From abea615596cbd76a4ffb156678c7e14d6291cfd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Tue, 10 Nov 2026 13:31:51 +0100 Subject: [PATCH 1/2] feat(python): add per-request redirect options and event hooks Redirects are followed by impit instead of reqwest, so they can be configured per request and the hooks can run on every hop. --- impit-node/src/lib.rs | 1 + impit-python/docs/api/request.rst | 9 + impit-python/docs/stub.py | 4 + impit-python/python/impit/__init__.py | 2 + impit-python/python/impit/impit.pyi | 121 +++++++++- impit-python/src/async_client.rs | 114 +++++++--- impit-python/src/client.rs | 122 +++++++--- impit-python/src/errors.rs | 4 + impit-python/src/hooks.rs | 189 ++++++++++++++++ impit-python/src/lib.rs | 11 +- impit-python/src/request.rs | 9 +- impit-python/test/async_client_test.py | 63 ++++++ impit-python/test/basic_client_test.py | 57 +++++ impit/src/errors.rs | 7 +- impit/src/impit.rs | 293 +++++++++++++++++++------ impit/src/request.rs | 25 ++- 16 files changed, 895 insertions(+), 136 deletions(-) create mode 100644 impit-python/docs/api/request.rst create mode 100644 impit-python/src/hooks.rs diff --git a/impit-node/src/lib.rs b/impit-node/src/lib.rs index c3d91e5e..cb152ca1 100644 --- a/impit-node/src/lib.rs +++ b/impit-node/src/lib.rs @@ -140,6 +140,7 @@ impl ImpitWrapper { Either::B(_) => None, }), http3_prior_knowledge: force_http3.unwrap_or_default(), + ..Default::default() }); let method = method.unwrap_or_default(); diff --git a/impit-python/docs/api/request.rst b/impit-python/docs/api/request.rst new file mode 100644 index 00000000..5bb03ff7 --- /dev/null +++ b/impit-python/docs/api/request.rst @@ -0,0 +1,9 @@ +Request +=================== + +.. .. toctree:: + +.. autoclass:: impit.Request + :members: + + diff --git a/impit-python/docs/stub.py b/impit-python/docs/stub.py index ee37bf27..39315337 100644 --- a/impit-python/docs/stub.py +++ b/impit-python/docs/stub.py @@ -6,11 +6,15 @@ class AsyncClient: class Client: pass +class Request: + pass + class Response: pass __all__ = [ "AsyncClient", "Client", + "Request", "Response", ] diff --git a/impit-python/python/impit/__init__.py b/impit-python/python/impit/__init__.py index 71c80b4d..3672ede7 100644 --- a/impit-python/python/impit/__init__.py +++ b/impit-python/python/impit/__init__.py @@ -23,6 +23,7 @@ ReadError, ReadTimeout, RemoteProtocolError, + Request, RequestError, RequestNotRead, Response, @@ -72,6 +73,7 @@ 'ReadError', 'ReadTimeout', 'RemoteProtocolError', + 'Request', 'RequestError', 'RequestNotRead', 'Response', diff --git a/impit-python/python/impit/impit.pyi b/impit-python/python/impit/impit.pyi index 36d77ae7..1d8a9752 100644 --- a/impit-python/python/impit/impit.pyi +++ b/impit-python/python/impit/impit.pyi @@ -6,7 +6,7 @@ from .headers import Headers from . import Browser from typing import Any -from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterable, Iterator from contextlib import AbstractAsyncContextManager, AbstractContextManager @@ -392,6 +392,27 @@ class Response: assert response.is_closed is True """ +class Request: + """Request passed to the ``request`` event hooks (see the ``event_hooks`` parameter of :class:`Client` and :class:`AsyncClient`). + + .. code-block:: python + + def log_request(request: Request) -> None: + print(f'{request.method} {request.url}') + + client = Client(event_hooks={'request': [log_request]}) + """ + + method: str + """HTTP method of the request (e.g., 'GET').""" + + url: str + """URL of the request.""" + + headers: Headers + """Headers of the request. Changes made to them apply to the sent request and the requests of the following redirects.""" + + class Client: """Synchronous HTTP client with browser impersonation capabilities. @@ -471,6 +492,12 @@ class Client: Useful for testing purposes or when you want to bind the client to a specific network interface. Can be an IP address in the format `xxx.xxx.xxx.xxx` (for IPv4) or `ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff` (for IPv6). + event_hooks: + + Functions to call for every request and response, including the ones of followed redirects, as in httpx. + + A dictionary with the ``"request"`` and ``"response"`` keys, each with a list of functions. Request hooks receive + a :class:`Request` and can modify its headers, response hooks receive a :class:`Response`. """ def __enter__(self) -> Client: @@ -494,6 +521,7 @@ class Client: cookies: Cookies | None = None, headers: dict[str, str] | None = None, local_address: str | None = None, + event_hooks: dict[str, list[Callable[..., Any]]] | None = None, ) -> None: """Initialize a synchronous HTTP client. @@ -513,6 +541,7 @@ class Client: headers and are overridden by per-request headers. Matching is case-insensitive. To remove an impersonated header, pass an empty string as the value. local_address: Local address to bind the client to. Useful for testing purposes or when you want to bind the client to a specific network interface. Can be an IP address in the format "xxx.xxx.xxx.xxx" (for IPv4) or "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" (for IPv6). + event_hooks: Functions to call for every request and response, including the ones of followed redirects. """ def get( @@ -523,6 +552,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make a GET request. @@ -533,6 +564,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def post( @@ -543,6 +576,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make a POST request. @@ -553,6 +588,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ @@ -564,6 +601,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make a PUT request. @@ -574,6 +613,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def patch( @@ -584,6 +625,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make a PATCH request. @@ -594,6 +637,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def delete( @@ -604,6 +649,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make a DELETE request. @@ -614,6 +661,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def head( @@ -624,6 +673,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make a HEAD request. @@ -634,6 +685,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def options( @@ -644,6 +697,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an OPTIONS request. @@ -654,6 +709,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def trace( @@ -664,6 +721,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make a TRACE request. @@ -674,6 +733,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def request( @@ -686,6 +747,8 @@ class Client: timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, stream: bool = False, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an HTTP request with the specified method. @@ -698,6 +761,8 @@ class Client: timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol stream: Whether to return a streaming response (default: False) + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def stream( @@ -709,6 +774,8 @@ class Client: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> AbstractContextManager[Response]: """Make a streaming request with the specified method. @@ -731,6 +798,8 @@ class Client: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ @@ -813,6 +882,14 @@ class AsyncClient: Useful for testing purposes or when you want to bind the client to a specific network interface. Can be an IP address in the format `xxx.xxx.xxx.xxx` (for IPv4) or `ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff` (for IPv6). + event_hooks: + + Functions to call for every request and response, including the ones of followed redirects, as in httpx. + + A dictionary with the ``"request"`` and ``"response"`` keys, each with a list of functions. Request hooks receive + a :class:`Request` and can modify its headers, response hooks receive a :class:`Response`. + + The functions have to be coroutine functions (``async def``). """ async def __aenter__(self) -> AsyncClient: @@ -835,6 +912,7 @@ class AsyncClient: cookies: Cookies | None = None, headers: dict[str, str] | None = None, local_address: str | None = None, + event_hooks: dict[str, list[Callable[..., Awaitable[Any]]]] | None = None, ) -> None: """Initialize an asynchronous HTTP client. @@ -854,6 +932,7 @@ class AsyncClient: headers and are overridden by per-request headers. Matching is case-insensitive. To remove an impersonated header, pass an empty string as the value. local_address: Local address to bind the client to. Useful for testing purposes or when you want to bind the client to a specific network interface. Can be an IP address in the format "xxx.xxx.xxx.xxx" (for IPv4) or "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" (for IPv6). + event_hooks: Functions to call for every request and response, including the ones of followed redirects. """ async def get( @@ -864,6 +943,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous GET request. @@ -874,6 +955,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ async def post( @@ -884,6 +967,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous POST request. @@ -894,6 +979,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ @@ -905,6 +992,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous PUT request. @@ -915,6 +1004,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ async def patch( @@ -925,6 +1016,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous PATCH request. @@ -935,6 +1028,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ async def delete( @@ -945,6 +1040,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous DELETE request. @@ -955,6 +1052,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ async def head( @@ -965,6 +1064,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous HEAD request. @@ -975,6 +1076,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ async def options( @@ -985,6 +1088,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous OPTIONS request. @@ -995,6 +1100,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ async def trace( @@ -1005,6 +1112,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous TRACE request. @@ -1015,6 +1124,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ async def request( @@ -1027,6 +1138,8 @@ class AsyncClient: timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, stream: bool = False, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> Response: """Make an asynchronous HTTP request with the specified method. @@ -1039,6 +1152,8 @@ class AsyncClient: timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol stream: Whether to return a streaming response (default: False) + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ def stream( @@ -1050,6 +1165,8 @@ class AsyncClient: headers: dict[str, str] | None = None, timeout: float | str | None = USE_CLIENT_DEFAULT, force_http3: bool | None = None, + follow_redirects: bool | None = None, + max_redirects: int | None = None, ) -> AbstractAsyncContextManager[Response]: """Make an asynchronous streaming request with the specified method. @@ -1072,6 +1189,8 @@ class AsyncClient: headers: HTTP headers for this request. Override both client-level and impersonation headers (case-insensitive). To remove an impersonated header, pass an empty string as the value timeout: Per-request timeout in seconds. Pass ``None`` to disable the timeout entirely. Defaults to ``USE_CLIENT_DEFAULT`` (inherits the client-level timeout). force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects. Defaults to the client's ``follow_redirects``. + max_redirects: Maximum number of redirects to follow. Defaults to the client's ``max_redirects``. """ diff --git a/impit-python/src/async_client.rs b/impit-python/src/async_client.rs index 990936b2..a85d1fd1 100644 --- a/impit-python/src/async_client.rs +++ b/impit-python/src/async_client.rs @@ -1,17 +1,20 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; -use either::{Either, Right}; +use either::{Either, Left, Right}; use impit::{ errors::ImpitError, impit::{Impit, ImpitBuilder}, - request::RequestOptions, + request::{RequestHooks, RequestOptions}, }; use pyo3::{ffi::c_str, prelude::*}; use crate::{ cookies::PythonCookieJar, errors::ImpitPyError, - request::{parse_timeout, to_body, RequestBody, USE_CLIENT_DEFAULT_SENTINEL}, + hooks::EventHooks, + request::{ + parse_timeout, redirect_behavior, to_body, RequestBody, USE_CLIENT_DEFAULT_SENTINEL, + }, response::ImpitPyResponse, }; @@ -19,6 +22,9 @@ use crate::{ pub(crate) struct AsyncClient { impit: Arc>, default_encoding: Option, + follow_redirects: bool, + max_redirects: u16, + event_hooks: EventHooks, } #[pymethods] @@ -41,7 +47,7 @@ impl AsyncClient { } #[new] - #[pyo3(signature = (browser=None, http3=None, proxy=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), verify=None, default_encoding=None, follow_redirects=None, max_redirects=Some(20), cookie_jar=None, cookies=None, headers=None, local_address=None))] + #[pyo3(signature = (browser=None, http3=None, proxy=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), verify=None, default_encoding=None, follow_redirects=None, max_redirects=Some(20), cookie_jar=None, cookies=None, headers=None, local_address=None, event_hooks=None))] pub fn new( py: Python<'_>, browser: Option, @@ -56,6 +62,7 @@ impl AsyncClient { cookies: Option>, headers: Option>, local_address: Option, + event_hooks: Option>>>, ) -> PyResult { let builder = ImpitBuilder::default(); @@ -87,12 +94,9 @@ impl AsyncClient { _ => builder, }; - let builder = match follow_redirects { - Some(true) => builder.with_redirect(impit::impit::RedirectBehavior::FollowRedirect( - max_redirects.unwrap_or(20).into(), - )), - _ => builder.with_redirect(impit::impit::RedirectBehavior::ManualRedirect), - }; + let follow_redirects = follow_redirects.unwrap_or(false); + let max_redirects = max_redirects.unwrap_or(20); + let builder = builder.with_redirect(redirect_behavior(follow_redirects, max_redirects)); let builder = match (cookie_jar, cookies) { (Some(_), Some(_)) => { @@ -126,11 +130,14 @@ impl AsyncClient { Ok(Self { impit: Arc::new(impit), + event_hooks: EventHooks::new(event_hooks, default_encoding.clone()), default_encoding, + follow_redirects, + max_redirects, }) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn get<'python>( &self, py: Python<'python>, @@ -140,6 +147,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { self.request( py, @@ -151,10 +160,12 @@ impl AsyncClient { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn head<'python>( &self, py: Python<'python>, @@ -164,6 +175,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { self.request( py, @@ -175,10 +188,12 @@ impl AsyncClient { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn post<'python>( &self, py: Python<'python>, @@ -188,6 +203,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { self.request( py, @@ -199,10 +216,12 @@ impl AsyncClient { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn patch<'python>( &self, py: Python<'python>, @@ -212,6 +231,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { self.request( py, @@ -223,10 +244,12 @@ impl AsyncClient { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn put<'python>( &self, py: Python<'python>, @@ -236,6 +259,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { self.request( py, @@ -247,10 +272,12 @@ impl AsyncClient { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn delete<'python>( &self, py: Python<'python>, @@ -260,6 +287,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { self.request( py, @@ -271,10 +300,12 @@ impl AsyncClient { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn options<'python>( &self, py: Python<'python>, @@ -284,6 +315,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { self.request( py, @@ -295,10 +328,12 @@ impl AsyncClient { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn trace<'python>( &self, py: Python<'python>, @@ -308,6 +343,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { self.request( py, @@ -319,10 +356,12 @@ impl AsyncClient { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn stream<'python>( &self, py: Python<'python>, @@ -333,6 +372,8 @@ impl AsyncClient { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { let response = self.request( py, @@ -344,6 +385,8 @@ impl AsyncClient { timeout, force_http3, Some(true), + follow_redirects, + max_redirects, )?; let fun: Py = PyModule::from_code( @@ -368,7 +411,7 @@ impl AsyncClient { Ok(wrapped_response.into_bound(py)) } - #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, stream=false))] + #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, stream=false, follow_redirects=None, max_redirects=None))] pub fn request<'python>( &self, py: Python<'python>, @@ -380,6 +423,8 @@ impl AsyncClient { timeout: Option>, force_http3: Option, stream: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { if let Some(content) = content { data = Some(content); @@ -387,12 +432,19 @@ impl AsyncClient { let body = to_body(data, &mut headers)?; + let hooks = self.event_hooks.for_request(py, true)?; + let timeout = parse_timeout(timeout)?; let options = RequestOptions { headers: headers.unwrap_or_default().into_iter().collect(), timeout, http3_prior_knowledge: force_http3.unwrap_or(false), + redirect: Some(redirect_behavior( + follow_redirects.unwrap_or(self.follow_redirects), + max_redirects.unwrap_or(self.max_redirects), + )), + hooks: hooks.clone().map(|hooks| hooks as Arc), }; let method_str = method.to_string(); @@ -400,7 +452,7 @@ impl AsyncClient { let stream_value = stream.unwrap_or(false); let impit = Arc::clone(&self.impit); - crate::exit_gate::future_into_py::<_, ImpitPyResponse>(py, async move { + crate::exit_gate::future_into_py(py, async move { let response = match method_str.to_lowercase().as_str() { "get" => impit.get(url, Some(body), Some(options)).await, "post" => impit.post(url, Some(body), Some(options)).await, @@ -413,14 +465,18 @@ impl AsyncClient { _ => Err(ImpitError::InvalidMethod(method_str.to_string())), }; - match response { - Ok(response) => { - ImpitPyResponse::from_async(response, default_encoding, stream_value) - .await - .map_err(|e| ImpitPyError(e).into()) - } - Err(err) => Err(ImpitPyError(err).into()), - } + let response = ImpitPyResponse::from_async( + response.map_err(ImpitPyError)?, + default_encoding, + stream_value, + ) + .await + .map_err(ImpitPyError)?; + + Ok(match hooks { + Some(hooks) => Right(hooks.on_response(response).await?), + None => Left(response), + }) }) } } diff --git a/impit-python/src/client.rs b/impit-python/src/client.rs index 708589b9..ccd3b93f 100644 --- a/impit-python/src/client.rs +++ b/impit-python/src/client.rs @@ -1,24 +1,30 @@ -use std::{collections::HashMap, time::Duration}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use either::{Either, Right}; use impit::{ errors::ImpitError, impit::{Impit, ImpitBuilder}, - request::RequestOptions, + request::{RequestHooks, RequestOptions}, }; use pyo3::{ffi::c_str, prelude::*}; use crate::{ cookies::PythonCookieJar, errors::ImpitPyError, - request::{parse_timeout, to_body, RequestBody, USE_CLIENT_DEFAULT_SENTINEL}, - response::{self, ImpitPyResponse}, + hooks::EventHooks, + request::{ + parse_timeout, redirect_behavior, to_body, RequestBody, USE_CLIENT_DEFAULT_SENTINEL, + }, + response::ImpitPyResponse, }; #[pyclass] pub(crate) struct Client { impit: Impit, default_encoding: Option, + follow_redirects: bool, + max_redirects: u16, + event_hooks: EventHooks, } #[pymethods] @@ -36,7 +42,7 @@ impl Client { } #[new] - #[pyo3(signature = (browser=None, http3=None, proxy=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), verify=None, default_encoding=None, follow_redirects=None, max_redirects=Some(20), cookie_jar=None, cookies=None, headers=None, local_address=None))] + #[pyo3(signature = (browser=None, http3=None, proxy=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), verify=None, default_encoding=None, follow_redirects=None, max_redirects=Some(20), cookie_jar=None, cookies=None, headers=None, local_address=None, event_hooks=None))] pub fn new( py: Python<'_>, browser: Option, @@ -51,6 +57,7 @@ impl Client { cookies: Option>, headers: Option>, local_address: Option, + event_hooks: Option>>>, ) -> PyResult { let builder = ImpitBuilder::default(); @@ -84,12 +91,9 @@ impl Client { _ => builder, }; - let builder = match follow_redirects { - Some(true) => builder.with_redirect(impit::impit::RedirectBehavior::FollowRedirect( - max_redirects.unwrap_or(20).into(), - )), - _ => builder.with_redirect(impit::impit::RedirectBehavior::ManualRedirect), - }; + let follow_redirects = follow_redirects.unwrap_or(false); + let max_redirects = max_redirects.unwrap_or(20); + let builder = builder.with_redirect(redirect_behavior(follow_redirects, max_redirects)); let builder = match headers { Some(headers) => builder.with_headers(headers.into_iter().collect::>()), @@ -122,12 +126,15 @@ impl Client { Ok(Self { impit, + event_hooks: EventHooks::new(event_hooks, default_encoding.clone()), default_encoding, + follow_redirects, + max_redirects, }) }) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn get( &self, py: Python<'_>, @@ -137,7 +144,9 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { self.request( py, "get", @@ -148,10 +157,12 @@ impl Client { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn head( &self, py: Python<'_>, @@ -161,7 +172,9 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { self.request( py, "head", @@ -172,10 +185,12 @@ impl Client { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn post( &self, py: Python<'_>, @@ -185,7 +200,9 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { self.request( py, "post", @@ -196,10 +213,12 @@ impl Client { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn patch( &self, py: Python<'_>, @@ -209,7 +228,9 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { self.request( py, "patch", @@ -220,10 +241,12 @@ impl Client { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn put( &self, py: Python<'_>, @@ -233,7 +256,9 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { self.request( py, "put", @@ -244,10 +269,12 @@ impl Client { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn delete( &self, py: Python<'_>, @@ -257,7 +284,9 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { self.request( py, "delete", @@ -268,10 +297,12 @@ impl Client { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn options( &self, py: Python<'_>, @@ -281,7 +312,9 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { self.request( py, "options", @@ -292,10 +325,12 @@ impl Client { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn trace( &self, py: Python<'_>, @@ -305,7 +340,9 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { self.request( py, "trace", @@ -316,10 +353,12 @@ impl Client { timeout, force_http3, Some(false), + follow_redirects, + max_redirects, ) } - #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false))] + #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, follow_redirects=None, max_redirects=None))] pub fn stream<'python>( &self, py: Python<'python>, @@ -330,6 +369,8 @@ impl Client { headers: Option>, timeout: Option>, force_http3: Option, + follow_redirects: Option, + max_redirects: Option, ) -> Result, PyErr> { let response = self.request( py, @@ -341,6 +382,8 @@ impl Client { timeout, force_http3, Some(true), + follow_redirects, + max_redirects, )?; let fun: Py = PyModule::from_code( @@ -365,7 +408,7 @@ impl Client { Ok(wrapped_response.into_bound(py)) } - #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, stream=false))] + #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), force_http3=false, stream=false, follow_redirects=None, max_redirects=None))] pub fn request( &self, py: Python<'_>, @@ -377,7 +420,9 @@ impl Client { timeout: Option>, force_http3: Option, stream: Option, - ) -> Result { + follow_redirects: Option, + max_redirects: Option, + ) -> PyResult> { if let Some(content) = content { data = Some(content); } @@ -385,6 +430,8 @@ impl Client { let body = to_body(data, &mut headers) .map_err(|e| ImpitPyError(ImpitError::BindingPassthroughError(e.to_string())))?; + let hooks = self.event_hooks.for_request(py, false)?; + let timeout = parse_timeout(timeout) .map_err(|e| ImpitPyError(ImpitError::BindingPassthroughError(e.to_string())))?; @@ -392,9 +439,14 @@ impl Client { headers: headers.unwrap_or_default().into_iter().collect(), timeout, http3_prior_knowledge: force_http3.unwrap_or(false), + redirect: Some(redirect_behavior( + follow_redirects.unwrap_or(self.follow_redirects), + max_redirects.unwrap_or(self.max_redirects), + )), + hooks: hooks.clone().map(|hooks| hooks as Arc), }; - py.detach(|| { + let response = py.detach(|| { pyo3_async_runtimes::tokio::get_runtime().block_on(async { let response = match method.to_lowercase().as_str() { "get" => self.impit.get(url, Some(body), Some(options)).await, @@ -419,6 +471,12 @@ impl Client { Err(err) => Err(ImpitPyError(err)), } }) - }) + })?; + + let response = Py::new(py, response)?; + if let Some(hooks) = hooks { + hooks.on_response_sync(py, &response)?; + } + Ok(response) } } diff --git a/impit-python/src/errors.rs b/impit-python/src/errors.rs index 66c60974..cd8fa387 100644 --- a/impit-python/src/errors.rs +++ b/impit-python/src/errors.rs @@ -112,6 +112,10 @@ impl From for pyo3::PyErr { ImpitPyError(ImpitError::InvalidHeaderValue(_)) => { LocalProtocolError::new_err(format!("{}", err.0)) } + ImpitPyError(ImpitError::HookError(err)) => match err.downcast::() { + Ok(err) => *err, + Err(err) => HTTPError::new_err(err.to_string()), + }, _ => HTTPError::new_err(format!("{}", err.0)), } } diff --git a/impit-python/src/hooks.rs b/impit-python/src/hooks.rs new file mode 100644 index 00000000..f01ea66f --- /dev/null +++ b/impit-python/src/hooks.rs @@ -0,0 +1,189 @@ +use std::{collections::HashMap, sync::Arc}; + +use futures::future::BoxFuture; +use impit::{ + errors::ImpitError, + request::{ImpitRequest, RequestHooks}, +}; +use pyo3::{prelude::*, types::PyBytes}; +use pyo3_async_runtimes::TaskLocals; +use reqwest::{ + header::{HeaderName, HeaderValue}, + Response, +}; + +use crate::response::ImpitPyResponse; + +/// The request passed to the `request` event hooks. Changes to its `headers` apply to the sent request. +#[pyclass(frozen)] +pub(crate) struct Request { + #[pyo3(get)] + method: String, + #[pyo3(get)] + url: String, + #[pyo3(get)] + headers: Py, +} + +/// The `event_hooks` of a client, called for every request and response of a redirect chain. +#[derive(Debug)] +pub(crate) struct EventHooks { + request: Vec>, + response: Vec>, + default_encoding: Option, + /// Set for `AsyncClient`, whose hooks are awaited. + locals: Option, +} + +impl EventHooks { + pub(crate) fn new( + hooks: Option>>>, + default_encoding: Option, + ) -> Self { + let mut hooks = hooks.unwrap_or_default(); + Self { + request: hooks.remove("request").unwrap_or_default(), + response: hooks.remove("response").unwrap_or_default(), + default_encoding, + locals: None, + } + } + + /// Returns the hooks for a single request, or `None` if there are no hooks. + pub(crate) fn for_request( + &self, + py: Python<'_>, + is_async: bool, + ) -> PyResult>> { + if self.request.is_empty() && self.response.is_empty() { + return Ok(None); + } + + Ok(Some(Arc::new(Self { + request: self.request.iter().map(|hook| hook.clone_ref(py)).collect(), + response: self + .response + .iter() + .map(|hook| hook.clone_ref(py)) + .collect(), + default_encoding: self.default_encoding.clone(), + locals: match is_async { + true => Some(TaskLocals::with_running_loop(py)?.copy_context(py)?), + false => None, + }, + }))) + } + + async fn call(&self, hooks: &[Py], argument: &Py) -> PyResult<()> { + for hook in hooks { + let result = Python::attach(|py| hook.call1(py, (argument,)))?; + if let Some(locals) = &self.locals { + Python::attach(|py| { + pyo3_async_runtimes::into_future_with_locals(locals, result.into_bound(py)) + })? + .await?; + } + } + Ok(()) + } + + /// Runs the `response` hooks of `Client` with the final response. + pub(crate) fn on_response_sync( + &self, + py: Python<'_>, + response: &Py, + ) -> PyResult<()> { + for hook in &self.response { + hook.call1(py, (response,))?; + } + Ok(()) + } + + /// Runs the `response` hooks of `AsyncClient` with the final response. + pub(crate) async fn on_response( + &self, + response: ImpitPyResponse, + ) -> PyResult> { + let response = Python::attach(|py| Py::new(py, response))?; + self.call(&self.response, response.as_any()).await?; + Ok(response) + } +} + +fn hook_error(err: PyErr) -> ImpitError { + ImpitError::HookError(Box::new(err)) +} + +impl RequestHooks for EventHooks { + fn on_request<'a>( + &'a self, + request: &'a mut ImpitRequest, + ) -> BoxFuture<'a, Result<(), ImpitError>> { + Box::pin(async move { + if self.request.is_empty() { + return Ok(()); + } + + let py_request = Python::attach(|py| { + let headers: Vec<_> = request + .headers + .iter() + .map(|(name, value)| { + ( + PyBytes::new(py, name.as_ref()), + PyBytes::new(py, value.as_bytes()), + ) + }) + .collect(); + Py::new( + py, + Request { + method: request.method.to_string(), + url: request.url.to_string(), + headers: py + .import("impit.headers")? + .getattr("Headers")? + .call1((headers,))? + .unbind(), + }, + ) + }) + .map_err(hook_error)?; + + self.call(&self.request, py_request.as_any()) + .await + .map_err(hook_error)?; + + let headers: Vec<(Vec, Vec)> = + Python::attach(|py| py_request.get().headers.bind(py).getattr("raw")?.extract()) + .map_err(hook_error)?; + request.headers = headers + .into_iter() + .map(|(name, value)| { + let name = HeaderName::from_bytes(&name).map_err(|_| { + ImpitError::InvalidHeaderName(String::from_utf8_lossy(&name).into_owned()) + })?; + let value = HeaderValue::from_bytes(&value).map_err(|_| { + ImpitError::InvalidHeaderValue(String::from_utf8_lossy(&value).into_owned()) + })?; + Ok((name, value)) + }) + .collect::>()?; + + Ok(()) + }) + } + + fn on_redirect(&self, response: Response) -> BoxFuture<'_, Result<(), ImpitError>> { + Box::pin(async move { + if self.response.is_empty() { + return Ok(()); + } + + let response = + ImpitPyResponse::from_async(response, self.default_encoding.clone(), false).await?; + self.on_response(response).await.map_err(hook_error)?; + Ok(()) + }) + } +} diff --git a/impit-python/src/lib.rs b/impit-python/src/lib.rs index 2617c6ba..248a3b23 100644 --- a/impit-python/src/lib.rs +++ b/impit-python/src/lib.rs @@ -7,6 +7,7 @@ mod cookies; mod errors; mod exit_gate; mod fingerprint; +mod hooks; mod request; mod response; @@ -20,6 +21,7 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; exit_gate::register(m)?; @@ -106,10 +108,10 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { follow_redirects: Option, max_redirects: Option, proxy: Option, - ) -> PyResult { - let client = Client::new(_py, None, None, proxy, Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), None, None, follow_redirects, max_redirects, cookie_jar, cookies, None, None); + ) -> PyResult> { + let client = Client::new(_py, None, None, proxy, Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), None, None, follow_redirects, max_redirects, cookie_jar, cookies, None, None, None); - Ok(client?.$name(_py, url, content, data, headers, timeout, force_http3)?) + client?.$name(_py, url, content, data, headers, timeout, force_http3, None, None) } m.add_function(wrap_pyfunction!($name, m)?)?; @@ -150,6 +152,7 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { cookies, None, None, + None, ); client?.stream( @@ -161,6 +164,8 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { headers, timeout, force_http3, + None, + None, ) } diff --git a/impit-python/src/request.rs b/impit-python/src/request.rs index 1e919f3b..dfbea315 100644 --- a/impit-python/src/request.rs +++ b/impit-python/src/request.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, io, time::Duration}; use bytes::Bytes; use either::{Either, Left, Right}; use futures::{stream, Stream}; -use impit::request::ImpitBody; +use impit::{impit::RedirectBehavior, request::ImpitBody}; use pyo3::{ exceptions::{PyStopAsyncIteration, PyStopIteration, PyTypeError}, types::{PyAnyMethods, PyMapping}, @@ -41,6 +41,13 @@ pub(crate) fn parse_timeout( } } +pub(crate) fn redirect_behavior(follow_redirects: bool, max_redirects: u16) -> RedirectBehavior { + match follow_redirects { + true => RedirectBehavior::FollowRedirect(max_redirects.into()), + false => RedirectBehavior::ManualRedirect, + } +} + use pyo3::FromPyObject; #[derive(FromPyObject)] diff --git a/impit-python/test/async_client_test.py b/impit-python/test/async_client_test.py index f1432985..bb70231d 100644 --- a/impit-python/test/async_client_test.py +++ b/impit-python/test/async_client_test.py @@ -13,7 +13,10 @@ Browser, Cookies, HTTPError, + HTTPStatusError, RemoteProtocolError, + Request, + Response, StreamClosed, StreamConsumed, TooManyRedirects, @@ -465,6 +468,66 @@ async def test_limit_redirects(self, browser: Browser) -> None: with pytest.raises(TooManyRedirects): await impit.get(redirect_url) + @pytest.mark.asyncio + async def test_per_request_redirect_options(self, browser: Browser) -> None: + redirect_url = get_httpbin_url('/absolute-redirect/3') + + impit = AsyncClient(browser=browser) + assert (await impit.get(redirect_url)).status_code == 302 + assert (await impit.get(redirect_url, follow_redirects=True)).status_code == 200 + async with impit.stream('GET', redirect_url, follow_redirects=True) as response: + assert response.status_code == 200 + with pytest.raises(TooManyRedirects): + await impit.request('GET', redirect_url, follow_redirects=True, max_redirects=2) + + impit = AsyncClient(browser=browser, follow_redirects=True) + assert (await impit.get(redirect_url, follow_redirects=False)).status_code == 302 + + @pytest.mark.asyncio + async def test_event_hooks(self, browser: Browser) -> None: + target_url = get_httpbin_url('/headers') + redirect_url = get_httpbin_url('/redirect-to', query={'url': target_url}) + urls: list[str] = [] + status_codes: list[int] = [] + + async def on_request(request: Request) -> None: + urls.append(request.url) + request.headers['x-hop'] = str(len(urls)) + + async def on_response(response: Response) -> None: + status_codes.append(response.status_code) + + impit = AsyncClient( + browser=browser, + follow_redirects=True, + event_hooks={'request': [on_request], 'response': [on_response]}, + ) + response = await impit.get(redirect_url) + + assert urls == [redirect_url, target_url] + assert status_codes == [302, 200] + assert response.json()['headers']['X-Hop'] == '2' + + @pytest.mark.asyncio + async def test_event_hook_errors_propagate(self, browser: Browser) -> None: + async def fail_on_get(request: Request) -> None: + if request.url.endswith('/get'): + raise ValueError(request.url) + + async def raise_for_status(response: Response) -> None: + response.raise_for_status() + + impit = AsyncClient( + browser=browser, + follow_redirects=True, + event_hooks={'request': [fail_on_get], 'response': [raise_for_status]}, + ) + + with pytest.raises(ValueError, match='/get'): + await impit.get(get_httpbin_url('/redirect-to', query={'url': get_httpbin_url('/get')})) + with pytest.raises(HTTPStatusError): + await impit.get(get_httpbin_url('/redirect-to', query={'url': get_httpbin_url('/status/404')})) + @pytest.mark.asyncio async def test_async_socket_server(self, browser: Browser) -> None: port_holder = [0] diff --git a/impit-python/test/basic_client_test.py b/impit-python/test/basic_client_test.py index 7707ba26..e43511fe 100644 --- a/impit-python/test/basic_client_test.py +++ b/impit-python/test/basic_client_test.py @@ -15,8 +15,11 @@ ConnectTimeout, Cookies, HTTPError, + HTTPStatusError, ReadTimeout, RemoteProtocolError, + Request, + Response, StreamClosed, StreamConsumed, TimeoutException, @@ -408,6 +411,60 @@ def test_limit_redirects(self, browser: Browser) -> None: with pytest.raises(TooManyRedirects): impit.get(redirect_url) + def test_per_request_redirect_options(self, browser: Browser) -> None: + redirect_url = get_httpbin_url('/absolute-redirect/3') + + impit = Client(browser=browser) + assert impit.get(redirect_url).status_code == 302 + assert impit.get(redirect_url, follow_redirects=True).status_code == 200 + with impit.stream('GET', redirect_url, follow_redirects=True) as response: + assert response.status_code == 200 + with pytest.raises(TooManyRedirects): + impit.request('GET', redirect_url, follow_redirects=True, max_redirects=2) + + impit = Client(browser=browser, follow_redirects=True) + assert impit.get(redirect_url, follow_redirects=False).status_code == 302 + + def test_event_hooks(self, browser: Browser) -> None: + target_url = get_httpbin_url('/headers') + redirect_url = get_httpbin_url('/redirect-to', query={'url': target_url}) + urls: list[str] = [] + status_codes: list[int] = [] + + def on_request(request: Request) -> None: + urls.append(request.url) + request.headers['x-hop'] = str(len(urls)) + + def on_response(response: Response) -> None: + status_codes.append(response.status_code) + + impit = Client( + browser=browser, + follow_redirects=True, + event_hooks={'request': [on_request], 'response': [on_response]}, + ) + response = impit.get(redirect_url) + + assert urls == [redirect_url, target_url] + assert status_codes == [302, 200] + assert response.json()['headers']['X-Hop'] == '2' + + def test_event_hook_errors_propagate(self, browser: Browser) -> None: + def fail_on_get(request: Request) -> None: + if request.url.endswith('/get'): + raise ValueError(request.url) + + impit = Client( + browser=browser, + follow_redirects=True, + event_hooks={'request': [fail_on_get], 'response': [Response.raise_for_status]}, + ) + + with pytest.raises(ValueError, match='/get'): + impit.get(get_httpbin_url('/redirect-to', query={'url': get_httpbin_url('/get')})) + with pytest.raises(HTTPStatusError): + impit.get(get_httpbin_url('/redirect-to', query={'url': get_httpbin_url('/status/404')})) + def test_thread_server(self, browser: Browser) -> None: port_holder = [0] thread = threading.Thread(target=thread_server, args=(port_holder,)) diff --git a/impit/src/errors.rs b/impit/src/errors.rs index 97d01570..0e188600 100644 --- a/impit/src/errors.rs +++ b/impit/src/errors.rs @@ -6,7 +6,6 @@ use thiserror::Error; #[derive(Default)] pub struct ErrorContext { pub timeout: Option, - pub max_redirects: Option, pub method: Option, pub protocol: Option, pub url: Option, @@ -96,6 +95,8 @@ pub enum ImpitError { InvalidHeaderValue(String), #[error("The internal HTTP library has thrown an error:\n{0}")] ReqwestError(String), + #[error("{0}")] + HookError(Box), } impl From for ImpitError { @@ -118,10 +119,6 @@ impl ImpitError { return ImpitError::TimeoutException(context.timeout.map(|x| x.as_millis())); } - if error.is_redirect() { - return ImpitError::TooManyRedirects(context.max_redirects); - } - if (error.is_body() || error.is_decode()) && (format!("{:?}", error).to_lowercase()).contains("unexpectedeof") { diff --git a/impit/src/impit.rs b/impit/src/impit.rs index 9a8164cc..5179226b 100644 --- a/impit/src/impit.rs +++ b/impit/src/impit.rs @@ -2,8 +2,20 @@ use tokio::sync::RwLock; use log::{debug, warn}; -use reqwest::{cookie::CookieStore, header::HeaderMap, Method, Response, Version}; -use std::{fmt::Debug, net::IpAddr, sync::Arc, time::Duration}; +use reqwest::{ + cookie::CookieStore, + header::{ + HeaderValue, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE, COOKIE, + LOCATION, PROXY_AUTHORIZATION, REFERER, TRANSFER_ENCODING, WWW_AUTHENTICATE, + }, + Method, Response, StatusCode, Version, +}; +use std::{ + fmt::Debug, + net::IpAddr, + sync::Arc, + time::{Duration, Instant}, +}; use url::Url; #[cfg(feature = "http3")] @@ -12,7 +24,7 @@ use crate::{ errors::{ErrorContext, ImpitError}, fingerprint::BrowserFingerprint, http_headers::HttpHeadersBuilder, - request::{ImpitBody, ImpitRequest, RequestOptions}, + request::{ImpitBody, ImpitRequest, RequestHooks, RequestOptions}, tls, }; @@ -30,13 +42,6 @@ pub struct Impit { config: ImpitBuilder, } -struct PreparedRequest { - method: Method, - url: Url, - headers: HeaderMap, - body: ImpitBody, -} - impl Default for Impit { fn default() -> Self { ImpitBuilder::::default().build().unwrap() @@ -320,14 +325,8 @@ impl Impit { client = client.local_address(ip_addr); } - match config.redirect { - RedirectBehavior::FollowRedirect(max) => { - client = client.redirect(reqwest::redirect::Policy::limited(max)); - } - RedirectBehavior::ManualRedirect => { - client = client.redirect(reqwest::redirect::Policy::none()); - } - } + // Redirects are followed by `Impit::send`, so they can be configured per request. + client = client.redirect(reqwest::redirect::Policy::none()); client .build() @@ -460,36 +459,34 @@ impl Impit { async fn execute_request( &self, client: &reqwest::Client, - prepared: &mut PreparedRequest, - timeout: Option, + request: &mut ImpitRequest, + timeout: Duration, h3: bool, ) -> Result { let mut req = client - .request(prepared.method.clone(), prepared.url.clone()) - .headers(prepared.headers.clone()); + .request(request.method.clone(), request.url.clone()) + .headers(request.headers.clone()) + .timeout(timeout); if h3 { req = req.version(Version::HTTP_3); } - if let Some(t) = timeout { - req = req.timeout(t); - } - - if let Some(body) = prepared.body.take() { + if let Some(body) = request.body.take() { req = req.body(body); } req.send().await } - async fn send( + /// Sends a single request of the redirect chain. `remaining` is the time left from `timeout`. + async fn send_once( &self, - request: ImpitRequest, - timeout: Option, - http3_prior_knowledge: Option, + request: &mut ImpitRequest, + timeout: Duration, + remaining: Duration, + http3_prior_knowledge: bool, ) -> Result { - let http3_prior_knowledge = http3_prior_knowledge.unwrap_or(false); if http3_prior_knowledge && self.config.max_http_version < Version::HTTP_3 { return Err(ImpitError::Http3Disabled); } @@ -506,21 +503,7 @@ impl Impit { &self.base_client }; - let max_redirects = match self.config.redirect { - RedirectBehavior::FollowRedirect(max) => max, - RedirectBehavior::ManualRedirect => 0, - }; - - let mut prepared = PreparedRequest { - method: request.method, - url: request.url, - headers: request.headers, - body: request.body, - }; - - let primary_result = self - .execute_request(client, &mut prepared, timeout, h3) - .await; + let primary_result = self.execute_request(client, request, remaining, h3).await; let response = match primary_result { Ok(resp) => resp, @@ -528,28 +511,27 @@ impl Impit { let primary_error = ImpitError::from( err, Some(ErrorContext { - timeout: Some(timeout.unwrap_or(self.config.request_timeout)), - max_redirects: Some(max_redirects), - method: Some(prepared.method.to_string()), - protocol: Some(prepared.url.scheme().to_string()), - url: Some(prepared.url.to_string()), + timeout: Some(timeout), + method: Some(request.method.to_string()), + protocol: Some(request.url.scheme().to_string()), + url: Some(request.url.to_string()), }), ); let fallback_client = self .vanilla_client .as_ref() - .filter(|_| primary_error.is_connect_error() && prepared.body.is_sendable()); + .filter(|_| primary_error.is_connect_error() && request.body.is_sendable()); let Some(vanilla_client) = fallback_client else { return Err(primary_error); }; debug!( "Primary request to {} failed with {primary_error}, retrying with vanilla client", - prepared.url + request.url ); match self - .execute_request(vanilla_client, &mut prepared, timeout, false) + .execute_request(vanilla_client, request, remaining, false) .await { Ok(resp) => resp, @@ -562,7 +544,7 @@ impl Impit { if !h3 { let engine_guard = self.h3_engine.read().await; if let Some(h3_engine) = engine_guard.as_ref() { - let host = prepared.url.host_str().unwrap_or_default(); + let host = request.url.host_str().unwrap_or_default(); h3_engine.set_h3_support(host, false).await; if let Some(alt_svc) = response.headers().get("Alt-Svc") { @@ -581,6 +563,46 @@ impl Impit { Ok(response) } + async fn send( + &self, + mut request: ImpitRequest, + timeout: Option, + http3_prior_knowledge: bool, + redirect: &RedirectBehavior, + hooks: Option<&dyn RequestHooks>, + ) -> Result { + let timeout = timeout.unwrap_or(self.config.request_timeout); + let start = Instant::now(); + let mut redirects = 0; + + loop { + if let Some(hooks) = hooks { + hooks.on_request(&mut request).await?; + } + + // The timeout bounds the whole redirect chain, not each request. + let remaining = timeout.saturating_sub(start.elapsed()); + let response = self + .send_once(&mut request, timeout, remaining, http3_prior_knowledge) + .await?; + + let RedirectBehavior::FollowRedirect(max_redirects) = *redirect else { + return Ok(response); + }; + if !follow_redirect(&mut request, &response)? { + return Ok(response); + } + if redirects == max_redirects { + return Err(ImpitError::TooManyRedirects(Some(max_redirects))); + } + redirects += 1; + + if let Some(hooks) = hooks { + hooks.on_redirect(response).await?; + } + } + } + async fn make_request( &self, method: Method, @@ -600,9 +622,17 @@ impl Impit { Some(None) => Some(Duration::MAX), Some(Some(d)) => Some(d), }; - let http3_prior_knowledge = request_options.http3_prior_knowledge; - self.send(request, timeout, Some(http3_prior_knowledge)) - .await + self.send( + request, + timeout, + request_options.http3_prior_knowledge, + request_options + .redirect + .as_ref() + .unwrap_or(&self.config.redirect), + request_options.hooks.as_deref(), + ) + .await } /// Makes a `GET` request to the specified URL. @@ -733,12 +763,151 @@ impl Impit { } } +/// Turns `request` into the request following the redirect in `response`, the same way `reqwest` does. +/// +/// Returns `false` and leaves `request` as is if `response` isn't a redirect that can be followed. +fn follow_redirect(request: &mut ImpitRequest, response: &Response) -> Result { + let drop_body = match response.status() { + StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND => request.method == Method::POST, + StatusCode::SEE_OTHER => true, + StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT => false, + _ => return Ok(false), + }; + + if !drop_body && !request.body.is_sendable() { + return Ok(false); + } + + let Some(url) = response + .headers() + .get(LOCATION) + .and_then(|location| std::str::from_utf8(location.as_bytes()).ok()) + .and_then(|location| request.url.join(location).ok()) + else { + return Ok(false); + }; + + if !matches!(url.scheme(), "http" | "https") { + return Err(ImpitError::UrlProtocolError(url.scheme().to_string())); + } + + if drop_body { + if request.method != Method::HEAD { + request.method = Method::GET; + } + request.body = ImpitBody::Empty; + for header in [ + CONTENT_TYPE, + CONTENT_LENGTH, + CONTENT_ENCODING, + TRANSFER_ENCODING, + ] { + request.headers.remove(header); + } + } + + let previous = &request.url; + if url.host_str() != previous.host_str() + || url.port_or_known_default() != previous.port_or_known_default() + || url.scheme() != previous.scheme() + { + for header in [AUTHORIZATION, COOKIE, PROXY_AUTHORIZATION, WWW_AUTHENTICATE] { + request.headers.remove(header); + } + request.headers.remove("cookie2"); + } + + if !(url.scheme() == "http" && previous.scheme() == "https") { + let mut referer = previous.clone(); + let _ = referer.set_username(""); + let _ = referer.set_password(None); + referer.set_fragment(None); + if let Ok(referer) = HeaderValue::from_str(referer.as_str()) { + request.headers.insert(REFERER, referer); + } + } + + request.url = url; + Ok(true) +} + #[cfg(test)] mod tests { - use reqwest::cookie::Jar; + use reqwest::{cookie::Jar, header::HeaderMap, Method, Response}; + + use super::{follow_redirect, Impit}; + use crate::{ + errors::ImpitError, + request::{ImpitBody, ImpitRequest, RequestOptions}, + }; + + fn request(method: Method, url: &str, headers: &[(&str, &str)]) -> ImpitRequest { + let headers: HeaderMap = headers + .iter() + .map(|(name, value)| (name.parse().unwrap(), value.parse().unwrap())) + .collect(); + ImpitRequest { + url: url.parse().unwrap(), + body: ImpitBody::from("body"), + headers, + method, + } + } - use super::Impit; - use crate::{errors::ImpitError, request::RequestOptions}; + fn redirect(status: u16, location: &str) -> Response { + hyper::Response::builder() + .status(status) + .header("location", location) + .body("") + .unwrap() + .into() + } + + #[test] + fn found_redirects_post_as_get() { + let mut request = request( + Method::POST, + "https://example.com/form", + &[("content-type", "text/plain")], + ); + + assert!(follow_redirect(&mut request, &redirect(302, "/done")).unwrap()); + assert_eq!(request.method, Method::GET); + assert_eq!(request.url.as_str(), "https://example.com/done"); + assert!(request.body.take().is_none()); + assert!(request.headers.get("content-type").is_none()); + assert_eq!(request.headers["referer"], "https://example.com/form"); + } + + #[test] + fn cross_origin_redirects_drop_credentials() { + let mut request = request( + Method::PUT, + "https://example.com/", + &[ + ("authorization", "secret"), + ("cookie", "a=b"), + ("x-custom", "1"), + ], + ); + + assert!(follow_redirect(&mut request, &redirect(307, "https://example.org/")).unwrap()); + assert_eq!(request.method, Method::PUT); + assert!(request.body.take().is_some()); + assert!(request.headers.get("authorization").is_none()); + assert!(request.headers.get("cookie").is_none()); + assert_eq!(request.headers["x-custom"], "1"); + } + + #[test] + fn sent_streamed_bodies_are_not_replayed() { + let mut request = request(Method::POST, "https://example.com/", &[]); + request.body = ImpitBody::Consumed; + + assert!(!follow_redirect(&mut request, &redirect(307, "/next")).unwrap()); + assert_eq!(request.url.as_str(), "https://example.com/"); + assert!(follow_redirect(&mut request, &redirect(303, "/next")).unwrap()); + } async fn get_with_http3_prior_knowledge(impit: &Impit) -> Result<(), ImpitError> { let options = RequestOptions { diff --git a/impit/src/request.rs b/impit/src/request.rs index 12e878aa..a1d6eed1 100644 --- a/impit/src/request.rs +++ b/impit/src/request.rs @@ -1,10 +1,12 @@ -use std::time::Duration; +use std::{fmt::Debug, sync::Arc, time::Duration}; use bytes::Bytes; -use futures_core::TryStream; -use reqwest::{header::HeaderMap, Method}; +use futures_core::{future::BoxFuture, TryStream}; +use reqwest::{header::HeaderMap, Method, Response}; use url::Url; +use crate::{errors::ImpitError, impit::RedirectBehavior}; + /// A struct that holds the request options. /// /// Unlike the [`ImpitBuilder`](crate::impit::ImpitBuilder) struct, these options are specific to a single request. @@ -25,6 +27,23 @@ pub struct RequestOptions { /// /// If [`ImpitBuilder::with_http3`](crate::impit::ImpitBuilder::with_http3) wasn't called, this option will cause [`ErrorType::Http3Disabled`](crate::impit::ErrorType::Http3Disabled) errors. pub http3_prior_knowledge: bool, + /// Overrides the redirect behavior set with [`ImpitBuilder::with_redirect`](crate::impit::ImpitBuilder::with_redirect) for this request. + pub redirect: Option, + /// Callbacks invoked for every request and redirect response of the redirect chain. + pub hooks: Option>, +} + +/// Callbacks invoked by [`Impit`](crate::impit::Impit) while sending a request and following its redirects. +pub trait RequestHooks: Debug + Send + Sync { + /// Called before each request of the redirect chain is sent. Changes made to `request` apply to the sent request + /// and are carried over to the following redirects. + fn on_request<'a>( + &'a self, + request: &'a mut ImpitRequest, + ) -> BoxFuture<'a, Result<(), ImpitError>>; + + /// Called with each redirect response that is followed. The last response of the chain is returned to the caller instead. + fn on_redirect(&self, response: Response) -> BoxFuture<'_, Result<(), ImpitError>>; } /// The body of a request. From d3d50c7d4350cdd981d83f54e6232fda8be4b040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Tue, 10 Nov 2026 13:52:33 +0100 Subject: [PATCH 2/2] refactor(python): drop the event hooks --- impit-python/docs/api/request.rst | 9 -- impit-python/docs/stub.py | 4 - impit-python/python/impit/__init__.py | 2 - impit-python/python/impit/impit.pyi | 41 +----- impit-python/src/async_client.rs | 35 ++--- impit-python/src/client.rs | 43 ++---- impit-python/src/errors.rs | 4 - impit-python/src/hooks.rs | 189 ------------------------- impit-python/src/lib.rs | 9 +- impit-python/test/async_client_test.py | 48 ------- impit-python/test/basic_client_test.py | 43 ------ impit/src/errors.rs | 2 - impit/src/impit.rs | 87 +++++++----- impit/src/request.rs | 23 +-- 14 files changed, 83 insertions(+), 456 deletions(-) delete mode 100644 impit-python/docs/api/request.rst delete mode 100644 impit-python/src/hooks.rs diff --git a/impit-python/docs/api/request.rst b/impit-python/docs/api/request.rst deleted file mode 100644 index 5bb03ff7..00000000 --- a/impit-python/docs/api/request.rst +++ /dev/null @@ -1,9 +0,0 @@ -Request -=================== - -.. .. toctree:: - -.. autoclass:: impit.Request - :members: - - diff --git a/impit-python/docs/stub.py b/impit-python/docs/stub.py index 39315337..ee37bf27 100644 --- a/impit-python/docs/stub.py +++ b/impit-python/docs/stub.py @@ -6,15 +6,11 @@ class AsyncClient: class Client: pass -class Request: - pass - class Response: pass __all__ = [ "AsyncClient", "Client", - "Request", "Response", ] diff --git a/impit-python/python/impit/__init__.py b/impit-python/python/impit/__init__.py index 3672ede7..71c80b4d 100644 --- a/impit-python/python/impit/__init__.py +++ b/impit-python/python/impit/__init__.py @@ -23,7 +23,6 @@ ReadError, ReadTimeout, RemoteProtocolError, - Request, RequestError, RequestNotRead, Response, @@ -73,7 +72,6 @@ 'ReadError', 'ReadTimeout', 'RemoteProtocolError', - 'Request', 'RequestError', 'RequestNotRead', 'Response', diff --git a/impit-python/python/impit/impit.pyi b/impit-python/python/impit/impit.pyi index 1d8a9752..4305cbf8 100644 --- a/impit-python/python/impit/impit.pyi +++ b/impit-python/python/impit/impit.pyi @@ -6,7 +6,7 @@ from .headers import Headers from . import Browser from typing import Any -from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterable, Iterator +from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator from contextlib import AbstractAsyncContextManager, AbstractContextManager @@ -392,27 +392,6 @@ class Response: assert response.is_closed is True """ -class Request: - """Request passed to the ``request`` event hooks (see the ``event_hooks`` parameter of :class:`Client` and :class:`AsyncClient`). - - .. code-block:: python - - def log_request(request: Request) -> None: - print(f'{request.method} {request.url}') - - client = Client(event_hooks={'request': [log_request]}) - """ - - method: str - """HTTP method of the request (e.g., 'GET').""" - - url: str - """URL of the request.""" - - headers: Headers - """Headers of the request. Changes made to them apply to the sent request and the requests of the following redirects.""" - - class Client: """Synchronous HTTP client with browser impersonation capabilities. @@ -492,12 +471,6 @@ class Client: Useful for testing purposes or when you want to bind the client to a specific network interface. Can be an IP address in the format `xxx.xxx.xxx.xxx` (for IPv4) or `ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff` (for IPv6). - event_hooks: - - Functions to call for every request and response, including the ones of followed redirects, as in httpx. - - A dictionary with the ``"request"`` and ``"response"`` keys, each with a list of functions. Request hooks receive - a :class:`Request` and can modify its headers, response hooks receive a :class:`Response`. """ def __enter__(self) -> Client: @@ -521,7 +494,6 @@ class Client: cookies: Cookies | None = None, headers: dict[str, str] | None = None, local_address: str | None = None, - event_hooks: dict[str, list[Callable[..., Any]]] | None = None, ) -> None: """Initialize a synchronous HTTP client. @@ -541,7 +513,6 @@ class Client: headers and are overridden by per-request headers. Matching is case-insensitive. To remove an impersonated header, pass an empty string as the value. local_address: Local address to bind the client to. Useful for testing purposes or when you want to bind the client to a specific network interface. Can be an IP address in the format "xxx.xxx.xxx.xxx" (for IPv4) or "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" (for IPv6). - event_hooks: Functions to call for every request and response, including the ones of followed redirects. """ def get( @@ -882,14 +853,6 @@ class AsyncClient: Useful for testing purposes or when you want to bind the client to a specific network interface. Can be an IP address in the format `xxx.xxx.xxx.xxx` (for IPv4) or `ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff` (for IPv6). - event_hooks: - - Functions to call for every request and response, including the ones of followed redirects, as in httpx. - - A dictionary with the ``"request"`` and ``"response"`` keys, each with a list of functions. Request hooks receive - a :class:`Request` and can modify its headers, response hooks receive a :class:`Response`. - - The functions have to be coroutine functions (``async def``). """ async def __aenter__(self) -> AsyncClient: @@ -912,7 +875,6 @@ class AsyncClient: cookies: Cookies | None = None, headers: dict[str, str] | None = None, local_address: str | None = None, - event_hooks: dict[str, list[Callable[..., Awaitable[Any]]]] | None = None, ) -> None: """Initialize an asynchronous HTTP client. @@ -932,7 +894,6 @@ class AsyncClient: headers and are overridden by per-request headers. Matching is case-insensitive. To remove an impersonated header, pass an empty string as the value. local_address: Local address to bind the client to. Useful for testing purposes or when you want to bind the client to a specific network interface. Can be an IP address in the format "xxx.xxx.xxx.xxx" (for IPv4) or "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" (for IPv6). - event_hooks: Functions to call for every request and response, including the ones of followed redirects. """ async def get( diff --git a/impit-python/src/async_client.rs b/impit-python/src/async_client.rs index a85d1fd1..96fdfa60 100644 --- a/impit-python/src/async_client.rs +++ b/impit-python/src/async_client.rs @@ -1,17 +1,16 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; -use either::{Either, Left, Right}; +use either::{Either, Right}; use impit::{ errors::ImpitError, impit::{Impit, ImpitBuilder}, - request::{RequestHooks, RequestOptions}, + request::RequestOptions, }; use pyo3::{ffi::c_str, prelude::*}; use crate::{ cookies::PythonCookieJar, errors::ImpitPyError, - hooks::EventHooks, request::{ parse_timeout, redirect_behavior, to_body, RequestBody, USE_CLIENT_DEFAULT_SENTINEL, }, @@ -24,7 +23,6 @@ pub(crate) struct AsyncClient { default_encoding: Option, follow_redirects: bool, max_redirects: u16, - event_hooks: EventHooks, } #[pymethods] @@ -47,7 +45,7 @@ impl AsyncClient { } #[new] - #[pyo3(signature = (browser=None, http3=None, proxy=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), verify=None, default_encoding=None, follow_redirects=None, max_redirects=Some(20), cookie_jar=None, cookies=None, headers=None, local_address=None, event_hooks=None))] + #[pyo3(signature = (browser=None, http3=None, proxy=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), verify=None, default_encoding=None, follow_redirects=None, max_redirects=Some(20), cookie_jar=None, cookies=None, headers=None, local_address=None))] pub fn new( py: Python<'_>, browser: Option, @@ -62,7 +60,6 @@ impl AsyncClient { cookies: Option>, headers: Option>, local_address: Option, - event_hooks: Option>>>, ) -> PyResult { let builder = ImpitBuilder::default(); @@ -130,7 +127,6 @@ impl AsyncClient { Ok(Self { impit: Arc::new(impit), - event_hooks: EventHooks::new(event_hooks, default_encoding.clone()), default_encoding, follow_redirects, max_redirects, @@ -432,8 +428,6 @@ impl AsyncClient { let body = to_body(data, &mut headers)?; - let hooks = self.event_hooks.for_request(py, true)?; - let timeout = parse_timeout(timeout)?; let options = RequestOptions { @@ -444,7 +438,6 @@ impl AsyncClient { follow_redirects.unwrap_or(self.follow_redirects), max_redirects.unwrap_or(self.max_redirects), )), - hooks: hooks.clone().map(|hooks| hooks as Arc), }; let method_str = method.to_string(); @@ -452,7 +445,7 @@ impl AsyncClient { let stream_value = stream.unwrap_or(false); let impit = Arc::clone(&self.impit); - crate::exit_gate::future_into_py(py, async move { + crate::exit_gate::future_into_py::<_, ImpitPyResponse>(py, async move { let response = match method_str.to_lowercase().as_str() { "get" => impit.get(url, Some(body), Some(options)).await, "post" => impit.post(url, Some(body), Some(options)).await, @@ -465,18 +458,14 @@ impl AsyncClient { _ => Err(ImpitError::InvalidMethod(method_str.to_string())), }; - let response = ImpitPyResponse::from_async( - response.map_err(ImpitPyError)?, - default_encoding, - stream_value, - ) - .await - .map_err(ImpitPyError)?; - - Ok(match hooks { - Some(hooks) => Right(hooks.on_response(response).await?), - None => Left(response), - }) + match response { + Ok(response) => { + ImpitPyResponse::from_async(response, default_encoding, stream_value) + .await + .map_err(|e| ImpitPyError(e).into()) + } + Err(err) => Err(ImpitPyError(err).into()), + } }) } } diff --git a/impit-python/src/client.rs b/impit-python/src/client.rs index ccd3b93f..364564c9 100644 --- a/impit-python/src/client.rs +++ b/impit-python/src/client.rs @@ -1,21 +1,20 @@ -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{collections::HashMap, time::Duration}; use either::{Either, Right}; use impit::{ errors::ImpitError, impit::{Impit, ImpitBuilder}, - request::{RequestHooks, RequestOptions}, + request::RequestOptions, }; use pyo3::{ffi::c_str, prelude::*}; use crate::{ cookies::PythonCookieJar, errors::ImpitPyError, - hooks::EventHooks, request::{ parse_timeout, redirect_behavior, to_body, RequestBody, USE_CLIENT_DEFAULT_SENTINEL, }, - response::ImpitPyResponse, + response::{self, ImpitPyResponse}, }; #[pyclass] @@ -24,7 +23,6 @@ pub(crate) struct Client { default_encoding: Option, follow_redirects: bool, max_redirects: u16, - event_hooks: EventHooks, } #[pymethods] @@ -42,7 +40,7 @@ impl Client { } #[new] - #[pyo3(signature = (browser=None, http3=None, proxy=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), verify=None, default_encoding=None, follow_redirects=None, max_redirects=Some(20), cookie_jar=None, cookies=None, headers=None, local_address=None, event_hooks=None))] + #[pyo3(signature = (browser=None, http3=None, proxy=None, timeout=Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), verify=None, default_encoding=None, follow_redirects=None, max_redirects=Some(20), cookie_jar=None, cookies=None, headers=None, local_address=None))] pub fn new( py: Python<'_>, browser: Option, @@ -57,7 +55,6 @@ impl Client { cookies: Option>, headers: Option>, local_address: Option, - event_hooks: Option>>>, ) -> PyResult { let builder = ImpitBuilder::default(); @@ -126,7 +123,6 @@ impl Client { Ok(Self { impit, - event_hooks: EventHooks::new(event_hooks, default_encoding.clone()), default_encoding, follow_redirects, max_redirects, @@ -146,7 +142,7 @@ impl Client { force_http3: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { self.request( py, "get", @@ -174,7 +170,7 @@ impl Client { force_http3: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { self.request( py, "head", @@ -202,7 +198,7 @@ impl Client { force_http3: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { self.request( py, "post", @@ -230,7 +226,7 @@ impl Client { force_http3: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { self.request( py, "patch", @@ -258,7 +254,7 @@ impl Client { force_http3: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { self.request( py, "put", @@ -286,7 +282,7 @@ impl Client { force_http3: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { self.request( py, "delete", @@ -314,7 +310,7 @@ impl Client { force_http3: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { self.request( py, "options", @@ -342,7 +338,7 @@ impl Client { force_http3: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { self.request( py, "trace", @@ -422,7 +418,7 @@ impl Client { stream: Option, follow_redirects: Option, max_redirects: Option, - ) -> PyResult> { + ) -> Result { if let Some(content) = content { data = Some(content); } @@ -430,8 +426,6 @@ impl Client { let body = to_body(data, &mut headers) .map_err(|e| ImpitPyError(ImpitError::BindingPassthroughError(e.to_string())))?; - let hooks = self.event_hooks.for_request(py, false)?; - let timeout = parse_timeout(timeout) .map_err(|e| ImpitPyError(ImpitError::BindingPassthroughError(e.to_string())))?; @@ -443,10 +437,9 @@ impl Client { follow_redirects.unwrap_or(self.follow_redirects), max_redirects.unwrap_or(self.max_redirects), )), - hooks: hooks.clone().map(|hooks| hooks as Arc), }; - let response = py.detach(|| { + py.detach(|| { pyo3_async_runtimes::tokio::get_runtime().block_on(async { let response = match method.to_lowercase().as_str() { "get" => self.impit.get(url, Some(body), Some(options)).await, @@ -471,12 +464,6 @@ impl Client { Err(err) => Err(ImpitPyError(err)), } }) - })?; - - let response = Py::new(py, response)?; - if let Some(hooks) = hooks { - hooks.on_response_sync(py, &response)?; - } - Ok(response) + }) } } diff --git a/impit-python/src/errors.rs b/impit-python/src/errors.rs index cd8fa387..66c60974 100644 --- a/impit-python/src/errors.rs +++ b/impit-python/src/errors.rs @@ -112,10 +112,6 @@ impl From for pyo3::PyErr { ImpitPyError(ImpitError::InvalidHeaderValue(_)) => { LocalProtocolError::new_err(format!("{}", err.0)) } - ImpitPyError(ImpitError::HookError(err)) => match err.downcast::() { - Ok(err) => *err, - Err(err) => HTTPError::new_err(err.to_string()), - }, _ => HTTPError::new_err(format!("{}", err.0)), } } diff --git a/impit-python/src/hooks.rs b/impit-python/src/hooks.rs deleted file mode 100644 index f01ea66f..00000000 --- a/impit-python/src/hooks.rs +++ /dev/null @@ -1,189 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use futures::future::BoxFuture; -use impit::{ - errors::ImpitError, - request::{ImpitRequest, RequestHooks}, -}; -use pyo3::{prelude::*, types::PyBytes}; -use pyo3_async_runtimes::TaskLocals; -use reqwest::{ - header::{HeaderName, HeaderValue}, - Response, -}; - -use crate::response::ImpitPyResponse; - -/// The request passed to the `request` event hooks. Changes to its `headers` apply to the sent request. -#[pyclass(frozen)] -pub(crate) struct Request { - #[pyo3(get)] - method: String, - #[pyo3(get)] - url: String, - #[pyo3(get)] - headers: Py, -} - -/// The `event_hooks` of a client, called for every request and response of a redirect chain. -#[derive(Debug)] -pub(crate) struct EventHooks { - request: Vec>, - response: Vec>, - default_encoding: Option, - /// Set for `AsyncClient`, whose hooks are awaited. - locals: Option, -} - -impl EventHooks { - pub(crate) fn new( - hooks: Option>>>, - default_encoding: Option, - ) -> Self { - let mut hooks = hooks.unwrap_or_default(); - Self { - request: hooks.remove("request").unwrap_or_default(), - response: hooks.remove("response").unwrap_or_default(), - default_encoding, - locals: None, - } - } - - /// Returns the hooks for a single request, or `None` if there are no hooks. - pub(crate) fn for_request( - &self, - py: Python<'_>, - is_async: bool, - ) -> PyResult>> { - if self.request.is_empty() && self.response.is_empty() { - return Ok(None); - } - - Ok(Some(Arc::new(Self { - request: self.request.iter().map(|hook| hook.clone_ref(py)).collect(), - response: self - .response - .iter() - .map(|hook| hook.clone_ref(py)) - .collect(), - default_encoding: self.default_encoding.clone(), - locals: match is_async { - true => Some(TaskLocals::with_running_loop(py)?.copy_context(py)?), - false => None, - }, - }))) - } - - async fn call(&self, hooks: &[Py], argument: &Py) -> PyResult<()> { - for hook in hooks { - let result = Python::attach(|py| hook.call1(py, (argument,)))?; - if let Some(locals) = &self.locals { - Python::attach(|py| { - pyo3_async_runtimes::into_future_with_locals(locals, result.into_bound(py)) - })? - .await?; - } - } - Ok(()) - } - - /// Runs the `response` hooks of `Client` with the final response. - pub(crate) fn on_response_sync( - &self, - py: Python<'_>, - response: &Py, - ) -> PyResult<()> { - for hook in &self.response { - hook.call1(py, (response,))?; - } - Ok(()) - } - - /// Runs the `response` hooks of `AsyncClient` with the final response. - pub(crate) async fn on_response( - &self, - response: ImpitPyResponse, - ) -> PyResult> { - let response = Python::attach(|py| Py::new(py, response))?; - self.call(&self.response, response.as_any()).await?; - Ok(response) - } -} - -fn hook_error(err: PyErr) -> ImpitError { - ImpitError::HookError(Box::new(err)) -} - -impl RequestHooks for EventHooks { - fn on_request<'a>( - &'a self, - request: &'a mut ImpitRequest, - ) -> BoxFuture<'a, Result<(), ImpitError>> { - Box::pin(async move { - if self.request.is_empty() { - return Ok(()); - } - - let py_request = Python::attach(|py| { - let headers: Vec<_> = request - .headers - .iter() - .map(|(name, value)| { - ( - PyBytes::new(py, name.as_ref()), - PyBytes::new(py, value.as_bytes()), - ) - }) - .collect(); - Py::new( - py, - Request { - method: request.method.to_string(), - url: request.url.to_string(), - headers: py - .import("impit.headers")? - .getattr("Headers")? - .call1((headers,))? - .unbind(), - }, - ) - }) - .map_err(hook_error)?; - - self.call(&self.request, py_request.as_any()) - .await - .map_err(hook_error)?; - - let headers: Vec<(Vec, Vec)> = - Python::attach(|py| py_request.get().headers.bind(py).getattr("raw")?.extract()) - .map_err(hook_error)?; - request.headers = headers - .into_iter() - .map(|(name, value)| { - let name = HeaderName::from_bytes(&name).map_err(|_| { - ImpitError::InvalidHeaderName(String::from_utf8_lossy(&name).into_owned()) - })?; - let value = HeaderValue::from_bytes(&value).map_err(|_| { - ImpitError::InvalidHeaderValue(String::from_utf8_lossy(&value).into_owned()) - })?; - Ok((name, value)) - }) - .collect::>()?; - - Ok(()) - }) - } - - fn on_redirect(&self, response: Response) -> BoxFuture<'_, Result<(), ImpitError>> { - Box::pin(async move { - if self.response.is_empty() { - return Ok(()); - } - - let response = - ImpitPyResponse::from_async(response, self.default_encoding.clone(), false).await?; - self.on_response(response).await.map_err(hook_error)?; - Ok(()) - }) - } -} diff --git a/impit-python/src/lib.rs b/impit-python/src/lib.rs index 248a3b23..4a2ecf3e 100644 --- a/impit-python/src/lib.rs +++ b/impit-python/src/lib.rs @@ -7,7 +7,6 @@ mod cookies; mod errors; mod exit_gate; mod fingerprint; -mod hooks; mod request; mod response; @@ -21,7 +20,6 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; exit_gate::register(m)?; @@ -108,10 +106,10 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { follow_redirects: Option, max_redirects: Option, proxy: Option, - ) -> PyResult> { - let client = Client::new(_py, None, None, proxy, Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), None, None, follow_redirects, max_redirects, cookie_jar, cookies, None, None, None); + ) -> PyResult { + let client = Client::new(_py, None, None, proxy, Some(Right(USE_CLIENT_DEFAULT_SENTINEL)), None, None, follow_redirects, max_redirects, cookie_jar, cookies, None, None); - client?.$name(_py, url, content, data, headers, timeout, force_http3, None, None) + Ok(client?.$name(_py, url, content, data, headers, timeout, force_http3, None, None)?) } m.add_function(wrap_pyfunction!($name, m)?)?; @@ -152,7 +150,6 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { cookies, None, None, - None, ); client?.stream( diff --git a/impit-python/test/async_client_test.py b/impit-python/test/async_client_test.py index bb70231d..e4ea4e50 100644 --- a/impit-python/test/async_client_test.py +++ b/impit-python/test/async_client_test.py @@ -13,10 +13,7 @@ Browser, Cookies, HTTPError, - HTTPStatusError, RemoteProtocolError, - Request, - Response, StreamClosed, StreamConsumed, TooManyRedirects, @@ -483,51 +480,6 @@ async def test_per_request_redirect_options(self, browser: Browser) -> None: impit = AsyncClient(browser=browser, follow_redirects=True) assert (await impit.get(redirect_url, follow_redirects=False)).status_code == 302 - @pytest.mark.asyncio - async def test_event_hooks(self, browser: Browser) -> None: - target_url = get_httpbin_url('/headers') - redirect_url = get_httpbin_url('/redirect-to', query={'url': target_url}) - urls: list[str] = [] - status_codes: list[int] = [] - - async def on_request(request: Request) -> None: - urls.append(request.url) - request.headers['x-hop'] = str(len(urls)) - - async def on_response(response: Response) -> None: - status_codes.append(response.status_code) - - impit = AsyncClient( - browser=browser, - follow_redirects=True, - event_hooks={'request': [on_request], 'response': [on_response]}, - ) - response = await impit.get(redirect_url) - - assert urls == [redirect_url, target_url] - assert status_codes == [302, 200] - assert response.json()['headers']['X-Hop'] == '2' - - @pytest.mark.asyncio - async def test_event_hook_errors_propagate(self, browser: Browser) -> None: - async def fail_on_get(request: Request) -> None: - if request.url.endswith('/get'): - raise ValueError(request.url) - - async def raise_for_status(response: Response) -> None: - response.raise_for_status() - - impit = AsyncClient( - browser=browser, - follow_redirects=True, - event_hooks={'request': [fail_on_get], 'response': [raise_for_status]}, - ) - - with pytest.raises(ValueError, match='/get'): - await impit.get(get_httpbin_url('/redirect-to', query={'url': get_httpbin_url('/get')})) - with pytest.raises(HTTPStatusError): - await impit.get(get_httpbin_url('/redirect-to', query={'url': get_httpbin_url('/status/404')})) - @pytest.mark.asyncio async def test_async_socket_server(self, browser: Browser) -> None: port_holder = [0] diff --git a/impit-python/test/basic_client_test.py b/impit-python/test/basic_client_test.py index e43511fe..c94832ec 100644 --- a/impit-python/test/basic_client_test.py +++ b/impit-python/test/basic_client_test.py @@ -15,11 +15,8 @@ ConnectTimeout, Cookies, HTTPError, - HTTPStatusError, ReadTimeout, RemoteProtocolError, - Request, - Response, StreamClosed, StreamConsumed, TimeoutException, @@ -425,46 +422,6 @@ def test_per_request_redirect_options(self, browser: Browser) -> None: impit = Client(browser=browser, follow_redirects=True) assert impit.get(redirect_url, follow_redirects=False).status_code == 302 - def test_event_hooks(self, browser: Browser) -> None: - target_url = get_httpbin_url('/headers') - redirect_url = get_httpbin_url('/redirect-to', query={'url': target_url}) - urls: list[str] = [] - status_codes: list[int] = [] - - def on_request(request: Request) -> None: - urls.append(request.url) - request.headers['x-hop'] = str(len(urls)) - - def on_response(response: Response) -> None: - status_codes.append(response.status_code) - - impit = Client( - browser=browser, - follow_redirects=True, - event_hooks={'request': [on_request], 'response': [on_response]}, - ) - response = impit.get(redirect_url) - - assert urls == [redirect_url, target_url] - assert status_codes == [302, 200] - assert response.json()['headers']['X-Hop'] == '2' - - def test_event_hook_errors_propagate(self, browser: Browser) -> None: - def fail_on_get(request: Request) -> None: - if request.url.endswith('/get'): - raise ValueError(request.url) - - impit = Client( - browser=browser, - follow_redirects=True, - event_hooks={'request': [fail_on_get], 'response': [Response.raise_for_status]}, - ) - - with pytest.raises(ValueError, match='/get'): - impit.get(get_httpbin_url('/redirect-to', query={'url': get_httpbin_url('/get')})) - with pytest.raises(HTTPStatusError): - impit.get(get_httpbin_url('/redirect-to', query={'url': get_httpbin_url('/status/404')})) - def test_thread_server(self, browser: Browser) -> None: port_holder = [0] thread = threading.Thread(target=thread_server, args=(port_holder,)) diff --git a/impit/src/errors.rs b/impit/src/errors.rs index 0e188600..72cdddb3 100644 --- a/impit/src/errors.rs +++ b/impit/src/errors.rs @@ -95,8 +95,6 @@ pub enum ImpitError { InvalidHeaderValue(String), #[error("The internal HTTP library has thrown an error:\n{0}")] ReqwestError(String), - #[error("{0}")] - HookError(Box), } impl From for ImpitError { diff --git a/impit/src/impit.rs b/impit/src/impit.rs index 5179226b..47c71154 100644 --- a/impit/src/impit.rs +++ b/impit/src/impit.rs @@ -5,8 +5,8 @@ use log::{debug, warn}; use reqwest::{ cookie::CookieStore, header::{ - HeaderValue, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE, COOKIE, - LOCATION, PROXY_AUTHORIZATION, REFERER, TRANSFER_ENCODING, WWW_AUTHENTICATE, + HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE, + COOKIE, LOCATION, PROXY_AUTHORIZATION, REFERER, TRANSFER_ENCODING, WWW_AUTHENTICATE, }, Method, Response, StatusCode, Version, }; @@ -24,7 +24,7 @@ use crate::{ errors::{ErrorContext, ImpitError}, fingerprint::BrowserFingerprint, http_headers::HttpHeadersBuilder, - request::{ImpitBody, ImpitRequest, RequestHooks, RequestOptions}, + request::{ImpitBody, ImpitRequest, RequestOptions}, tls, }; @@ -42,6 +42,13 @@ pub struct Impit { config: ImpitBuilder, } +struct PreparedRequest { + method: Method, + url: Url, + headers: HeaderMap, + body: ImpitBody, +} + impl Default for Impit { fn default() -> Self { ImpitBuilder::::default().build().unwrap() @@ -459,20 +466,23 @@ impl Impit { async fn execute_request( &self, client: &reqwest::Client, - request: &mut ImpitRequest, - timeout: Duration, + prepared: &mut PreparedRequest, + timeout: Option, h3: bool, ) -> Result { let mut req = client - .request(request.method.clone(), request.url.clone()) - .headers(request.headers.clone()) - .timeout(timeout); + .request(prepared.method.clone(), prepared.url.clone()) + .headers(prepared.headers.clone()); if h3 { req = req.version(Version::HTTP_3); } - if let Some(body) = request.body.take() { + if let Some(t) = timeout { + req = req.timeout(t); + } + + if let Some(body) = prepared.body.take() { req = req.body(body); } @@ -482,7 +492,7 @@ impl Impit { /// Sends a single request of the redirect chain. `remaining` is the time left from `timeout`. async fn send_once( &self, - request: &mut ImpitRequest, + prepared: &mut PreparedRequest, timeout: Duration, remaining: Duration, http3_prior_knowledge: bool, @@ -493,17 +503,19 @@ impl Impit { let h3 = http3_prior_knowledge || self - .should_use_h3(request.url.host_str().unwrap_or_default()) + .should_use_h3(prepared.url.host_str().unwrap_or_default()) .await; let client = if h3 { - debug!("Using QUIC for request to {}", request.url); + debug!("Using QUIC for request to {}", prepared.url); self.h3_client.as_ref().unwrap_or(&self.base_client) } else { - debug!("{} doesn't seem to have HTTP3 support", request.url); + debug!("{} doesn't seem to have HTTP3 support", prepared.url); &self.base_client }; - let primary_result = self.execute_request(client, request, remaining, h3).await; + let primary_result = self + .execute_request(client, prepared, Some(remaining), h3) + .await; let response = match primary_result { Ok(resp) => resp, @@ -512,26 +524,26 @@ impl Impit { err, Some(ErrorContext { timeout: Some(timeout), - method: Some(request.method.to_string()), - protocol: Some(request.url.scheme().to_string()), - url: Some(request.url.to_string()), + method: Some(prepared.method.to_string()), + protocol: Some(prepared.url.scheme().to_string()), + url: Some(prepared.url.to_string()), }), ); let fallback_client = self .vanilla_client .as_ref() - .filter(|_| primary_error.is_connect_error() && request.body.is_sendable()); + .filter(|_| primary_error.is_connect_error() && prepared.body.is_sendable()); let Some(vanilla_client) = fallback_client else { return Err(primary_error); }; debug!( "Primary request to {} failed with {primary_error}, retrying with vanilla client", - request.url + prepared.url ); match self - .execute_request(vanilla_client, request, remaining, false) + .execute_request(vanilla_client, prepared, Some(remaining), false) .await { Ok(resp) => resp, @@ -544,7 +556,7 @@ impl Impit { if !h3 { let engine_guard = self.h3_engine.read().await; if let Some(h3_engine) = engine_guard.as_ref() { - let host = request.url.host_str().unwrap_or_default(); + let host = prepared.url.host_str().unwrap_or_default(); h3_engine.set_h3_support(host, false).await; if let Some(alt_svc) = response.headers().get("Alt-Svc") { @@ -565,41 +577,39 @@ impl Impit { async fn send( &self, - mut request: ImpitRequest, + request: ImpitRequest, timeout: Option, http3_prior_knowledge: bool, redirect: &RedirectBehavior, - hooks: Option<&dyn RequestHooks>, ) -> Result { let timeout = timeout.unwrap_or(self.config.request_timeout); let start = Instant::now(); let mut redirects = 0; - loop { - if let Some(hooks) = hooks { - hooks.on_request(&mut request).await?; - } + let mut prepared = PreparedRequest { + method: request.method, + url: request.url, + headers: request.headers, + body: request.body, + }; + loop { // The timeout bounds the whole redirect chain, not each request. let remaining = timeout.saturating_sub(start.elapsed()); let response = self - .send_once(&mut request, timeout, remaining, http3_prior_knowledge) + .send_once(&mut prepared, timeout, remaining, http3_prior_knowledge) .await?; let RedirectBehavior::FollowRedirect(max_redirects) = *redirect else { return Ok(response); }; - if !follow_redirect(&mut request, &response)? { + if !follow_redirect(&mut prepared, &response)? { return Ok(response); } if redirects == max_redirects { return Err(ImpitError::TooManyRedirects(Some(max_redirects))); } redirects += 1; - - if let Some(hooks) = hooks { - hooks.on_redirect(response).await?; - } } } @@ -630,7 +640,6 @@ impl Impit { .redirect .as_ref() .unwrap_or(&self.config.redirect), - request_options.hooks.as_deref(), ) .await } @@ -766,7 +775,7 @@ impl Impit { /// Turns `request` into the request following the redirect in `response`, the same way `reqwest` does. /// /// Returns `false` and leaves `request` as is if `response` isn't a redirect that can be followed. -fn follow_redirect(request: &mut ImpitRequest, response: &Response) -> Result { +fn follow_redirect(request: &mut PreparedRequest, response: &Response) -> Result { let drop_body = match response.status() { StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND => request.method == Method::POST, StatusCode::SEE_OTHER => true, @@ -835,18 +844,18 @@ fn follow_redirect(request: &mut ImpitRequest, response: &Response) -> Result ImpitRequest { + fn request(method: Method, url: &str, headers: &[(&str, &str)]) -> PreparedRequest { let headers: HeaderMap = headers .iter() .map(|(name, value)| (name.parse().unwrap(), value.parse().unwrap())) .collect(); - ImpitRequest { + PreparedRequest { url: url.parse().unwrap(), body: ImpitBody::from("body"), headers, diff --git a/impit/src/request.rs b/impit/src/request.rs index a1d6eed1..9799c725 100644 --- a/impit/src/request.rs +++ b/impit/src/request.rs @@ -1,11 +1,11 @@ -use std::{fmt::Debug, sync::Arc, time::Duration}; +use std::time::Duration; use bytes::Bytes; -use futures_core::{future::BoxFuture, TryStream}; -use reqwest::{header::HeaderMap, Method, Response}; +use futures_core::TryStream; +use reqwest::{header::HeaderMap, Method}; use url::Url; -use crate::{errors::ImpitError, impit::RedirectBehavior}; +use crate::impit::RedirectBehavior; /// A struct that holds the request options. /// @@ -29,21 +29,6 @@ pub struct RequestOptions { pub http3_prior_knowledge: bool, /// Overrides the redirect behavior set with [`ImpitBuilder::with_redirect`](crate::impit::ImpitBuilder::with_redirect) for this request. pub redirect: Option, - /// Callbacks invoked for every request and redirect response of the redirect chain. - pub hooks: Option>, -} - -/// Callbacks invoked by [`Impit`](crate::impit::Impit) while sending a request and following its redirects. -pub trait RequestHooks: Debug + Send + Sync { - /// Called before each request of the redirect chain is sent. Changes made to `request` apply to the sent request - /// and are carried over to the following redirects. - fn on_request<'a>( - &'a self, - request: &'a mut ImpitRequest, - ) -> BoxFuture<'a, Result<(), ImpitError>>; - - /// Called with each redirect response that is followed. The last response of the chain is returned to the caller instead. - fn on_redirect(&self, response: Response) -> BoxFuture<'_, Result<(), ImpitError>>; } /// The body of a request.