diff --git a/README.md b/README.md index 9d21a9d..fc70d33 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,33 @@ drive the page through `driver.browsing_context` and `driver.script` with an explicit context, created first as above. Pass `args=["--protocol", "cdp"]` to serve CDP on the same port as well. +## Respecting robots.txt + +The browser can enforce `robots.txt` for you. It is off by default, matching +the `lightpanda` binary's own default, and every wrapper takes browser flags +through `args=`: + +```python +Browser(args=["--obey-robots"]) # also AsyncBrowser +CDPServer(args=["--obey-robots"]) # also BiDiServer, and the async twins +run_script("saved.js", args=["--obey-robots"]) +``` + +A request the site disallows then fails rather than being sent: a tool call +raises `ToolError: navigation failed: RobotsBlocked`, and `run_script` raises +`ScriptError`. + +One thing to know before turning it on, because it is easy to mistake for a +bug: the rule is applied to **every** request, not just the page you asked for. +Plenty of sites disallow the directory their own assets live in, so a page you +are allowed to fetch can load with its scripts and styles missing, and render +blank or empty. That is `robots.txt` being honoured, not a failure — the site +is asking you not to fetch those files. If a page comes back strangely empty +under `--obey-robots`, read the site's `robots.txt` before assuming otherwise. + +Whether a given scrape is acceptable is not a question `robots.txt` alone +answers: a site's terms of service can forbid what its `robots.txt` permits. + ## How the bindings work Every browser tool is a `Session` method, typed and documented in your IDE. diff --git a/lightpanda/__init__.py b/lightpanda/__init__.py index ad09e69..f68d60b 100644 --- a/lightpanda/__init__.py +++ b/lightpanda/__init__.py @@ -24,6 +24,13 @@ Chrome DevTools Protocol server and hands you the endpoint to connect to (see its docs for an example). For Selenium, ``BiDiServer`` serves WebDriver BiDi the same way and hands you the ``command_executor`` URL. + +The browser can enforce ``robots.txt``, which is off by default. Pass +``args=["--obey-robots"]`` to any of these and a disallowed request fails +instead of being sent, surfacing as whatever your client raises: a +:class:`ToolError` here, a navigation error in Playwright or Selenium. Note +that the rule applies to every request, not only the page you asked for, so a +permitted page whose assets sit under a disallowed path will load without them. """ __docformat__ = "google" diff --git a/lightpanda/_methods.py b/lightpanda/_methods.py index e44a9d3..9307c2c 100644 --- a/lightpanda/_methods.py +++ b/lightpanda/_methods.py @@ -80,7 +80,7 @@ def find_element(self, *, role: str | None = None, name: str | None = None) -> A Args: role: Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox'). - name: Optional accessible name substring to match (case-insensitive). + name: Optional accessible name to match, case-insensitive: a substring, or a JavaScript regex literal such as /sign (in|up)/ (unanchored; flags i, m, s, u accepted; case-insensitive even without i, prefix (?-i) to make it case-sensitive). """ return self.call("findElement", role=role, name=name) def get_cookies(self, *, url: str | None = None, all: bool | None = None) -> Any: @@ -335,7 +335,7 @@ async def find_element(self, *, role: str | None = None, name: str | None = None Args: role: Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox'). - name: Optional accessible name substring to match (case-insensitive). + name: Optional accessible name to match, case-insensitive: a substring, or a JavaScript regex literal such as /sign (in|up)/ (unanchored; flags i, m, s, u accepted; case-insensitive even without i, prefix (?-i) to make it case-sensitive). """ return await self.call("findElement", role=role, name=name) async def get_cookies(self, *, url: str | None = None, all: bool | None = None) -> Any: diff --git a/lightpanda/_serve.py b/lightpanda/_serve.py index 39e6cdd..6c9f72d 100644 --- a/lightpanda/_serve.py +++ b/lightpanda/_serve.py @@ -39,8 +39,9 @@ def __init__( bundled in the package, then ``PATH``. env: Extra environment variables for the spawned process. verbose: Let the browser's own logging through to stderr. - args: Extra ``lightpanda serve`` flags; pass ``port=`` rather - than ``--port``. + args: Extra ``lightpanda serve`` flags, e.g. + ``["--obey-robots"]`` to enforce ``robots.txt``; pass + ``port=`` rather than ``--port``. port: Pin the listening port. Defaults to a free one. """ self._proc, self._port = _spawn( diff --git a/lightpanda/async_browser.py b/lightpanda/async_browser.py index 3b45fb5..5408e64 100644 --- a/lightpanda/async_browser.py +++ b/lightpanda/async_browser.py @@ -163,9 +163,12 @@ async def run_script_async( env: dict[str, str] | None = None, binary: str | os.PathLike | None = None, timeout: float | None = None, + args: Sequence[str] = (), ) -> str: """Async variant of :func:`lightpanda.run_script` (runs in a worker thread).""" - return await asyncio.to_thread(run_script, script, env=env, binary=binary, timeout=timeout) + return await asyncio.to_thread( + run_script, script, env=env, binary=binary, timeout=timeout, args=args + ) __all__ = ["AsyncBrowser", "AsyncSession", "run_script_async"] diff --git a/lightpanda/browser.py b/lightpanda/browser.py index 6522b21..9a64fb9 100644 --- a/lightpanda/browser.py +++ b/lightpanda/browser.py @@ -172,6 +172,7 @@ def __init__( raising :class:`ProtocolError`. verbose: Let the browser's own logging through to stderr. args: Extra CLI flags for the spawned browser process, e.g. + ``["--obey-robots"]`` to enforce ``robots.txt``, ``["--http-cache-dir", path]`` or cookie flags. """ self._client = Client(binary=binary, env=env, timeout=timeout, verbose=verbose, args=args) @@ -214,15 +215,18 @@ def run_script( env: dict[str, str] | None = None, binary: str | os.PathLike | None = None, timeout: float | None = None, + args: Sequence[str] = (), ) -> str: """Replay a saved lightpanda script (no LLM) and return its stdout. ``env`` entries (e.g. ``LP_*`` placeholder values) are added to the - child's environment. Raises :class:`ScriptError` on a non-zero exit. + child's environment. ``args`` are extra CLI flags for the browser, e.g. + ``["--obey-robots"]`` to enforce ``robots.txt``. Raises + :class:`ScriptError` on a non-zero exit. """ path = Path(script) proc = subprocess.run( - [str(find_binary(binary)), "run", str(path)], + [str(find_binary(binary)), "run", *args, str(path)], env=os.environ | (env or {}), capture_output=True, text=True, diff --git a/tests/test_browser.py b/tests/test_browser.py index 3386942..29ae81d 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -8,7 +8,7 @@ import pytest from conftest import alive -from lightpanda import Browser, LightpandaError, ToolError, run_script, client +from lightpanda import Browser, LightpandaError, ScriptError, ToolError, run_script, client def test_goto_and_markdown(browser, fixture_url): @@ -117,6 +117,14 @@ def test_run_script(binary, fixture_url, tmp_path): run_script(script, env={"LP_TEST_URL": f"{fixture_url}/index.html"}, binary=binary) +def test_run_script_forwards_args(binary, tmp_path): + # A flag the binary rejects proves `args` reaches it, without needing a network. + script = tmp_path / "unused.js" + script.write_text("const page = new Page();\n") + with pytest.raises(ScriptError): + run_script(script, binary=binary, args=["--not-a-real-flag"]) + + def test_find_binary_skips_own_console_script(tmp_path, monkeypatch): # `uv run` puts the venv's bin dir first on PATH, where the package's # `lightpanda` entry point shadows a real binary further along.