From c8b24aba0dfbc3fae9cfd9488baf8c0f07a928cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 23 Sep 2026 12:29:03 +0200 Subject: [PATCH 1/3] Document robots.txt support, and let run_script pass browser flags The browser has had --obey-robots since before these bindings existed, and nothing here mentioned it, so nobody using the package would find it. It is now a README section and appears in the pdoc reference, on the landing page and on each class that takes args=. The section spends most of its length on the part that surprises people: the rule applies to every request, not just the page you asked for. Sites commonly disallow the directory their own assets live in, so a permitted page can load with its scripts blocked and render blank. That is robots.txt working, and without saying so the obvious conclusion is that the flag is broken. run_script was the one entry point that could not enable it, taking no flags at all; it and run_script_async now accept args= like everything else. Not added: a named obey_robots= parameter. Every other browser flag goes through args=, and promoting one creates a second way to say the same thing plus a conflict to resolve when both are passed. --- README.md | 27 +++++++++++++++++++++++++++ lightpanda/__init__.py | 6 ++++++ lightpanda/_serve.py | 5 +++-- lightpanda/async_browser.py | 5 ++++- lightpanda/browser.py | 8 ++++++-- tests/test_browser.py | 10 +++++++++- 6 files changed, 55 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9d21a9d..fc70d33 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,33 @@ drive the page through `driver.browsing_context` and `driver.script` with an explicit context, created first as above. Pass `args=["--protocol", "cdp"]` to serve CDP on the same port as well. +## Respecting robots.txt + +The browser can enforce `robots.txt` for you. It is off by default, matching +the `lightpanda` binary's own default, and every wrapper takes browser flags +through `args=`: + +```python +Browser(args=["--obey-robots"]) # also AsyncBrowser +CDPServer(args=["--obey-robots"]) # also BiDiServer, and the async twins +run_script("saved.js", args=["--obey-robots"]) +``` + +A request the site disallows then fails rather than being sent: a tool call +raises `ToolError: navigation failed: RobotsBlocked`, and `run_script` raises +`ScriptError`. + +One thing to know before turning it on, because it is easy to mistake for a +bug: the rule is applied to **every** request, not just the page you asked for. +Plenty of sites disallow the directory their own assets live in, so a page you +are allowed to fetch can load with its scripts and styles missing, and render +blank or empty. That is `robots.txt` being honoured, not a failure — the site +is asking you not to fetch those files. If a page comes back strangely empty +under `--obey-robots`, read the site's `robots.txt` before assuming otherwise. + +Whether a given scrape is acceptable is not a question `robots.txt` alone +answers: a site's terms of service can forbid what its `robots.txt` permits. + ## How the bindings work Every browser tool is a `Session` method, typed and documented in your IDE. diff --git a/lightpanda/__init__.py b/lightpanda/__init__.py index ad09e69..1be98b8 100644 --- a/lightpanda/__init__.py +++ b/lightpanda/__init__.py @@ -24,6 +24,12 @@ Chrome DevTools Protocol server and hands you the endpoint to connect to (see its docs for an example). For Selenium, ``BiDiServer`` serves WebDriver BiDi the same way and hands you the ``command_executor`` URL. + +The browser can enforce ``robots.txt``, which is off by default. Pass +``args=["--obey-robots"]`` to any of these, and a disallowed request fails +with :class:`ToolError` instead of being sent. Note that the rule applies to +every request, not only the page you asked for, so a permitted page whose +assets sit under a disallowed path will load without them. """ __docformat__ = "google" diff --git a/lightpanda/_serve.py b/lightpanda/_serve.py index 39e6cdd..6c9f72d 100644 --- a/lightpanda/_serve.py +++ b/lightpanda/_serve.py @@ -39,8 +39,9 @@ def __init__( bundled in the package, then ``PATH``. env: Extra environment variables for the spawned process. verbose: Let the browser's own logging through to stderr. - args: Extra ``lightpanda serve`` flags; pass ``port=`` rather - than ``--port``. + args: Extra ``lightpanda serve`` flags, e.g. + ``["--obey-robots"]`` to enforce ``robots.txt``; pass + ``port=`` rather than ``--port``. port: Pin the listening port. Defaults to a free one. """ self._proc, self._port = _spawn( diff --git a/lightpanda/async_browser.py b/lightpanda/async_browser.py index 3b45fb5..5408e64 100644 --- a/lightpanda/async_browser.py +++ b/lightpanda/async_browser.py @@ -163,9 +163,12 @@ async def run_script_async( env: dict[str, str] | None = None, binary: str | os.PathLike | None = None, timeout: float | None = None, + args: Sequence[str] = (), ) -> str: """Async variant of :func:`lightpanda.run_script` (runs in a worker thread).""" - return await asyncio.to_thread(run_script, script, env=env, binary=binary, timeout=timeout) + return await asyncio.to_thread( + run_script, script, env=env, binary=binary, timeout=timeout, args=args + ) __all__ = ["AsyncBrowser", "AsyncSession", "run_script_async"] diff --git a/lightpanda/browser.py b/lightpanda/browser.py index 6522b21..9a64fb9 100644 --- a/lightpanda/browser.py +++ b/lightpanda/browser.py @@ -172,6 +172,7 @@ def __init__( raising :class:`ProtocolError`. verbose: Let the browser's own logging through to stderr. args: Extra CLI flags for the spawned browser process, e.g. + ``["--obey-robots"]`` to enforce ``robots.txt``, ``["--http-cache-dir", path]`` or cookie flags. """ self._client = Client(binary=binary, env=env, timeout=timeout, verbose=verbose, args=args) @@ -214,15 +215,18 @@ def run_script( env: dict[str, str] | None = None, binary: str | os.PathLike | None = None, timeout: float | None = None, + args: Sequence[str] = (), ) -> str: """Replay a saved lightpanda script (no LLM) and return its stdout. ``env`` entries (e.g. ``LP_*`` placeholder values) are added to the - child's environment. Raises :class:`ScriptError` on a non-zero exit. + child's environment. ``args`` are extra CLI flags for the browser, e.g. + ``["--obey-robots"]`` to enforce ``robots.txt``. Raises + :class:`ScriptError` on a non-zero exit. """ path = Path(script) proc = subprocess.run( - [str(find_binary(binary)), "run", str(path)], + [str(find_binary(binary)), "run", *args, str(path)], env=os.environ | (env or {}), capture_output=True, text=True, diff --git a/tests/test_browser.py b/tests/test_browser.py index 3386942..b0fd70e 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -8,7 +8,7 @@ import pytest from conftest import alive -from lightpanda import Browser, LightpandaError, ToolError, run_script, client +from lightpanda import Browser, LightpandaError, ScriptError, ToolError, run_script, client def test_goto_and_markdown(browser, fixture_url): @@ -117,6 +117,14 @@ def test_run_script(binary, fixture_url, tmp_path): run_script(script, env={"LP_TEST_URL": f"{fixture_url}/index.html"}, binary=binary) +def test_run_script_forwards_args(binary, tmp_path): + # A flag the binary rejects proves `args` reaches it, without needing a network. + script = tmp_path / "visit.js" + script.write_text("const page = new Page();\n") + with pytest.raises(ScriptError): + run_script(script, binary=binary, args=["--not-a-real-flag"]) + + def test_find_binary_skips_own_console_script(tmp_path, monkeypatch): # `uv run` puts the venv's bin dir first on PATH, where the package's # `lightpanda` entry point shadows a real binary further along. From e13ceb8e59cedad7288cff711b8f2c1e509e4b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 23 Sep 2026 12:29:11 +0200 Subject: [PATCH 2/3] Regenerate _methods.py: findElement now documents regex name matching Unrelated to the robots change, but the committed file was stale: the browser gained regex matching for findElement's name argument and the regenerated docstring never landed here. CI regenerates against nightly and fails when this file drifts, so it would have failed on the next push regardless. --- lightpanda/_methods.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lightpanda/_methods.py b/lightpanda/_methods.py index e44a9d3..9307c2c 100644 --- a/lightpanda/_methods.py +++ b/lightpanda/_methods.py @@ -80,7 +80,7 @@ def find_element(self, *, role: str | None = None, name: str | None = None) -> A Args: role: Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox'). - name: Optional accessible name substring to match (case-insensitive). + name: Optional accessible name to match, case-insensitive: a substring, or a JavaScript regex literal such as /sign (in|up)/ (unanchored; flags i, m, s, u accepted; case-insensitive even without i, prefix (?-i) to make it case-sensitive). """ return self.call("findElement", role=role, name=name) def get_cookies(self, *, url: str | None = None, all: bool | None = None) -> Any: @@ -335,7 +335,7 @@ async def find_element(self, *, role: str | None = None, name: str | None = None Args: role: Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox'). - name: Optional accessible name substring to match (case-insensitive). + name: Optional accessible name to match, case-insensitive: a substring, or a JavaScript regex literal such as /sign (in|up)/ (unanchored; flags i, m, s, u accepted; case-insensitive even without i, prefix (?-i) to make it case-sensitive). """ return await self.call("findElement", role=role, name=name) async def get_cookies(self, *, url: str | None = None, all: bool | None = None) -> Any: From 5772f61ba054d5739fdf2f573b1e87441910e725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 23 Sep 2026 12:47:26 +0200 Subject: [PATCH 3/3] Do not name ToolError on the pdoc landing page Two of the four classes that paragraph points at are driven through Playwright or Selenium, where a blocked request surfaces as that client's navigation error, never as a ToolError. Say what happens instead of naming one exception the reader may never see. Also rename the test fixture, which never visits anything: the flag is rejected before the script runs. --- lightpanda/__init__.py | 9 +++++---- tests/test_browser.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lightpanda/__init__.py b/lightpanda/__init__.py index 1be98b8..f68d60b 100644 --- a/lightpanda/__init__.py +++ b/lightpanda/__init__.py @@ -26,10 +26,11 @@ BiDi the same way and hands you the ``command_executor`` URL. The browser can enforce ``robots.txt``, which is off by default. Pass -``args=["--obey-robots"]`` to any of these, and a disallowed request fails -with :class:`ToolError` instead of being sent. Note that the rule applies to -every request, not only the page you asked for, so a permitted page whose -assets sit under a disallowed path will load without them. +``args=["--obey-robots"]`` to any of these and a disallowed request fails +instead of being sent, surfacing as whatever your client raises: a +:class:`ToolError` here, a navigation error in Playwright or Selenium. Note +that the rule applies to every request, not only the page you asked for, so a +permitted page whose assets sit under a disallowed path will load without them. """ __docformat__ = "google" diff --git a/tests/test_browser.py b/tests/test_browser.py index b0fd70e..29ae81d 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -119,7 +119,7 @@ def test_run_script(binary, fixture_url, tmp_path): def test_run_script_forwards_args(binary, tmp_path): # A flag the binary rejects proves `args` reaches it, without needing a network. - script = tmp_path / "visit.js" + script = tmp_path / "unused.js" script.write_text("const page = new Page();\n") with pytest.raises(ScriptError): run_script(script, binary=binary, args=["--not-a-real-flag"])