Document robots.txt support, and let run_script pass browser flags

README.md

@@ -118,6 +118,33 @@ drive the page through `driver.browsing_context` and `driver.script` with an
 explicit context, created first as above. Pass `args=["--protocol", "cdp"]`
 to serve CDP on the same port as well.
 
+## Respecting robots.txt
+
+The browser can enforce `robots.txt` for you. It is off by default, matching
+the `lightpanda` binary's own default, and every wrapper takes browser flags
+through `args=`:
+
+```python
+Browser(args=["--obey-robots"])                    # also AsyncBrowser
+CDPServer(args=["--obey-robots"])                  # also BiDiServer, and the async twins
+run_script("saved.js", args=["--obey-robots"])
+```
+
+A request the site disallows then fails rather than being sent: a tool call
+raises `ToolError: navigation failed: RobotsBlocked`, and `run_script` raises
+`ScriptError`.
+
+One thing to know before turning it on, because it is easy to mistake for a
+bug: the rule is applied to **every** request, not just the page you asked for.
+Plenty of sites disallow the directory their own assets live in, so a page you
+are allowed to fetch can load with its scripts and styles missing, and render
+blank or empty. That is `robots.txt` being honoured, not a failure — the site
+is asking you not to fetch those files. If a page comes back strangely empty
+under `--obey-robots`, read the site's `robots.txt` before assuming otherwise.
+
+Whether a given scrape is acceptable is not a question `robots.txt` alone
+answers: a site's terms of service can forbid what its `robots.txt` permits.
+
 ## How the bindings work
 
 Every browser tool is a `Session` method, typed and documented in your IDE.

lightpanda/__init__.py

@@ -24,6 +24,13 @@
 Chrome DevTools Protocol server and hands you the endpoint to connect to
 (see its docs for an example). For Selenium, ``BiDiServer`` serves WebDriver
 BiDi the same way and hands you the ``command_executor`` URL.
+
+The browser can enforce ``robots.txt``, which is off by default. Pass
+``args=["--obey-robots"]`` to any of these and a disallowed request fails
+instead of being sent, surfacing as whatever your client raises: a
+:class:`ToolError` here, a navigation error in Playwright or Selenium. Note
+that the rule applies to every request, not only the page you asked for, so a
+permitted page whose assets sit under a disallowed path will load without them.
 """
 
 __docformat__ = "google"

lightpanda/_methods.py

@@ -80,7 +80,7 @@ def find_element(self, *, role: str | None = None, name: str | None = None) -> A
 
         Args:
             role: Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox').
-            name: Optional accessible name substring to match (case-insensitive).
+            name: Optional accessible name to match, case-insensitive: a substring, or a JavaScript regex literal such as /sign (in|up)/ (unanchored; flags i, m, s, u accepted; case-insensitive even without i, prefix (?-i) to make it case-sensitive).
         """
         return self.call("findElement", role=role, name=name)
     def get_cookies(self, *, url: str | None = None, all: bool | None = None) -> Any:
@@ -335,7 +335,7 @@ async def find_element(self, *, role: str | None = None, name: str | None = None
 
         Args:
             role: Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox').
-            name: Optional accessible name substring to match (case-insensitive).
+            name: Optional accessible name to match, case-insensitive: a substring, or a JavaScript regex literal such as /sign (in|up)/ (unanchored; flags i, m, s, u accepted; case-insensitive even without i, prefix (?-i) to make it case-sensitive).
         """
         return await self.call("findElement", role=role, name=name)
     async def get_cookies(self, *, url: str | None = None, all: bool | None = None) -> Any:

lightpanda/_serve.py

@@ -39,8 +39,9 @@ def __init__(
                 bundled in the package, then ``PATH``.
             env: Extra environment variables for the spawned process.
             verbose: Let the browser's own logging through to stderr.
-            args: Extra ``lightpanda serve`` flags; pass ``port=`` rather
-                than ``--port``.
+            args: Extra ``lightpanda serve`` flags, e.g.
+                ``["--obey-robots"]`` to enforce ``robots.txt``; pass
+                ``port=`` rather than ``--port``.
             port: Pin the listening port. Defaults to a free one.
         """
         self._proc, self._port = _spawn(

lightpanda/async_browser.py

@@ -163,9 +163,12 @@ async def run_script_async(
     env: dict[str, str] | None = None,
     binary: str | os.PathLike | None = None,
     timeout: float | None = None,
+    args: Sequence[str] = (),
 ) -> str:
     """Async variant of :func:`lightpanda.run_script` (runs in a worker thread)."""
-    return await asyncio.to_thread(run_script, script, env=env, binary=binary, timeout=timeout)
+    return await asyncio.to_thread(
+        run_script, script, env=env, binary=binary, timeout=timeout, args=args
+    )
 
 
 __all__ = ["AsyncBrowser", "AsyncSession", "run_script_async"]

lightpanda/browser.py

@@ -172,6 +172,7 @@ def __init__(
                 raising :class:`ProtocolError`.
             verbose: Let the browser's own logging through to stderr.
             args: Extra CLI flags for the spawned browser process, e.g.
+                ``["--obey-robots"]`` to enforce ``robots.txt``,
                 ``["--http-cache-dir", path]`` or cookie flags.
         """
         self._client = Client(binary=binary, env=env, timeout=timeout, verbose=verbose, args=args)
@@ -214,15 +215,18 @@ def run_script(
     env: dict[str, str] | None = None,
     binary: str | os.PathLike | None = None,
     timeout: float | None = None,
+    args: Sequence[str] = (),
 ) -> str:
     """Replay a saved lightpanda script (no LLM) and return its stdout.
 
     ``env`` entries (e.g. ``LP_*`` placeholder values) are added to the
-    child's environment. Raises :class:`ScriptError` on a non-zero exit.
+    child's environment. ``args`` are extra CLI flags for the browser, e.g.
+    ``["--obey-robots"]`` to enforce ``robots.txt``. Raises
+    :class:`ScriptError` on a non-zero exit.
     """
     path = Path(script)
     proc = subprocess.run(
-        [str(find_binary(binary)), "run", str(path)],
+        [str(find_binary(binary)), "run", *args, str(path)],
         env=os.environ | (env or {}),
         capture_output=True,
         text=True,

tests/test_browser.py

@@ -8,7 +8,7 @@
 import pytest
 
 from conftest import alive
-from lightpanda import Browser, LightpandaError, ToolError, run_script, client
+from lightpanda import Browser, LightpandaError, ScriptError, ToolError, run_script, client
 
 
 def test_goto_and_markdown(browser, fixture_url):
@@ -117,6 +117,14 @@ def test_run_script(binary, fixture_url, tmp_path):
     run_script(script, env={"LP_TEST_URL": f"{fixture_url}/index.html"}, binary=binary)
 
 
+def test_run_script_forwards_args(binary, tmp_path):
+    # A flag the binary rejects proves `args` reaches it, without needing a network.
+    script = tmp_path / "unused.js"
+    script.write_text("const page = new Page();\n")
+    with pytest.raises(ScriptError):
+        run_script(script, binary=binary, args=["--not-a-real-flag"])
+
+
 def test_find_binary_skips_own_console_script(tmp_path, monkeypatch):
     # `uv run` puts the venv's bin dir first on PATH, where the package's
     # `lightpanda` entry point shadows a real binary further along.