diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore new file mode 100644 index 000000000..1b96e0b0a --- /dev/null +++ b/bindings/python/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.egg-info/ +build/ +dist/ +docs/ +.venv/ +lightpanda/lightpanda +lightpanda/lightpanda.exe diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 000000000..ec57c68e1 --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,43 @@ +# lightpanda for Python + +Scrape and automate the web from Python with the [Lightpanda](https://lightpanda.io) +headless browser — an order of magnitude lighter than Chrome. The wheel bundles +the browser binary; there is no separate install step. + +```python +from lightpanda import Browser + +with Browser() as b: + page = b.new_session() + page.goto(url="https://example.com") + data = page.extract(schema={"title": "h1"}) +``` + +Every browser tool is a `Session` method (both `waitForSelector` and +`wait_for_selector` work), with kwargs matching the tool's documented +arguments. Replay a saved lightpanda agent script without any LLM: + +```python +from lightpanda import run_script + +run_script("hn.lp.js", env={"LP_HN_USERNAME": "me"}) +``` + +The package also puts the full `lightpanda` CLI on PATH (agent REPL, fetch, +serve). + +## Development + +The runtime binary is resolved from `LIGHTPANDA_BIN`, the package directory, +then PATH. For a repo checkout: build with `zig build` and run tests with + +```bash +uv run --group dev pytest tests +``` + +Regenerate the tool methods (`lightpanda/_methods.py`) and the API docs: + +```bash +uv run --no-project python scripts/generate_methods.py +uv run --no-project --with pdoc python scripts/build_docs.py # writes docs/ +``` diff --git a/bindings/python/lightpanda/__init__.py b/bindings/python/lightpanda/__init__.py new file mode 100644 index 000000000..99c03d74e --- /dev/null +++ b/bindings/python/lightpanda/__init__.py @@ -0,0 +1,24 @@ +"""Lightpanda for Python: a lightweight headless browser. + +```python +from lightpanda import Browser + +with Browser() as b: + page = b.new_session() + page.goto(url="https://example.com") + data = page.extract(schema={"title": "h1"}) +``` +""" + +from .browser import Browser, Session, run_script +from .errors import LightpandaError, ProtocolError, ScriptError, ToolError + +__all__ = [ + "Browser", + "Session", + "run_script", + "LightpandaError", + "ProtocolError", + "ScriptError", + "ToolError", +] diff --git a/bindings/python/lightpanda/_methods.py b/bindings/python/lightpanda/_methods.py new file mode 100644 index 000000000..955d6418f --- /dev/null +++ b/bindings/python/lightpanda/_methods.py @@ -0,0 +1,139 @@ +"""Generated by scripts/generate_methods.py — do not edit. + +One method per lightpanda browser tool, signatures derived from the tool +JSON schemas. camelCase aliases mirror the underlying tool names. +""" + +from __future__ import annotations + +from typing import Any + + +class SessionMethods: + def call(self, tool: str, **kwargs: Any) -> Any: + raise NotImplementedError + + def click(self, *, selector: str | None = None, backendNodeId: int | None = None) -> Any: + """Click on an interactive element. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Returns the current page URL and title after the click.""" + return self.call("click", selector=selector, backendNodeId=backendNodeId) + def console_logs(self) -> Any: + """Get buffered console.log/warn/error messages from the current page. Returns all messages since last call and clears the buffer.""" + return self.call("consoleLogs") + + consoleLogs = console_logs + def detect_forms(self, *, url: str | None = None, timeout: int | None = None) -> Any: + """Detect all forms on the page and return their structure including fields, types, and required status. If a url is provided, it navigates to that url first.""" + return self.call("detectForms", url=url, timeout=timeout) + + detectForms = detect_forms + def evaluate(self, *, script: str, url: str | None = None, timeout: int | None = None, save: str | None = None) -> Any: + """Evaluate JavaScript in the current page context — an escape hatch for page-side logic the dedicated tools can't express; prefer `extract` for data and click/fill/etc. for actions. It runs in the page, so it cannot see the agent script's variables or builtins — interpolate any value into the `script` string. A bare trailing expression yields its value; top-level `await` and `return` are supported (the body then runs as an async function, so use `return` to produce a value). Objects and arrays return as JSON, so no `JSON.stringify` is needed. If a url is provided, it navigates there first. The `globalThis.lp` object exposes a Session-scoped bridge store: values written via `lp.foo = ...` auto-sync at end of evaluate, surviving navigation; values previously set via `/extract save=` or `/evaluate save=` appear as `lp.`.""" + return self.call("evaluate", script=script, url=url, timeout=timeout, save=save) + def extract(self, *, schema: str | dict | list, save: str | None = None) -> Any: + """Extract structured data from the current page (navigate first). `schema` is a JSON object (passed as a string) mapping output field names to CSS-selector specs. It is NOT a JSON Schema — no "type"/"properties" wrappers; the keys ARE your output fields. Value shapes: + "" → first match's text (trimmed; null if no match) + [""] → every match's text (string[]) + {"selector":"","attr":""} → first match's attribute value (href/src resolved to absolute URLs) + [{"selector":"","attr":""}] → every match's attribute (string[]) + [{"selector":"","fields":{…}}] → one object per match; field selectors resolve relative to that match and accept any shape above ("" = the match's own text; nest arrays for per-item sub-lists) +Add "limit": N inside any array's object spec to cap matches. +Every extracted value is a string or null — parse numbers downstream. An empty array is a valid result, but if ALL top-level keys miss, the call errors: inspect the page (tree/markdown) and retry with corrected selectors. +Finish data tasks with extract — it is the only read recorded as a replayable `extract(...)` script call; answers lifted from `markdown` text in chat are not. + +Examples (schema → result): + {"karma": "#karma"} → {"karma":"42"} + {"items": [".story .title"]} → {"items":["Title 1","Title 2"]} + {"top3": [{"selector":".story .title","limit":3}]} → {"top3":["A","B","C"]} + {"links": [{"selector":"a.title","attr":"href"}]} → {"links":["https://site/a","https://site/b"]} + {"stories": [{"selector":".athing","fields":{"title":".titleline","rank":".rank"}}]} → {"stories":[{"title":"Foo","rank":"1"}]}""" + return self.call("extract", schema=schema, save=save) + def fill(self, *, value: str, selector: str | None = None, backendNodeId: int | None = None) -> Any: + """Fill text into an input element. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId.""" + return self.call("fill", value=value, selector=selector, backendNodeId=backendNodeId) + def find_element(self, *, role: str | None = None, name: str | None = None) -> Any: + """Find interactive elements by role and/or accessible name. Returns matching elements with their backend node IDs. Useful for locating specific elements without parsing the full semantic tree.""" + return self.call("findElement", role=role, name=name) + + findElement = find_element + def get_cookies(self, *, url: str | None = None, all: bool | None = None) -> Any: + """Cookies stored in the browser. Defaults to cookies whose domain matches the current page's host. Pass `url=` to filter for another host, or `all=true` to dump every cookie regardless of host. Useful for debugging authentication and session state.""" + return self.call("getCookies", url=url, all=all) + + getCookies = get_cookies + def get_env(self, *, name: str | None = None) -> Any: + """With `name`: read an LP_* env var (other namespaces report as not set) — for non-secret config only (base URLs, flags). Without `name`: list LP_* names that are set (no values) — safe credential discovery. For secrets, pass `$LP_*` placeholders in tool args; never request a credential by name (the value would land in your context).""" + return self.call("getEnv", name=name) + + getEnv = get_env + def get_url(self) -> Any: + """Current page URL. The browser may already have a page loaded (command, replayed script) not visible in this conversation — call this before assuming nothing is loaded when the user references the current page/site. Also useful to verify a navigation or detect a redirect.""" + return self.call("getUrl") + + getUrl = get_url + def goto(self, *, url: str, timeout: int | None = None) -> Any: + """Navigate to a specified URL and load the page in memory so it can be reused later for info extraction.""" + return self.call("goto", url=url, timeout=timeout) + def hover(self, *, selector: str | None = None, backendNodeId: int | None = None) -> Any: + """Hover over an element, triggering mouseover and mouseenter events. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Useful for menus, tooltips, and hover states.""" + return self.call("hover", selector=selector, backendNodeId=backendNodeId) + def html(self, *, selector: str | None = None, backendNodeId: int | None = None, url: str | None = None, timeout: int | None = None) -> Any: + """Raw HTML for the document or, with `selector`/`backendNodeId`, a single node's outerHTML. Verbose; use only when you need attributes that markdown discards.""" + return self.call("html", selector=selector, backendNodeId=backendNodeId, url=url, timeout=timeout) + def interactive_elements(self, *, url: str | None = None, timeout: int | None = None) -> Any: + """Extract interactive elements from the opened page. If a url is provided, it navigates to that url first.""" + return self.call("interactiveElements", url=url, timeout=timeout) + + interactiveElements = interactive_elements + def links(self, *, url: str | None = None, timeout: int | None = None) -> Any: + """Extract all links in the opened page as JSON objects with `text` (visible anchor text), `href` (resolved URL), and `backendNodeId` (pass to click/nodeDetails). If a url is provided, it navigates to that url first.""" + return self.call("links", url=url, timeout=timeout) + def markdown(self, *, selector: str | None = None, backendNodeId: int | None = None, maxBytes: int | None = None, url: str | None = None, timeout: int | None = None) -> Any: + """Render the page (or a subtree) as markdown. Scope with `selector` or `backendNodeId` to read just the relevant region — full-page markdown is the last resort. Use `maxBytes` to cap long pages.""" + return self.call("markdown", selector=selector, backendNodeId=backendNodeId, maxBytes=maxBytes, url=url, timeout=timeout) + def node_details(self, *, backendNodeId: int) -> Any: + """Details for a node by backendNodeId: a ready-to-use CSS `selector` that resolves to the node (the first match, as click/fill resolve it), plus tag, role, name, interactivity, disabled, value, input type, placeholder, href, id, class, checked, select options. The canonical way to turn a tree backendNodeId into a CSS selector.""" + return self.call("nodeDetails", backendNodeId=backendNodeId) + + nodeDetails = node_details + def press(self, *, key: str, selector: str | None = None, backendNodeId: int | None = None) -> Any: + """Press a keyboard key, dispatching keydown and keyup events. Use key names like 'Enter', 'Tab', 'Escape', 'ArrowDown', 'Backspace', or single characters like 'a', '1'. Common shorthand is normalized: 'enter'/'return' → 'Enter', 'esc' → 'Escape', 'up'/'down'/'left'/'right' → 'Arrow*', 'space' → ' '. Pressing 'Enter' on a form input or submit button triggers implicit form submission.""" + return self.call("press", key=key, selector=selector, backendNodeId=backendNodeId) + def scroll(self, *, backendNodeId: int | None = None, x: int | None = None, y: int | None = None) -> Any: + """Scroll the page or a specific element. Returns the scroll position and current page URL and title.""" + return self.call("scroll", backendNodeId=backendNodeId, x=x, y=y) + def search(self, *, query: str, timeout: int | None = None) -> Any: + """Run a web search and return results as markdown. When TAVILY_API_KEY is set, queries the Tavily Search API and returns a numbered list of {title, url, snippet}. Otherwise (or on Tavily failure) falls back to scraping the DuckDuckGo HTML endpoint — degraded results, may rate-limit on bursty traffic. Prefer this over goto-ing google.com/search directly (Google blocks the browser on User-Agent/TLS). Browser state after this call is unspecified — to interact with a result, use `goto` with its URL; do not assume the browser DOM matches the results page.""" + return self.call("search", query=query, timeout=timeout) + def select_option(self, *, value: str, selector: str | None = None, backendNodeId: int | None = None) -> Any: + """Select an option in a