From cecedc0e2b012f9c585483680fc4fcba00079732 Mon Sep 17 00:00:00 2001 From: aadithyan rajesh Date: Tue, 1 Sep 2026 18:52:05 +0530 Subject: [PATCH 1/4] chore: configure standalone Haystack package --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 6 +++++- pyproject.toml | 33 +++++++++++++++++++++------------ 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5c16f97..e429b98 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2026-present AUTHOR +# SPDX-FileCopyrightText: 2026-present Context.dev # # SPDX-License-Identifier: Apache-2.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b338f3a..f73d24e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2026-present AUTHOR +# SPDX-FileCopyrightText: 2026-present Context.dev # # SPDX-License-Identifier: Apache-2.0 @@ -44,5 +44,9 @@ jobs: if: matrix.python-version == '3.10' && runner.os == 'Linux' run: hatch run fmt-check + - name: Type check + if: matrix.python-version == '3.10' && runner.os == 'Linux' + run: hatch run test:types + - name: Run tests run: hatch run test:all diff --git a/pyproject.toml b/pyproject.toml index 747b062..29ddcd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2026-present AUTHOR +# SPDX-FileCopyrightText: 2026-present Context.dev # # SPDX-License-Identifier: Apache-2.0 @@ -7,19 +7,22 @@ requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" [project] -name = "example-haystack" # TODO: Replace with your package name, e.g. "deepset-ai-haystack" +name = "context-dev-haystack" dynamic = ["version"] -description = "A custom Haystack component" # TODO: Replace with your description +description = "Haystack components for Context.dev live web search, scraping, and crawling" readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" keywords = [ "haystack", - # TODO: Add relevant keywords for your integration + "context-dev", + "web-search", + "web-scraping", + "web-crawling", + "rag", ] authors = [ - # TODO: Replace with your name and email - { name = "AUTHOR", email = "your@email.com" }, + { name = "Context.dev" }, ] classifiers = [ "Development Status :: 4 - Beta", @@ -35,15 +38,16 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "haystack-ai", - # TODO: Add your integration-specific dependencies here + "haystack-ai>=2.24.1", + "httpx>=0.27.0", + "requests>=2.32.0", ] [project.urls] -# TODO: Replace with your repository URL -Documentation = "https://github.com/your-org/example-haystack#readme" -Issues = "https://github.com/your-org/example-haystack/issues" -Source = "https://github.com/your-org/example-haystack" +Homepage = "https://context.dev" +Documentation = "https://github.com/context-dot-dev/context-haystack#readme" +Issues = "https://github.com/context-dot-dev/context-haystack/issues" +Source = "https://github.com/context-dot-dev/context-haystack" [tool.hatch.version] source = "vcs" @@ -65,8 +69,12 @@ fmt-check = "ruff check {args:.} && ruff format --check {args:.}" [tool.hatch.envs.test] dependencies = [ + "mypy", "pytest", + "pytest-asyncio", "pytest-cov", + "pytest-mock", + "types-requests", ] [tool.hatch.envs.test.scripts] @@ -74,6 +82,7 @@ unit = 'pytest -m "not integration" {args:tests}' integration = 'pytest -m "integration" {args:tests}' all = "pytest {args:tests}" cov = "pytest --cov=haystack_integrations {args:tests}" +types = "mypy src tests" [tool.ruff] line-length = 120 From 6af9f264994b017702ad6325f91bc7e354c327b4 Mon Sep 17 00:00:00 2001 From: aadithyan rajesh Date: Tue, 1 Sep 2026 18:55:12 +0530 Subject: [PATCH 2/4] feat: add Context.dev web components --- .../components/example/__init__.py | 8 - .../components/example/example_component.py | 46 ---- .../components/fetchers/context/__init__.py | 9 + .../fetchers/context/context_crawler.py | 145 +++++++++++++ .../fetchers/context/context_fetcher.py | 138 ++++++++++++ .../components/websearch/context/__init__.py | 8 + .../websearch/context/context_websearch.py | 199 ++++++++++++++++++ src/haystack_integrations/context/__init__.py | 7 + src/haystack_integrations/context/_client.py | 157 ++++++++++++++ 9 files changed, 663 insertions(+), 54 deletions(-) delete mode 100644 src/haystack_integrations/components/example/__init__.py delete mode 100644 src/haystack_integrations/components/example/example_component.py create mode 100644 src/haystack_integrations/components/fetchers/context/__init__.py create mode 100644 src/haystack_integrations/components/fetchers/context/context_crawler.py create mode 100644 src/haystack_integrations/components/fetchers/context/context_fetcher.py create mode 100644 src/haystack_integrations/components/websearch/context/__init__.py create mode 100644 src/haystack_integrations/components/websearch/context/context_websearch.py create mode 100644 src/haystack_integrations/context/__init__.py create mode 100644 src/haystack_integrations/context/_client.py diff --git a/src/haystack_integrations/components/example/__init__.py b/src/haystack_integrations/components/example/__init__.py deleted file mode 100644 index 0989a98..0000000 --- a/src/haystack_integrations/components/example/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present AUTHOR -# -# SPDX-License-Identifier: Apache-2.0 - -# TODO: Rename the import to match your component class. -from .example_component import ExampleComponent - -__all__ = ["ExampleComponent"] diff --git a/src/haystack_integrations/components/example/example_component.py b/src/haystack_integrations/components/example/example_component.py deleted file mode 100644 index 6b286e1..0000000 --- a/src/haystack_integrations/components/example/example_component.py +++ /dev/null @@ -1,46 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present AUTHOR -# -# SPDX-License-Identifier: Apache-2.0 - -from haystack import component - - -# TODO: Rename this class and update the output types and run method to match your use case. -@component -class ExampleComponent: - """ - A custom Haystack component. - - Usage: - ```python - from haystack_integrations.components.example import ExampleComponent - - component = ExampleComponent() - result = component.run(input_text="Hello, world!") - ``` - """ - - def __init__(self, param: str = "default") -> None: - """ - Initialize the component. - - :param param: An example parameter. - """ - self.param = param - - @component.output_types(output=str) - def run(self, input_text: str) -> dict[str, str]: - """ - Process the input and return results. - - :param input_text: The text to process. - :returns: A dictionary with the following keys: - - `output`: The processed text. - """ - # TODO: Implement your component logic here. - result = input_text - return {"output": result} - - # NOTE: Custom `to_dict` and `from_dict` methods are only needed if the default serialization doesn't work - # for your component (e.g. it has non-serializable attributes). For details, see: - # https://docs.haystack.deepset.ai/docs/serialization#default-serialization-behavior diff --git a/src/haystack_integrations/components/fetchers/context/__init__.py b/src/haystack_integrations/components/fetchers/context/__init__.py new file mode 100644 index 0000000..128d110 --- /dev/null +++ b/src/haystack_integrations/components/fetchers/context/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +from haystack_integrations.components.fetchers.context.context_crawler import ContextCrawler +from haystack_integrations.components.fetchers.context.context_fetcher import ContextFetcher +from haystack_integrations.context import ContextError + +__all__ = ["ContextCrawler", "ContextError", "ContextFetcher"] diff --git a/src/haystack_integrations/components/fetchers/context/context_crawler.py b/src/haystack_integrations/components/fetchers/context/context_crawler.py new file mode 100644 index 0000000..3a0bf72 --- /dev/null +++ b/src/haystack_integrations/components/fetchers/context/context_crawler.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +from typing import Any + +from haystack import Document, component +from haystack.utils import Secret + +from haystack_integrations.context._client import ( + DEFAULT_API_KEY, + DEFAULT_API_URL, + request_context, + request_context_async, +) + + +@component +class ContextCrawler: + """ + Crawl one or more websites with Context.dev and return each page as a Haystack Document. + + Crawls are bounded to one page by default to prevent accidental credit consumption. Increase `maxPages` through + `crawl_params` when a pipeline needs a larger site ingest. Create an API key in the + [Context.dev dashboard](https://context.dev/dashboard/api-keys) and set it as `CONTEXT_API_KEY`. + + ### Usage example + + ```python + from haystack_integrations.components.fetchers.context import ContextCrawler + + crawler = ContextCrawler(crawl_params={"maxPages": 10, "maxDepth": 2}) + result = crawler.run(urls=["https://docs.haystack.deepset.ai"]) + documents = result["documents"] + ``` + """ + + def __init__( + self, + api_key: Secret = DEFAULT_API_KEY, + *, + crawl_params: dict[str, Any] | None = None, + api_url: str = DEFAULT_API_URL, + timeout: int = 120, + max_retries: int = 3, + ) -> None: + """ + Initialize the Context.dev crawler component. + + :param api_key: Context.dev API key. Defaults to the `CONTEXT_API_KEY` environment variable. + :param crawl_params: Additional parameters passed to the Context.dev Crawl API. + :param api_url: Base URL for the Context.dev API. + :param timeout: Request timeout in seconds. + :param max_retries: Maximum number of retry attempts on transient failures. + """ + self.api_key = api_key + self.crawl_params = crawl_params + self.api_url = api_url + self.timeout = timeout + self.max_retries = max_retries + + @component.output_types(documents=list[Document]) + def run( + self, + urls: list[str], + crawl_params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """ + Crawl the given URLs and return successful pages as Documents. + + :param urls: Starting URLs to crawl. + :param crawl_params: Optional per-run replacement for init-time crawl parameters. + :returns: A dictionary containing `documents`. + """ + documents = [document for url in urls for document in self._crawl_url(url, crawl_params)] + return {"documents": documents} + + @component.output_types(documents=list[Document]) + async def run_async( + self, + urls: list[str], + crawl_params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """ + Asynchronously crawl the given URLs and return successful pages as Documents. + + :param urls: Starting URLs to crawl concurrently. + :param crawl_params: Optional per-run replacement for init-time crawl parameters. + :returns: A dictionary containing `documents`. + """ + crawls = await asyncio.gather(*(self._crawl_url_async(url, crawl_params) for url in urls)) + documents = [document for crawl in crawls for document in crawl] + return {"documents": documents} + + def _crawl_url(self, url: str, crawl_params: dict[str, Any] | None) -> list[Document]: + response = request_context( + api_key=self.api_key, + api_url=self.api_url, + method="POST", + path="/web/crawl", + json=self._request_body(url, crawl_params), + timeout=self.timeout, + max_retries=self.max_retries, + ) + return self._documents_from_response(response) + + async def _crawl_url_async(self, url: str, crawl_params: dict[str, Any] | None) -> list[Document]: + response = await request_context_async( + api_key=self.api_key, + api_url=self.api_url, + method="POST", + path="/web/crawl", + json=self._request_body(url, crawl_params), + timeout=self.timeout, + max_retries=self.max_retries, + ) + return self._documents_from_response(response) + + def _request_body(self, url: str, crawl_params: dict[str, Any] | None) -> dict[str, Any]: + params = (crawl_params if crawl_params is not None else self.crawl_params or {}).copy() + defaults = { + "maxPages": 1, + "useMainContentOnly": True, + "includeLinks": True, + "includeImages": False, + } + return {**defaults, **params, "url": url} + + @staticmethod + def _documents_from_response(response: dict[str, Any]) -> list[Document]: + results = response.get("results", []) + if not isinstance(results, list): + return [] + + documents: list[Document] = [] + for result in results: + if not isinstance(result, dict): + continue + metadata = result.get("metadata", {}) + if not isinstance(metadata, dict) or not metadata.get("success", False): + continue + content = result.get("markdown", "") + documents.append(Document(content=content if isinstance(content, str) else "", meta=metadata.copy())) + return documents diff --git a/src/haystack_integrations/components/fetchers/context/context_fetcher.py b/src/haystack_integrations/components/fetchers/context/context_fetcher.py new file mode 100644 index 0000000..25729ba --- /dev/null +++ b/src/haystack_integrations/components/fetchers/context/context_fetcher.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +from typing import Any + +from haystack import Document, component +from haystack.utils import Secret + +from haystack_integrations.context._client import ( + DEFAULT_API_KEY, + DEFAULT_API_URL, + request_context, + request_context_async, +) + + +@component +class ContextFetcher: + """ + Fetch known URLs as clean Markdown with Context.dev. + + The component converts each URL into a Haystack Document and preserves page metadata. Create an API key in the + [Context.dev dashboard](https://context.dev/dashboard/api-keys) and set it as `CONTEXT_API_KEY`. + + ### Usage example + + ```python + from haystack_integrations.components.fetchers.context import ContextFetcher + + fetcher = ContextFetcher() + result = fetcher.run(urls=["https://haystack.deepset.ai"]) + documents = result["documents"] + ``` + """ + + def __init__( + self, + api_key: Secret = DEFAULT_API_KEY, + *, + scrape_params: dict[str, Any] | None = None, + api_url: str = DEFAULT_API_URL, + timeout: int = 60, + max_retries: int = 3, + ) -> None: + """ + Initialize the Context.dev fetcher component. + + :param api_key: Context.dev API key. Defaults to the `CONTEXT_API_KEY` environment variable. + :param scrape_params: Additional query parameters passed to the Context.dev Markdown Scrape API. + :param api_url: Base URL for the Context.dev API. + :param timeout: Request timeout in seconds. + :param max_retries: Maximum number of retry attempts on transient failures. + """ + self.api_key = api_key + self.scrape_params = scrape_params + self.api_url = api_url + self.timeout = timeout + self.max_retries = max_retries + + @component.output_types(documents=list[Document]) + def run( + self, + urls: list[str], + scrape_params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """ + Fetch the given URLs and return their Markdown as Documents. + + :param urls: URLs to fetch. + :param scrape_params: Optional per-run replacement for init-time scrape parameters. + :returns: A dictionary containing `documents`. + """ + documents = [self._fetch_url(url, scrape_params) for url in urls] + return {"documents": documents} + + @component.output_types(documents=list[Document]) + async def run_async( + self, + urls: list[str], + scrape_params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """ + Asynchronously fetch the given URLs and return their Markdown as Documents. + + :param urls: URLs to fetch concurrently. + :param scrape_params: Optional per-run replacement for init-time scrape parameters. + :returns: A dictionary containing `documents`. + """ + documents = await asyncio.gather(*(self._fetch_url_async(url, scrape_params) for url in urls)) + return {"documents": list(documents)} + + def _fetch_url(self, url: str, scrape_params: dict[str, Any] | None) -> Document: + response = request_context( + api_key=self.api_key, + api_url=self.api_url, + method="GET", + path="/web/scrape/markdown", + params=self._request_params(url, scrape_params), + timeout=self.timeout, + max_retries=self.max_retries, + ) + return self._document_from_response(response, url) + + async def _fetch_url_async(self, url: str, scrape_params: dict[str, Any] | None) -> Document: + response = await request_context_async( + api_key=self.api_key, + api_url=self.api_url, + method="GET", + path="/web/scrape/markdown", + params=self._request_params(url, scrape_params), + timeout=self.timeout, + max_retries=self.max_retries, + ) + return self._document_from_response(response, url) + + def _request_params(self, url: str, scrape_params: dict[str, Any] | None) -> dict[str, Any]: + params = (scrape_params if scrape_params is not None else self.scrape_params or {}).copy() + defaults = { + "useMainContentOnly": True, + "includeLinks": True, + "includeImages": False, + } + return {**defaults, **params, "url": url} + + @staticmethod + def _document_from_response(response: dict[str, Any], requested_url: str) -> Document: + metadata = response.get("metadata", {}) + meta = metadata.copy() if isinstance(metadata, dict) else {} + meta.update( + { + "url": response.get("url", requested_url), + "content_length": response.get("contentLength"), + } + ) + content = response.get("markdown", "") + return Document(content=content if isinstance(content, str) else "", meta=meta) diff --git a/src/haystack_integrations/components/websearch/context/__init__.py b/src/haystack_integrations/components/websearch/context/__init__.py new file mode 100644 index 0000000..69e2b7e --- /dev/null +++ b/src/haystack_integrations/components/websearch/context/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +from haystack_integrations.components.websearch.context.context_websearch import ContextWebSearch +from haystack_integrations.context import ContextError + +__all__ = ["ContextError", "ContextWebSearch"] diff --git a/src/haystack_integrations/components/websearch/context/context_websearch.py b/src/haystack_integrations/components/websearch/context/context_websearch.py new file mode 100644 index 0000000..c95d891 --- /dev/null +++ b/src/haystack_integrations/components/websearch/context/context_websearch.py @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, Literal + +from haystack import Document, component +from haystack.utils import Secret + +from haystack_integrations.context._client import ( + DEFAULT_API_KEY, + DEFAULT_API_URL, + request_context, + request_context_async, +) + +Freshness = Literal["last_24_hours", "last_week", "last_month", "last_year"] +MAX_RESULTS = 100 + + +@component +class ContextWebSearch: + """ + Search the live web with Context.dev and return ranked Haystack Documents. + + Context.dev combines relevance-ranked web search with optional Markdown extraction. Create an API key in the + [Context.dev dashboard](https://context.dev/dashboard/api-keys) and set it as `CONTEXT_API_KEY`. + + ### Usage example + + ```python + from haystack_integrations.components.websearch.context import ContextWebSearch + + websearch = ContextWebSearch(top_k=5) + result = websearch.run(query="What is Haystack by deepset?") + documents = result["documents"] + links = result["links"] + ``` + """ + + def __init__( + self, + api_key: Secret = DEFAULT_API_KEY, + *, + top_k: int = 10, + include_domains: list[str] | None = None, + exclude_domains: list[str] | None = None, + freshness: Freshness | None = None, + country: str | None = None, + include_markdown: bool = False, + search_params: dict[str, Any] | None = None, + api_url: str = DEFAULT_API_URL, + timeout: int = 30, + max_retries: int = 3, + ) -> None: + """ + Initialize the Context.dev web search component. + + :param api_key: Context.dev API key. Defaults to the `CONTEXT_API_KEY` environment variable. + :param top_k: Maximum number of results to return. Must be between 1 and 100. + :param include_domains: Only return results from these domains. + :param exclude_domains: Exclude results from these domains. + :param freshness: Restrict results to a recent time window. + :param country: Two-letter country code for geographically focused results. + :param include_markdown: Fetch the full Markdown content for each search result. + :param search_params: Additional parameters passed to the Context.dev Search API. + :param api_url: Base URL for the Context.dev API. + :param timeout: Request timeout in seconds. + :param max_retries: Maximum number of retry attempts on transient failures. + """ + if not 1 <= top_k <= MAX_RESULTS: + msg = "top_k must be between 1 and 100." + raise ValueError(msg) + self.api_key = api_key + self.top_k = top_k + self.include_domains = include_domains + self.exclude_domains = exclude_domains + self.freshness = freshness + self.country = country + self.include_markdown = include_markdown + self.search_params = search_params + self.api_url = api_url + self.timeout = timeout + self.max_retries = max_retries + + @component.output_types(documents=list[Document], links=list[str]) + def run( + self, + query: str, + top_k: int | None = None, + search_params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """ + Search the web and return results as Documents. + + :param query: Search query. + :param top_k: Optional per-run override for the number of results returned. + :param search_params: Optional per-run replacement for init-time search parameters. + :returns: A dictionary containing `documents` and `links`. + """ + effective_top_k = self._effective_top_k(top_k) + response = request_context( + api_key=self.api_key, + api_url=self.api_url, + method="POST", + path="/web/search", + json=self._request_body(query, effective_top_k, search_params), + timeout=self.timeout, + max_retries=self.max_retries, + ) + return self._parse_response(response, effective_top_k) + + @component.output_types(documents=list[Document], links=list[str]) + async def run_async( + self, + query: str, + top_k: int | None = None, + search_params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """ + Asynchronously search the web and return results as Documents. + + :param query: Search query. + :param top_k: Optional per-run override for the number of results returned. + :param search_params: Optional per-run replacement for init-time search parameters. + :returns: A dictionary containing `documents` and `links`. + """ + effective_top_k = self._effective_top_k(top_k) + response = await request_context_async( + api_key=self.api_key, + api_url=self.api_url, + method="POST", + path="/web/search", + json=self._request_body(query, effective_top_k, search_params), + timeout=self.timeout, + max_retries=self.max_retries, + ) + return self._parse_response(response, effective_top_k) + + def _request_body( + self, + query: str, + top_k: int, + search_params: dict[str, Any] | None, + ) -> dict[str, Any]: + body = (search_params if search_params is not None else self.search_params or {}).copy() + optional_params = { + "includeDomains": self.include_domains, + "excludeDomains": self.exclude_domains, + "freshness": self.freshness, + "country": self.country, + } + body.update({name: value for name, value in optional_params.items() if value is not None}) + body.update( + { + "query": query, + "numResults": max(10, top_k), + "markdownOptions": { + "enabled": self.include_markdown, + "useMainContentOnly": True, + "includeLinks": True, + "includeImages": False, + }, + } + ) + return body + + def _effective_top_k(self, top_k: int | None) -> int: + effective_top_k = self.top_k if top_k is None else top_k + if not 1 <= effective_top_k <= MAX_RESULTS: + msg = "top_k must be between 1 and 100." + raise ValueError(msg) + return effective_top_k + + @staticmethod + def _parse_response(response: dict[str, Any], top_k: int) -> dict[str, Any]: + documents: list[Document] = [] + links: list[str] = [] + results = response.get("results", []) + if not isinstance(results, list): + return {"documents": documents, "links": links} + + for result in results[:top_k]: + if not isinstance(result, dict): + continue + url = result.get("url", "") + markdown = result.get("markdown", {}) + markdown_content = markdown.get("markdown") if isinstance(markdown, dict) else None + content = markdown_content if isinstance(markdown_content, str) else result.get("description", "") + meta = { + "title": result.get("title", ""), + "url": url, + "relevance": result.get("relevance"), + "markdown_code": markdown.get("code") if isinstance(markdown, dict) else None, + } + documents.append(Document(content=content, meta=meta)) + if isinstance(url, str) and url: + links.append(url) + return {"documents": documents, "links": links} diff --git a/src/haystack_integrations/context/__init__.py b/src/haystack_integrations/context/__init__.py new file mode 100644 index 0000000..fb92327 --- /dev/null +++ b/src/haystack_integrations/context/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +from haystack_integrations.context._client import ContextError + +__all__ = ["ContextError"] diff --git a/src/haystack_integrations/context/_client.py b/src/haystack_integrations/context/_client.py new file mode 100644 index 0000000..e5762a2 --- /dev/null +++ b/src/haystack_integrations/context/_client.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +from importlib.metadata import PackageNotFoundError, version +from typing import Any + +import httpx +import requests +from haystack import ComponentError +from haystack.utils import Secret +from haystack.utils.requests_utils import async_request_with_retry, request_with_retry + +API_KEY_ENV_VAR = "CONTEXT_API_KEY" +DEFAULT_API_URL = "https://api.context.dev/v1" +DEFAULT_API_KEY = Secret.from_env_var(API_KEY_ENV_VAR) + +try: + _VERSION = version("context-dev-haystack") +except PackageNotFoundError: # pragma: no cover + _VERSION = "0.0.0-dev" + +USER_AGENT = f"context-dev-haystack/{_VERSION}" + + +class ContextError(ComponentError): + """An error occurred while calling the Context.dev API.""" + + +def request_context( + *, + api_key: Secret, + api_url: str, + method: str, + path: str, + timeout: int, + max_retries: int, + params: dict[str, Any] | None = None, + json: dict[str, Any] | None = None, +) -> dict[str, Any]: + """ + Call a Context.dev API endpoint. + + :param api_key: Context.dev API key. + :param api_url: Base URL for the Context.dev API. + :param method: HTTP method. + :param path: Endpoint path relative to the API base URL. + :param timeout: Request timeout in seconds. + :param max_retries: Maximum number of retry attempts on transient failures. + :param params: Optional query parameters. + :param json: Optional JSON request body. + :returns: Parsed JSON response. + :raises ContextError: If the request fails or the API returns a non-object response. + """ + try: + response = request_with_retry( + attempts=max_retries, + method=method, + url=_endpoint_url(api_url, path), + headers=_headers(api_key), + params=_query_params(params), + json=json, + timeout=timeout, + ) + except (httpx.HTTPError, requests.RequestException) as error: + raise ContextError(_error_message(error)) from error + return _response_json(response) + + +async def request_context_async( + *, + api_key: Secret, + api_url: str, + method: str, + path: str, + timeout: int, + max_retries: int, + params: dict[str, Any] | None = None, + json: dict[str, Any] | None = None, +) -> dict[str, Any]: + """ + Asynchronously call a Context.dev API endpoint. + + :param api_key: Context.dev API key. + :param api_url: Base URL for the Context.dev API. + :param method: HTTP method. + :param path: Endpoint path relative to the API base URL. + :param timeout: Request timeout in seconds. + :param max_retries: Maximum number of retry attempts on transient failures. + :param params: Optional query parameters. + :param json: Optional JSON request body. + :returns: Parsed JSON response. + :raises ContextError: If the request fails or the API returns a non-object response. + """ + try: + response = await async_request_with_retry( + attempts=max_retries, + method=method, + url=_endpoint_url(api_url, path), + headers=_headers(api_key), + params=_query_params(params), + json=json, + timeout=timeout, + ) + except httpx.HTTPError as error: + raise ContextError(_error_message(error)) from error + return _response_json(response) + + +def _endpoint_url(api_url: str, path: str) -> str: + return f"{api_url.rstrip('/')}/{path.lstrip('/')}" + + +def _headers(api_key: Secret) -> dict[str, str]: + return { + "Accept": "application/json", + "Authorization": f"Bearer {api_key.resolve_value()}", + "User-Agent": USER_AGENT, + } + + +def _query_params(params: dict[str, Any] | None) -> list[tuple[str, str]] | None: + if params is None: + return None + return [pair for name, value in params.items() for pair in _query_value(name, value)] + + +def _query_value(name: str, value: object) -> list[tuple[str, str]]: + if value is None: + return [] + if isinstance(value, dict): + return [pair for key, item in value.items() for pair in _query_value(f"{name}[{key}]", item)] + if isinstance(value, (list, tuple)): + return [pair for item in value for pair in _query_value(name, item)] + if isinstance(value, bool): + return [(name, "true" if value else "false")] + return [(name, str(value))] + + +def _response_json(response: httpx.Response | requests.Response) -> dict[str, Any]: + try: + payload = response.json() + except ValueError as error: + msg = "Context.dev API returned an invalid JSON response." + raise ContextError(msg) from error + if not isinstance(payload, dict): + msg = "Context.dev API returned an invalid JSON response." + raise ContextError(msg) + return payload + + +def _error_message(error: httpx.HTTPError | requests.RequestException) -> str: + message = f"An error occurred while calling the Context.dev API. Error: {error}" + response = getattr(error, "response", None) + if response is not None: + return f"{message}, Response: {response.text}" + return message From d27edcc0f6930b1ac39ed08b508d4a1161c99830 Mon Sep 17 00:00:00 2001 From: aadithyan rajesh Date: Tue, 1 Sep 2026 18:58:36 +0530 Subject: [PATCH 3/4] test: cover Context.dev components --- pyproject.toml | 4 +- tests/test_context_client.py | 178 ++++++++++++++++++++++++++++++++ tests/test_context_crawler.py | 135 ++++++++++++++++++++++++ tests/test_context_fetcher.py | 123 ++++++++++++++++++++++ tests/test_context_websearch.py | 165 +++++++++++++++++++++++++++++ tests/test_example.py | 40 ------- 6 files changed, 603 insertions(+), 42 deletions(-) create mode 100644 tests/test_context_client.py create mode 100644 tests/test_context_crawler.py create mode 100644 tests/test_context_fetcher.py create mode 100644 tests/test_context_websearch.py delete mode 100644 tests/test_example.py diff --git a/pyproject.toml b/pyproject.toml index 29ddcd7..09c7bbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,10 +147,10 @@ ban-relative-imports = "parents" "examples/**/*" = ["D", "T201", "ANN"] [tool.mypy] -install_types = true -non_interactive = true check_untyped_defs = true disallow_incomplete_defs = true +explicit_package_bases = true +mypy_path = "src" [[tool.mypy.overrides]] module = ["haystack.*"] diff --git a/tests/test_context_client.py b/tests/test_context_client.py new file mode 100644 index 0000000..7cab082 --- /dev/null +++ b/tests/test_context_client.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest +import requests +from haystack.utils import Secret + +from haystack_integrations.context._client import USER_AGENT, ContextError, request_context, request_context_async + + +def test_request_context_builds_authenticated_request() -> None: + response = MagicMock() + response.json.return_value = {"results": []} + + with patch("haystack_integrations.context._client.request_with_retry", return_value=response) as request: + result = request_context( + api_key=Secret.from_token("test-key"), + api_url="https://example.test/v1/", + method="POST", + path="/web/search", + json={"query": "Haystack"}, + timeout=30, + max_retries=2, + ) + + assert result == {"results": []} + assert USER_AGENT.startswith("context-dev-haystack/") + assert request.call_args.kwargs == { + "attempts": 2, + "method": "POST", + "url": "https://example.test/v1/web/search", + "headers": { + "Accept": "application/json", + "Authorization": "Bearer test-key", + "User-Agent": USER_AGENT, + }, + "params": None, + "json": {"query": "Haystack"}, + "timeout": 30, + } + + +@pytest.mark.asyncio +async def test_request_context_async_builds_authenticated_request() -> None: + response = MagicMock() + response.json.return_value = {"success": True} + + with patch( + "haystack_integrations.context._client.async_request_with_retry", + new=AsyncMock(return_value=response), + ) as request: + result = await request_context_async( + api_key=Secret.from_token("test-key"), + api_url="https://example.test/v1", + method="GET", + path="web/scrape/markdown", + params={"url": "https://example.com"}, + timeout=60, + max_retries=3, + ) + + assert result == {"success": True} + request.assert_awaited_once() + assert request.call_args.kwargs["url"] == "https://example.test/v1/web/scrape/markdown" + + +def test_request_context_wraps_http_errors() -> None: + request = httpx.Request("POST", "https://example.test/v1/web/search") + response = httpx.Response(401, text="invalid api key", request=request) + error = httpx.HTTPStatusError("401 Unauthorized", request=request, response=response) + + with ( + patch("haystack_integrations.context._client.request_with_retry", side_effect=error), + pytest.raises(ContextError, match="invalid api key") as exc_info, + ): + request_context( + api_key=Secret.from_token("bad-key"), + api_url="https://example.test/v1", + method="POST", + path="/web/search", + timeout=30, + max_retries=1, + ) + + assert exc_info.value.__cause__ is error + + +def test_request_context_wraps_requests_http_errors() -> None: + response = requests.Response() + response.status_code = 400 + response._content = b"invalid query" + error = requests.HTTPError("400 Bad Request", response=response) + + with ( + patch("haystack_integrations.context._client.request_with_retry", side_effect=error), + pytest.raises(ContextError, match="invalid query") as exc_info, + ): + request_context( + api_key=Secret.from_token("test-key"), + api_url="https://example.test/v1", + method="GET", + path="/web/scrape/markdown", + timeout=30, + max_retries=1, + ) + + assert exc_info.value.__cause__ is error + + +def test_request_context_serializes_nested_query_parameters() -> None: + response = MagicMock() + response.json.return_value = {"success": True} + + with patch("haystack_integrations.context._client.request_with_retry", return_value=response) as request: + request_context( + api_key=Secret.from_token("test-key"), + api_url="https://example.test/v1", + method="GET", + path="/web/scrape/markdown", + params={ + "includeLinks": True, + "includeSelectors": ["main", "article"], + "pdf": {"start": 2, "ocr": False}, + "unset": None, + }, + timeout=30, + max_retries=1, + ) + + assert request.call_args.kwargs["params"] == [ + ("includeLinks", "true"), + ("includeSelectors", "main"), + ("includeSelectors", "article"), + ("pdf[start]", "2"), + ("pdf[ocr]", "false"), + ] + + +def test_request_context_rejects_non_object_json() -> None: + response = MagicMock() + response.json.return_value = [] + + with ( + patch("haystack_integrations.context._client.request_with_retry", return_value=response), + pytest.raises(ContextError, match="invalid JSON response"), + ): + request_context( + api_key=Secret.from_token("test-key"), + api_url="https://example.test/v1", + method="GET", + path="/web/scrape/markdown", + timeout=30, + max_retries=1, + ) + + +def test_request_context_wraps_invalid_json() -> None: + response = MagicMock() + response.json.side_effect = ValueError("invalid json") + + with ( + patch("haystack_integrations.context._client.request_with_retry", return_value=response), + pytest.raises(ContextError, match="invalid JSON response") as exc_info, + ): + request_context( + api_key=Secret.from_token("test-key"), + api_url="https://example.test/v1", + method="GET", + path="/web/scrape/markdown", + timeout=30, + max_retries=1, + ) + + assert isinstance(exc_info.value.__cause__, ValueError) diff --git a/tests/test_context_crawler.py b/tests/test_context_crawler.py new file mode 100644 index 0000000..2bb9fd2 --- /dev/null +++ b/tests/test_context_crawler.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +import os +from copy import deepcopy +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from haystack import Document +from haystack.core.serialization import component_from_dict, component_to_dict +from haystack.utils import Secret + +from haystack_integrations.components.fetchers.context import ContextCrawler + +CRAWL_RESPONSE = { + "results": [ + { + "markdown": "# Haystack", + "metadata": { + "url": "https://haystack.deepset.ai", + "title": "Haystack", + "crawlDepth": 0, + "statusCode": 200, + "success": True, + }, + }, + { + "markdown": "", + "metadata": { + "url": "https://haystack.deepset.ai/missing", + "title": "", + "crawlDepth": 1, + "statusCode": 404, + "success": False, + }, + }, + ], + "metadata": {"numUrls": 2, "numSucceeded": 1, "numFailed": 1}, +} + + +class TestContextCrawler: + def test_init_defaults(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CONTEXT_API_KEY", "test-key") + crawler = ContextCrawler() + + assert crawler.api_key.resolve_value() == "test-key" + assert crawler.crawl_params is None + assert crawler.timeout == 120 + + def test_serialization_roundtrip(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CONTEXT_API_KEY", "test-key") + crawler = ContextCrawler(crawl_params={"maxPages": 5, "maxDepth": 2}, timeout=90) + + data = component_to_dict(crawler, name="crawler") + restored = component_from_dict(ContextCrawler, data, name="crawler") + + assert restored.crawl_params == {"maxPages": 5, "maxDepth": 2} + assert restored.timeout == 90 + assert restored.api_key.resolve_value() == "test-key" + + @patch("haystack_integrations.components.fetchers.context.context_crawler.request_context") + def test_run_returns_only_successful_pages(self, request_context: MagicMock) -> None: + request_context.return_value = CRAWL_RESPONSE + crawl_params = {"maxPages": 5, "maxDepth": 2} + original_params = deepcopy(crawl_params) + crawler = ContextCrawler(api_key=Secret.from_token("test-key"), crawl_params=crawl_params) + + result = crawler.run(urls=["https://haystack.deepset.ai"]) + + assert len(result["documents"]) == 1 + assert isinstance(result["documents"][0], Document) + assert result["documents"][0].content == "# Haystack" + assert result["documents"][0].meta["success"] is True + assert request_context.call_args.kwargs["json"] == { + "url": "https://haystack.deepset.ai", + "maxPages": 5, + "maxDepth": 2, + "useMainContentOnly": True, + "includeLinks": True, + "includeImages": False, + } + assert crawl_params == original_params + + @patch("haystack_integrations.components.fetchers.context.context_crawler.request_context") + def test_run_defaults_to_one_page(self, request_context: MagicMock) -> None: + request_context.return_value = CRAWL_RESPONSE + crawler = ContextCrawler(api_key=Secret.from_token("test-key")) + + crawler.run(urls=["https://haystack.deepset.ai"]) + + assert request_context.call_args.kwargs["json"]["maxPages"] == 1 + + @patch("haystack_integrations.components.fetchers.context.context_crawler.request_context_async") + @pytest.mark.asyncio + async def test_run_async_crawls_concurrently(self, request_context: AsyncMock) -> None: + request_context.side_effect = AsyncMock(return_value=CRAWL_RESPONSE) + crawler = ContextCrawler(api_key=Secret.from_token("test-key")) + + result = await crawler.run_async(urls=["https://haystack.deepset.ai", "https://docs.haystack.deepset.ai"]) + + assert len(result["documents"]) == 2 + assert request_context.await_count == 2 + + def test_parse_response_handles_invalid_results(self) -> None: + assert ContextCrawler._documents_from_response({"results": None}) == [] + assert ContextCrawler._documents_from_response({"results": [None, "invalid"]}) == [] + + @pytest.mark.skipif( + not os.environ.get("CONTEXT_API_KEY"), + reason="Export CONTEXT_API_KEY to run integration tests.", + ) + @pytest.mark.integration + def test_run_integration(self) -> None: + crawler = ContextCrawler(crawl_params={"maxPages": 1, "maxDepth": 0}) + result = crawler.run(urls=["https://haystack.deepset.ai"]) + + assert len(result["documents"]) == 1 + assert result["documents"][0].content + assert result["documents"][0].meta["success"] is True + + @pytest.mark.skipif( + not os.environ.get("CONTEXT_API_KEY"), + reason="Export CONTEXT_API_KEY to run integration tests.", + ) + @pytest.mark.integration + @pytest.mark.asyncio + async def test_run_async_integration(self) -> None: + crawler = ContextCrawler(crawl_params={"maxPages": 1, "maxDepth": 0}) + result = await crawler.run_async(urls=["https://haystack.deepset.ai"]) + + assert len(result["documents"]) == 1 + assert result["documents"][0].content + assert result["documents"][0].meta["success"] is True diff --git a/tests/test_context_fetcher.py b/tests/test_context_fetcher.py new file mode 100644 index 0000000..ca218dd --- /dev/null +++ b/tests/test_context_fetcher.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +import os +from copy import deepcopy +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from haystack import Document +from haystack.core.serialization import component_from_dict, component_to_dict +from haystack.utils import Secret + +from haystack_integrations.components.fetchers.context import ContextFetcher + +SCRAPE_RESPONSE = { + "success": True, + "markdown": "# Haystack", + "contentLength": 10, + "url": "https://haystack.deepset.ai", + "metadata": {"title": "Haystack", "description": "Build AI applications."}, +} + + +class TestContextFetcher: + def test_init_defaults(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CONTEXT_API_KEY", "test-key") + fetcher = ContextFetcher() + + assert fetcher.api_key.resolve_value() == "test-key" + assert fetcher.scrape_params is None + assert fetcher.timeout == 60 + + def test_serialization_roundtrip(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CONTEXT_API_KEY", "test-key") + fetcher = ContextFetcher(scrape_params={"includeImages": True}, timeout=45) + + data = component_to_dict(fetcher, name="fetcher") + restored = component_from_dict(ContextFetcher, data, name="fetcher") + + assert restored.scrape_params == {"includeImages": True} + assert restored.timeout == 45 + assert restored.api_key.resolve_value() == "test-key" + + @patch("haystack_integrations.components.fetchers.context.context_fetcher.request_context") + def test_run_fetches_each_url(self, request_context: MagicMock) -> None: + request_context.return_value = SCRAPE_RESPONSE + scrape_params = {"maxAgeMs": 0} + original_params = deepcopy(scrape_params) + fetcher = ContextFetcher(api_key=Secret.from_token("test-key"), scrape_params=scrape_params) + + result = fetcher.run(urls=["https://haystack.deepset.ai", "https://docs.haystack.deepset.ai"]) + + assert len(result["documents"]) == 2 + assert all(isinstance(document, Document) for document in result["documents"]) + assert result["documents"][0].content == "# Haystack" + assert result["documents"][0].meta == { + "title": "Haystack", + "description": "Build AI applications.", + "url": "https://haystack.deepset.ai", + "content_length": 10, + } + assert request_context.call_count == 2 + assert request_context.call_args_list[0].kwargs["params"] == { + "url": "https://haystack.deepset.ai", + "useMainContentOnly": True, + "includeLinks": True, + "includeImages": False, + "maxAgeMs": 0, + } + assert scrape_params == original_params + + @patch("haystack_integrations.components.fetchers.context.context_fetcher.request_context") + def test_run_runtime_params_replace_init_params(self, request_context: MagicMock) -> None: + request_context.return_value = SCRAPE_RESPONSE + fetcher = ContextFetcher( + api_key=Secret.from_token("test-key"), + scrape_params={"maxAgeMs": 1000}, + ) + + fetcher.run(urls=["https://haystack.deepset.ai"], scrape_params={"includeImages": True}) + + params = request_context.call_args.kwargs["params"] + assert params["includeImages"] is True + assert "maxAgeMs" not in params + + @patch("haystack_integrations.components.fetchers.context.context_fetcher.request_context_async") + @pytest.mark.asyncio + async def test_run_async_fetches_concurrently(self, request_context: AsyncMock) -> None: + request_context.side_effect = AsyncMock(return_value=SCRAPE_RESPONSE) + fetcher = ContextFetcher(api_key=Secret.from_token("test-key")) + + result = await fetcher.run_async(urls=["https://haystack.deepset.ai", "https://docs.haystack.deepset.ai"]) + + assert len(result["documents"]) == 2 + assert request_context.await_count == 2 + + @pytest.mark.skipif( + not os.environ.get("CONTEXT_API_KEY"), + reason="Export CONTEXT_API_KEY to run integration tests.", + ) + @pytest.mark.integration + def test_run_integration(self) -> None: + fetcher = ContextFetcher() + result = fetcher.run(urls=["https://haystack.deepset.ai"]) + + assert len(result["documents"]) == 1 + assert result["documents"][0].content + assert result["documents"][0].meta["url"] == "https://haystack.deepset.ai" + + @pytest.mark.skipif( + not os.environ.get("CONTEXT_API_KEY"), + reason="Export CONTEXT_API_KEY to run integration tests.", + ) + @pytest.mark.integration + @pytest.mark.asyncio + async def test_run_async_integration(self) -> None: + fetcher = ContextFetcher() + result = await fetcher.run_async(urls=["https://haystack.deepset.ai"]) + + assert len(result["documents"]) == 1 + assert result["documents"][0].content + assert result["documents"][0].meta["url"] == "https://haystack.deepset.ai" diff --git a/tests/test_context_websearch.py b/tests/test_context_websearch.py new file mode 100644 index 0000000..103d596 --- /dev/null +++ b/tests/test_context_websearch.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +import os +from copy import deepcopy +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from haystack import Document +from haystack.core.serialization import component_from_dict, component_to_dict +from haystack.utils import Secret + +from haystack_integrations.components.websearch.context import ContextWebSearch + +SEARCH_RESPONSE = { + "query": "Haystack", + "results": [ + { + "url": "https://haystack.deepset.ai", + "title": "Haystack", + "description": "Build production-ready AI applications.", + "relevance": "high", + "markdown": {"markdown": None, "code": "NOT_REQUESTED"}, + }, + { + "url": "https://docs.haystack.deepset.ai", + "title": "Haystack documentation", + "description": "Haystack documentation.", + "relevance": "medium", + "markdown": {"markdown": "# Haystack docs", "code": "SUCCESS"}, + }, + ], +} + + +class TestContextWebSearch: + def test_init_defaults(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CONTEXT_API_KEY", "test-key") + websearch = ContextWebSearch() + + assert websearch.api_key.resolve_value() == "test-key" + assert websearch.top_k == 10 + assert websearch.include_markdown is False + assert websearch.api_url == "https://api.context.dev/v1" + + def test_init_rejects_invalid_top_k(self) -> None: + with pytest.raises(ValueError, match="between 1 and 100"): + ContextWebSearch(api_key=Secret.from_token("test-key"), top_k=0) + + def test_serialization_roundtrip(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CONTEXT_API_KEY", "test-key") + websearch = ContextWebSearch( + top_k=5, + include_domains=["haystack.deepset.ai"], + include_markdown=True, + search_params={"tags": ["haystack"]}, + ) + + data = component_to_dict(websearch, name="websearch") + restored = component_from_dict(ContextWebSearch, data, name="websearch") + + assert restored.top_k == 5 + assert restored.include_domains == ["haystack.deepset.ai"] + assert restored.include_markdown is True + assert restored.search_params == {"tags": ["haystack"]} + assert restored.api_key.resolve_value() == "test-key" + + @patch("haystack_integrations.components.websearch.context.context_websearch.request_context") + def test_run_builds_request_and_returns_documents(self, request_context: MagicMock) -> None: + request_context.return_value = SEARCH_RESPONSE + search_params = {"tags": ["haystack"]} + original_params = deepcopy(search_params) + websearch = ContextWebSearch( + api_key=Secret.from_token("test-key"), + top_k=2, + include_domains=["deepset.ai"], + exclude_domains=["example.com"], + freshness="last_week", + country="us", + include_markdown=True, + search_params=search_params, + ) + + result = websearch.run(query="Haystack") + + request_body = request_context.call_args.kwargs["json"] + assert request_body == { + "query": "Haystack", + "numResults": 10, + "includeDomains": ["deepset.ai"], + "excludeDomains": ["example.com"], + "freshness": "last_week", + "country": "us", + "tags": ["haystack"], + "markdownOptions": { + "enabled": True, + "useMainContentOnly": True, + "includeLinks": True, + "includeImages": False, + }, + } + assert search_params == original_params + assert result["links"] == ["https://haystack.deepset.ai", "https://docs.haystack.deepset.ai"] + assert all(isinstance(document, Document) for document in result["documents"]) + assert result["documents"][0].content == "Build production-ready AI applications." + assert result["documents"][1].content == "# Haystack docs" + assert result["documents"][1].meta["markdown_code"] == "SUCCESS" + + @patch("haystack_integrations.components.websearch.context.context_websearch.request_context") + def test_run_runtime_params_replace_init_params(self, request_context: MagicMock) -> None: + request_context.return_value = SEARCH_RESPONSE + websearch = ContextWebSearch( + api_key=Secret.from_token("test-key"), + top_k=10, + search_params={"tags": ["init"]}, + ) + + result = websearch.run(query="Haystack", top_k=1, search_params={"tags": ["run"]}) + + assert request_context.call_args.kwargs["json"]["tags"] == ["run"] + assert len(result["documents"]) == 1 + + @patch("haystack_integrations.components.websearch.context.context_websearch.request_context_async") + @pytest.mark.asyncio + async def test_run_async(self, request_context: AsyncMock) -> None: + request_context.side_effect = AsyncMock(return_value=SEARCH_RESPONSE) + websearch = ContextWebSearch(api_key=Secret.from_token("test-key"), top_k=1) + + result = await websearch.run_async(query="Haystack") + + assert len(result["documents"]) == 1 + assert result["links"] == ["https://haystack.deepset.ai"] + request_context.assert_awaited_once() + + def test_parse_response_skips_invalid_results(self) -> None: + result = ContextWebSearch._parse_response({"results": [None, "invalid"]}, top_k=10) + assert result == {"documents": [], "links": []} + + @pytest.mark.skipif( + not os.environ.get("CONTEXT_API_KEY"), + reason="Export CONTEXT_API_KEY to run integration tests.", + ) + @pytest.mark.integration + def test_run_integration(self) -> None: + websearch = ContextWebSearch(top_k=3) + result = websearch.run(query="What is Haystack by deepset?") + + assert len(result["documents"]) == 3 + assert len(result["links"]) == 3 + assert all(document.content for document in result["documents"]) + + @pytest.mark.skipif( + not os.environ.get("CONTEXT_API_KEY"), + reason="Export CONTEXT_API_KEY to run integration tests.", + ) + @pytest.mark.integration + @pytest.mark.asyncio + async def test_run_async_integration(self) -> None: + websearch = ContextWebSearch(top_k=3) + result = await websearch.run_async(query="What is Haystack by deepset?") + + assert len(result["documents"]) == 3 + assert len(result["links"]) == 3 + assert all(document.content for document in result["documents"]) diff --git a/tests/test_example.py b/tests/test_example.py deleted file mode 100644 index a4dff1a..0000000 --- a/tests/test_example.py +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present AUTHOR -# -# SPDX-License-Identifier: Apache-2.0 - -# TODO: Replace these example tests with tests for your own component(s). - -from haystack.core.serialization import component_from_dict, component_to_dict - -from haystack_integrations.components.example import ExampleComponent - - -class TestExampleComponent: - def test_init_default(self): - component = ExampleComponent() - assert component.param == "default" - - def test_init_custom_param(self): - component = ExampleComponent(param="custom") - assert component.param == "custom" - - def test_run(self): - component = ExampleComponent() - result = component.run(input_text="Hello, world!") - assert result == {"output": "Hello, world!"} - - def test_to_dict(self): - component = ExampleComponent(param="custom") - data = component_to_dict(component, "ExampleComponent") - assert data == { - "type": "haystack_integrations.components.example.example_component.ExampleComponent", - "init_parameters": {"param": "custom"}, - } - - def test_from_dict(self): - data = { - "type": "haystack_integrations.components.example.example_component.ExampleComponent", - "init_parameters": {"param": "custom"}, - } - deserialized = component_from_dict(ExampleComponent, data, "ExampleComponent") - assert deserialized.param == "custom" From 7b72a34e8cabdbc9a2b40759457b413c8c1a2339 Mon Sep 17 00:00:00 2001 From: aadithyan rajesh Date: Tue, 1 Sep 2026 19:00:21 +0530 Subject: [PATCH 4/4] docs: add usage and contribution guides --- CONTRIBUTING.md | 38 ++++++++++++ README.md | 114 +++++++++++++++++++---------------- examples/context_pipeline.py | 14 +++++ examples/example.py | 12 ---- 4 files changed, 115 insertions(+), 63 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 examples/context_pipeline.py delete mode 100644 examples/example.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..6fac404 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,38 @@ +# Contributing + +Thanks for contributing to the Context.dev Haystack integration. + +## Setup + +This project uses [Hatch](https://hatch.pypa.io/) for environments, formatting, tests, and builds. + +```bash +pip install hatch +hatch --version +``` + +## Checks + +Run the same checks used by CI before opening a pull request: + +```bash +hatch run fmt-check +hatch run test:types +hatch run test:unit +hatch run test:cov +``` + +Integration tests call the live Context.dev API and consume credits. They are skipped unless `CONTEXT_API_KEY` is set: + +```bash +export CONTEXT_API_KEY="your-api-key" +hatch run test:integration +``` + +## Pull requests + +Keep changes focused, add tests for behavior changes, and use Conventional Commit titles such as `feat: add a component option` or `fix: preserve response metadata`. + +## Releases + +Maintainers publish releases by pushing a semantic version tag such as `v0.1.0`. The release workflow builds the source distribution and wheel, then publishes both to PyPI. diff --git a/README.md b/README.md index e72d49d..c0e6af4 100644 --- a/README.md +++ b/README.md @@ -1,80 +1,92 @@ -# Custom Component Template +# Context.dev for Haystack -A template repository for creating custom [Haystack](https://haystack.deepset.ai/) components and publishing them as standalone Python packages. +[![PyPI](https://img.shields.io/pypi/v/context-dev-haystack)](https://pypi.org/project/context-dev-haystack/) +[![Python](https://img.shields.io/pypi/pyversions/context-dev-haystack)](https://pypi.org/project/context-dev-haystack/) +[![Test](https://github.com/context-dot-dev/context-haystack/actions/workflows/test.yml/badge.svg)](https://github.com/context-dot-dev/context-haystack/actions/workflows/test.yml) +[![License](https://img.shields.io/github/license/context-dot-dev/context-haystack)](LICENSE) -For more details, see the Haystack documentation on [creating custom components](https://docs.haystack.deepset.ai/docs/custom-components) and [creating custom document stores](https://docs.haystack.deepset.ai/docs/creating-custom-document-stores). +Haystack components for live web search, webpage and YouTube transcript retrieval, and bounded website crawling with [Context.dev](https://context.dev). -## How to use this template +## Installation -1. Click **[Use this template](https://github.com/deepset-ai/custom-component/generate)** to create a new repository. +```bash +pip install context-dev-haystack +``` -2. **Rename the package directory** from `src/haystack_integrations/components/example/` to match your integration. See [Namespace convention](#namespace-convention) below for the correct path. +Create an API key in the [Context.dev dashboard](https://context.dev/dashboard/api-keys), then export it: -3. **Update `pyproject.toml`** — search for `TODO` comments and replace: - - `name`: your package name, following the `-haystack` convention (e.g. `opensearch-haystack`) - - `description`, `authors`, `keywords`, `project.urls` - - `dependencies`: add your integration-specific dependencies - - `tool.hatch.version.raw-options`: if you renamed directories, the version path is still derived from git tags so no change is needed here +```bash +export CONTEXT_API_KEY="your-api-key" +``` -4. **Add your component code** in the renamed directory and export your classes from `__init__.py`. +## Components -5. **Add tests** in `tests/` — see the skeleton in `tests/test_example.py`. +| Component | Purpose | Import | +| --- | --- | --- | +| `ContextWebSearch` | Search the live web and return ranked Haystack Documents and source links | `haystack_integrations.components.websearch.context` | +| `ContextFetcher` | Fetch webpages or YouTube videos as clean Markdown Documents | `haystack_integrations.components.fetchers.context` | +| `ContextCrawler` | Crawl websites into Documents with explicit page and depth limits | `haystack_integrations.components.fetchers.context` | -6. **Search for all `TODO` comments** across the project and address them. +All components support both `run()` and `run_async()`, Haystack serialization, custom timeouts, and retry configuration. -Check out the [video walkthrough](https://www.youtube.com/watch?v=SWC0QecAMcI) for a step-by-step guide on how to use this template. +## Search the live web -## Namespace convention +```python +from haystack_integrations.components.websearch.context import ContextWebSearch -Haystack integrations use the `haystack_integrations` namespace package. The directory structure under `src/` determines the import path for your component. +search = ContextWebSearch(top_k=5, include_markdown=True) +result = search.run(query="Recent advances in retrieval-augmented generation") -**Components** (converters, embedders, generators, rankers, etc.) use: +documents = result["documents"] +links = result["links"] ``` -src/haystack_integrations/components/// -``` -Import path: `from haystack_integrations.components.. import MyComponent` -Common component types: `converters`, `embedders`, `generators`, `rankers`, `retrievers`, `connectors`, `tools`, `websearch` +Use `include_domains`, `exclude_domains`, `freshness`, and `country` to constrain results. Extra Context.dev Search API fields can be supplied through `search_params`. -**Document stores** use a separate namespace: -``` -src/haystack_integrations/document_stores// +## Fetch webpages or YouTube transcripts + +```python +from haystack_integrations.components.fetchers.context import ContextFetcher + +fetcher = ContextFetcher() +result = fetcher.run( + urls=[ + "https://haystack.deepset.ai", + "https://www.youtube.com/watch?v=UF8uR6Z6KLc", + ] +) + +documents = result["documents"] ``` -Import path: `from haystack_integrations.document_stores. import MyDocumentStore` -## Development +Each URL becomes a Haystack `Document`. Webpages contain clean Markdown and page metadata; supported YouTube URLs return timestamped transcript Markdown. -This project uses [Hatch](https://hatch.pypa.io/) for build and environment management. +## Crawl a website -```bash -# Install Hatch -pip install hatch - -# Format and lint -hatch run fmt # auto-fix -hatch run fmt-check # check only - -# Run tests -hatch run test:unit # unit tests only -hatch run test:integration # integration tests only -hatch run test:all # all tests -hatch run test:cov # with coverage +```python +from haystack_integrations.components.fetchers.context import ContextCrawler + +crawler = ContextCrawler(crawl_params={"maxPages": 25, "maxDepth": 2}) +result = crawler.run(urls=["https://docs.haystack.deepset.ai"]) + +documents = result["documents"] ``` -## Publishing to PyPI +`ContextCrawler` defaults to one page to prevent accidental credit consumption. Set `maxPages` explicitly for larger crawls. + +## Async usage -This template includes a GitHub Actions workflow that publishes your package to PyPI when you push a version tag. +```python +result = await search.run_async(query="Haystack agents") +documents = result["documents"] +``` -1. **Add a `PYPI_API_TOKEN` secret** to your repository settings (Settings > Secrets and variables > Actions). +The fetcher and crawler process multiple input URLs concurrently in their async methods. -2. **Create a version tag** and push it: - ```bash - git tag v0.1.0 - git push origin v0.1.0 - ``` +## Development -The release workflow will build and publish the package automatically. +See [CONTRIBUTING.md](CONTRIBUTING.md) for the Hatch-based development and release workflow. ## License -`Apache-2.0` - See [LICENSE](LICENSE) for details. +Apache-2.0. See [LICENSE](LICENSE). diff --git a/examples/context_pipeline.py b/examples/context_pipeline.py new file mode 100644 index 0000000..7c9766f --- /dev/null +++ b/examples/context_pipeline.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2026-present Context.dev +# +# SPDX-License-Identifier: Apache-2.0 + +from haystack import Pipeline + +from haystack_integrations.components.websearch.context import ContextWebSearch + +pipeline = Pipeline() +pipeline.add_component("search", ContextWebSearch(top_k=5, include_markdown=True)) + +result = pipeline.run({"search": {"query": "What is Haystack by deepset?"}}) +for document in result["search"]["documents"]: + print(document.meta["url"]) diff --git a/examples/example.py b/examples/example.py deleted file mode 100644 index dda8191..0000000 --- a/examples/example.py +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present AUTHOR -# -# SPDX-License-Identifier: Apache-2.0 - -from haystack_integrations.components.example import ExampleComponent - -# This is a minimal example showing how to use the component. -# Replace this with a usage example that demonstrates your component's functionality. -component = ExampleComponent(param="my_param") -result = component.run(input_text="Hello, world!") - -print(result)