From f664ac0344aa910c7e9d8160aa8ff234d808499b Mon Sep 17 00:00:00 2001 From: "open-monitor-open-swe[bot]" Date: Tue, 11 Aug 2026 19:08:47 +0000 Subject: [PATCH] feat: add Tavily search backend --- README.md | 3 +- devika.py | 2 +- docs/Installation/search_engine.md | 7 +- requirements.txt | 1 + sample.config.toml | 1 + src/agents/agent.py | 9 ++- src/agents/coder/prompt.jinja2 | 12 ++- src/browser/search.py | 48 +++++++++++- src/config.py | 3 + tests/test_tavily_search.py | 109 ++++++++++++++++++++++++++++ ui/src/routes/settings/+page.svelte | 1 + 11 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 tests/test_tavily_search.py diff --git a/README.md b/README.md index d3cec019..da8e17c3 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ when you first time run Devika, it will create a `config.toml` file for you in t - `BING`: Your Bing Search API key for web searching capabilities. - `GOOGLE_SEARCH`: Your Google Search API key for web searching capabilities. - `GOOGLE_SEARCH_ENGINE_ID`: Your Google Search Engine ID for web searching using Google. + - `TAVILY`: Your Tavily API key for web searching capabilities. - `OPENAI`: Your OpenAI API key for accessing GPT models. - `GEMINI`: Your Gemini API key for accessing Gemini models. - `CLAUDE`: Your Anthropic API key for accessing Claude models. @@ -153,7 +154,7 @@ when you first time run Devika, it will create a `config.toml` file for you in t - `OLLAMA`: The Ollama API endpoint for accessing Local LLMs. - `OPENAI`: The OpenAI API endpoint for accessing OpenAI models. -Make sure to keep your API keys secure and do not share them publicly. For setting up the Bing and Google search API keys, follow the instructions in the [search engine setup](docs/Installation/search_engine.md) +Make sure to keep your API keys secure and do not share them publicly. For setting up the Bing, Google, and Tavily search API keys, follow the instructions in the [search engine setup](docs/Installation/search_engine.md) ## Contributing diff --git a/devika.py b/devika.py index 961b792a..87560353 100644 --- a/devika.py +++ b/devika.py @@ -61,7 +61,7 @@ def test_connect(data): def data(): project = manager.get_project_list() models = LLM().list_models() - search_engines = ["Bing", "Google", "DuckDuckGo"] + search_engines = ["Bing", "Google", "DuckDuckGo", "Tavily"] return jsonify({"projects": project, "models": models, "search_engines": search_engines}) diff --git a/docs/Installation/search_engine.md b/docs/Installation/search_engine.md index 0334457d..c71303f6 100644 --- a/docs/Installation/search_engine.md +++ b/docs/Installation/search_engine.md @@ -1,6 +1,6 @@ # search Engine setup -To use the search engine capabilities of Devika, you need to set up the search engine API keys. Currently, Devika supports Bing, Google and DuckDuckGo search engines. If you want to use duckduckgo, you don't need to set up any API keys. +To use the search engine capabilities of Devika, you need to set up the search engine API keys. Devika supports Bing, Google, DuckDuckGo, and Tavily search engines. DuckDuckGo does not require an API key. For Bing and Google search engines, you need to set up the API keys. Here's how you can do it: @@ -31,3 +31,8 @@ For Bing and Google search engines, you need to set up the API keys. Here's how - click on the `Add` button. ![alt text](images/google-2.png) - After creating the engine. Copy the `Search Engine ID` and paste it in the API_Endpoints field with the name `GOOGLE_SEARCH_ENGINE_ID` in the `config.toml` file in the root directory of Devika or you can set it via UI. + +## Tavily Search API +- Create a Tavily account and generate an API key from the [Tavily dashboard](https://app.tavily.com/). +- Copy the key into the `TAVILY` field under `API_KEYS` in `config.toml`, or set it through the Devika UI. +- Select `Tavily` as the search engine when starting a task. Tavily returns multiple ranked sources and relevant content directly to the coding agent. diff --git a/requirements.txt b/requirements.txt index 91666960..41bf0c51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,3 +31,4 @@ orjson gevent gevent-websocket curl_cffi +tavily-python==0.7.27 diff --git a/sample.config.toml b/sample.config.toml index 62cde107..83686d91 100644 --- a/sample.config.toml +++ b/sample.config.toml @@ -10,6 +10,7 @@ REPOS_DIR = "data/repos" BING = "" GOOGLE_SEARCH = "" GOOGLE_SEARCH_ENGINE_ID = "" +TAVILY = "" CLAUDE = "" OPENAI = "" GEMINI = "" diff --git a/src/agents/agent.py b/src/agents/agent.py index 2018337e..9eebaab1 100644 --- a/src/agents/agent.py +++ b/src/agents/agent.py @@ -17,7 +17,7 @@ from src.bert.sentence import SentenceBert from src.memory import KnowledgeBase -from src.browser.search import BingSearch, GoogleSearch, DuckDuckGoSearch +from src.browser.search import BingSearch, GoogleSearch, DuckDuckGoSearch, TavilySearch from src.browser import Browser from src.browser import start_interaction from src.filesystem import ReadCode @@ -85,6 +85,8 @@ def search_queries(self, queries: list, project_name: str) -> dict: web_search = BingSearch() elif self.engine == "google": web_search = GoogleSearch() + elif self.engine == "tavily": + web_search = TavilySearch() else: web_search = DuckDuckGoSearch() @@ -98,6 +100,11 @@ def search_queries(self, queries: list, project_name: str) -> dict: # results[query] = knowledge # continue + if self.engine == "tavily": + results[query] = web_search.search(query) + self.logger.info(f"got the search results for : {query}") + continue + loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) diff --git a/src/agents/coder/prompt.jinja2 b/src/agents/coder/prompt.jinja2 index c6336a03..be3c3219 100644 --- a/src/agents/coder/prompt.jinja2 +++ b/src/agents/coder/prompt.jinja2 @@ -10,15 +10,25 @@ Context From User: Context From Knowledge Base: -{% if not knowledge_base_context %} +{% if not search_results %} No context found. {% else %} {% for query, result in search_results.items() %} Query: {{ query }} +{% if result is string %} Result: ``` {{ result }} ``` +{% else %} +Sources: +{% for source in result %} +- Title: {{ source.title }} + URL: {{ source.url }} + Score: {{ source.score }} + Content: {{ source.content }} +{% endfor %} +{% endif %} --- {% endfor %} {% endif %} diff --git a/src/browser/search.py b/src/browser/search.py index 030fa5fe..b3a275f1 100644 --- a/src/browser/search.py +++ b/src/browser/search.py @@ -1,10 +1,16 @@ -import requests -from src.config import Config - +import logging import re -from urllib.parse import unquote from html import unescape +from urllib.parse import unquote + import orjson +import requests +from tavily import TavilyClient + +from src.config import Config + + +logger = logging.getLogger(__name__) class BingSearch: @@ -30,6 +36,40 @@ def get_first_link(self): return self.query_result["webPages"]["value"][0]["url"] +class TavilySearch: + def __init__(self): + self.config = Config() + self.client = TavilyClient( + api_key=self.config.get_tavily_api_key(), + client_name="open-monitor/stitionai/devika", + ) + self.query_result = [] + + def search(self, query): + try: + response = self.client.search( + query=query, + max_results=5, + include_answer=False, + include_raw_content=False, + ) + results = response.get("results", []) if isinstance(response, dict) else [] + self.query_result = [ + { + "title": result.get("title", ""), + "url": result.get("url", ""), + "content": result.get("content", ""), + "score": result.get("score"), + } + for result in results + if isinstance(result, dict) and result.get("url") + ] + except Exception as error: + logger.warning("Tavily search failed for %r: %s", query, error) + self.query_result = [] + return self.query_result + + class GoogleSearch: def __init__(self): self.config = Config() diff --git a/src/config.py b/src/config.py index a3303118..0acf670b 100644 --- a/src/config.py +++ b/src/config.py @@ -57,6 +57,9 @@ def get_google_search_engine_id(self): def get_google_search_api_endpoint(self): return self.config["API_ENDPOINTS"]["GOOGLE"] + def get_tavily_api_key(self): + return self.config["API_KEYS"]["TAVILY"] + def get_ollama_api_endpoint(self): return self.config["API_ENDPOINTS"]["OLLAMA"] diff --git a/tests/test_tavily_search.py b/tests/test_tavily_search.py new file mode 100644 index 00000000..a220e341 --- /dev/null +++ b/tests/test_tavily_search.py @@ -0,0 +1,109 @@ +import importlib.util +import sys +import types + + +browser_package = types.ModuleType("src.browser") +browser_package.__path__ = [] +sys.modules["src.browser"] = browser_package +config_module = types.ModuleType("src.config") +config_module.Config = object +sys.modules["src.config"] = config_module + +spec = importlib.util.spec_from_file_location("src.browser.search", "src/browser/search.py") +search_module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = search_module +spec.loader.exec_module(search_module) + + +class ConfigStub: + def get_tavily_api_key(self): + return "test-key" + + +def test_tavily_search_normalizes_results_and_attributes_client(monkeypatch): + client_args = {} + search_args = {} + + class ClientStub: + def __init__(self, **kwargs): + client_args.update(kwargs) + + def search(self, **kwargs): + search_args.update(kwargs) + return { + "results": [ + { + "title": "First source", + "url": "https://example.com/one", + "content": "First content", + "score": 0.91, + }, + { + "title": "Second source", + "url": "https://example.com/two", + "content": "Second content", + "score": 0.72, + }, + {"title": "Missing URL"}, + "invalid", + ] + } + + monkeypatch.setattr(search_module, "Config", ConfigStub) + monkeypatch.setattr(search_module, "TavilyClient", ClientStub) + + search = search_module.TavilySearch() + + assert search.search("test query") == [ + { + "title": "First source", + "url": "https://example.com/one", + "content": "First content", + "score": 0.91, + }, + { + "title": "Second source", + "url": "https://example.com/two", + "content": "Second content", + "score": 0.72, + }, + ] + assert client_args == { + "api_key": "test-key", + "client_name": "open-monitor/stitionai/devika", + } + assert search_args == { + "query": "test query", + "max_results": 5, + "include_answer": False, + "include_raw_content": False, + } + + +def test_tavily_search_returns_empty_results_for_invalid_response(monkeypatch): + class ClientStub: + def __init__(self, **kwargs): + pass + + def search(self, **kwargs): + return {"results": [{"title": "No URL"}, "invalid"]} + + monkeypatch.setattr(search_module, "Config", ConfigStub) + monkeypatch.setattr(search_module, "TavilyClient", ClientStub) + + assert search_module.TavilySearch().search("test query") == [] + + +def test_tavily_search_isolates_provider_errors(monkeypatch): + class ClientStub: + def __init__(self, **kwargs): + pass + + def search(self, **kwargs): + raise RuntimeError("provider unavailable") + + monkeypatch.setattr(search_module, "Config", ConfigStub) + monkeypatch.setattr(search_module, "TavilyClient", ClientStub) + + assert search_module.TavilySearch().search("test query") == [] diff --git a/ui/src/routes/settings/+page.svelte b/ui/src/routes/settings/+page.svelte index 3db5ecde..49e23e87 100644 --- a/ui/src/routes/settings/+page.svelte +++ b/ui/src/routes/settings/+page.svelte @@ -47,6 +47,7 @@ "BING": settings["API_KEYS"]["BING"], "GOOGLE_SEARCH": settings["API_KEYS"]["GOOGLE_SEARCH"], "GOOGLE_SEARCH_ENGINE_ID": settings["API_KEYS"]["GOOGLE_SEARCH_ENGINE_ID"], + "TAVILY": settings["API_KEYS"]["TAVILY"], "CLAUDE": settings["API_KEYS"]["CLAUDE"], "OPENAI": settings["API_KEYS"]["OPENAI"], "GEMINI": settings["API_KEYS"]["GEMINI"],