feat: add Tavily search backend

README.md

@@ -140,6 +140,7 @@ when you first time run Devika, it will create a `config.toml` file for you in t
    - `BING`: Your Bing Search API key for web searching capabilities.
    - `GOOGLE_SEARCH`: Your Google Search API key for web searching capabilities.
    - `GOOGLE_SEARCH_ENGINE_ID`: Your Google Search Engine ID for web searching using Google.
+   - `TAVILY`: Your Tavily API key for web searching capabilities.
    - `OPENAI`: Your OpenAI API key for accessing GPT models.
    - `GEMINI`: Your Gemini API key for accessing Gemini models.
    - `CLAUDE`: Your Anthropic API key for accessing Claude models.
@@ -153,7 +154,7 @@ when you first time run Devika, it will create a `config.toml` file for you in t
    - `OLLAMA`: The Ollama API endpoint for accessing Local LLMs.
    - `OPENAI`: The OpenAI API endpoint for accessing OpenAI models.
 
-Make sure to keep your API keys secure and do not share them publicly. For setting up the Bing and Google search API keys, follow the instructions in the [search engine setup](docs/Installation/search_engine.md)
+Make sure to keep your API keys secure and do not share them publicly. For setting up the Bing, Google, and Tavily search API keys, follow the instructions in the [search engine setup](docs/Installation/search_engine.md)
 
 
 ## Contributing

devika.py

@@ -61,7 +61,7 @@ def test_connect(data):
 def data():
     project = manager.get_project_list()
     models = LLM().list_models()
-    search_engines = ["Bing", "Google", "DuckDuckGo"]
+    search_engines = ["Bing", "Google", "DuckDuckGo", "Tavily"]
     return jsonify({"projects": project, "models": models, "search_engines": search_engines})
 
 

docs/Installation/search_engine.md

@@ -1,6 +1,6 @@
 # search Engine setup
 
-To use the search engine capabilities of Devika, you need to set up the search engine API keys. Currently, Devika supports Bing, Google and DuckDuckGo search engines. If you want to use duckduckgo, you don't need to set up any API keys.
+To use the search engine capabilities of Devika, you need to set up the search engine API keys. Devika supports Bing, Google, DuckDuckGo, and Tavily search engines. DuckDuckGo does not require an API key.
 
 For Bing and Google search engines, you need to set up the API keys. Here's how you can do it:
 
@@ -31,3 +31,8 @@ For Bing and Google search engines, you need to set up the API keys. Here's how
 - click on the `Add` button.
 ![alt text](images/google-2.png)
 - After creating the engine. Copy the `Search Engine ID` and paste it in the API_Endpoints field with the name `GOOGLE_SEARCH_ENGINE_ID` in the `config.toml` file in the root directory of Devika or you can set it via UI.
+
+## Tavily Search API
+- Create a Tavily account and generate an API key from the [Tavily dashboard](https://app.tavily.com/).
+- Copy the key into the `TAVILY` field under `API_KEYS` in `config.toml`, or set it through the Devika UI.
+- Select `Tavily` as the search engine when starting a task. Tavily returns multiple ranked sources and relevant content directly to the coding agent.

requirements.txt

@@ -31,3 +31,4 @@ orjson
 gevent
 gevent-websocket
 curl_cffi
+tavily-python==0.7.27

sample.config.toml

@@ -10,6 +10,7 @@ REPOS_DIR = "data/repos"
 BING = "<YOUR_BING_API_KEY>"
 GOOGLE_SEARCH = "<YOUR_GOOGLE_SEARCH_API_KEY>"
 GOOGLE_SEARCH_ENGINE_ID = "<YOUR_GOOGLE_SEARCH_ENGINE_ID>"
+TAVILY = "<YOUR_TAVILY_API_KEY>"
 CLAUDE = "<YOUR_CLAUDE_API_KEY>"
 OPENAI = "<YOUR_OPENAI_API_KEY>"
 GEMINI = "<YOUR_GEMINI_API_KEY>"

src/agents/agent.py

@@ -17,7 +17,7 @@
 
 from src.bert.sentence import SentenceBert
 from src.memory import KnowledgeBase
-from src.browser.search import BingSearch, GoogleSearch, DuckDuckGoSearch
+from src.browser.search import BingSearch, GoogleSearch, DuckDuckGoSearch, TavilySearch
 from src.browser import Browser
 from src.browser import start_interaction
 from src.filesystem import ReadCode
@@ -85,6 +85,8 @@ def search_queries(self, queries: list, project_name: str) -> dict:
             web_search = BingSearch()
         elif self.engine == "google":
             web_search = GoogleSearch()
+        elif self.engine == "tavily":
+            web_search = TavilySearch()
         else:
             web_search = DuckDuckGoSearch()
 
@@ -98,6 +100,11 @@ def search_queries(self, queries: list, project_name: str) -> dict:
             #     results[query] = knowledge
             #     continue
 
+            if self.engine == "tavily":
+                results[query] = web_search.search(query)
+                self.logger.info(f"got the search results for : {query}")
+                continue
+
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
 

src/agents/coder/prompt.jinja2

@@ -10,15 +10,25 @@ Context From User:
 
 Context From Knowledge Base:
 
-{% if not knowledge_base_context %}
+{% if not search_results %}
 No context found.
 {% else %}
 {% for query, result in search_results.items() %}
 Query: {{ query }}
+{% if result is string %}
 Result:
 ```
 {{ result }}
 ```
+{% else %}
+Sources:
+{% for source in result %}
+- Title: {{ source.title }}
+  URL: {{ source.url }}
+  Score: {{ source.score }}
+  Content: {{ source.content }}
+{% endfor %}
+{% endif %}
 ---
 {% endfor %}
 {% endif %}

src/browser/search.py

@@ -1,10 +1,16 @@
-import requests
-from src.config import Config
-
+import logging
 import re
-from urllib.parse import unquote
 from html import unescape
+from urllib.parse import unquote
+
 import orjson
+import requests
+from tavily import TavilyClient
+
+from src.config import Config
+
+
+logger = logging.getLogger(__name__)
 
 
 class BingSearch:
@@ -30,6 +36,40 @@ def get_first_link(self):
         return self.query_result["webPages"]["value"][0]["url"]
 
 
+class TavilySearch:
+    def __init__(self):
+        self.config = Config()
+        self.client = TavilyClient(
+            api_key=self.config.get_tavily_api_key(),
+            client_name="open-monitor/stitionai/devika",
+        )
+        self.query_result = []
+
+    def search(self, query):
+        try:
+            response = self.client.search(
+                query=query,
+                max_results=5,
+                include_answer=False,
+                include_raw_content=False,
+            )
+            results = response.get("results", []) if isinstance(response, dict) else []
+            self.query_result = [
+                {
+                    "title": result.get("title", ""),
+                    "url": result.get("url", ""),
+                    "content": result.get("content", ""),
+                    "score": result.get("score"),
+                }
+                for result in results
+                if isinstance(result, dict) and result.get("url")
+            ]
+        except Exception as error:
+            logger.warning("Tavily search failed for %r: %s", query, error)
+            self.query_result = []
+        return self.query_result
+
+
 class GoogleSearch:
     def __init__(self):
         self.config = Config()

src/config.py

@@ -57,6 +57,9 @@ def get_google_search_engine_id(self):
     def get_google_search_api_endpoint(self):
         return self.config["API_ENDPOINTS"]["GOOGLE"]
 
+    def get_tavily_api_key(self):
+        return self.config["API_KEYS"]["TAVILY"]
+
     def get_ollama_api_endpoint(self):
         return self.config["API_ENDPOINTS"]["OLLAMA"]
 

tests/test_tavily_search.py

@@ -0,0 +1,109 @@
+import importlib.util
+import sys
+import types
+
+
+browser_package = types.ModuleType("src.browser")
+browser_package.__path__ = []
+sys.modules["src.browser"] = browser_package
+config_module = types.ModuleType("src.config")
+config_module.Config = object
+sys.modules["src.config"] = config_module
+
+spec = importlib.util.spec_from_file_location("src.browser.search", "src/browser/search.py")
+search_module = importlib.util.module_from_spec(spec)
+sys.modules[spec.name] = search_module
+spec.loader.exec_module(search_module)
+
+
+class ConfigStub:
+    def get_tavily_api_key(self):
+        return "test-key"
+
+
+def test_tavily_search_normalizes_results_and_attributes_client(monkeypatch):
+    client_args = {}
+    search_args = {}
+
+    class ClientStub:
+        def __init__(self, **kwargs):
+            client_args.update(kwargs)
+
+        def search(self, **kwargs):
+            search_args.update(kwargs)
+            return {
+                "results": [
+                    {
+                        "title": "First source",
+                        "url": "https://example.com/one",
+                        "content": "First content",
+                        "score": 0.91,
+                    },
+                    {
+                        "title": "Second source",
+                        "url": "https://example.com/two",
+                        "content": "Second content",
+                        "score": 0.72,
+                    },
+                    {"title": "Missing URL"},
+                    "invalid",
+                ]
+            }
+
+    monkeypatch.setattr(search_module, "Config", ConfigStub)
+    monkeypatch.setattr(search_module, "TavilyClient", ClientStub)
+
+    search = search_module.TavilySearch()
+
+    assert search.search("test query") == [
+        {
+            "title": "First source",
+            "url": "https://example.com/one",
+            "content": "First content",
+            "score": 0.91,
+        },
+        {
+            "title": "Second source",
+            "url": "https://example.com/two",
+            "content": "Second content",
+            "score": 0.72,
+        },
+    ]
+    assert client_args == {
+        "api_key": "test-key",
+        "client_name": "open-monitor/stitionai/devika",
+    }
+    assert search_args == {
+        "query": "test query",
+        "max_results": 5,
+        "include_answer": False,
+        "include_raw_content": False,
+    }
+
+
+def test_tavily_search_returns_empty_results_for_invalid_response(monkeypatch):
+    class ClientStub:
+        def __init__(self, **kwargs):
+            pass
+
+        def search(self, **kwargs):
+            return {"results": [{"title": "No URL"}, "invalid"]}
+
+    monkeypatch.setattr(search_module, "Config", ConfigStub)
+    monkeypatch.setattr(search_module, "TavilyClient", ClientStub)
+
+    assert search_module.TavilySearch().search("test query") == []
+
+
+def test_tavily_search_isolates_provider_errors(monkeypatch):
+    class ClientStub:
+        def __init__(self, **kwargs):
+            pass
+
+        def search(self, **kwargs):
+            raise RuntimeError("provider unavailable")
+
+    monkeypatch.setattr(search_module, "Config", ConfigStub)
+    monkeypatch.setattr(search_module, "TavilyClient", ClientStub)
+
+    assert search_module.TavilySearch().search("test query") == []

ui/src/routes/settings/+page.svelte

@@ -47,6 +47,7 @@
       "BING": settings["API_KEYS"]["BING"],
       "GOOGLE_SEARCH": settings["API_KEYS"]["GOOGLE_SEARCH"],
       "GOOGLE_SEARCH_ENGINE_ID": settings["API_KEYS"]["GOOGLE_SEARCH_ENGINE_ID"],
+      "TAVILY": settings["API_KEYS"]["TAVILY"],
       "CLAUDE": settings["API_KEYS"]["CLAUDE"],
       "OPENAI": settings["API_KEYS"]["OPENAI"],
       "GEMINI": settings["API_KEYS"]["GEMINI"],