duckduckgo search prototype

main.py

@@ -55,6 +55,7 @@ def initialize():
         # msgs_keep_start = 5,
         # msgs_keep_end = 10,
         # max_tool_response_length = 3000,
+        # response_timeout_seconds = 60,
         code_exec_docker_enabled = True,
         # code_exec_docker_name = "agent-zero-exe",
         # code_exec_docker_image = "frdel/agent-zero-exe:latest",

prompts/agent.memory.md

@@ -1,5 +1,4 @@
 # Memories
 - following are your memories on the current topic
-- you may find some of them helpful to solve the current task
 
 {{memories}}

prompts/fw.code_no_output.md

@@ -1,3 +1,5 @@
-No output or error was returned.
-If you require output from the tool, you have to use use console printing in your code.
-Otherwise proceed.
+~~~json
+{
+    "system_warning": "No output or error was returned. If you require output from the tool, you have to use use console printing in your code. Otherwise proceed."
+}
+~~~

prompts/fw.code_runtime_wrong.md

@@ -1 +1,5 @@
-The runtime "{{runtime}}" is not supported, available options are "terminal", "python" and "nodejs".
+~~~json
+{
+    "system_warning": "The runtime '{{runtime}}' is not supported, available options are 'terminal', 'python', 'nodejs' and 'output'."
+}
+~~~

prompts/fw.error.md

@@ -1,2 +1,5 @@
-An error has occured due to your last message:
-{{error}}
+~~~json
+{
+    "system_error": "{{error}}"
+}
+~~~

prompts/fw.intervention.md

@@ -1 +1,5 @@
-INTERVENTION: {{user_message}}
+~~~json
+{
+    "user_intervention": "{{user_message}}"
+}
+~~~

prompts/fw.memories_deleted.md

@@ -1 +1,5 @@
-Memories deleted: {{memory_count}}
+~~~json
+{
+    "memories_deleted": "{{memory_count}}"
+}
+~~~

prompts/fw.memories_not_found.md

@@ -1 +1,5 @@
-No memories found for specified query: {{query}}
+~~~json
+{
+    "memory": "No memories found for specified query: {{query}}"
+}
+~~~

prompts/fw.memorized.md

@@ -1 +0,0 @@
-Information saved to memory.

prompts/fw.memory_saved.md

@@ -1 +1,5 @@
-Memory has been saved with id {{memory_id}}.
+~~~json
+{
+    "memory": "Memory has been saved with id {{memory_id}}."
+}
+~~~

prompts/fw.msg_cleanup.md

@@ -1,6 +1,7 @@
 # Provide a JSON summary of given messages
 - From the messages you are given, write a summary of key points in the conversation.
 - Include important aspects and remove unnecessary details.
+- Keep necessary information like file names, URLs, keys etc.
 
 # Expected output format
 ~~~json

prompts/tool.knowledge.response.md

@@ -0,0 +1,6 @@
+~~~json
+{
+    "online_sources": "{{online_sources}}",
+    "memory": "{{memory}}",
+}
+~~~

python/helpers/duckduckgo_search.py

@@ -0,0 +1,30 @@
+# from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
+
+# def search(query: str, results = 5, region = "wt-wt", time="y") -> str:
+#     # Create an instance with custom parameters
+#     api = DuckDuckGoSearchAPIWrapper(
+#         region=region,  # Set the region for search results
+#         safesearch="off",  # Set safesearch level (options: strict, moderate, off)
+#         time=time,  # Set time range (options: d, w, m, y)
+#         max_results=results  # Set maximum number of results to return
+#     )
+#     # Perform a search
+#     result = api.run(query)
+#     return result
+
+from duckduckgo_search import DDGS
+
+def search(query: str, results = 5, region = "wt-wt", time="y") -> list[str]:
+
+    ddgs = DDGS()
+    src = ddgs.text(
+        query,
+        region=region,  # Specify region 
+        safesearch="off",  # SafeSearch setting
+        timelimit=time,  # Time limit (y = past year)
+        max_results=results  # Number of results to return
+    )
+    results = []
+    for s in src:
+        results.append(str(s))
+    return results

python/helpers/vector_db.py

@@ -28,6 +28,7 @@ class VectorDB:
             self.store, 
             namespace=getattr(embeddings_model, 'model', getattr(embeddings_model, 'model_name', "default")) )
 
+
         self.db = Chroma(embedding_function=self.embedder,persist_directory=db_cache)
         
         
@@ -35,7 +36,7 @@ class VectorDB:
         return self.db.similarity_search(query,results)
     
     def search_similarity_threshold(self, query, results=3, threshold=0.5):
-        return self.db.search(query,search_type="similarity_score_threshold",score_threshold=threshold)
+        return self.db.search(query, search_type="similarity_score_threshold", k=results, score_threshold=threshold)
 
     def search_max_rel(self, query, results=3):
         return self.db.max_marginal_relevance_search(query,results)

python/tools/knowledge_tool.py

@@ -1,5 +1,9 @@
+import os
 from agent import Agent
 from . import online_knowledge_tool
+from python.helpers import perplexity_search
+from python.helpers import duckduckgo_search
+
 from . import memory_tool
 import concurrent.futures
 
@@ -12,12 +16,25 @@ class Knowledge(Tool):
     def execute(self, question="", **kwargs):
         with concurrent.futures.ThreadPoolExecutor() as executor:
             # Schedule the two functions to be run in parallel
-            future_online = executor.submit(online_knowledge_tool.process_question, question)
+
+            # perplexity search, if API provided
+            if os.getenv("API_KEY_PERPLEXITY"):
+                perplexity = executor.submit(perplexity_search.perplexity_search, question)
+            else: perplexity = None
+
+            # duckduckgo search
+            duckduckgo = executor.submit(duckduckgo_search.search, question)
+
+            # memory search
             future_memory = executor.submit(memory_tool.search, self.agent, question)
 
             # Wait for both functions to complete
-            online_result = future_online.result()
+            perplexity_result = (perplexity.result() if perplexity else "") or ""
+            duckduckgo_result = duckduckgo.result()
             memory_result = future_memory.result()
 
-        result = f"# Online sources:\n{online_result}\n\n# Memory:\n{memory_result}"
-        return Response(message=result, break_loop=False)
+        msg = files.read_file("prompts/tool.knowledge.response.md", 
+                              online_sources = perplexity_result + "\n\n" + str(duckduckgo_result),
+                              memory = memory_result )
+
+        return Response(message=msg, break_loop=False)

requirements.txt

@@ -11,4 +11,5 @@ webcolors==24.6.0
 sentence-transformers==3.0.1
 pytimedinput==2.0.1
 docker==7.1.0
-paramiko==3.4.0
+paramiko==3.4.0
+duckduckgo_search==6.1.12