Instruments

Added instruments - loading to memory, injecting to system prompt, example yt_download

.gitignore

@@ -38,4 +38,18 @@ knowledge/**/*.*
 
 # Explicitly allow the default folder and its contents
 !knowledge/default/
-!knowledge/default/**
+!knowledge/default/**
+
+# Ignore everything in the "instruments" directory
+instruments/*
+
+# Do not ignore subdirectories (so we can track .gitkeep)
+!instruments/*/
+
+# Ignore all files within subdirectories (except .gitkeep)
+instruments/**/*.*
+!instruments/**/.gitkeep
+
+# Explicitly allow the default folder and its contents
+!instruments/default/
+!instruments/default/**

agent.py

@@ -111,7 +111,8 @@ class AgentConfig:
     )
     code_exec_docker_volumes: dict[str, dict[str, str]] = field(
         default_factory=lambda: {
-            files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"}
+            files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},
+            files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},
         }
     )
     code_exec_ssh_enabled: bool = True

initialize.py

@@ -1,5 +1,6 @@
 import models
 from agent import AgentConfig
+from python.helpers import files
 
 def initialize():
     
@@ -47,7 +48,10 @@ def initialize():
         # code_exec_docker_name = "agent-zero-exe",
         # code_exec_docker_image = "frdel/agent-zero-exe:latest",
         # code_exec_docker_ports = { "22/tcp": 50022 }
-        # code_exec_docker_volumes = { files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"} }
+        # code_exec_docker_volumes = { 
+            # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},
+            # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},
+            #                         },
         code_exec_ssh_enabled = True,
         # code_exec_ssh_addr = "localhost",
         # code_exec_ssh_port = 50022,

instruments/default/yt_download/yt_download.md

@@ -0,0 +1,6 @@
+# Problem
+Download a YouTube video
+# Solution
+1. cd to the desired location to download
+2. Run instrument "bash /instruments/default/yt_download/yt_download.sh <url>" with your video URL
+3. Wait for the terminal to finish

instruments/default/yt_download/yt_download.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Install yt-dlp and ffmpeg
+sudo apt-get update && sudo apt-get install -y yt-dlp ffmpeg
+
+# Download the best video and audio, and merge them
+yt-dlp -f bestvideo+bestaudio --merge-output-format mp4 "$1"

knowledge/default/solutions/get_current_time.md

@@ -0,0 +1,13 @@
+# Problem
+User asked for current time in timezone
+# Solution
+Use code_execution_tool with following python code adjusted for your timezone
+~~~python
+from datetime import datetime
+import pytz
+
+timezone = pytz.timezone('America/New_York')
+current_time = datetime.now(timezone)
+
+print("Current time in New York:", current_time)
+~~~

knowledge/default/solutions/yt_download.md

@@ -1,6 +0,0 @@
-# Problem
-Download a YouTube video
-# Solution
-1. If you don't have exact URL, use knowledge_tool to get it
-2. Pip install yt-dlp and ffmpeg
-3. Download the video using yt-dlp command: 'yt-dlp YT_URL', replace YT_URL with your video URL.

prompts/default/agent.system.instruments.md

@@ -0,0 +1,4 @@
+# Instruments
+- following are instruments that could possibly be used:
+
+{{instruments}}

prompts/default/agent.system.main.solving.md

@@ -3,7 +3,7 @@
 - Explain each step using your thoughts argument.
 
 0. Outline the plan by repeating these instructions.
-1. Check the memory output of your knowledge_tool. Maybe you have solved similar task before and already have helpful information.
+1. Check your memories, solutions and instruments. Prefer using instruments when possible.
 2. Check the online sources output of your knowledge_tool. 
     - Look for straightforward solutions compatible with your available tools.
     - Always look for opensource python/nodejs/terminal tools and packages first.

prompts/default/agent.system.main.tips.md

@@ -14,6 +14,10 @@
 - Communication is the key to succesfull solution.
 - NEVER delegate your whole task, only parts of it.
 
+## Instruments
+- Instruments are programs you can utilize to solve tasks
+- Instrument descriptions are injected into the prompt and can be executed with code_execution_tool
+
 ## Tips and tricks
 - Focus on python/nodejs/linux libraries when searching for solutions. You can use them with your tools and make solutions easy.
 - Sometimes you don't need tools, some things can be determined.

python/extensions/message_loop_prompts/_50_recall_memories.py

@@ -60,12 +60,12 @@ class RecallMemories(Extension):
         # log the short result
         if not isinstance(memories, list) or len(memories) == 0:
             log_item.update(
-                heading="No useful memories found.",
+                heading="No useful memories found",
             )
             return
         else:
             log_item.update(
-                heading=f"\n\n{len(memories)} memories found.",
+                heading=f"{len(memories)} memories found",
             )
 
         # concatenate memory.page_content in memories:

python/extensions/message_loop_prompts/_51_recall_solutions.py

@@ -7,7 +7,8 @@ class RecallSolutions(Extension):
 
     INTERVAL = 3
     HISTORY = 5
-    RESULTS = 3
+    SOLUTIONS_COUNT = 2
+    INSTRUMENTS_COUNT = 2
     THRESHOLD = 0.1
 
     async def execute(self, loop_data: LoopData = LoopData(), **kwargs):
@@ -19,74 +20,78 @@ class RecallSolutions(Extension):
 
     async def search_solutions(self, loop_data: LoopData, **kwargs):
         # try:
-            # show temp info message
-            self.agent.context.log.log(
-                type="info", content="Searching memory for solutions...", temp=True
+        # show temp info message
+        self.agent.context.log.log(
+            type="info", content="Searching memory for solutions...", temp=True
+        )
+
+        # show full util message, this will hide temp message immediately if turned on
+        log_item = self.agent.context.log.log(
+            type="util",
+            heading="Searching memory for solutions...",
+        )
+
+        # get system message and chat history for util llm
+        msgs_text = self.agent.concat_messages(
+            self.agent.history[-RecallSolutions.HISTORY :]
+        )  # only last X messages
+        system = self.agent.read_prompt(
+            "memory.solutions_query.sys.md", history=msgs_text
+        )
+
+        # log query streamed by LLM
+        def log_callback(content):
+            log_item.stream(query=content)
+
+        # call util llm to summarize conversation
+        query = await self.agent.call_utility_llm(
+            system=system, msg=loop_data.message, callback=log_callback
+        )
+
+        # get solutions database
+        db = await Memory.get(self.agent)
+
+        solutions = await db.search_similarity_threshold(
+            query=query,
+            limit=RecallSolutions.SOLUTIONS_COUNT,
+            threshold=RecallSolutions.THRESHOLD,
+            filter=f"area == '{Memory.Area.SOLUTIONS.value}'",
+        )
+        instruments = await db.search_similarity_threshold(
+            query=query,
+            limit=RecallSolutions.INSTRUMENTS_COUNT,
+            threshold=RecallSolutions.THRESHOLD,
+            filter=f"area == '{Memory.Area.INSTRUMENTS.value}'",
+        )
+
+        log_item.update(
+            heading=f"{len(instruments)} instruments, {len(solutions)} solutions found",
+        )
+
+        if instruments:
+            instruments_text = ""
+            for instrument in instruments:
+                instruments_text += instrument.page_content + "\n\n"
+            instruments_text = instruments_text.strip()
+            log_item.update(instruments=instruments_text)
+            instruments_prompt = self.agent.read_prompt(
+                "agent.system.instruments.md", instruments=instruments_text
             )
+            loop_data.system.append(instruments_prompt)
 
-            # show full util message, this will hide temp message immediately if turned on
-            log_item = self.agent.context.log.log(
-                type="util",
-                heading="Searching memory for solutions...",
-            )
-
-            # get system message and chat history for util llm
-            msgs_text = self.agent.concat_messages(
-                self.agent.history[-RecallSolutions.HISTORY :]
-            )  # only last X messages
-            system = self.agent.read_prompt(
-                "memory.solutions_query.sys.md", history=msgs_text
-            )
-
-            # log query streamed by LLM
-            def log_callback(content):
-                log_item.stream(query=content)
-
-            # call util llm to summarize conversation
-            query = await self.agent.call_utility_llm(
-                system=system, msg=loop_data.message, callback=log_callback
-            )
-
-            # get solutions database
-            db = await Memory.get(self.agent)
-
-            solutions = await db.search_similarity_threshold(
-                query=query,
-                limit=RecallSolutions.RESULTS,
-                threshold=RecallSolutions.THRESHOLD,
-                filter=f"area == '{Memory.Area.SOLUTIONS.value}'"
-            )
-
-            # log the short result
-            if not isinstance(solutions, list) or len(solutions) == 0:
-                log_item.update(
-                    heading="No successful solution memories found.",
-                )
-                return
-            else:
-                log_item.update(
-                    heading=f"\n\n{len(solutions)} successful solution memories found.",
-                )
-
-            # concatenate solution.page_content in solutions:
+        if solutions:
             solutions_text = ""
             for solution in solutions:
                 solutions_text += solution.page_content + "\n\n"
             solutions_text = solutions_text.strip()
-
-            # log the full results
             log_item.update(solutions=solutions_text)
-
-            # place to prompt
             solutions_prompt = self.agent.read_prompt(
                 "agent.system.solutions.md", solutions=solutions_text
             )
-
-            # append to system message
             loop_data.system.append(solutions_prompt)
 
-        # except Exception as e:
-        #     err = errors.format_error(e)
-        #     self.agent.context.log.log(
-        #         type="error", heading="Recall solutions extension error:", content=err
-        #     )
+    # except Exception as e:
+    #     err = errors.format_error(e)
+    #     self.agent.context.log.log(
+    #         type="error", heading="Recall solutions extension error:", content=err
+    #     )

python/helpers/knowledge_import.py

@@ -34,11 +34,14 @@ def calculate_checksum(file_path: str) -> str:
 
 
 def load_knowledge(
-    log_item: LogItem | None, knowledge_dir: str, index: Dict[str, KnowledgeImport]
+    log_item: LogItem | None,
+    knowledge_dir: str,
+    index: Dict[str, KnowledgeImport],
+    metadata: dict[str, Any] = {},
+    filename_pattern: str = "**/*",
 ) -> Dict[str, KnowledgeImport]:
-    knowledge_dir = files.get_abs_path("knowledge",knowledge_dir)
 
-    from python.helpers.memory import Memory
+    # from python.helpers.memory import Memory
 
     # Mapping file extensions to corresponding loader classes
     file_types_loaders = {
@@ -47,62 +50,67 @@ def load_knowledge(
         "csv": CSVLoader,
         "html": UnstructuredHTMLLoader,
         "json": JSONLoader,
-        "md": UnstructuredMarkdownLoader,
+        # "md": UnstructuredMarkdownLoader,
+        "md": TextLoader,
     }
 
     cnt_files = 0
     cnt_docs = 0
 
-    for area in Memory.Area:
-        subdir = files.get_abs_path(knowledge_dir, area.value)
+    # for area in Memory.Area:
+    #     subdir = files.get_abs_path(knowledge_dir, area.value)
 
-        if not os.path.exists(subdir):
-            os.makedirs(subdir)
-            continue
+    # if not os.path.exists(knowledge_dir):
+    #     os.makedirs(knowledge_dir)
+    #     continue
 
-        # Fetch all files in the directory with specified extensions
-        kn_files = glob.glob(subdir + "/**/*", recursive=True)
-        if kn_files:
-            print(f"Found {len(kn_files)} knowledge files in {subdir}, processing...")
-            if log_item:
-                log_item.stream(
-                    progress=f"\nFound {len(kn_files)} knowledge files in {subdir}, processing...",
-                )
+    # Fetch all files in the directory with specified extensions
+    kn_files = glob.glob(knowledge_dir + "/" + filename_pattern, recursive=True)
+    kn_files = [f for f in kn_files if os.path.isfile(f)]
 
-        for file_path in kn_files:
-            ext = file_path.split(".")[-1].lower()
-            if ext in file_types_loaders:
-                checksum = calculate_checksum(file_path)
-                file_key = file_path  # os.path.relpath(file_path, knowledge_dir)
-
-                # Load existing data from the index or create a new entry
-                file_data = index.get(file_key, {})
-
-                if file_data.get("checksum") == checksum:
-                    file_data["state"] = "original"
-                else:
-                    file_data["state"] = "changed"
-
-                if file_data["state"] == "changed":
-                    file_data["checksum"] = checksum
-                    loader_cls = file_types_loaders[ext]
-                    loader = loader_cls(
-                        file_path,
-                        **(
-                            text_loader_kwargs
-                            if ext in ["txt", "csv", "html", "md"]
-                            else {}
-                        ),
-                    )
-                    file_data["documents"] = loader.load_and_split()
-                    for doc in file_data["documents"]:
-                        doc.metadata["area"] = area.value
-                    cnt_files += 1
-                    cnt_docs += len(file_data["documents"])
-                    # print(f"Imported {len(file_data['documents'])} documents from {file_path}")
-
-                # Update the index
-                index[file_key] = file_data  # type: ignore
+    if kn_files:
+        print(
+            f"Found {len(kn_files)} knowledge files in {knowledge_dir}, processing..."
+        )
+        if log_item:
+            log_item.stream(
+                progress=f"\nFound {len(kn_files)} knowledge files in {knowledge_dir}, processing...",
+            )
+
+    for file_path in kn_files:
+        ext = file_path.split(".")[-1].lower()
+        if ext in file_types_loaders:
+            checksum = calculate_checksum(file_path)
+            file_key = file_path  # os.path.relpath(file_path, knowledge_dir)
+
+            # Load existing data from the index or create a new entry
+            file_data = index.get(file_key, {})
+
+            if file_data.get("checksum") == checksum:
+                file_data["state"] = "original"
+            else:
+                file_data["state"] = "changed"
+
+            if file_data["state"] == "changed":
+                file_data["checksum"] = checksum
+                loader_cls = file_types_loaders[ext]
+                loader = loader_cls(
+                    file_path,
+                    **(
+                        text_loader_kwargs
+                        if ext in ["txt", "csv", "html", "md"]
+                        else {}
+                    ),
+                )
+                file_data["documents"] = loader.load_and_split()
+                for doc in file_data["documents"]:
+                    doc.metadata = {**doc.metadata, **metadata}
+                cnt_files += 1
+                cnt_docs += len(file_data["documents"])
+                # print(f"Imported {len(file_data['documents'])} documents from {file_path}")
+
+            # Update the index
+            index[file_key] = file_data  # type: ignore
 
     # loop index where state is not set and mark it as removed
     for file_key, file_data in index.items():

python/helpers/memory.py

@@ -21,20 +21,23 @@ from python.helpers.log import Log, LogItem
 from enum import Enum
 from agent import Agent
 
+
 class MyFaiss(FAISS):
-    #override aget_by_ids
+    # override aget_by_ids
     def get_by_ids(self, ids: Sequence[str], /) -> List[Document]:
         # return all self.docstore._dict[id] in ids
-        return [self.docstore._dict[id] for id in ids if id in self.docstore._dict] #type: ignore
+        return [self.docstore._dict[id] for id in ids if id in self.docstore._dict]  # type: ignore
 
     async def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:
         return self.get_by_ids(ids)
 
+
 class Memory:
 
     class Area(Enum):
         MAIN = "main"
         SOLUTIONS = "solutions"
+        INSTRUMENTS = "instruments"
 
     index: dict[str, "MyFaiss"] = {}
 
@@ -130,7 +133,7 @@ class Memory:
                 # normalize_L2=True,
                 relevance_score_fn=Memory._cosine_normalizer,
             )
-        return db # type: ignore
+        return db  # type: ignore
 
     def __init__(
         self,
@@ -160,8 +163,8 @@ class Memory:
             with open(index_path, "r") as f:
                 index = json.load(f)
 
-        for kn_dir in kn_dirs:
-            index = knowledge_import.load_knowledge(log_item, kn_dir, index)
+        # preload knowledge folders
+        index = self._preload_knowledge_folders(log_item, kn_dirs, index)
 
         for file in index:
             if index[file]["state"] in ["changed", "removed"] and index[file].get(
@@ -187,6 +190,33 @@ class Memory:
         with open(index_path, "w") as f:
             json.dump(index, f)
 
+    def _preload_knowledge_folders(
+        self,
+        log_item: LogItem | None,
+        kn_dirs: list[str],
+        index: dict[str, knowledge_import.KnowledgeImport],
+    ):
+        # load knowledge folders, subfolders by area
+        for kn_dir in kn_dirs:
+            for area in Memory.Area:
+                index = knowledge_import.load_knowledge(
+                    log_item,
+                    files.get_abs_path("knowledge", kn_dir, area.value),
+                    index,
+                    {"area": area.value},
+                )
+
+        # load instruments descriptions
+        index = knowledge_import.load_knowledge(
+            log_item,
+            files.get_abs_path("instruments"),
+            index,
+            {"area": Memory.Area.INSTRUMENTS.value},
+            filename_pattern="**/*.md",
+        )
+
+        return index
+
     async def search_similarity_threshold(
         self, query: str, limit: int, threshold: float, filter: str = ""
     ):
@@ -235,9 +265,9 @@ class Memory:
 
     async def delete_documents_by_ids(self, ids: list[str]):
         # aget_by_ids is not yet implemented in faiss, need to do a workaround
-        rem_docs =self.db.get_by_ids(ids) # existing docs to remove (prevents error)
+        rem_docs = self.db.get_by_ids(ids)  # existing docs to remove (prevents error)
         if rem_docs:
-            rem_ids = [doc.metadata["id"] for doc in rem_docs] # ids to remove
+            rem_ids = [doc.metadata["id"] for doc in rem_docs]  # ids to remove
             await self.db.adelete(ids=rem_ids)
 
         if rem_docs:

python/tools/code_execution_tool.py

@@ -36,7 +36,7 @@ class CodeExecution(Tool):
             response = await self.execute_terminal_command(self.args["code"])
         elif runtime == "output":
             response = await self.get_terminal_output(
-                wait_with_output=5, wait_without_output=20
+                wait_with_output=5, wait_without_output=60
             )
         elif runtime == "reset":
             response = await self.reset_terminal()
@@ -137,7 +137,7 @@ class CodeExecution(Tool):
         return await self.get_terminal_output()
 
     async def get_terminal_output(
-        self, wait_with_output=3, wait_without_output=10, max_exec_time=15
+        self, wait_with_output=3, wait_without_output=10, max_exec_time=60
     ):
         idle = 0
         SLEEP_TIME = 0.1