feature(#23): updated prompts to categorify the input and applied for autogpt to get its achievement.

Brain/src/common/assembler.py

@@ -57,7 +57,7 @@ class Assembler:
 
     """mapping result type into json
     {
-        "program": sms | contacts | browser | select_item_detail_info, 
+        "program": sms | contacts | browser | selectitemdetailinfo, 
         "content": string
     }
     """

Brain/src/common/program_type.py

@@ -10,15 +10,15 @@ class ProgramType:
     MESSAGE = "message"
 
     class BrowserType:
-        OPEN_TAB = "open_tab"
-        OPEN_TAB_SEARCH = "open_tab_search"
-        CLOSE_TAB = "close_tab"
-        PREVIOUS_PAGE = "previous_page"
-        NEXT_PAGE = "next_page"
-        SCROLL_UP = "scroll_up"
-        SCROLL_DOWN = "scroll_down"
-        SCROLL_TOP = "scroll_top"
-        SCROLL_BOTTOM = "scroll_bottom"
-        SELECT_ITEM_DETAIL_INFO = "select_item_detail_info"
-        SELECT_ITEM = "select_item"
+        OPEN_TAB = "opentab"
+        OPEN_TAB_SEARCH = "opentabsearch"
+        CLOSE_TAB = "closetab"
+        PREVIOUS_PAGE = "previouspage"
+        NEXT_PAGE = "nextpage"
+        SCROLL_UP = "scrollup"
+        SCROLL_DOWN = "scrolldown"
+        SCROLL_TOP = "scrolltop"
+        SCROLL_BOTTOM = "scrollbottom"
+        SELECT_ITEM_DETAIL_INFO = "selectitemdetailinfo"
+        SELECT_ITEM = "selectitem"
         MESSAGE = "message"

Brain/src/common/utils.py

@@ -40,6 +40,10 @@ AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN")
 HUGGINGFACEHUB_API_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN")
 
 
+# actions flag :True -> Directly with Category_prompt
+ACTION_FLAG = True
+
+
 class ProgramType:
     BROWSER = "browser"
     ALERT = "alert"
@@ -47,6 +51,7 @@ class ProgramType:
     SMS = "sms"
     CONTACT = "contact"
     MESSAGE = "message"
+    AUTO_TASK = "autotask"
 
 
 # validate json format

Brain/src/rising_plugin/guardrails-config/actions/actions.py

@@ -17,6 +17,7 @@ import os
 import json
 import numpy as np
 
+from Brain.src.service.auto_task_service import AutoTaskService
 from Brain.src.service.train_service import TrainService
 from langchain.docstore.document import Document
 
@@ -26,6 +27,7 @@ from Brain.src.common.utils import (
     COMMAND_BROWSER_OPEN,
     PINECONE_INDEX_NAME,
     DEFAULT_GPT_MODEL,
+    ACTION_FLAG,
 )
 from Brain.src.model.req_model import ReqModel
 from Brain.src.model.requests.request_model import BasicReq
@@ -45,6 +47,7 @@ from Brain.src.rising_plugin.llm.llms import (
     GPT_4,
     FALCON_7B,
     GPT_LLM_MODELS,
+    CATEGORY_PROMPT,
 )
 
 from Brain.src.rising_plugin.pinecone_engine import (
@@ -80,6 +83,8 @@ query is json string with below format
 async def general_question(query):
     """init falcon model"""
     falcon_llm = FalconLLM()
+    autotask_service = AutoTaskService()
+    document_id = ""
     docs = []
 
     """step 0-->: parsing parms from the json query"""
@@ -89,16 +94,23 @@ async def general_question(query):
         raise BrainException(BrainException.JSON_PARSING_ISSUE_MSG)
     query = json_query["query"]
     image_search = json_query["image_search"]
-    page_content = json_query["page_content"]
-    document_id = json_query["document_id"]
+
     setting = ReqModel(json_query["setting"])
     is_browser = json_query["is_browser"]
-
-    docs.append(Document(page_content=page_content, metadata=""))
-    """ 1. calling gpt model to categorize for all message"""
-    chain_data = get_llm_chain(model=DEFAULT_GPT_MODEL, setting=setting).run(
-        input_documents=docs, question=query
-    )
+    if ACTION_FLAG:
+        docs.append(Document(page_content=CATEGORY_PROMPT, metadata=""))
+        # temperature shouldbe 0.
+        chain_data = get_llm_chain(
+            model=DEFAULT_GPT_MODEL, setting=setting, temperature=0.0
+        ).run(input_documents=docs, question=query)
+    else:
+        document_id = json_query["document_id"]
+        page_content = json_query["page_content"]
+        docs.append(Document(page_content=page_content, metadata=""))
+        """ 1. calling gpt model to categorize for all message"""
+        chain_data = get_llm_chain(model=DEFAULT_GPT_MODEL, setting=setting).run(
+            input_documents=docs, question=query
+        )
     try:
         result = json.loads(chain_data)
         # check image query with only its text
@@ -110,10 +122,7 @@ async def general_question(query):
         """ 2. check program is message to handle it with falcon llm """
         if result["program"] == "message":
             if is_browser:
-                result["program"] = "ask_website"
-            else:
-                # """FALCON_7B:"""
-                result["content"] = falcon_llm.query(question=query)
+                result["program"] = "askwebsite"
         return json.dumps(result)
     except ValueError as e:
         # Check sms and browser query
@@ -123,7 +132,5 @@ async def general_question(query):
             return json.dumps({"program": "browser", "content": "https://google.com"})
 
         if is_browser:
-            return json.dumps({"program": "ask_website", "content": ""})
-        return json.dumps(
-            {"program": "message", "content": falcon_llm.query(question=query)}
-        )
+            return json.dumps({"program": "askwebsite", "content": ""})
+        return json.dumps({"program": "message", "content": chain_data})

Brain/src/rising_plugin/llm/llms.py

@@ -20,6 +20,38 @@ GPT_LLM_MODELS = [GPT_3_5_TURBO, GPT_4, GPT_4_32K]
 """exception message"""
 EXCEPTION_MSG = f"The model is not correct. It should be in {LLM_MODELS}"
 
+
+"""prompt"""
+CATEGORY_PROMPT = """
+If user is going to say about a image with its description to search, please answer belowing json format. {"program": "image", "content": "description of the image that user is going to search"}
+If user is going to ask about a image, please answer belowing json format. {"program": "image", "content": "description of the image that user is going to search"}
+If user said that send sms or text, please answer belowing json format. {"program": "sms", "content": "ask who"}
+If user said that compose, write, or create an sms message, please answer belowing json format. {"program": "sms", "content": "ask who"}
+If user said that search contact with its description such as display name or phone number, please answer belowing json format. {"program": "contact", "content": "description of the contact that user is going to search"}
+If user said that open a tab, go to a tab, or open a page, please answer belowing json format. {"program": "opentab", "content": ""}
+If user said that open a tab and search, go to a tab and search, or open a page and search, please answer belowing json format. {"program": "opentabsearch", "content": "keyword that user is going to search"}
+If user said that close a tab, please answer belowing json format. {"program": "close_tab", "content": ""}
+If user said that launch a browser or open a browser, please answer belowing json format. {"program": "browser", "content": "https://google.com"}
+If user said that go to a previous page, or open a previous page, please answer belowing json format. {"program": "previouspage", "content": ""}
+If user said that go to a next page, or open a next page, please answer belowing json format. {"program": "nextpage", "content": ""}
+If user said that scroll up, scroll up page, or page scroll up, please answer belowing json format. {"program": "scrollup", "content": ""}
+If user said that scroll down, scroll down page, page scroll down, please answer belowing json format. {"program": "scrolldown", "content": ""}
+If user said that scroll top, scroll top page, or scroll top of page, please answer belowing json format. {"program": "scrolltop", "content": ""}
+If user said that scroll bottom, scroll bottom page, or scroll bottom of page, please answer belowing json format. {"program": "scrollbottom", "content": ""}
+If user is going to select an item, an article or a website with its description in a web browser, please answer belowing json format. {"program": "selectitemdetailinfo", "content": "the description of an item, an article or a website in a browser"}
+If user said that ask about the content in website, for example, if users ask something like 'what is #query in this website?' or 'Can you tell me about #query based on this website?', please answer belowing json format. {"program": "askwebsite", "content": ""}
+If user said that open a website using web browsers, please answer belowing json format. The url user is going to open can exist or not. If user doesn\\'t say exact url and want to open some sites, you have to find the best proper url. If user didn\\'t say any url and you can't find proper url, please set website url to "https://www.google.com". {"program": "browser", "url": "website url that user is going to open"}
+If users are going to ask something based on the data of website, please answer belowing json format. {"program": "askwebsite", "content": ""}
+If user is going to set or create alarm with time and label, please answer belowing json format.\n {"program": "alarm", "content": {"type":"create", "time":"please set time as 24-hours format that user is going to set. If user did not provide any alarm time, set "0:0"", "label":"please set label that user is going to set. If user did not provide any label, set "alarm""}}\n This is example data.\n User: Set an alarm.\n AI: {"program":"alarm", "content": {"type":"create", "time":"0:0", "label":"alarm"}}\n User: Set an alarm with label as "wake up".\n AI: {"program":"alarm", "content": {"type":"create", "time":"0:0", "label":"wake up"}}\n User: Set an alarm for 5:00 AM.\n AI: {"program":"alarm", "content": {"type":"create", "time":"5:00", "label":"alarm"}}\n User: Set an alarm for 5:00 PM with label as "wake up".\n AI: {"program":"alarm", "content": {"type":"create", "time":"17:00", "label":"wake up"}}
+If user is going to read email or message, please answer belowing json format. {"program": "reademail", "content": ""}
+If user is going to send email or message, please answer belowing json format. {"program": "sendemail", "content": ""}
+If user is going to compose, write, or create an email, please answer belowing json format. {"program": "writeemail", "content": ""}
+User wants to help organization in achieving its goals, which bridges the gap between the user is (present) and where he/she wants to go (future), or that is deciding in advance what to do, how to do when to do it and by whom it is to be done. Also the description ensures in thoughts and actions, work is carried on smoothly without any confusion and misunderstanding. \n And it can be done for the future and the future is full of uncertainties. \n So it looks like to make a decision as well. It helps make rational decisions by choosing the best most profitable alternative which may bring lower cost, adaptable to the organization and situations. \n If user is going to say about planning strategy, or something like that , please answer belowing json format. {"program": "autotask", "content": ""}
+If all of above is not correct, please give the most appropriate answer to the user's question. Please answer belowing json format. {"program":"message", "content":"your answer"}
+If users are going to know about  real-time capabilities such as News, Weather, Stocks, Booking, Planning, or etc, then please answer belowing json format. {"program": "autotask", "content": ""}
+If your answer is not correct with the program type which mentioned in this rules, please answer belowing json format. {"program": "autotask", "content": ""}
+"""
+
 """validate model"""
 
 
@@ -61,7 +93,9 @@ def get_llm(
     """check model"""
     llm = GptLLM(openai_key=setting.openai_key)
     if model == GPT_3_5_TURBO or model == GPT_4 or model == GPT_4_32K:
-        llm = GptLLM(model=model, openai_key=setting.openai_key)
+        llm = GptLLM(
+            model=model, openai_key=setting.openai_key, temperature=temperature
+        )
     elif model == FALCON_7B:
         llm = FalconLLM(temperature=temperature, max_new_tokens=max_new_tokens)
     return llm

Brain/src/rising_plugin/risingplugin.py

@@ -28,6 +28,7 @@ from ..common.utils import (
     DEFAULT_GPT_MODEL,
     parseJsonFromCompletion,
     PINECONE_INDEX_NAME,
+    ACTION_FLAG,
 )
 from .image_embedding import (
     query_image_text,
@@ -59,30 +60,46 @@ def llm_rails(
     image_search: bool = True,
     is_browser: bool = False,
 ) -> Any:
-    """step 0: convert string to json"""
-    index = init_pinecone(index_name=PINECONE_INDEX_NAME, setting=setting)
-    train_service = TrainService(firebase_app=firebase_app, setting=setting)
+    if not ACTION_FLAG:
+        """step 0: convert string to json"""
+        index = init_pinecone(index_name=PINECONE_INDEX_NAME, setting=setting)
+        train_service = TrainService(firebase_app=firebase_app, setting=setting)
 
-    """step 1: handle with gpt-4"""
+        """step 1: handle with gpt-4"""
 
-    query_result = get_embed(data=query, setting=setting)
-    try:
-        relatedness_data = index.query(
-            vector=query_result,
-            top_k=1,
-            include_values=False,
-            namespace=train_service.get_pinecone_index_train_namespace(),
+        query_result = get_embed(data=query, setting=setting)
+        try:
+            relatedness_data = index.query(
+                vector=query_result,
+                top_k=1,
+                include_values=False,
+                namespace=train_service.get_pinecone_index_train_namespace(),
+            )
+        except Exception as ex:
+            raise BrainException(code=508, message=responses[508])
+        if len(relatedness_data["matches"]) == 0:
+            return str({"program": "message", "content": ""})
+        document_id = relatedness_data["matches"][0]["id"]
+
+        document = train_service.read_one_document(document_id)
+        page_content = document["page_content"]
+
+        return rails_app.generate(
+            messages=[
+                {
+                    "role": "user",
+                    "content": rails_input_with_args(
+                        setting=setting,
+                        query=query,
+                        image_search=image_search,
+                        page_content=page_content,
+                        document_id=document_id,
+                        is_browser=is_browser,
+                    ),
+                }
+            ]
         )
-    except Exception as ex:
-        raise BrainException(code=508, message=responses[508])
-    if len(relatedness_data["matches"]) == 0:
-        return str({"program": "message", "content": ""})
-    document_id = relatedness_data["matches"][0]["id"]
-
-    document = train_service.read_one_document(document_id)
-    page_content = document["page_content"]
-
-    message = rails_app.generate(
+    return rails_app.generate(
         messages=[
             {
                 "role": "user",
@@ -90,14 +107,11 @@ def llm_rails(
                     setting=setting,
                     query=query,
                     image_search=image_search,
-                    page_content=page_content,
-                    document_id=document_id,
                     is_browser=is_browser,
                 ),
             }
         ]
     )
-    return message
 
 
 def processLargeText(
@@ -309,9 +323,9 @@ def rails_input_with_args(
     setting: ReqModel,
     query: str,
     image_search: bool,
-    page_content: str,
-    document_id: str,
     is_browser: bool,
+    page_content: str = "",
+    document_id: str = "",
 ) -> str:
     # convert json with params for rails.
     json_query_with_params = {

Brain/src/router/api.py

@@ -30,6 +30,7 @@ from Brain.src.rising_plugin.image_embedding import embed_image_text, query_imag
 from Brain.src.logs import logger
 from Brain.src.model.basic_model import BasicModel
 from Brain.src.model.feedback_model import FeedbackModel
+from Brain.src.service.auto_task_service import AutoTaskService
 from Brain.src.service.command_service import CommandService
 from Brain.src.service.contact_service import ContactsService
 from Brain.src.service.feedback_service import FeedbackService
@@ -88,6 +89,13 @@ def construct_blueprint_api() -> APIRouter:
         # check contact querying
         try:
             contacts_service = ContactsService(setting=setting)
+            if result["program"] == ProgramType.AUTO_TASK:
+                auto_task_service = AutoTaskService()
+                result["content"] = auto_task_service.ask_task_with_autogpt(
+                    query=query, firebase_app=firebase_app, setting=setting
+                )
+                return assembler.to_response(200, "", result)
+
             if result["program"] == ProgramType.CONTACT:
                 # querying contacts to getting its expected results
                 contacts_results = contacts_service.query_contacts(