Web Search and Multi-Modality

.github/workflows/deployment.yml

@@ -55,7 +55,7 @@ jobs:
         env:
           IMAGE_TAG: ${{ github.sha }}
         run: |
-          docker build --build-arg FASTAPI_KEY=${{secrets.FASTAPI_KEY}} --build-arg OPENAI_API_KEY=${{secrets.OPENAI_API_KEY}} -t ${{inputs.ecr_url}}:$IMAGE_TAG .
+          docker build --build-arg FASTAPI_KEY=${{secrets.FASTAPI_KEY}} --build-arg BING_API_KEY=${{secrets.BING_API_KEY}} --build-arg OPENAI_API_KEY=${{secrets.OPENAI_API_KEY}} -t ${{inputs.ecr_url}}:$IMAGE_TAG .
           docker push ${{inputs.ecr_url}}:$IMAGE_TAG
           echo "image=${{inputs.ecr_url}}:$IMAGE_TAG" >> $GITHUB_OUTPUT
 

Dockerfile

@@ -29,12 +29,14 @@ ARG FASTAPI_KEY
 ARG OPENAI_API_KEY
 ARG OPENAI_GENERAL_ASSISTANT
 ARG SENTRY_DSN
+ARG BING_API_KEY
 
 ENV FASTAPI_KEY=$FASTAPI_KEY
 ENV OPENAI_API_KEY=$OPENAI_API_KEY
 ENV OPENAI_GENERAL_ASSISTANT=$OPENAI_GENERAL_ASSISTANT
 ENV PYTHONUNBUFFERED=1
 ENV SENTRY_DSN=$SENTRY_DSN
+ENV BING_API_KEY=$BING_API_KEY
 
 # Set the working directory
 # WORKDIR /code

app/assistants.py

@@ -16,173 +16,14 @@ import pytz
 from app.exceptions import AssistantError, BaseOurcoachException, OpenAIRequestError, UtilsError
 from app.utils import get_booked_gg_sessions, get_growth_guide, get_growth_guide_summary, get_user_subscriptions, get_users_mementos, print_log
 from app.flows import FOLLOW_UP_STATE, GENERAL_COACHING_STATE, MICRO_ACTION_STATE, REFLECTION_STATE
+from app.web_search import SearchEngine
 
 load_dotenv()
 
 logger = logging.getLogger(__name__)
 OURCOACH_DASHBOARD_URL = os.getenv("OURCOACH_DASHBOARD_URL")
 
-import requests
 
-class SearchEngine:
-    BING_API_KEY = os.getenv("BING_API_KEY")
-    BING_ENDPOINT = 'https://api.bing.microsoft.com/v7.0'
-
-    @staticmethod
-    def search(feedback_type_name, search_term):
-        """
-        Public method to perform a search based on the feedback type.
-        """
-        search_methods = {
-            "Resource Links": SearchEngine._search_relevant_links,
-            "Book/Podcast Recommendations": SearchEngine._search_books_or_podcasts,
-            # "Success Stories/Testimonials": SearchEngine._search_success_stories,
-            "Inspirational Stories or Case Studies": SearchEngine._search_inspirational_stories,
-            "Fun Facts": SearchEngine._search_fun_facts,
-            # "Visual Content (Images, Infographics)": SearchEngine._search_visual_content,
-            "Personalised Recommendations": SearchEngine._search_personalized_recommendations
-        }
-        
-        search_method = search_methods.get(feedback_type_name)
-        
-        if search_method:
-            return search_method(search_term)
-        else:
-            return (feedback_type_name, search_term)
-
-    @staticmethod
-    def _search_relevant_links(search_term):
-        """
-        Uses Bing Web Search API to search for relevant links.
-        """
-        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
-        params = {'q': search_term, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
-        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
-        if response.status_code == 200:
-            data = response.json()
-            links = []
-            if 'webPages' in data and 'value' in data['webPages']:
-                for result in data['webPages']['value']:
-                    links.append(result)
-                return links
-        return ["No relevant links found."]
-
-    @staticmethod
-    def _search_books_or_podcasts(search_term):
-        """
-        Uses Bing Web Search API to search for books or podcasts.
-        """
-        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
-        query = f"{search_term} book OR podcast"
-        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
-        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
-        if response.status_code == 200:
-            data = response.json()
-            recommendations = []
-            if 'webPages' in data and 'value' in data['webPages']:
-                for result in data['webPages']['value']:
-                    title = result.get('name', 'Unknown Title')
-                    url = result.get('url', '')
-                    recommendations.append(f"{title}: {url}")
-                return recommendations
-        return ["No book or podcast recommendations found."]
-
-    @staticmethod
-    def _search_success_stories(search_term):
-        """
-        Uses Bing Web Search API to search for success stories.
-        """
-        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
-        query = f"{search_term} success stories"
-        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
-        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
-        if response.status_code == 200:
-            data = response.json()
-            stories = []
-            if 'webPages' in data and 'value' in data['webPages']:
-                for result in data['webPages']['value']:
-                    title = result.get('name', 'Unknown Title')
-                    url = result.get('url', '')
-                    stories.append(f"{title}: {url}")
-                return stories
-        return ["No success stories found."]
-
-    @staticmethod
-    def _search_inspirational_stories(search_term):
-        """
-        Uses Bing Web Search API to search for inspirational stories or case studies.
-        """
-        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
-        query = f"{search_term} inspirational stories OR case studies"
-        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
-        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
-        if response.status_code == 200:
-            data = response.json()
-            stories = []
-            if 'webPages' in data and 'value' in data['webPages']:
-                for result in data['webPages']['value']:
-                    title = result.get('name', 'Unknown Title')
-                    url = result.get('url', '')
-                    stories.append(f"{title}: {url}")
-                return stories
-        return ["No inspirational stories found."]
-
-    @staticmethod
-    def _search_fun_facts(search_term):
-        """
-        Uses Bing Web Search API to search for fun facts related to personal growth.
-        """
-        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
-        query = f"{search_term} fun facts"
-        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
-        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
-        if response.status_code == 200:
-            data = response.json()
-            facts = []
-            if 'webPages' in data and 'value' in data['webPages']:
-                for result in data['webPages']['value']:
-                    snippet = result.get('snippet', '')
-                    facts.append(snippet)
-                return facts
-        return ["No fun facts found."]
-
-    @staticmethod
-    def _search_visual_content(search_term):
-        """
-        Uses Bing Image Search API to search for images or infographics.
-        """
-        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
-        params = {'q': search_term, 'count': 3}
-        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/images/search", headers=headers, params=params)
-        if response.status_code == 200:
-            data = response.json()
-            images = []
-            if 'value' in data:
-                for result in data['value']:
-                    image_url = result.get('contentUrl', '')
-                    images.append(image_url)
-                return images
-        return ["No visual content found."]
-
-    @staticmethod
-    def _search_personalized_recommendations(search_term):
-        """
-        Uses Bing Web Search API to provide personalized recommendations.
-        """
-        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
-        query = f"tips for {search_term}"
-        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
-        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
-        if response.status_code == 200:
-            data = response.json()
-            recommendations = []
-            if 'webPages' in data and 'value' in data['webPages']:
-                for result in data['webPages']['value']:
-                    title = result.get('name', 'Unknown Title')
-                    url = result.get('url', '')
-                    recommendations.append(f"{title}: {url}")
-                return recommendations
-        return ["No personalized recommendations found."]
 
 class FeedbackContent:
     def __init__(self, content, role):
@@ -569,11 +410,11 @@ class Assistant:
                         "tool_call_id": tool.id,
                         "output": "Generate a final coach message (feedback message) using these 3 feedback types (together with the stated emoji at the beginning of each feedback): " + str(sample_feedbacks)
                     })
-                elif tool.function.name == "search_resource":
+                elif tool.function.name == "search_web":
                     type = json.loads(tool.function.arguments)['resource_type']
                     query = json.loads(tool.function.arguments)['query']
                     logger.info(f"Getting microaction theme: {type} - {query}", extra={"user_id": self.cm.user.user_id, "endpoint": "get_microaction_theme"})
-                    relevant_context = SearchEngine.search(type, query)
+                    relevant_context = SearchEngine.search(type, query, self.cm.user.user_id)
                     logger.info(f"Finish Getting microaction theme: {relevant_context}", extra={"user_id": self.cm.user.user_id, "endpoint": "get_microaction_theme"})
                     tool_outputs.append({
                         "tool_call_id": tool.id,
@@ -676,7 +517,7 @@ class Assistant:
                     user_info = '** If the user asks for their progress, also include a link to their Revelation Dashboard: {OURCOACH_DASHBOARD_URL} so that they can find out more **\n\n'
                     if category == "personal":
                         user_info += f"** Personal Information **\n\n{self.cm.user.user_info}"
-                        user_info += f"\n\n** User's Mantra This Week**\n\n{self.cm.user.mantra}"
+                        user_info += f"\n\n** User's Mantra This Week:**\n\n{self.cm.user.mantra or 'Mantra not available.'}"
                     elif category == "challenges":
                         user_info += f"** User's Challenges (prioritise ONGOING challenges) **\n\n{self.cm.user.challenges}\n\nLet the user know that ongoing challenges from their growth guide will be integrated into their day-to-day interaction."
                     elif category == "recommended_actions":

app/cache.py

@@ -34,7 +34,7 @@ def upload_file_to_s3(filename):
         s3_client = session.client('s3')
         with open(filename, "rb") as f:
             ## Upload to Production Folder
-            s3_client.upload_fileobj(f, bucket, f'staging/users/{user_id}.pkl')
+            s3_client.upload_fileobj(f, bucket, f'dev/users/{user_id}.pkl')
         logger.info(f"File {filename} uploaded successfully to S3", extra={'user_id': user_id, 'endpoint': function_name})
         
         os.remove(filename)

app/conversation_manager.py

@@ -1,3 +1,4 @@
+import base64
 import os
 import openai
 import pandas as pd
@@ -10,7 +11,9 @@ from datetime import datetime
 
 import dotenv
 
-from app.utils import id_to_persona
+from app.utils import get_user_local_timezone, id_to_persona
+from PIL import Image
+import io
 dotenv.load_dotenv()
 
 OURCOACH_DASHBOARD_URL = os.getenv("OURCOACH_DASHBOARD_URL")
@@ -27,7 +30,13 @@ class ConversationManager:
         self.assistants = {'general': Assistant(asst_id, self), 'intro': Assistant('asst_baczEK65KKvPWIUONSzdYH8j', self)}
 
         self.client = client
-        self.state = {'date': pd.Timestamp.now(tz='UTC').strftime("%Y-%m-%d %a %H:%M:%S")}
+
+        # get users local timezone
+        timezone = get_user_local_timezone(user.user_id)
+
+        # convert UTC to user's timezone
+        local_time = pd.Timestamp.now(tz='UTC').tz_convert(timezone)
+        self.state = {'date': local_time.strftime("%Y-%m-%d %a %H:%M:%S")}
 
         self.current_thread = self.create_thread()
         self.daily_thread = None
@@ -70,6 +79,8 @@ class ConversationManager:
                                 {id_to_persona(self.user.asst_id)}, always adhere to your choosen persona by incorporating it conversationally.
                                 You represent a coach at ourcoach. You may refer to you Knowledgebase (ourcoach FAQ) for all information related to ourcoach.
                                 ** Branding ** Always stylize ourcoach as 'ourcoach' instead of 'OurCoach' or 'Ourcoach', regardless of any grammatical errors.
+                                **You truly believe you are your persona**
+                                **You always try to be engaging and personal with your coachee in order to get them to continue interacting with you daily**
                                 -------------------------------------------
                                 You are coaching:
                                 \n\n{user_interaction_guidelines}\n\n\
@@ -103,11 +114,51 @@ class ConversationManager:
         return message
     
     @catch_error
-    def _run_current_thread(self, text, thread=None, hidden=False):
+    def _run_current_thread(self, text, thread=None, hidden=False, media=None):
+        image_context = ""
+        if media:
+            logger.info(f"Describing Media", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
+            # assert media is a base64 encoded string
+            try:
+                # scale down media to 512x512
+                img_data = base64.b64decode(media)
+                img = Image.open(io.BytesIO(img_data))
+                img = img.resize((512, 512), Image.Resampling.LANCZOS)  # Better quality resizing
+                buffered = io.BytesIO()
+                img.save(buffered, format="JPEG", quality=85)  # Specify format and quality
+                media = base64.b64encode(buffered.getvalue()).decode()
+
+            except Exception as e:
+                raise ConversationManagerError(user_id=self.user.user_id, message="Invalid base64 image data", e=str(e))
+            logger.info(f"Media is valid", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
+            response = self.client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": f"""Describe this image to an LLM for it to have rich and complete context. 
+                            {'Lock in on things related to the users message:'+text if text else text}""",
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/jpeg;base64,{media}", "detail": "low"},
+                        },
+                    ],
+                }
+            ])
+
+            image_context = response.choices[0].message.content
+            logger.info(f"Image context: {image_context}", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
+            
         if thread is None:
             thread = self.current_thread
         logger.warning(f"{self}", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
-        logger.info(f"User Message: {text}", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
+
+        if image_context:
+            text = f"{text}\n\n[MEDIA ATTACHMENT] User attached an image about: {image_context}"
 
         # need to select assistant
         if self.intro_done:
@@ -117,7 +168,7 @@ class ConversationManager:
             logger.info(f"Running intro assistant", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
             run, just_finished_intro, message = self.assistants['intro'].process(thread, text) 
         
-        logger.info(f"Run {run.id} {run.status} just finished intro: {just_finished_intro}", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
+        logger.info(f"Run done, status={run.status} just finished intro: {just_finished_intro}", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
         
         if 'message' in run.metadata:
             info = run.metadata['message']
@@ -136,21 +187,21 @@ class ConversationManager:
 
         if hidden:
             self.client.beta.threads.messages.delete(message_id=message.id, thread_id=thread.id)
-            logger.info(f"Deleted hidden message: {message}", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
+            logger.info(f"Deleted hidden message", extra={"user_id": self.user.user_id, "endpoint": "run_current_thread"})
         
         return self._get_current_thread_history(remove_system_message=False)[-1], run 
     
     @catch_error
     def _send_and_replace_message(self, text, replacement_msg=None):
-        logger.info(f"Sending hidden message: {text}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
+        logger.info(f"Sending hidden message", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
         response, _ = self._run_current_thread(text, hidden=True)
 
         # check if there is a replacement message
         if replacement_msg:
-            logger.info(f"Adding replacement message: {replacement_msg}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
+            logger.info(f"Adding replacement message", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
             # get the last message
             last_msg = list(self.client.beta.threads.messages.list(self.current_thread.id, order="asc"))[-1]
-            logger.info(f"Last message: {last_msg}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
+            # logger.info(f"Last message: {last_msg}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
             response = last_msg.content[0].text.value
 
             # delete the last message
@@ -203,13 +254,13 @@ class ConversationManager:
                         Be mindful of this information at all times in order to
                         be as personalised as possible when conversing. Ensure to
                         follow the conversation guidelines and flow provided.""", "role":"assistant"}] + messages[-29:]
-        logger.info(f"Current Thread Messages: {messages}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
+        # logger.info(f"Current Thread Messages: {messages}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
         
         temp_thread = self.client.beta.threads.create(messages=messages)
-        logger.info(f"Created Temp Thread: {temp_thread}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
+        # logger.info(f"Created Temp Thread: {temp_thread}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
         
         if add_to_main:
-            logger.info(f"Adding message to main thread: {text}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
+            logger.info(f"Adding message to main thread", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
             self.add_message_to_thread(self.current_thread.id, "assistant", text)
 
         self.add_message_to_thread(temp_thread.id, "user", text)
@@ -220,7 +271,7 @@ class ConversationManager:
 
         # delete temp thread
         self.client.beta.threads.delete(temp_thread.id)
-        logger.info(f"Deleted Temp Thread: {temp_thread}", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
+        logger.info(f"Deleted Temp Thread", extra={"user_id": self.user.user_id, "endpoint": "send_hidden_message"})
 
         return response
     
@@ -229,19 +280,18 @@ class ConversationManager:
         if old_thread is None:
             old_thread = self.current_thread
 
-        # create a new thread
-        messages = [msg for msg in self._get_current_thread_history(remove_system_message=False) if not msg['content'].startswith("[hidden]")]
-        if len(messages) >= 29:
-            messages = messages[-29:]
-        logger.info(f"Current Thread Messages: {messages}", extra={"user_id": self.user.user_id, "endpoint": "delete_hidden_messages"})
-
-        new_thread = self.client.beta.threads.create(messages=messages)
+        # find all hidden messages to delete
+        to_delete = [{'id': msg.id,'content': msg.content[0].text.value, 'role': msg.role} for msg in self.client.beta.threads.messages.list(self.current_thread.id, order="desc") if msg.content[0].text.value.startswith("[hidden]")]
+        logger.info(f"Marked for deletion= {to_delete}", extra={"user_id": self.user.user_id, "endpoint": "delete_hidden_messaged"})
 
-        # delete old thread
-        self.client.beta.threads.delete(old_thread.id)
-
-        # set current thread
-        self.current_thread = new_thread
+        for msg in to_delete:
+            # delete msg
+            deleted_message = self.client.beta.threads.messages.delete(
+                message_id=msg['id'],
+                thread_id=self.current_thread.id,
+            )
+            logger.info(f"Deleted Message = {deleted_message}", extra={"user_id": self.user.user_id, "endpoint": "delete_hidden_messaged"})
+        return True
     
     @catch_error
     def cancel_run(self, run, thread = None):

app/flows.py

@@ -59,7 +59,8 @@ Step 1:
 **First Message: Micro-Action Suggestion**
 
 - Propose one clear, actionable task for the day.
-- Ensure it is relevant, easy to begin, and framed positively.
+- Ensure it is relevant, timely, easy to begin, and framed positively.
+- You can call search_web(...) to get better context or timely recommendations to incorporate into your response.
 - Avoid repeating previous actions.
 - Your message must be concise! (use Whatsapp texting length)
 - Wait for the user's response
@@ -87,6 +88,7 @@ Step 1:
 
 - **If the user has decided to try out the micro-action:**
   - Encourage them to proceed with the task (Your message must be concise!)
+  - Ok coach, if you feel like its appropriate i.e, the quality of the microaction can be enhanced via a picture or the microaction accountability tracking can be enhaced by receiving picture, let the user know that you can track their progress and provide coaching even through picture (but dont always ask for pictures, thats kinda creepy).
   - Do **not** ask any questions at this point.
   - After the user confirms they've completed the micro-action:
     - Acknowledge their effort.
@@ -107,7 +109,7 @@ Step 1:
 **Key Rules for Micro-Actions**
 
 - **Personalized and Achievable:** Align with the user’s progress, keeping early actions (Day 1–5) simple and gradually increasing difficulty (Day 6+).
-- **Resource Suggestions:** Avoid recommending resources on Day 1; may do so sparingly on later days if relevant.
+- **Resource Suggestions:** You can call search_web(...) to get better context or timely recommendations.
 - **Guided Coaching:** Provide micro-actions as if you naturally guide the user—never state or imply a function call.
 
 ---
@@ -117,16 +119,15 @@ Step 1:
 - **Warm and Encouraging:** Mirror a friendly, personable texting style.
 - **Simple and Succinct:** Use conversational, human-like language in WhatsApp texting length.
 - **Action-Oriented:** Focus on immediate, actionable steps rather than abstract suggestions.
-
+- **Coach Mindset:** You have entered focussed mind state akin to that of elite coaches. While being encouraging and friendly, you should lightly push and challenge the user when appropriate.
 ---
 
 **Interaction Principles**
 
 1. **Simplicity:** Actions should feel easy and immediately doable.
 2. **Relevance:** Align each action with the user’s goal and progress.
-3. **Encouragement:** Motivate the user without judgment or pressure.
-4. **Feedback:** Reinforce positive habits subtly to build momentum.
-5. **Tone:** Casual, personable, and focused on meaningful progress.
+3. **Feedback:** Reinforce positive habits subtly to build momentum.
+4. **Tone:** Casual, personable, and focused on meaningful progress.
 
 ---
 

app/main.py

@@ -1,4 +1,4 @@
-from fastapi import FastAPI, HTTPException, Security, Query, status, Request, Depends
+from fastapi import FastAPI, HTTPException, Security, Query, status, Request, Depends, File, UploadFile, Form
 from fastapi.responses import FileResponse, StreamingResponse, JSONResponse
 from fastapi.security import APIKeyHeader
 import openai
@@ -28,6 +28,9 @@ import pickle
 from app.exceptions import *
 import re
 import sentry_sdk
+import base64
+from io import BytesIO
+from PIL import Image
 
 load_dotenv()
 AWS_ACCESS_KEY = os.getenv('AWS_ACCESS_KEY')
@@ -35,18 +38,18 @@ AWS_SECRET_KEY = os.getenv('AWS_SECRET_KEY')
 REGION = os.getenv('AWS_REGION')
 SENTRY_DSN = os.getenv('SENTRY_DSN')
 
-sentry_sdk.init(
-    dsn=SENTRY_DSN,
-    # Set traces_sample_rate to 1.0 to capture 100%
-    # of transactions for tracing.
-    traces_sample_rate=1.0,
-    _experiments={
-        # Set continuous_profiling_auto_start to True
-        # to automatically start the profiler on when
-        # possible.
-        "continuous_profiling_auto_start": True,
-    },
-)
+# sentry_sdk.init(
+#     dsn=SENTRY_DSN,
+#     # Set traces_sample_rate to 1.0 to capture 100%
+#     # of transactions for tracing.
+#     traces_sample_rate=1.0,
+#     _experiments={
+#         # Set continuous_profiling_auto_start to True
+#         # to automatically start the profiler on when
+#         # possible.
+#         "continuous_profiling_auto_start": True,
+#     },
+# )
 
 
 
@@ -278,6 +281,7 @@ class CreateUserItem(BaseModel):
 class ChatItem(BaseModel):
     user_id: str
     message: str
+    image64: str = None
 
 class PersonaItem(BaseModel):
     user_id: str
@@ -668,7 +672,7 @@ async def add_ai_message(
 ):
     user_id = request.user_id
     message = request.message
-    logger.info(f"Adding AI response: {message}", extra={"user_id": user_id, "endpoint": "/add_ai_message"})
+    logger.info(f"Adding AI response", extra={"user_id": user_id, "endpoint": "/add_ai_message"})
     print_log("INFO", "Adding AI response", extra={"user_id": user_id, "endpoint": "/add_ai_message"})
  
     user = get_user(user_id)
@@ -836,7 +840,9 @@ async def chat(
     logger.info("Processing chat request", extra={"user_id": request.user_id, "endpoint": "/chat"})
     user = get_user(request.user_id)
 
-    response = user.send_message(request.message)
+    if request.image64:
+        logger.info(f"Processing image", extra={"user_id": request.user_id, "endpoint": "/chat"})
+    response = user.send_message(request.message, request.image64)
     logger.info(f"Assistant response generated", extra={"user_id": request.user_id, "endpoint": "/chat"})
     return {"response": response}
     
@@ -1109,3 +1115,42 @@ async def get_recent_booking(
     if 'conn' in locals():
         conn.close() 
     return {"booking_id": booking_id}
+
+@app.post("/answer_image_question")
+async def answer_image_question(
+    question: str = Form(...),
+    file: UploadFile = File(None),
+    image_base64: str = Form(None)
+):
+    if file:
+        contents = await file.read()
+        # convert to base64 string
+        image_base64 = base64.b64encode(contents).decode('utf-8')
+    
+    client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))
+    response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "What is in this image?",
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"},
+                },
+            ],
+        }
+    ])
+
+    return {"response": response.choices[0]}
+    
+@app.post("/upload_image")
+async def upload_image(file: UploadFile = File(...)):
+    # process or save the file
+    contents = await file.read()
+    image = Image.open(BytesIO(contents))
+    return {"filename": file.filename, "info": "Image uploaded successfully"}

app/user.py

@@ -6,6 +6,7 @@ import pandas as pd
 from datetime import datetime, timezone
 import json
 from app.assistants import Assistant 
+from app.exceptions import DBError
 import glob
 import pickle  # Replace dill with pickle
 import random
@@ -328,9 +329,33 @@ class User:
         return False
 
     @catch_error
-    def set_mantra(self, mantra):
-        logger.info(f"Setting mantra: {mantra}", extra={"user_id": self.user_id, "endpoint": "set_mantra"})
-        self.mantra = mantra
+    def set_mantra(self):
+        ### To save mantra in database to user object
+        logger.info(f"Getting mantra from user...", extra={"user_id": self.user_id, "endpoint": "get_mantra"})
+        user_id = self.user_id
+        db_params = {
+            'dbname': 'ourcoach',
+            'user': 'ourcoach',
+            'password': 'hvcTL3kN3pOG5KteT17T',
+            'host': 'staging-ourcoach.cx8se8o0iaiy.ap-southeast-1.rds.amazonaws.com',
+            'port': '5432'
+        }
+        try:
+            with psycopg2.connect(**db_params) as conn:
+                with conn.cursor() as cursor:
+                    query = sql.SQL("SELECT mantra FROM {table} WHERE user_id = %s").format(table=sql.Identifier('public', 'user_growth_status'))
+                    cursor.execute(query, (user_id,))
+                    row = cursor.fetchone()
+                    if (row):
+                        colnames = [desc[0] for desc in cursor.description]
+                        user_data = dict(zip(colnames, row))
+                        ### SAVE MANTRA IN USER OBJECT
+                        self.mantra = user_data['mantra']
+                    else:
+                        logger.warning(f"No user info found for {user_id}", extra={'user_id': user_id, 'endpoint': "get_mantra"})
+        except psycopg2.Error as e:
+            logger.error(f"Database error while retrieving user info for {user_id}: {e}", extra={'user_id': user_id, 'endpoint': "get_mantra"})
+            raise DBError(user_id=user_id, message="Error retrieving user info", code="SQLError", e=str(e))
 
     @catch_error
     def set_goal(self, goal, goal_area, add=True, completed=False):
@@ -457,8 +482,10 @@ class User:
         return self.conversations.current_thread
 
     @catch_error
-    def send_message(self, text):
-        response, run = self.conversations._run_current_thread(text)
+    def send_message(self, text, media=None):
+        if media:
+            logger.info(f"Sending message with media", extra={"user_id": self.user_id, "endpoint": "send_message"})
+        response, run = self.conversations._run_current_thread(text, media=media)
         message = run.metadata.get("message", "No message")
         logger.info(f"Message: {message}", extra={"user_id": self.user_id, "endpoint": "user_send_message"})
 
@@ -475,6 +502,8 @@ class User:
             # Move to the next action
             self.growth_plan.next()
 
+            response['add_one'] = True
+
         elif message == "change_goal":
             # send the change goal prompt
             logger.info("Sending change goal message...", extra={"user_id": self.user_id, "endpoint": "user_send_message"})
@@ -726,8 +755,8 @@ class User:
         return [response]
 
     @catch_error
-    def do_theme(self, theme, date, day, last_msg_is_answered = True):
-        logger.info(f"Doing theme: {theme}", extra={"user_id": self.user_id, "endpoint": "do_theme"})
+    def do_theme(self, theme, date, day, last_msg_is_answered = True, extra=None):
+        logger.info(f"Doing theme: {theme}, extra={extra}", extra={"user_id": self.user_id, "endpoint": "do_theme"})
         
         # Add 1 day to cumulative_plan_day
         self.cumulative_plan_day += 1
@@ -833,20 +862,23 @@ class User:
         else:
             # Remind the user that they can book a Growth Guide session if they have not done one yet after the FINAL_SUMMARY_STATE
             if self.growth_plan.previous()['coachingTheme'] == "FINAL_SUMMARY_STATE":
-                formatted_message = f"[IMPORTANT] The user has not booked a Growth Guide session yet. Remind them that they can book one through their Revelation Dahsboard: {OURCOACH_DASHBOARD_URL} to get more personalized advice and guidance!\n\n" + formatted_message
+                if day != 1:
+                    formatted_message = f"[IMPORTANT] The user has not booked a Growth Guide session yet. Remind them that they can book one through their Revelation Dahsboard: {OURCOACH_DASHBOARD_URL} to get more personalized advice and guidance!\n\n" + formatted_message
         
         prompt = f"""** It is a new day: {date} ({day}) 10:00:00 ** 
 
-        (If the day is a public holiday (e.g., Christmas, New Year, or other significant occasions), customize your message to reflect the context appropriately, acknowledging the holiday or its significance.)
+        (If the day is a public holiday (e.g., Christmas, New Year, the user's Birthday or other significant occasions), customize your message to reflect the context appropriately, acknowledging the holiday or its significance.)
 
         **Before we start,**
         Has the user answered your last question? : {last_msg_is_answered}
         If the answer above is "True", you may proceed to do the instruction below
-        If the answer above is "False", you must immediately ask something like (but warmer) "Hey <user name>, I've noticed that you haven't answered my latest question. Do you still want to continue the growth plan?". If the user says "no", then ask if they want to set a new goal (therefore later, call the change_goal() function)
+        If the answer above is "False", take a deep breath coach, and utilizing your ability as an elite coach with the users best interest in mind, think whether it would be more appropriate to follow up the unanswered question with the user or continue with todays theme below. However if the user indicates that they want to set a new goal (call the change_goal() function)
         But if the user says "yes", then proceed to do the instruction below.
 
         Today is day {self.cumulative_plan_day} of the user's growth journey (out of {final_day} days). You may (or may not) mention this occasionally in your first message of the day.
         
+        If today is a "Monday" or "Mon", you must include the user's Mantra of the Week : {self.mantra} to your first message of the day (include with today-theme's first message below)
+{extra if extra else ''}
         Today's Theme:
         {formatted_message}
         """
@@ -869,6 +901,8 @@ class User:
         self.conversations.delete_hidden_messages()
         # update the date in the state
         self.conversations.state['date'] = date
+        # update mantra
+        self.set_mantra()
 
         action = self.growth_plan.current()
 
@@ -918,7 +952,20 @@ class User:
         else:
             last_msg_is_answered = True
 
-        response, prompt = self.do_theme(theme, date, action['day'], last_msg_is_answered)        
+        extra="Greet the user with a creative good morning message!"
+
+        # Check if user didnt respond (last 2 messages in conversation history are from the AI) on FINAL_SUMMARY_STATE
+        messages = self.get_messages()
+        logger.info(f"Last 2 messages are from: 1) {messages[-1]['role']} and 2) {messages[-2]['role']}")
+        if (messages[-1]['role'] == "assistant" and messages[-2]['role'] == "assistant") and self.growth_plan.previous()['coachingTheme'] == "FINAL_SUMMARY_STATE":
+            self.extend_growth_plan()
+            extra += """\nDang, the user did not indicate what they wanted to do yesterday. 
+            This is not a good sign as it may indicate that the user is going to dropoff. 
+            We have automatically extended their growth plan for them, let the user know this and also integrate upselling the ourcoach platform and subscription as much as you can
+            in the first message you send to the user.."""
+            
+
+        response, prompt = self.do_theme(theme, date, action['day'], last_msg_is_answered, extra=extra)        
 
         # add today's reminders to response to schedule
         # response['reminders'] = all reminders which date is today (so all the reminders that BE has to queue today)

app/utils.py

@@ -327,7 +327,7 @@ def generate_html(json_data, coach_name='Growth Guide', booking_id = None):
         s3_client = session.client('s3')
         with open(path_to_upload, "rb") as f:
             ## Upload to Production Folder
-            s3_client.upload_fileobj(f, bucket, f'staging/pre_gg_reports/{filename}.pdf')
+            s3_client.upload_fileobj(f, bucket, f'dev/pre_gg_reports/{filename}.pdf')
         logger.info(f"File {filename} uploaded successfully to S3", extra={'booking_id': booking_id, 'endpoint': function_name})
         
         # Removing files
@@ -984,7 +984,8 @@ def get_user_life_status(user_id):
     mantra = json.loads(response.choices[0].message.content)["mantra_of_the_week"]
     
     # Update the users mantra
-    user.set_mantra(mantra)
+    # user.set_mantra(mantra)
+    # We remove because we want the mantra to be updated weekly (by backend), not updated everytime we call this endpoint/func
 
     cumulative_life_score = {
             "overall": user.personal_growth_score + user.career_growth_score + user.relationship_score + user.mental_well_being_score + user.health_and_wellness_score,
@@ -1304,7 +1305,7 @@ def download_file_from_s3(filename, bucket):
         s3_client = session.client('s3')
         with open(file_path, 'wb') as f:
             ## Upload to Production Folder
-            s3_client.download_fileobj(bucket, f"staging/users/{filename}", f)
+            s3_client.download_fileobj(bucket, f"dev/users/{filename}", f)
         logger.info(f"File {filename} downloaded successfully from S3", extra={'user_id': user_id, 'endpoint': function_name})
         return True
     except Exception as e:

app/web_search.py

@@ -0,0 +1,255 @@
+import requests
+from dotenv import load_dotenv
+import os
+import logging
+
+logger = logging.getLogger(__name__)
+
+load_dotenv()
+
+class SearchEngine:
+    BING_API_KEY = os.getenv("BING_API_KEY")
+    BING_ENDPOINT = 'https://api.bing.microsoft.com/v7.0'
+
+    @staticmethod
+    def search(feedback_type_name, search_term, user_id):
+        logger.info(f"User {user_id}: Initiating search for type '{feedback_type_name}' with term '{search_term}'")
+        """
+        Public method to perform a search based on the feedback type.
+        """
+        search_methods = {
+            "General": SearchEngine._search_general,
+            "Resource Links": SearchEngine._search_relevant_links,
+            "Book/Podcast Recommendations": SearchEngine._search_books_or_podcasts,
+            "Inspirational Stories or Case Studies": SearchEngine._search_inspirational_stories,
+            "Fun Facts": SearchEngine._search_fun_facts,
+            "Personalised Recommendations": SearchEngine._search_personalized_recommendations,
+            "Videos": SearchEngine._search_videos
+        }
+        
+        search_method = search_methods.get(feedback_type_name)
+        
+        if search_method:
+            return search_method(search_term)
+        else:
+            return (feedback_type_name, search_term)
+
+    @staticmethod
+    def _search_relevant_links(search_term):
+        logger.debug(f"Searching relevant links for term: {search_term}")
+        """
+        Uses Bing Web Search API to search for relevant links.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        params = {'q': search_term, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Web Search API.")
+            data = response.json()
+            links = []
+            if 'webPages' in data and 'value' in data['webPages']:
+                for result in data['webPages']['value']:
+                    links.append(result)
+                return links
+        else:
+            logger.error(f"Bing Web Search API returned status code {response.status_code}")
+        return ["No relevant links found."]
+
+    @staticmethod
+    def _search_books_or_podcasts(search_term):
+        logger.debug(f"Searching books or podcasts for term: {search_term}")
+        """
+        Uses Bing Web Search API to search for books or podcasts.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        query = f"{search_term} book OR podcast"
+        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Web Search API for books/podcasts.")
+            data = response.json()
+            recommendations = []
+            if 'webPages' in data and 'value' in data['webPages']:
+                for result in data['webPages']['value']:
+                    title = result.get('name', 'Unknown Title')
+                    url = result.get('url', '')
+                    recommendations.append(f"{title}: {url}")
+                return recommendations
+        else:
+            logger.error(f"Bing Web Search API returned status code {response.status_code} for books/podcasts search.")
+        return ["No book or podcast recommendations found."]
+
+    @staticmethod
+    def _search_success_stories(search_term):
+        logger.debug(f"Searching success stories for term: {search_term}")
+        """
+        Uses Bing Web Search API to search for success stories.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        query = f"{search_term} success stories"
+        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Web Search API for success stories.")
+            data = response.json()
+            stories = []
+            if 'webPages' in data and 'value' in data['webPages']:
+                for result in data['webPages']['value']:
+                    title = result.get('name', 'Unknown Title')
+                    url = result.get('url', '')
+                    stories.append(f"{title}: {url}")
+                return stories
+        else:
+            logger.error(f"Bing Web Search API returned status code {response.status_code} for success stories search.")
+        return ["No success stories found."]
+
+    @staticmethod
+    def _search_inspirational_stories(search_term):
+        logger.debug(f"Searching inspirational stories or case studies for term: {search_term}")
+        """
+        Uses Bing Web Search API to search for inspirational stories or case studies.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        query = f"{search_term} inspirational stories OR case studies"
+        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Web Search API for inspirational stories.")
+            data = response.json()
+            stories = []
+            if 'webPages' in data and 'value' in data['webPages']:
+                for result in data['webPages']['value']:
+                    title = result.get('name', 'Unknown Title')
+                    url = result.get('url', '')
+                    stories.append(f"{title}: {url}")
+                return stories
+        else:
+            logger.error(f"Bing Web Search API returned status code {response.status_code} for inspirational stories search.")
+        return ["No inspirational stories found."]
+
+    @staticmethod
+    def _search_fun_facts(search_term):
+        logger.debug(f"Searching fun facts for term: {search_term}")
+        """
+        Uses Bing Web Search API to search for fun facts related to personal growth.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        query = f"{search_term} fun facts"
+        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Web Search API for fun facts.")
+            data = response.json()
+            facts = []
+            if 'webPages' in data and 'value' in data['webPages']:
+                for result in data['webPages']['value']:
+                    snippet = result.get('snippet', '')
+                    facts.append(snippet)
+                return facts
+        else:
+            logger.error(f"Bing Web Search API returned status code {response.status_code} for fun facts search.")
+        return ["No fun facts found."]
+
+    @staticmethod
+    def _search_visual_content(search_term):
+        logger.debug(f"Searching visual content for term: {search_term}")
+        """
+        Uses Bing Image Search API to search for images or infographics.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        params = {'q': search_term, 'count': 3}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/images/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Image Search API.")
+            data = response.json()
+            images = []
+            if 'value' in data:
+                for result in data['value']:
+                    image_url = result.get('contentUrl', '')
+                    images.append(image_url)
+                return images
+        else:
+            logger.error(f"Bing Image Search API returned status code {response.status_code} for visual content search.")
+        return ["No visual content found."]
+
+    @staticmethod
+    def _search_personalized_recommendations(search_term):
+        logger.debug(f"Searching personalized recommendations for term: {search_term}")
+        """
+        Uses Bing Web Search API to provide personalized recommendations.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        query = f"tips for {search_term}"
+        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Web Search API for personalized recommendations.")
+            data = response.json()
+            recommendations = []
+            if 'webPages' in data and 'value' in data['webPages']:
+                for result in data['webPages']['value']:
+                    title = result.get('name', 'Unknown Title')
+                    url = result.get('url', '')
+                    recommendations.append(f"{title}: {url}")
+                return recommendations
+        else:
+            logger.error(f"Bing Web Search API returned status code {response.status_code} for personalized recommendations search.")
+        return ["No personalized recommendations found."]
+
+    @staticmethod
+    def _search_videos(search_term):
+        logger.debug(f"Searching videos for term: {search_term}")
+        """
+        Uses Bing Video Search API to search for videos, prioritizing YouTube results.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        query = f"site:youtube.com {search_term}"
+        params = {'q': query, 'textDecorations': True, 'textFormat': 'HTML', 'count': 3}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/videos/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Video Search API.")
+            data = response.json()
+            videos = []
+            if 'value' in data:
+                for result in data['value']:
+                    title = result.get('name', 'Unknown Title')
+                    url = result.get('contentUrl', '')
+                    # Prioritize YouTube results
+                    if 'youtube.com' in url.lower():
+                        videos.append(f"{title}: {url}")
+                    if len(videos) >= 3:
+                        break
+                # If we don't have enough YouTube results, add other video results
+                if len(videos) < 3:
+                    for result in data['value']:
+                        title = result.get('name', 'Unknown Title')
+                        url = result.get('contentUrl', '')
+                        if url not in [v.split(': ')[1] for v in videos]:
+                            videos.append(f"{title}: {url}")
+                        if len(videos) >= 3:
+                            break
+                return videos
+        else:
+            logger.error(f"Bing Video Search API returned status code {response.status_code} for video search.")
+        return ["No video content found."]
+
+    @staticmethod
+    def _search_general(search_term):
+        logger.debug(f"Performing a general search for term: {search_term}")
+        """
+        Uses Bing Web Search API to perform a general search.
+        """
+        headers = {'Ocp-Apim-Subscription-Key': SearchEngine.BING_API_KEY}
+        params = {'q': search_term, 'textDecorations': True, 'textFormat': 'HTML', 'count': 5}
+        response = requests.get(f"{SearchEngine.BING_ENDPOINT}/search", headers=headers, params=params)
+        if response.status_code == 200:
+            logger.debug("Received successful response from Bing Web Search API for general search.")
+            data = response.json()
+            results = []
+            if 'webPages' in data and 'value' in data['webPages']:
+                for result in data['webPages']['value']:
+                    results.append(result)
+                return results
+        else:
+            logger.error(f"Bing Web Search API returned status code {response.status_code} for general search.")
+        return ["No results found for general search."]