Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import numpy as np | |
| # BOTH THE FUNCTIONS MAY BE EXECUTING SIMULTANEOUSLY | |
| import torch | |
| import matplotlib.pyplot as plt | |
| import torchvision.transforms as T | |
| import torch.nn as nn | |
| import torchvision.models as models | |
| import torch.nn.functional as F | |
| from google import genai | |
| from google.genai import types | |
| import time | |
| from collections import deque | |
| from loguru import logger | |
| from transformers import CLIPTokenizer, CLIPProcessor, CLIPModel | |
| import cv2 | |
| print("-----------------------------------------------------------------------------------------------") | |
| logger.add('main.log') | |
| API_KEY = 'AIzaSyBEEJ8oGnE-GhGExxTHNzWz-YPEoblO4Cc' | |
| client = genai.Client(api_key=API_KEY) | |
| chat = client.chats.create(model="gemini-2.5-flash") | |
| # CLIP ------------------- TEXT EMBEDDING PART | |
| ls = ['angry face', 'disgust face', 'fear face', 'happy face', 'neutral face', 'sad face' ,'surprise face', 'anxiety face'] | |
| ls = ['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad', 'surprise'] | |
| ls = [ | |
| "a close-up photo of a person with furrowed brows, clenched jaw, and intense glaring eyes, expressing strong anger", | |
| "a person with a wrinkled nose, curled upper lip, and a look of revulsion, clearly showing disgust", | |
| "a frightened person with widened eyes, raised eyebrows, and slightly open mouth, looking fearful and startled", | |
| "a joyful person with a big bright smile, crinkled eyes, and a cheerful face, radiating happiness", | |
| "a portrait of a person with a relaxed face, neutral lips, and no particular emotion, showing a calm and composed expression", | |
| "a person with teary eyes, downward gaze, and frowning lips, visibly sad and emotionally down", | |
| "a surprised person with mouth wide open, eyebrows raised high, and eyes widened in shock or amazement", | |
| "a person with tense facial muscles, furrowed brow, and worried eyes, clearly showing anxiety and stress" | |
| ] | |
| model_name = "openai/clip-vit-base-patch32" | |
| model = CLIPModel.from_pretrained(model_name) | |
| model.load_state_dict(torch.load("clip_model_state_dict.pth", map_location=torch.device('cpu'))) | |
| tokenizer = CLIPTokenizer.from_pretrained(model_name) | |
| inputs = tokenizer(ls, padding = True, return_tensors = "pt") | |
| input_ids = inputs.input_ids | |
| with torch.no_grad(): | |
| text_embeddings = model.get_text_features(**inputs) | |
| # NOW LETS DEAL WITH IMAGES EMBEDDINGS | |
| processor = CLIPProcessor.from_pretrained(model_name) | |
| # PROMPTING PART BEGINS | |
| base_prompt = prompt = f""" | |
| You are an emotionally intelligent AI chatbot designed to support users based on their current emotional state. Your primary objective is to understand and adapt to the user's emotional needs in order to provide the most helpful, empathetic, and appropriate response. | |
| You will be provided with: | |
| 1. The **emotion detected** on the user's face on last 5 imgs. | |
| 2. The **user's query or message**. | |
| 3. Specific **guidelines** on how to respond based on the detected emotion. | |
| Please strictly follow these instructions based on the user's emotional state: | |
| - **Neutral**: Maintain a balanced and informative tone. Keep responses factual and clear without emotional bias. | |
| - **Happiness**: Respond in a cheerful and engaging manner. Celebrate the user's good mood and use positive, affirming language. | |
| - **Surprise**: Acknowledge the user's surprise. Match their energy, provide context, and clarify any unexpected aspects if necessary. | |
| - **Sadness**: Be empathetic and gentle. Offer comfort, encouragement, and support. Use soft, understanding language to uplift the user. | |
| - **Anger**: Stay calm and respectful. Acknowledge the frustration, avoid confrontational language, and gently offer help or solutions. | |
| - **Disgust**: Show understanding and avoid reinforcing the negative emotion. Shift the conversation to a neutral or constructive tone if appropriate. | |
| - **Fear**: Offer reassurance and use a calm, composed tone. Provide clear, supportive information that can reduce uncertainty or distress. | |
| - **Anxiety**: Respond with patience and kindness. Use calming language, avoid overwhelming the user, and guide them with simple, encouraging responses. | |
| Always personalize your reply to match both the **user's query** and their **emotional state**. Your responses should reflect emotional intelligence, compassion, and clarity. | |
| Here is the information you have: | |
| - List of Emotions person was having over last 5 imgs: {{emotion}} | |
| - User Query: {{query}} | |
| - history : {{history}} | |
| - Respond using the corresponding emotion guideline from above. | |
| be careful while reading from emotions as many times neutral gets classified as sad so in this cases cross check from previous chats and accordingly decide the emotion | |
| don't ever tell user that they are seeming sad or anything else as it may provoke some emotions which they might not even be having | |
| """ | |
| maxlen = 5 | |
| class EmotionSample: | |
| def __init__(self, timestamp, frame): | |
| self.timestamp = timestamp # Time frame was captured | |
| self.frame = frame # Raw or cropped face image | |
| buffer = deque(maxlen=maxlen) | |
| # frame processing function | |
| def process_frame(frame): | |
| """Processes each frame from the webcam.""" | |
| # Replace this with your actual processing logic | |
| if frame is not None: | |
| # Example: Convert to grayscale | |
| sample = EmotionSample(timestamp=time.time(), frame=frame) #it@canamgroup.com, hrhead hradmin | |
| buffer.append(sample) | |
| else: | |
| return None | |
| # chatting function | |
| def echo(message, history): | |
| # all the main code for responding will be here | |
| logger.info(history) | |
| images = [] | |
| for i in range(len(buffer)): | |
| img = buffer.popleft().frame | |
| images.append(img) | |
| if not images or len(images) == 0: | |
| return "No image provided! Please upload an image." | |
| image_inputs = processor(images=images, return_tensors="pt") | |
| with torch.no_grad(): | |
| image_embeddings = model.get_image_features(**image_inputs) | |
| txt_image_similarity = F.cosine_similarity(text_embeddings[:, None, :], image_embeddings[None, :, :], dim=2).cpu().numpy() | |
| best_label_indices = np.argmax(txt_image_similarity, axis=0) | |
| all_emotions = [ls[idx] for idx in best_label_indices] | |
| logger.info(f"Emotion : {all_emotions}") | |
| # querying | |
| response = chat.send_message(prompt.format(emotion=all_emotions, query=message, history = history)) | |
| return response.text | |
| # with gr.Blocks() as demo: | |
| # with gr.Row(): | |
| # input_img = gr.Image(sources=["webcam"], streaming=True, label="Webcam Input", visible=True,height=360, width=320) | |
| # input_img.stream(process_frame, [input_img], []) | |
| # chatbot = gr.Chatbot(placeholder="<strong>Your Personal Emotional ChatBot </strong><br>Ask Me Anything") | |
| # gr.ChatInterface(fn=echo, type="messages", chatbot=chatbot, examples=["hello", "hola", "namaste"], title="Emotional ChatBot 🤔🤔") | |
| # demo.launch() | |
| with gr.Blocks() as demo: | |
| with gr.Row(): | |
| input_img = gr.Image( | |
| sources=["webcam"], | |
| streaming=True, | |
| label="Webcam Input", | |
| visible=True, | |
| height=360, | |
| width=320 | |
| ) | |
| input_img.stream(process_frame, [input_img], []) | |
| chatbot = gr.Chatbot( | |
| placeholder="Your Personal Emotional ChatBot - Ask Me Anything" | |
| ) | |
| gr.ChatInterface( | |
| fn=echo, | |
| type="messages", | |
| chatbot=chatbot, | |
| examples=["hello", "hola", "namaste"], | |
| title="Emotional ChatBot 🤔🤔" | |
| ) | |
| demo.launch() |