Spaces:
Sleeping
Sleeping
File size: 2,689 Bytes
52e408e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
import gradio as gr
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
import torch
from PIL import Image
import numpy as np
import librosa
# Load T5 model for simplification
tokenizer = AutoTokenizer.from_pretrained("t5-base")
model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
# Dummy function for stress detection from voice (replace with your actual model)
def detect_stress_from_voice(audio_path):
# For demo, let's randomly return 'low' or 'high' stress
# You will replace this with real stress detection logic
return "high"
# Dummy function for stress detection from face image (replace with your actual model)
def detect_stress_from_face(image):
# For demo, randomly return 'low' or 'high' stress
return "high"
def simplify_task(task, stress_level):
if stress_level == "low":
return task # No simplification needed if stress is low
input_text = "simplify: " + task
inputs = tokenizer.encode(input_text, return_tensors="pt", max_length=512, truncation=True)
outputs = model.generate(inputs, max_length=60, num_beams=4, early_stopping=True)
simplified_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
return simplified_text
def assistant(voice, face_image, task):
# Step 1: Detect stress from voice and face image
voice_stress = detect_stress_from_voice(voice.name)
face_stress = detect_stress_from_face(face_image)
# Combine stress signals (simple majority vote)
stress_level = "high" if (voice_stress == "high" or face_stress == "high") else "low"
# Step 2: Simplify the task based on stress level
simplified = simplify_task(task, stress_level)
# Return stress level and simplified task
return f"Detected Stress Level: {stress_level.capitalize()}", simplified
with gr.Blocks() as demo:
gr.Markdown("# Context-Aware Multimodal Assistant")
gr.Markdown("Upload your voice recording and face image, then type your task below.")
voice_input = gr.Audio(label="Upload your voice recording (.wav)", type="filepath")
face_input = gr.Image(label="Upload your face image")
task_input = gr.Textbox(label="📝 What are you trying to do or say?", placeholder="E.g. I need help writing a message to my manager.")
output_stress = gr.Textbox(label="🧠 Stress Level Detected", interactive=False)
output_simplified = gr.Textbox(label="💬 Simplified Task / Message", interactive=False)
submit_btn = gr.Button("Simplify Task")
submit_btn.click(
fn=assistant,
inputs=[voice_input, face_input, task_input],
outputs=[output_stress, output_simplified]
)
if __name__ == "__main__":
demo.launch()
|