Spaces:
Runtime error
Runtime error
File size: 3,543 Bytes
f06c4ab a136b37 f06c4ab a136b37 f06c4ab 3facc07 f06c4ab 3facc07 f06c4ab 3facc07 f06c4ab 3facc07 f06c4ab 3facc07 f06c4ab 3facc07 f06c4ab a136b37 f06c4ab | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | # app.py
from flask import Flask, request, jsonify, render_template
from flask_cors import CORS
import base64
import tempfile
import os
app = Flask(__name__)
CORS(app)
# Initialize components
medical_agent = MedicalAgent()
sign_translator = SignLanguageTranslator()
sign_generator = SignLanguageGenerator()
speech_processor = SpeechProcessor()
# Add sample medical knowledge
medical_knowledge = [
"Headache can be caused by tension, migraine, or sinus issues",
"Common headache symptoms include throbbing pain, sensitivity to light",
"Headache duration and location help diagnose the type",
"Migraine often includes nausea and light sensitivity",
"Tension headaches typically cause band-like pressure around head"
]
medical_agent.rag.add_medical_knowledge(medical_knowledge)
@app.route('/')
def index():
return render_template('index.html')
@app.route('/api/process_sign_language', methods=['POST'])
def process_sign_language():
"""Process sign language video and return agent response"""
try:
video_data = request.json['video_data'] # Base64 encoded video frame
frame = decode_video_frame(video_data)
# Convert sign language to text
patient_text = sign_translator.process_video_frame(frame)
# Process with medical agent
agent_response = medical_agent.process_patient_input(patient_text)
if agent_response['type'] == 'question':
# Generate sign language for the question
sign_animation = sign_generator.text_to_sign_animation(
agent_response['content']
)
return jsonify({
'type': 'question',
'text': agent_response['content'],
'sign_animation': sign_animation,
'question_count': agent_response['question_count']
})
else:
# Send summary to doctor via TTS
tts_audio = speech_processor.text_to_speech(
agent_response['content'],
"summary.wav"
)
return jsonify({
'type': 'summary',
'text': agent_response['content'],
'audio': tts_audio
})
except Exception as e:
return jsonify({'error': str(e)}), 500
@app.route('/api/process_doctor_audio', methods=['POST'])
def process_doctor_audio():
"""Process doctor's audio question"""
try:
audio_data = request.json['audio_data']
# Save audio temporarily
with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as f:
f.write(base64.b64decode(audio_data))
audio_path = f.name
# Convert speech to text
doctor_text = speech_processor.speech_to_text(audio_path)
# Process with agent
patient_question = medical_agent.process_doctor_question(doctor_text)
# Generate sign language
sign_animation = sign_generator.text_to_sign_animation(patient_question)
os.unlink(audio_path) # Clean up
return jsonify({
'question': patient_question,
'sign_animation': sign_animation
})
except Exception as e:
return jsonify({'error': str(e)}), 500
def decode_video_frame(video_data):
"""Decode base64 video frame"""
# Implementation depends on your frontend format
pass
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True) |