# app.py from flask import Flask, request, jsonify, render_template from flask_cors import CORS import base64 import tempfile import os app = Flask(__name__) CORS(app) # Initialize components medical_agent = MedicalAgent() sign_translator = SignLanguageTranslator() sign_generator = SignLanguageGenerator() speech_processor = SpeechProcessor() # Add sample medical knowledge medical_knowledge = [ "Headache can be caused by tension, migraine, or sinus issues", "Common headache symptoms include throbbing pain, sensitivity to light", "Headache duration and location help diagnose the type", "Migraine often includes nausea and light sensitivity", "Tension headaches typically cause band-like pressure around head" ] medical_agent.rag.add_medical_knowledge(medical_knowledge) @app.route('/') def index(): return render_template('index.html') @app.route('/api/process_sign_language', methods=['POST']) def process_sign_language(): """Process sign language video and return agent response""" try: video_data = request.json['video_data'] # Base64 encoded video frame frame = decode_video_frame(video_data) # Convert sign language to text patient_text = sign_translator.process_video_frame(frame) # Process with medical agent agent_response = medical_agent.process_patient_input(patient_text) if agent_response['type'] == 'question': # Generate sign language for the question sign_animation = sign_generator.text_to_sign_animation( agent_response['content'] ) return jsonify({ 'type': 'question', 'text': agent_response['content'], 'sign_animation': sign_animation, 'question_count': agent_response['question_count'] }) else: # Send summary to doctor via TTS tts_audio = speech_processor.text_to_speech( agent_response['content'], "summary.wav" ) return jsonify({ 'type': 'summary', 'text': agent_response['content'], 'audio': tts_audio }) except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/process_doctor_audio', methods=['POST']) def process_doctor_audio(): """Process doctor's audio question""" try: audio_data = request.json['audio_data'] # Save audio temporarily with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as f: f.write(base64.b64decode(audio_data)) audio_path = f.name # Convert speech to text doctor_text = speech_processor.speech_to_text(audio_path) # Process with agent patient_question = medical_agent.process_doctor_question(doctor_text) # Generate sign language sign_animation = sign_generator.text_to_sign_animation(patient_question) os.unlink(audio_path) # Clean up return jsonify({ 'question': patient_question, 'sign_animation': sign_animation }) except Exception as e: return jsonify({'error': str(e)}), 500 def decode_video_frame(video_data): """Decode base64 video frame""" # Implementation depends on your frontend format pass if __name__ == '__main__': app.run(host='0.0.0.0', port=5000, debug=True)