Update app.py

app.py

@@ -1,270 +1,217 @@
+import os
+import uuid
 import requests
 import json
 import random
-from gradio_client import Client
+import re
+from pydub import AudioSegment
 import gradio as gr
-from dotenv import load_dotenv
-import os
-import tempfile
 import speech_recognition as sr
-import io
-import soundfile as sf
-
-# Load environment variables
-load_dotenv()
-
-# إعدادات API
-API_URL = "https://api.deepseek.com/v1/chat/completions"
-API_KEY = os.getenv("DEEPSEEK_API_KEY")
-HF_TOKEN = os.getenv("HUGGINGFACE_TOKEN")
-
-# إعداد TTS
-TTS_MODEL = os.getenv("TTS_MODEL", "KindSynapse/Youssef-Ahmed-Private-Text-To-Speech-Unlimited")
-TTS_CLIENT = Client(TTS_MODEL, hf_token=HF_TOKEN)
-TTS_PASSWORD = os.getenv("TTS_PASSWORD")
-TTS_VOICE = os.getenv("TTS_VOICE", "coral")
-TTS_SEED = int(os.getenv("TTS_SEED", "12345"))
-
-# إعداد Speech Recognition
-recognizer = sr.Recognizer()
-
-# التحقق من وجود المتغيرات المطلوبة
-required_env_vars = {
-    "DEEPSEEK_API_KEY": API_KEY,
-    "HUGGINGFACE_TOKEN": HF_TOKEN,
-    "TTS_PASSWORD": TTS_PASSWORD
-}
 
-for var_name, var_value in required_env_vars.items():
-    if not var_value:
-        raise ValueError(f"Missing required environment variable: {var_name}")
+def log_step(msg):
+    print(f"[EVAL_SPOKEN] {msg}")
 
-# البرومبت الرئيسي للشات بوت
-MAIN_SYSTEM_PROMPT = {
-    "role": "system",
-    "content": (
-        "You are Sam, a friendly and encouraging English conversation tutor. "
-        "Your responses must be in JSON with these keys: "
-        "'response': Your main response to the user, "
-        "'corrections': Grammar or pronunciation corrections if needed, "
-        "'vocabulary': Suggested alternative words or phrases, "
-        "'level': Assessment of user's English level (beginner/intermediate/advanced), "
-        "'encouragement': A motivating comment. "
-        "\n\nGuidelines:"
-        "\n1. Adapt your language to their level"
-        "\n2. Keep conversations natural and engaging"
-        "\n3. Focus on their interests and context"
-        "\n4. Be patient and supportive"
-        "\n5. Provide gentle corrections"
-        "\n6. Suggest vocabulary improvements naturally"
-        "\n7. Keep responses clear and structured"
-    )
-}
-
-# برومبت خاص بالترحيب (مختصر)
-WELCOME_SYSTEM_PROMPT = {
-    "role": "system",
-    "content": (
-        "You are Sam, a friendly English tutor. Create a short, warm welcome message (2-3 sentences max) that: "
-        "1) Introduces yourself briefly "
-        "2) Asks for the user's name and what they'd like to practice. "
-        "Make it casual and friendly. Return ONLY the greeting in JSON format with a single key 'greeting'."
-        "Example: {'greeting': 'Hi! I'm Sam, your English buddy. What's your name and what would you like to practice today? 😊'}"
-    )
-}
+def convert_to_wav(input_path, output_path=None):
+    if output_path is None:
+        output_path = os.path.join("uploads", f"converted_{uuid.uuid4()}.wav")
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    log_step(f"Converting audio from {input_path} to WAV format")
+    try:
+        audio = AudioSegment.from_file(input_path)
+        audio.export(output_path, format="wav")
+        log_step(f"Audio successfully converted to {output_path}")
+        return output_path
+    except Exception as e:
+        error_msg = f"Failed to convert audio: {e}"
+        log_step(f"ERROR in audio conversion: {error_msg}")
+        raise RuntimeError(error_msg)
+
+def img_detector(model, url):
+    api_keys = [
+        os.getenv("OPENROUTER_API_KEY_1"),
+        os.getenv("OPENROUTER_API_KEY_2"),
+        os.getenv("OPENROUTER_API_KEY_3"),
+        os.getenv("OPENROUTER_API_KEY_4"),
+    ]
+    api_keys = [k for k in api_keys if k]
+    errors = []
+    for api_key in api_keys:
+        try:
+            response = requests.post(
+                url="https://openrouter.ai/api/v1/chat/completions",
+                headers={
+                    "Authorization": f"Bearer {api_key}",
+                    "Content-Type": "application/json",
+                },
+                data=json.dumps({
+                    "model": model,
+                    "messages": [
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": "What is appear in this image? Please provide a detailed description."},
+                                {"type": "image_url", "image_url": {"url": url}}
+                            ]
+                        }
+                    ]
+                })
+            )
+            if response.status_code == 200:
+                data = response.json()
+                if 'choices' in data and len(data['choices']) > 0:
+                    return data['choices'][0]['message']['content']
+                else:
+                    errors.append(f"API key {api_key[:8]}...: No choices in response.")
+            else:
+                errors.append(f"API key {api_key[:8]}...: Status {response.status_code}")
+        except Exception as e:
+            errors.append(f"API key {api_key[:8]}...: Exception {e}")
+    return f"All VLM API requests failed: {' | '.join(errors)}"
+
+def transcribe_audio(audio_path):
+    recognizer = sr.Recognizer()
+    with sr.AudioFile(audio_path) as source:
+        audio = recognizer.record(source)
+    try:
+        text = recognizer.recognize_google(audio, language='en-US')
+        return text
+    except sr.UnknownValueError:
+        return "Could not understand the audio."
+    except sr.RequestError as e:
+        return f"Could not request results; {e}"
+
+def evaluate_spoken_english(vlm_description, transcript_input, lang="english"):
+    """
+    Evaluate spoken English based on a VLM image description and a transcript.
+    Returns a dict with feedback and scores.
+    """
+    DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY")
+    if not DEEPSEEK_API_KEY:
+        raise EnvironmentError("Missing DEEPSEEK_API_KEY in environment.")
+
+    prompt = f"""
+      You are an expert spoken English tutor evaluating learners' spoken English skills based on CEFR (Common European Framework of Reference for Languages) standards. Your evaluation must consider key criteria such as:
+
+      - Pronunciation and stress
+      - Fluency and rhythm
+      - Vocabulary range and appropriateness
+      - Coherence and structure
+      - Grammar (as inferred from the transcript)
+
+      Important: The transcript below has **no punctuation**, as it is auto-generated from speech. Focus on what can be evaluated reliably based on the content and structure.
+
+      The learner was shown the following image (described by a vision-language model):
+      ---
+      {vlm_description}
+      ---
+
+      Then the learner described the image by voice. This is the auto-generated transcript:
+      ---
+      {transcript_input}
+      ---
+
+      Please evaluate the learner’s spoken English and return a JSON response including:
+      1. "relevance_score": Score out of 100 for how relevant the speech was to the image.
+      2. "fluency_score": Score out of 100 for fluency and smoothness of speech.
+      3. "pronunciation_feedback": Identify any pronunciation issues or common errors.
+      4. "mistakes": A list of grammar or vocabulary issues inferred from the transcript.
+      5. "corrected_transcript": A corrected version of the transcript with appropriate punctuation and grammar.
+      6. "learning_level": Estimated CEFR level (A1–C2).
+      7. "tips": Actionable learning advice tailored to the learner’s weaknesses.
+      8. "highlight": Something strong or impressive in the learner’s speaking (e.g., good phrasing or word choice).
+      9. "motivational_comment": A short, encouraging note to boost the learner’s confidence.
+
+      Respond in clean JSON format only.
+      """
 
-class EnglishTutor:
-    def __init__(self):
-        self.chat_history = []
-        self.user_info = {
-            "name": None,
-            "level": None,
-            "interests": None,
-            "goals": None
-        }
-        # Initialize with welcome message
-        self.chat_history = [MAIN_SYSTEM_PROMPT]
-    
-    def get_welcome_message(self):
-        """توليد رسالة ترحيب فريدة"""
+    try:
         response = requests.post(
-            API_URL,
-            headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
+            "https://api.deepseek.com/v1/chat/completions",
+            headers={"Authorization": f"Bearer {DEEPSEEK_API_KEY}"},
             json={
                 "model": "deepseek-chat",
-                "messages": [WELCOME_SYSTEM_PROMPT],
+                "messages": [
+                    {
+                        "role": "system",
+                        "content": "You are a certified spoken English tutor helping learners improve pronunciation, fluency, and confidence. You follow CEFR guidelines and speak warmly to encourage learners."
+                    },
+                    {
+                        "role": "user",
+                        "content": prompt.strip()
+                    }
+                ],
                 "temperature": random.uniform(0.9, 1),
-                "response_format": {"type": "json_object"}
+                "max_tokens": 1500
             }
         )
-        welcome_json = json.loads(response.json()["choices"][0]["message"]["content"])
-        return welcome_json["greeting"]
 
-    def get_bot_response(self, user_message):
-        """معالجة رسالة المستخدم والحصول على رد"""
-        self.chat_history.append({"role": "user", "content": user_message})
-        
-        response = requests.post(
-            API_URL,
-            headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
-            json={
-                "model": "deepseek-chat",
-                "messages": self.chat_history,
-                "temperature": random.uniform(0.9, 1.0),
-                "response_format": {"type": "json_object"}
+        if response.status_code == 200:
+            data = response.json()
+            if "choices" in data and len(data["choices"]) > 0:
+                content = data["choices"][0]["message"]["content"]
+                clean_json_text = re.sub(r"```json|```", "", content).strip()
+
+                try:
+                    return json.loads(clean_json_text)
+                except json.JSONDecodeError as decode_err:
+                    return {
+                        "relevance_score": 0,
+                        "fluency_score": 0,
+                        "pronunciation_feedback": "N/A",
+                        "mistakes": [],
+                        "corrected_transcript": "N/A",
+                        "learning_level": "Unknown",
+                        "highlight": "N/A",
+                        "motivational_comment": "N/A",
+                        "tips": [f"Invalid JSON format: {str(decode_err)}", "Raw content:", clean_json_text]
+                    }
+        else:
+            return {
+                "relevance_score": 0,
+                "fluency_score": 0,
+                "pronunciation_feedback": "N/A",
+                "mistakes": [],
+                "corrected_transcript": "N/A",
+                "learning_level": "Unknown",
+                "highlight": "N/A",
+                "motivational_comment": "N/A",
+                "tips": [f"API Error: {response.status_code}"]
             }
-        )
-        
-        bot_message = response.json()["choices"][0]["message"]["content"]
-        bot_json = json.loads(bot_message)
-        
-        # تحديث معلومات المستخدم إذا وجدت
-        if "level" in bot_json:
-            self.user_info["level"] = bot_json["level"]
-        
-        self.chat_history.append({"role": "assistant", "content": bot_message})
-        return bot_json
-
-    def text_to_speech(self, text):
-        """تحويل نص إلى صوت مع مراعاة المبتدئين في اللغة الإنجليزية"""
-        # تنظيف النص من أي علامات إضافية أو نصوص زائدة
-        text = text.strip()
-        if text.startswith('"') and text.endswith('"'):
-            text = text[1:-1]
-            
-        tts_prompt = text
-        tts_emotion = "Warm, encouraging, and clear with a friendly and supportive tone."
-
-        return TTS_CLIENT.predict(
-            password=TTS_PASSWORD,
-            prompt=tts_prompt,
-            voice=TTS_VOICE,
-            emotion=tts_emotion,
-            use_random_seed=True,
-            specific_seed=TTS_SEED,
-            api_name="/text_to_speech_app"
-        )
-
-# Create a single instance of EnglishTutor
-tutor = EnglishTutor()
 
-def format_response(response_dict):
-    """Format the response dictionary into a nice HTML string"""
-    html = f"<div style='font-size: 16px;'>"
-    html += f"<p>{response_dict['response']}</p>"
-    
-    if response_dict['corrections']:
-        html += f"<p><b>✍️ Corrections:</b> {response_dict['corrections']}</p>"
-    
-    if response_dict['vocabulary']:
-        html += f"<p><b>📚 Vocabulary:</b> {response_dict['vocabulary']}</p>"
-    
-    if response_dict['encouragement']:
-        html += f"<p><b>🌟 Encouragement:</b> {response_dict['encouragement']}</p>"
-    
-    html += "</div>"
-    return html
-
-def speech_to_text(audio_path):
-    """Convert speech to text using speech_recognition"""
-    try:
-        # Load audio file
-        with sr.AudioFile(audio_path) as source:
-            # Record the audio file
-            audio = recognizer.record(source)
-            # Use Google Speech Recognition
-            text = recognizer.recognize_google(audio)
-            return text
     except Exception as e:
-        print(f"Error in speech recognition: {str(e)}")
-        return None
-
-def chat(audio, history):
-    """Handle chat interactions"""
-    if audio is None:
-        # Return empty response if no audio
-        return history, None
-    
-    # Convert audio to WAV format for speech recognition
-    audio_data = audio[1]  # Get the numpy array
-    sample_rate = audio[0]  # Get the sample rate
-    
-    # Save as temporary WAV file
-    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_wav:
-        sf.write(temp_wav.name, audio_data, sample_rate)
-        # Convert speech to text
-        audio_text = speech_to_text(temp_wav.name)
-    
-    # Clean up temporary file
-    os.unlink(temp_wav.name)
-    
-    if not audio_text:
-        return history, None
-    
-    # Get bot response
-    response = tutor.get_bot_response(audio_text)
-    
-    # Generate audio for the main response
-    audio_path = tutor.text_to_speech(response["response"])[0]
-    
-    # Format the complete response
-    formatted_response = format_response(response)
-    
-    # Update history in the correct format for gr.Chatbot
-    history = history or []
-    history.append((audio_text, formatted_response))
-    
-    return history, audio_path
-
-def show_welcome():
-    """Show welcome message on startup"""
-    welcome = tutor.get_welcome_message()
-    audio_path = tutor.text_to_speech(welcome)[0]
-    return [(None, welcome)], audio_path
+        return {
+            "relevance_score": 0,
+            "fluency_score": 0,
+            "pronunciation_feedback": "N/A",
+            "mistakes": [],
+            "corrected_transcript": "N/A",
+            "learning_level": "Unknown",
+            "highlight": "N/A",
+            "motivational_comment": "N/A",
+            "tips": [f"Exception occurred: {str(e)}"]
+        }
 
-# Create Gradio interface
-with gr.Blocks(css="footer {display: none}") as demo:
-    gr.Markdown("# 🤖 Sam - Your English Tutor")
-    gr.Markdown("Welcome to your personalized English learning session! Click the microphone and start speaking!")
-    
-    chatbot = gr.Chatbot(
-        show_label=False,
-        height=400,
-        type="messages"
-    )
-    
-    with gr.Row():
-        audio_input = gr.Audio(
-            source="microphone",
-            type="numpy",
-            label="Speak here",
-            show_label=True
-        )
-        audio_output = gr.Audio(
-            label="Sam's Voice",
-            show_label=True,
-            type="filepath"
-        )
-    
-    # Handle audio input
-    audio_input.stop_recording(
-        fn=chat,
-        inputs=[audio_input, chatbot],
-        outputs=[chatbot, audio_output],
-        queue=False
-    )
-    
-    # Show welcome message on page load
-    demo.load_event(
-        fn=show_welcome,
-        inputs=None,
-        outputs=[chatbot, audio_output]
-    )
+def gradio_spoken_eval(image_url, audio_file, model="meta-llama/llama-3.2-11b-vision-instruct:free"):
+    log_step(f"Received image_url={image_url}, audio_file={audio_file}")
+    wav_path = convert_to_wav(audio_file)
+    transcript = transcribe_audio(wav_path)
+    log_step(f"Transcript: {transcript}")
+    vlm_desc = img_detector(model, image_url)
+    log_step(f"VLM Description: {vlm_desc}")
+    result = evaluate_spoken_english(vlm_desc, transcript)
+    return json.dumps(result, indent=2, ensure_ascii=False)
+
+iface = gr.Interface(
+    fn=gradio_spoken_eval,
+    inputs=[
+        gr.Textbox(label="Image URL"),
+        gr.Audio(type="filepath", label="Audio File (any format)"),
+        gr.Dropdown(choices=["meta-llama/llama-3.2-11b-vision-instruct:free", "google/gemini-2.0-flash-exp:free"], value="meta-llama/llama-3.2-11b-vision-instruct:free", label="Vision Model")
+    ],
+    outputs=gr.Code(label="Evaluation JSON", language="json"),
+    title="Spoken English Evaluation",
+    description="Upload an image URL and an audio file describing the image. The system will transcribe your speech, analyze it, and return a detailed evaluation."
+)
 
-# Launch the interface
 if __name__ == "__main__":
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=7860,
-        share=False
-    )
+    iface.launch()