Upload 4 files

asr_api.py

@@ -0,0 +1,20 @@
+import os
+import requests
+
+OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY", "")
+
+def transcribe_with_openrouter(audio_path):
+    """Transcribe audio using OpenRouter API (speech-to-text)."""
+    if not OPENROUTER_API_KEY:
+        return "[No OpenRouter API key set]"
+    url = "https://openrouter.ai/api/v1/audio/transcriptions"
+    headers = {"Authorization": f"Bearer {OPENROUTER_API_KEY}"}
+    files = {"file": open(audio_path, "rb")}
+    data = {"model": "whisper-large-v3"}
+    try:
+        response = requests.post(url, headers=headers, files=files, data=data, timeout=60)
+        response.raise_for_status()
+        result = response.json()
+        return result.get("text", "")
+    except Exception as e:
+        return f"[ASR API error: {e}]"

llm.py

@@ -0,0 +1,64 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from user_data import load_user_data, save_user_data
+from phonetics import analyze_audio_phonetically, extract_phonemes
+
+model_name = "BeastGokul/Nika-1.5B"
+llm_tokenizer = AutoTokenizer.from_pretrained(model_name)
+llm_model = AutoModelForCausalLM.from_pretrained(model_name)
+
+SYSTEM_PROMPT = """You are a specialized pronunciation assistant for non-native English speakers.\nYour job is to provide targeted, actionable feedback based on the user's speech or description.\n\nWhen analyzing pronunciation:\n1. Identify at most 2 specific phonemes or pronunciation patterns that need improvement\n2. Explain how the sound is correctly formed (tongue position, lip movement, etc.)\n3. Suggest one simple, targeted exercise for practice\n4. Be encouraging and note any improvements from previous sessions\n5. Use simple language appropriate for language learners\n\nWhen provided with phonetic analysis data, incorporate this information into your feedback.\n"""
+
+def get_llm_feedback(audio=None, text=None, reference_text=None, user_id="default", transcribe_func=None):
+    user_data = load_user_data(user_id)
+    # Process audio if provided
+    if audio:
+        from user_data import save_audio
+        audio_path = save_audio(audio, user_id)
+        # Transcribe if no text was provided
+        if not text and transcribe_func:
+            text = transcribe_func(audio_path)
+        # Get phonetic analysis
+        phonetic_analysis = analyze_audio_phonetically(audio_path, reference_text)
+        phonetic_info = f"""
+Phonetic analysis:\n- Detected phonemes: {phonetic_analysis['detected_phonemes']}\n"""
+        if reference_text:
+            phonetic_info += f"- Reference phonemes: {phonetic_analysis.get('reference_phonemes', 'N/A')}\n"
+    else:
+        audio_path = None
+        phonetic_info = ""
+    # Get user history context
+    history_context = ""
+    if user_data["practice_sessions"]:
+        phoneme_counts = {p: data["practice_count"] for p, data in user_data["phoneme_progress"].items()}
+        challenging = sorted(phoneme_counts.items(), key=lambda x: x[1], reverse=True)[:3]
+        history_context = f"""
+User has practiced {len(user_data['practice_sessions'])} times before.\nCommon challenging phonemes: {', '.join([p for p, _ in challenging])}.\n"""
+    # Build prompt for LLM
+    if text:
+        user_input = f"I said: '{text}'"
+        if reference_text and reference_text != text:
+            user_input += f". I was trying to say: '{reference_text}'"
+    else:
+        user_input = "Please analyze my pronunciation."
+    full_prompt = f"""{SYSTEM_PROMPT}\n\nUser history:\n{history_context}\n\n{phonetic_info}\n\nUser: {user_input}\n"""
+    # Get LLM response
+    inputs = llm_tokenizer(full_prompt, return_tensors="pt").to(llm_model.device)
+    import torch
+    with torch.no_grad():
+        outputs = llm_model.generate(
+            **inputs, 
+            max_new_tokens=200,
+            temperature=0.7,
+            top_p=0.9,
+            do_sample=True
+        )
+    response = llm_tokenizer.decode(outputs[0], skip_special_tokens=True)
+    try:
+        response = response.split("Assistant: ")[-1].strip()
+    except:
+        pass
+    # Track the session if audio was provided
+    if audio_path:
+        from user_data import track_practice_session
+        track_practice_session(user_id, audio_path, text, reference_text, response)
+    return response, text

phonetics.py

@@ -0,0 +1,37 @@
+import numpy as np
+import librosa
+from phonemizer import phonemize
+
+def extract_phonemes(text):
+    """Convert text to phonemes"""
+    return phonemize(text, language='en-us', backend='espeak', strip=True)
+
+def analyze_audio_phonetically(audio_path, reference_text=None, wav2vec_processor=None, wav2vec_model=None):
+    """Perform phonetic analysis of the audio compared to reference text (optional, only if local model loaded)"""
+    if not wav2vec_processor or not wav2vec_model:
+        return {"detected_phonemes": "[Phoneme analysis not available]"}
+    audio, sr = librosa.load(audio_path, sr=16000)
+    inputs = wav2vec_processor(audio, sampling_rate=16000, return_tensors="pt")
+    import torch
+    with torch.no_grad():
+        logits = wav2vec_model(inputs.input_values).logits
+    predicted_ids = torch.argmax(logits, dim=-1)
+    phoneme_sequence = wav2vec_processor.batch_decode(predicted_ids)[0]
+    result = {"detected_phonemes": phoneme_sequence}
+    if reference_text:
+        reference_phonemes = extract_phonemes(reference_text)
+        result["reference_phonemes"] = reference_phonemes
+        result["analysis"] = "Phoneme comparison would be performed here"
+    return result
+
+def extract_pronunciation_embedding(audio_path, hubert_processor=None, hubert_model=None):
+    """Extract pronunciation embedding for comparison purposes (optional, only if local model loaded)"""
+    if not hubert_model or not hubert_processor:
+        return None
+    audio, sr = librosa.load(audio_path, sr=16000)
+    inputs = hubert_processor(audio, sampling_rate=16000, return_tensors="pt")
+    import torch
+    with torch.no_grad():
+        outputs = hubert_model(**inputs)
+    embedding = outputs.last_hidden_state.mean(dim=1).cpu().numpy()
+    return embedding

user_data.py

@@ -0,0 +1,28 @@
+import json
+import os
+from datetime import datetime
+
+def get_user_data_path(user_id="default"):
+    return f"user_data/{user_id}_data.json"
+
+def load_user_data(user_id="default"):
+    file_path = get_user_data_path(user_id)
+    if os.path.exists(file_path):
+        with open(file_path, "r") as f:
+            return json.load(f)
+    return {
+        "profile": {
+            "native_language": "",
+            "challenge_sounds": [],
+            "practice_count": 0,
+            "joined_date": datetime.now().strftime("%Y-%m-%d")
+        },
+        "practice_sessions": [],
+        "phoneme_progress": {},
+        "word_progress": {},
+        "goals": []
+    }
+
+def save_user_data(data, user_id="default"):
+    with open(get_user_data_path(user_id), "w") as f:
+        json.dump(data, f, indent=2)