from transformers import AutoTokenizer, AutoModelForCausalLM #from user_data import load_user_data, save_user_data from phonetics import analyze_audio_phonetically, extract_phonemes model_name = "BeastGokul/Nika-1.5B" llm_tokenizer = AutoTokenizer.from_pretrained(model_name) llm_model = AutoModelForCausalLM.from_pretrained(model_name) SYSTEM_PROMPT = """You are a specialized pronunciation assistant for non-native English speakers.\nYour job is to provide targeted, actionable feedback based on the user's speech or description.\n\nWhen analyzing pronunciation:\n1. Identify at most 2 specific phonemes or pronunciation patterns that need improvement\n2. Explain how the sound is correctly formed (tongue position, lip movement, etc.)\n3. Suggest one simple, targeted exercise for practice\n4. Be encouraging and note any improvements from previous sessions\n5. Use simple language appropriate for language learners\n\nWhen provided with phonetic analysis data, incorporate this information into your feedback.\n""" def get_llm_feedback(audio=None, text=None, reference_text=None, user_id="default", transcribe_func=None): user_data = load_user_data(user_id) # Process audio if provided if audio: from user_data import save_audio audio_path = save_audio(audio, user_id) # Transcribe if no text was provided if not text and transcribe_func: text = transcribe_func(audio_path) # Get phonetic analysis phonetic_analysis = analyze_audio_phonetically(audio_path, reference_text) phonetic_info = f""" Phonetic analysis:\n- Detected phonemes: {phonetic_analysis['detected_phonemes']}\n""" if reference_text: phonetic_info += f"- Reference phonemes: {phonetic_analysis.get('reference_phonemes', 'N/A')}\n" else: audio_path = None phonetic_info = "" # Get user history context history_context = "" if user_data["practice_sessions"]: phoneme_counts = {p: data["practice_count"] for p, data in user_data["phoneme_progress"].items()} challenging = sorted(phoneme_counts.items(), key=lambda x: x[1], reverse=True)[:3] history_context = f""" User has practiced {len(user_data['practice_sessions'])} times before.\nCommon challenging phonemes: {', '.join([p for p, _ in challenging])}.\n""" # Build prompt for LLM if text: user_input = f"I said: '{text}'" if reference_text and reference_text != text: user_input += f". I was trying to say: '{reference_text}'" else: user_input = "Please analyze my pronunciation." full_prompt = f"""{SYSTEM_PROMPT}\n\nUser history:\n{history_context}\n\n{phonetic_info}\n\nUser: {user_input}\n""" # Get LLM response inputs = llm_tokenizer(full_prompt, return_tensors="pt").to(llm_model.device) import torch with torch.no_grad(): outputs = llm_model.generate( **inputs, max_new_tokens=200, temperature=0.7, top_p=0.9, do_sample=True ) response = llm_tokenizer.decode(outputs[0], skip_special_tokens=True) try: response = response.split("Assistant: ")[-1].strip() except: pass # Track the session if audio was provided if audio_path: from user_data import track_practice_session track_practice_session(user_id, audio_path, text, reference_text, response) return response, text