import warnings warnings.filterwarnings("ignore") import os os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" import gradio as gr import matplotlib.pyplot as plt import numpy as np import pandas as pd import torch import torchaudio from captum.attr import Saliency from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor, pipeline, logging as hf_logging hf_logging.set_verbosity_error() MODEL_ID = "matheetharanadshyan/ipd-submission-model" LLM_ID = "Qwen/Qwen2.5-0.5B-Instruct" SAMPLE_RATE = 16000 PRACTICE_PHRASES = { "Simple Words": [ "Hello", "Carrot", "Apple", "Water", "Happy", "Good Morning" ], } ALL_PHRASES = [] for category, phrases in PRACTICE_PHRASES.items(): ALL_PHRASES.extend(phrases) ALL_PHRASES.append("Custom Phrase...") device = torch.device( "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu" ) print(f"Loading Models On {device}...") processor = Wav2Vec2Processor.from_pretrained(MODEL_ID) model = Wav2Vec2ForCTC.from_pretrained(MODEL_ID).to(device).eval() llm_pipe = pipeline("text-generation", model=LLM_ID, device=device) print("Models Loaded Successfully!") def preprocess_audio(audio_path): speech, sr = torchaudio.load(audio_path) if speech.shape[0] > 1: speech = torch.mean(speech, dim=0, keepdim=True) if sr != SAMPLE_RATE: speech = torchaudio.transforms.Resample(sr, SAMPLE_RATE)(speech) speech = trim_silence(speech) speech = speech / (torch.max(torch.abs(speech)) + 1e-8) speech = torch.cat([speech[:, :1], speech[:, 1:] - 0.97 * speech[:, :-1]], dim=1) return speech def trim_silence(audio, threshold=0.01, frame_length=1024): audio_abs = torch.abs(audio.squeeze()) start_idx, end_idx = 0, len(audio_abs) for i in range(0, len(audio_abs) - frame_length, frame_length // 2): if torch.mean(audio_abs[i:i+frame_length]) > threshold: start_idx = max(0, i - frame_length) break for i in range(len(audio_abs) - frame_length, 0, -frame_length // 2): if torch.mean(audio_abs[i:i+frame_length]) > threshold: end_idx = min(len(audio_abs), i + frame_length * 2) break return audio[:, start_idx:end_idx] def get_alignment(target, actual): n, m = len(target), len(actual) dp = [[0] * (m + 1) for _ in range(n + 1)] for i in range(n + 1): dp[i][0] = i for j in range(m + 1): dp[0][j] = j for i in range(1, n + 1): for j in range(1, m + 1): cost = 0 if target[i-1] == actual[j-1] else 1 dp[i][j] = min(dp[i-1][j] + 1, dp[i][j-1] + 1, dp[i-1][j-1] + cost) align, i, j = [], n, m while i > 0 or j > 0: if i > 0 and j > 0 and target[i-1] == actual[j-1]: align.append((target[i-1], actual[j-1], "✓ Correct")) i, j = i - 1, j - 1 elif i > 0 and j > 0 and dp[i][j] == dp[i-1][j-1] + 1: align.append((target[i-1], actual[j-1], "✗ Substitution")) i, j = i - 1, j - 1 elif j > 0 and (i == 0 or dp[i][j] == dp[i][j-1] + 1): align.append(("—", actual[j-1], "⊕ Insertion")) j -= 1 else: align.append((target[i-1], "—", "⊖ Deletion")) i -= 1 return align[::-1] def calculate_accuracy(alignment): total = len(alignment) correct = sum(1 for _, _, s in alignment if "Correct" in s) subs = sum(1 for _, _, s in alignment if "Substitution" in s) ins = sum(1 for _, _, s in alignment if "Insertion" in s) dels = sum(1 for _, _, s in alignment if "Deletion" in s) return { "accuracy": (correct / total * 100) if total > 0 else 0, "wer": min(((subs + ins + dels) / max(total - ins, 1)) * 100, 100), "correct": correct, "total": total } def create_saliency_plot(grads, transcription=""): grads_norm = grads / (np.max(grads) + 1e-8) threshold = np.percentile(grads_norm, 85) grads_smooth = np.convolve(grads_norm, np.ones(5)/5, mode='same') fig, ax = plt.subplots(figsize=(10, 3), facecolor='#ffffff') time_axis = np.arange(len(grads_norm)) / 50 ax.fill_between(time_axis, grads_smooth, color='#3b82f6', alpha=0.15) ax.plot(time_axis, grads_smooth, color='#3b82f6', alpha=0.9, linewidth=2) important_mask = grads_smooth > threshold ax.fill_between(time_axis, grads_smooth, where=important_mask, color='#3b82f6', alpha=0.4, label='High importance') ax.set_facecolor('#ffffff') for spine in ax.spines.values(): spine.set_visible(False) ax.tick_params(colors='#6b7280', which='both', labelsize=9, length=0) ax.set_xlabel('Time (seconds)', color='#374151', fontsize=10, fontweight='500') ax.set_ylabel('Model Attention', color='#374151', fontsize=10, fontweight='500') ax.set_xlim(0, time_axis[-1] if len(time_axis) > 0 else 1) ax.set_ylim(0, 1.1) ax.grid(True, axis='y', alpha=0.2, linestyle='-', linewidth=0.5, color='#d1d5db') if transcription: ax.set_title(f'Speech Pattern Analysis', fontsize=11, fontweight='600', color='#111827', pad=10) plt.tight_layout(pad=1.5) return fig def generate_feedback(target_txt, alignment, metrics): errors = [] for target, actual, status in alignment: if "Substitution" in status: errors.append(f"said '{actual}' instead of '{target}'") elif "Deletion" in status: errors.append(f"omitted '{target}'") elif "Insertion" in status: errors.append(f"added extra '{actual}'") accuracy = metrics['accuracy'] if accuracy >= 90: score_color = "excellent" elif accuracy >= 70: score_color = "good" elif accuracy >= 50: score_color = "improving" else: score_color = "keep practicing" score_text = f"Score: {accuracy:.0f}%\n**{metrics['correct']}/{metrics['total']}** words correct — {score_color}!\n\n" if not errors: return score_text + "**Perfect pronunciation!** All words were clearly articulated. Great job!" prompt = f"""<|im_start|>system You are a friendly speech coach. Provide ONE specific, actionable tip to improve pronunciation. Focus on mouth position, tongue placement, or breathing. Keep under 40 words. Be encouraging.<|im_end|> <|im_start|>user Target phrase: "{target_txt}" Errors made: {"; ".join(errors[:4])} Accuracy: {metrics['accuracy']:.0f}%<|im_end|> <|im_start|>assistant """ try: response = llm_pipe(prompt, max_new_tokens=70, max_length=None, do_sample=True, temperature=0.7) feedback = response[0]['generated_text'].split("assistant\n")[-1].strip() if feedback and feedback[-1] not in '.!?': feedback = feedback.rsplit('.', 1)[0] + '.' if '.' in feedback else feedback except Exception: if any("Substitution" in e for _, _, e in alignment): feedback = "Try speaking more slowly and focus on each syllable. Exaggerate mouth movements to improve clarity." elif any("Deletion" in e for _, _, e in alignment): feedback = "Some words were missed. Try pausing briefly between words and speaking at a steady pace." else: feedback = "Focus on speaking clearly and at a comfortable pace. Record yourself and compare to improve." return score_text + f"**Tip:** {feedback}" def get_accuracy_color(accuracy): """Return a color based on accuracy level.""" if accuracy >= 90: return "#10b981" elif accuracy >= 70: return "#f59e0b" elif accuracy >= 50: return "#f97316" else: return "#ef4444" def analyze(audio, selected, custom): target_txt = (custom.strip() if selected == "Custom Phrase..." and custom else selected).strip() if not target_txt or target_txt == "Custom Phrase...": return ( "", None, "**Please select or enter a target phrase** to begin your practice session.", None ) if not audio: return ( "", None, f"**Ready to analyze:** \"{target_txt}\"\n\nClick the microphone button above to record yourself saying this phrase.", None ) try: speech = preprocess_audio(audio) if speech.shape[1] < 1600: return ( "", None, "**Recording too short**\n\nPlease record for at least 1 second. Speak the entire phrase clearly.", None ) inputs = processor( speech.squeeze().numpy(), sampling_rate=SAMPLE_RATE, return_tensors="pt" ).input_values.to(device) inputs.requires_grad = True with torch.no_grad(): logits = model(inputs).logits pred_ids = torch.argmax(logits, dim=-1) transcription = processor.batch_decode(pred_ids)[0].lower().strip() if not transcription: return ( "", None, "**No speech detected**\n\nPlease speak louder and closer to the microphone. Make sure your microphone is working.", None ) alignment = get_alignment(target_txt.lower().split(), transcription.split()) metrics = calculate_accuracy(alignment) df = pd.DataFrame(alignment, columns=["Expected", "Heard", "Status"]) inputs_grad = processor( speech.squeeze().numpy(), sampling_rate=SAMPLE_RATE, return_tensors="pt" ).input_values.to(device) inputs_grad.requires_grad = True def forward_fn(x): return torch.gather(model(x).logits, 2, pred_ids.unsqueeze(-1)).squeeze(-1).sum(dim=-1) grads = Saliency(forward_fn).attribute(inputs_grad).abs().squeeze().detach().cpu().numpy() fig = create_saliency_plot(grads, transcription) feedback = generate_feedback(target_txt, alignment, metrics) return transcription.upper(), fig, feedback, df except Exception as e: import traceback traceback.print_exc() return ( "", None, f"**Analysis Error**\n\nSomething went wrong: {str(e)}\n\nPlease try recording again.", None ) css = """ * { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important; } footer { visibility: hidden; } .gradio-container { max-width: 640px !important; margin: 32px auto !important; padding: 32px !important; } .app-header { text-align: center; margin-bottom: 32px; } .app-title { font-size: 1.5rem; font-weight: 600; color: #111; margin: 0 0 4px; } .app-subtitle { font-size: 0.875rem; color: #666; margin: 0; } .section-title { font-size: 0.7rem; font-weight: 600; color: #888; text-transform: uppercase; letter-spacing: 0.05em; margin: 24px 0 8px; } .analyze-button { background: #111 !important; color: #fff !important; font-weight: 500 !important; font-size: 0.875rem !important; padding: 10px 20px !important; border: none !important; border-radius: 6px !important; width: 100% !important; margin-top: 16px !important; } .analyze-button:hover { background: #333 !important; } .transcription-output textarea { text-align: center !important; font-weight: 600 !important; font-size: 1.25rem !important; padding: 16px !important; background: #f9f9f9 !important; border: 1px solid #eee !important; border-radius: 6px !important; } .feedback-output { background: #fefce8 !important; border: 1px solid #fde047 !important; border-radius: 6px !important; padding: 16px !important; line-height: 1.6 !important; font-size: 0.875rem !important; } .dataframe-container table { font-size: 0.8rem !important; } .dataframe-container thead th { background: #f5f5f5 !important; font-size: 0.7rem !important; text-transform: uppercase; padding: 8px 12px !important; } .dataframe-container tbody td { padding: 8px 12px !important; } .gr-box, .gr-panel, .gr-form, .gr-group, .block, .wrap, .contain { background: transparent !important; border: none !important; box-shadow: none !important; } """ with gr.Blocks(css=css, theme=gr.themes.Soft()) as demo: gr.HTML("""
An Adaptive Explainable AI Framework For Pronunciation Training