Update pipeline.py

pipeline.py

@@ -5,7 +5,9 @@ import json
 import math
 import torch
 import librosa
+import ffmpeg
 import numpy as np
+import soundfile as sf
 import mediapipe as mp
 from PIL import Image
 from transformers import AutoImageProcessor, AutoModelForImageClassification, pipeline
@@ -13,299 +15,653 @@ from sentence_transformers import SentenceTransformer, CrossEncoder
 from sklearn.metrics.pairwise import cosine_similarity
 from mediapipe.tasks import python
 from mediapipe.tasks.python import vision
-import warnings
 
-# Suppress unnecessary warnings for a cleaner console output
+# Ignore unnecessary warnings
+import warnings
 warnings.filterwarnings("ignore", category=UserWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 
-# Set device to GPU if available
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-# --- Configuration & Mappings ---
-
-# Tone Mapping: 0: Hesitant, 1: Confident, 2: Unstable, 3: Natural
 TONE_MAPPING = {
     "Hesitant": 0,
     "Confident": 1,
     "Unstable": 2,
     "Natural": 3,
-    "Excited": 3  # Excitement is treated as a high-energy Natural/Positive state
-}
-
-# Emotion Valence-Arousal coordinates for the Emotion Wheel
-emotion_va = {
-    "happy": (0.8, 0.2), "fear": (0.2, 0.8), "angry": (-0.7, 0.65),
-    "sad": (-0.65, -0.55), "surprise": (0.1, -0.75), "disgust": (0.6, -0.4), "neutral": (0.0, 0.0)
+    "Excited": 3
 }
 
-# Static data for drawing the Emotion Ring labels
-EMOTION_RING = [
-    ("Happy", 0, 0.84), ("Surprise", 45, 0.84), ("Fear", 100, 0.84),
-    ("Sad", 160, 0.84), ("Disgust", 215, 0.84), ("Angry", 270, 0.84)
-]
-
-# --- Model Initialization ---
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-# Download Mediapipe Task file if not exists
+# 2. Download and Initialize Mediapipe once (Global)
 MODEL_PATH = "face_landmarker.task"
 if not os.path.exists(MODEL_PATH):
-    print("[INFO] Downloading Mediapipe Face Landmarker model...")
     os.system(f"wget -O {MODEL_PATH} -q https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task")
 
-# NLP Models
+# 3. Initialize Models
 asr = pipeline("automatic-speech-recognition", model="openai/whisper-small", device=0 if torch.cuda.is_available() else -1)
 semantic_model = SentenceTransformer("all-MiniLM-L6-v2")
 cross_encoder = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
 
-# Visual Emotion Model
 FACE_MODEL_NAME = "dima806/facial_emotions_image_detection"
 face_processor = AutoImageProcessor.from_pretrained(FACE_MODEL_NAME)
 face_model = AutoModelForImageClassification.from_pretrained(FACE_MODEL_NAME).to(device).eval()
 
-# --- Audio Analysis Modules ---
+# Emotion Mapping for Wheel
+emotion_va = {
+    "happy": (0.8, 0.2), "fear": (0.2, 0.8), "angry": (-0.7, 0.65),
+    "sad": (-0.65, -0.55), "surprise": (0.1, -0.75), "disgust": (0.6, -0.4), "neutral": (0.0, 0.0)
+}
+EMOTION_RING = [
+    ("Happy", 0, 0.84), ("Surprise", 45, 0.84), ("Fear", 100, 0.84),
+    ("Sad", 160, 0.84), ("Disgust", 215, 0.84), ("Angry", 270, 0.84)
+]
+
+##Utility functions
+
+def normalize(v, mn, mx):
+    return np.clip((v - mn) / (mx - mn), 0, 1) if mx - mn != 0 else 0.0
+
+def extract_audio(v_in, a_out):
+    ffmpeg.input(v_in).output(a_out, ac=1, ar=16000).overwrite_output().run(quiet=True)
+
+def merge_audio_video(v_in, a_in, v_out):
+    ffmpeg.output(ffmpeg.input(v_in).video, ffmpeg.input(a_in).audio, v_out, vcodec="libx264", acodec="aac").overwrite_output().run(quiet=True)
+
+def draw_face_box(frame, x, y, w, h, emotion_name=""):
+    color, th, cl = (0, 255, 100), 2, 20  # Green color
+    cv2.rectangle(frame, (x, y), (x+w, y+h), color, 1)
+
+    # Add emotion name above face box
+    if emotion_name:
+        cv2.putText(
+            frame,
+            emotion_name.upper(),
+            (x + 10, y - 15),
+            cv2.FONT_HERSHEY_DUPLEX,
+            0.7,
+            (0, 255, 100),
+            2,
+            cv2.LINE_AA
+        )
+
+    # Corners
+    for px, py, dx, dy in [(x,y,cl,0), (x,y,0,cl), (x+w,y,-cl,0), (x+w,y,0,cl), (x,y+h,cl,0), (x,y+h,0,-cl), (x+w,y+h,-cl,0), (x+w,y+h,0,-cl)]:
+        cv2.line(frame, (px, py), (px+dx, py+dy), color, 5)
+    return frame
+
+def compute_eye_contact_ratio(frame, landmarks):
+    h, w, _ = frame.shape
+    def ear(idx):
+        p = [np.array([landmarks[i].x * w, landmarks[i].y * h]) for i in idx]
+        return (np.linalg.norm(p[1]-p[5]) + np.linalg.norm(p[2]-p[4])) / (2.0 * np.linalg.norm(p[0]-p[3]))
+    avg_ear = (ear([33, 160, 158, 133, 153, 144]) + ear([362, 385, 387, 263, 373, 380])) / 2.0
+    return min(max(avg_ear * 3, 0), 1)
+
+def analyze_face_emotion(frame):
+    img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+    inputs = face_processor(images=img, return_tensors="pt").to(device)
+    with torch.no_grad():
+        outputs = face_model(**inputs)
+    probs = torch.nn.functional.softmax(outputs.logits, dim=-1)[0]
+    return {face_model.config.id2label[i].lower(): float(probs[i]) for i in range(len(probs))}
+
+##Audio analysis
 
 def extract_audio_features(y, sr):
-    """Calculates physical audio properties like pitch, jitter, and energy."""
     duration = librosa.get_duration(y=y, sr=sr)
     if duration == 0:
         return {"pitch_std": 0, "jitter": 0, "energy_std": 0, "pause_ratio": 0, "speech_rate": 0}
 
-    # Pitch tracking using YIN algorithm
+    # Pitch & Jitter
     f0 = librosa.yin(y, fmin=75, fmax=300, sr=sr)
     f0 = f0[~np.isnan(f0)]
     pitch_std = np.std(f0) if len(f0) else 0
     jitter = np.mean(np.abs(np.diff(f0)) / np.maximum(f0[:-1], 1e-6)) if len(f0) > 1 else 0
 
-    # Energy tracking (RMS)
+    # Energy
     rms = librosa.feature.rms(y=y)[0]
     energy_std = np.std(rms)
 
-    # Speech vs Pause detection
     intervals = librosa.effects.split(y, top_db=20)
     speech_duration = sum((e - s) for s, e in intervals) / sr
     pause_ratio = 1 - (speech_duration / duration) if duration > 0 else 0
 
-    # Speech rate (Word onsets per second)
+    # Speech Rate
     oenv = librosa.onset.onset_strength(y=y, sr=sr)
     onsets = librosa.onset.onset_detect(onset_envelope=oenv, sr=sr)
     speech_rate = len(onsets) / duration if duration > 0 else 0
 
     return {
-        "pitch_std": pitch_std, "jitter": jitter, "energy_std": energy_std,
-        "pause_ratio": pause_ratio, "speech_rate": speech_rate
+        "pitch_std": pitch_std,
+        "jitter": jitter,
+        "energy_std": energy_std,
+        "pause_ratio": pause_ratio,
+        "speech_rate": speech_rate
     }
 
+
 def compute_audio_scores(features, baseline=None):
-    """Translates raw audio features into behavioral scores and tone IDs."""
+    """
+    Fairness-aware audio scoring with personal baseline comparison
+    """
+    # Use standard defaults if no baseline provided
     if baseline is None:
         baseline = {"pitch_std": 30.0, "energy_std": 0.05, "jitter": 0.02, "pause_ratio": 0.2, "speech_rate": 4.0}
 
-    # Relative ratios compared to personal baseline
-    p_ratio = features["pitch_std"] / max(baseline["pitch_std"], 1e-6)
-    e_ratio = features["energy_std"] / max(baseline["energy_std"], 1e-6)
-    r_ratio = features["speech_rate"] / max(baseline["speech_rate"], 1e-6)
+    # Calculate Relative Ratios (Current / Baseline)
+    pitch_ratio = features["pitch_std"] / max(baseline["pitch_std"], 1e-6)
+    energy_ratio = features["energy_std"] / max(baseline["energy_std"], 1e-6)
+    rate_ratio = features["speech_rate"] / max(baseline["speech_rate"], 1e-6)
+
+    # Stress Score (Relative)
+    pitch_dev = abs(1 - pitch_ratio)
+    energy_dev = abs(1 - energy_ratio)
+    stress_val = (pitch_dev * 0.4 + energy_dev * 0.4 + features["jitter"] * 0.2) * 150
+    stress = np.clip(stress_val + 20, 0, 100)
+
+    # Clarity Score (Relative)
+    pause_dev = max(0, features["pause_ratio"] - baseline["pause_ratio"])
+    clarity = 100 - (pause_dev * 120 + features["jitter"] * 400)
 
-    # Behavioral Formulas
-    stress = np.clip((abs(1 - p_ratio) * 0.4 + abs(1 - e_ratio) * 0.4 + features["jitter"] * 0.2) * 150 + 20, 0, 100)
-    clarity = np.clip(100 - (max(0, features["pause_ratio"] - baseline["pause_ratio"]) * 120 + features["jitter"] * 400), 0, 100)
-    conf_audio = np.clip(100 - (abs(1 - r_ratio) * 40 + abs(1 - e_ratio) * 30 + features["pause_ratio"] * 50), 0, 100)
+    # Confidence Score (Relative)
+    rate_dev = abs(1 - rate_ratio)
+    confidence_audio = 100 - (rate_dev * 40 + energy_dev * 30 + features["pause_ratio"] * 50)
 
-    # Tone Classification
+    # Tone classification based on relative shifts
     tones = {
-        "Confident": conf_audio,
+        "Confident": confidence_audio,
         "Hesitant": features["pause_ratio"] * 150,
+        "Excited": (energy_ratio - 1) * 100 if energy_ratio > 1 else 0,
         "Unstable": stress,
-        "Natural": 100 - (abs(1 - p_ratio) * 60 + abs(1 - r_ratio) * 40)
+        "Natural": 100 - (pitch_dev * 60 + rate_dev * 40)
     }
-    dom_tone_name = max(tones, key=tones.get)
-    
+
+    dominant_tone = max(tones, key=tones.get)
+
     return {
-        "confidence_audio": round(float(conf_audio), 2),
-        "clarity": round(float(clarity), 2),
-        "stress": round(float(stress), 2),
+        "confidence_audio": round(float(np.clip(confidence_audio, 0, 100)), 2),
+        "clarity": round(float(np.clip(clarity, 0, 100)), 2),
+        "stress": round(float(np.clip(stress, 0, 100)), 2),
         "pauses": round(float(features["pause_ratio"] * 100), 2),
-        "tone_of_voice": TONE_MAPPING.get(dom_tone_name, 3) # Backend Integer ID
+        "tone_of_voice": TONE_MAPPING.get(dominant_tone, 3)
     }
 
-# --- Visual Analysis & Drawing ---
+def analyze_audio_segment(audio_path, baseline=None):
+    """
+    Main entry point for audio segment analysis
+    """
+    y, sr = librosa.load(audio_path, sr=16000)
+    features = extract_audio_features(y, sr)
+    return compute_audio_scores(features, baseline)
+
+
+##Text analysis
+
+def get_user_answer(audio_path):
+    """Transcribe audio using Whisper"""
+    result = asr(audio_path, chunk_length_s=20)
+    return result["text"].strip()
+
+
+def compute_similarity_score(user_answer, ideal_answer):
+    emb = semantic_model.encode([user_answer, ideal_answer])
+    sim = cosine_similarity([emb[0]], [emb[1]])[0][0]
+    score = float(sim * 100)
+    return round(max(0, score), 2)
+
+def compute_relevance_score(question, user_answer):
+    raw_score = cross_encoder.predict([(question, user_answer)])[0]
+    prob = 1 / (1 + np.exp(-raw_score))
+    score = float(prob * 100)
+    return round(max(0, score), 2)
+
+##Video
+
+# Eye indices
+LEFT_EYE = [33, 160, 158, 133, 153, 144]
+RIGHT_EYE = [362, 385, 387, 263, 373, 380]
 
+# Eye Contact Function
 def compute_eye_contact_ratio(frame, landmarks):
-    """Calculates Eye Aspect Ratio (EAR) to estimate focus/contact."""
-    if not landmarks: return 0.5
+    """
+    Compute eye contact ratio from detected face landmarks
+    """
+
+    if not landmarks:
+        return 0.5
+
     h, w, _ = frame.shape
-    def ear(idx):
-        p = [np.array([landmarks[i].x * w, landmarks[i].y * h]) for i in idx]
-        return (np.linalg.norm(p[1]-p[5]) + np.linalg.norm(p[2]-p[4])) / (2.0 * np.linalg.norm(p[0]-p[3]))
-    
-    avg_ear = (ear([33, 160, 158, 133, 153, 144]) + ear([362, 385, 387, 263, 373, 380])) / 2.0
-    return min(max(avg_ear * 3, 0), 1)
+
+    def ear(indices):
+        points = [
+            np.array([
+                landmarks[i].x * w,
+                landmarks[i].y * h
+            ])
+            for i in indices
+        ]
+
+        v1 = np.linalg.norm(points[1] - points[5])
+        v2 = np.linalg.norm(points[2] - points[4])
+        h_dist = np.linalg.norm(points[0] - points[3])
+
+        return (v1 + v2) / (2.0 * h_dist)
+
+    ear_left = ear(LEFT_EYE)
+    ear_right = ear(RIGHT_EYE)
+
+    avg_ear = (ear_left + ear_right) / 2.0
+
+    eye_score = min(max(avg_ear * 3, 0), 1)
+
+    return eye_score
 
 def analyze_face_emotion(frame):
-    """Predicts emotion probabilities using Vision Transformer (ViT)."""
+    """
+    Predict facial emotion probabilities from single frame
+    """
+
+    # Convert BGR to RGB
     rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-    img = Image.fromarray(rgb)
-    inputs = face_processor(images=img, return_tensors="pt").to(device)
+    image = Image.fromarray(rgb)
+
+    # Preprocess
+    inputs = face_processor(images=image, return_tensors="pt").to(device)
+
     with torch.no_grad():
         outputs = face_model(**inputs)
+
     probs = torch.nn.functional.softmax(outputs.logits, dim=-1)[0]
     labels = face_model.config.id2label
-    return {labels[i].lower(): float(probs[i]) for i in range(len(probs))}
 
-def draw_face_ui(frame, x, y, w, h, emotion_label):
-    """Draws professional bounding box and active emotion label."""
+    emotion_probs = {
+        labels[i].lower(): float(probs[i])
+        for i in range(len(probs))
+    }
+
+    return emotion_probs
+
+def draw_face_box(frame, x, y, w, h, emotion_label="Neutral"):
+    """
+    Draw face bounding box with emotion label above it
+    """
+
+    # Green color for face box
     color = (0, 255, 0)
-    cv2.rectangle(frame, (x, y), (x+w, y+h), color, 1)
-    # Corners
-    c_len = 20
-    for px, py, dx, dy in [(x,y,c_len,0), (x,y,0,c_len), (x+w,y,-c_len,0), (x+w,y,0,c_len), (x,y+h,c_len,0), (x,y+h,0,-c_len), (x+w,y+h,-c_len,0), (x+w,y+h,0,-c_len)]:
+
+    thickness = 2
+    corner_len = 22
+
+    # Main rectangle
+    cv2.rectangle(frame, (x, y), (x+w, y+h), color, thickness)
+
+    # Decorative corner lines
+    for (px, py, dx, dy) in [
+        (x, y, corner_len, 0), (x, y, 0, corner_len),
+        (x+w, y, -corner_len, 0), (x+w, y, 0, corner_len), 
+        (x, y+h, corner_len, 0), (x, y+h, 0, -corner_len),
+        (x+w, y+h, -corner_len, 0), (x+w, y+h, 0, -corner_len),
+    ]:
         cv2.line(frame, (px, py), (px+dx, py+dy), color, 4)
-    cv2.putText(frame, emotion_label.upper(), (x, y-10), cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2)
+
+    # Draw emotion text above the face box
+    label_text = emotion_label.capitalize()
+
+    (tw, th), _ = cv2.getTextSize(
+        label_text,
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.7,
+        2
+    )
+
+    text_x = x + (w - tw) // 2
+    text_y = y - 10
+
+    cv2.putText(
+        frame,
+        label_text,
+        (text_x, text_y),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.7,
+        (0, 255, 0),
+        2,
+        cv2.LINE_AA
+    )
+
     return frame
 
-def draw_emotion_wheel(panel, center, radius, valence, arousal, dominant_emo):
-    """Renders the Valence-Arousal circular UI."""
+def compute_valence_arousal_from_probs(emotion_probs):
+    """Computing Valence and Arousal from emotion probabilities"""
+    v, a, total = 0.0, 0.0, 0.0
+
+    for emo, score in emotion_probs.items():
+        emo = emo.lower()
+        if emo in emotion_va:
+            v += emotion_va[emo][0] * score
+            a += emotion_va[emo][1] * score
+            total += score
+
+    if total == 0:
+        return 0.0, 0.0
+
+    return v / total, a / total
+
+def draw_full_emotion_wheel(panel, center, radius, valence, arousal,
+                            dominant_emotion="neutral"):
     cx, cy = center
-    cv2.circle(panel, center, radius, (60, 60, 85), 2) # Outer Ring
-    # Labels
-    for label, angle, rf in EMOTION_RING:
-        rad = math.radians(angle)
-        lx, ly = int(cx + rf*radius*math.cos(rad)), int(cy - rf*radius*math.sin(rad))
-        color = (0, 255, 200) if label.lower() == dominant_emo else (180, 180, 180)
-        cv2.putText(panel, label, (lx-20, ly), cv2.FONT_HERSHEY_SIMPLEX, 0.35, color, 1)
-    # Animated Dot
-    dx, dy = int(cx + valence*radius*0.8), int(cy - arousal*radius*0.8)
-    cv2.circle(panel, (dx, dy), 8, (255, 230, 60), -1)
+
+    # Circle background
+    cv2.circle(panel, center, radius + 5, (15, 15, 25), -1)
+    cv2.circle(panel, center, radius,     (60, 60, 85),  2)
+    for rf in [0.33, 0.66]:
+        cv2.circle(panel, center, int(radius * rf), (35, 35, 50), 1)
+
+    # Drawing dividing lines between emotions
+    for angle_deg in range(0, 360, 60):
+        rad = math.radians(angle_deg)
+        x1 = int(cx + radius * math.cos(rad))
+        y1 = int(cy - radius * math.sin(rad))
+        cv2.line(panel, (cx, cy), (x1, y1), (40, 40, 60), 1)
+
+    # Drawing emotion labels
+    ef, es, et = cv2.FONT_HERSHEY_SIMPLEX, 0.40, 1
+    for emotion_data in EMOTION_RING:
+        if emotion_data[1] is None:
+            continue
+
+        label, angle_deg, rf = emotion_data
+        rad = math.radians(angle_deg)
+        lx  = int(cx + rf * radius * math.cos(rad))
+        ly  = int(cy - rf * radius * math.sin(rad))
+        (tw, th), _ = cv2.getTextSize(label, ef, es, et)
+        tx, ty = lx - tw//2, ly + th//2
+
+        # Highlight active emotion
+        if label.lower() == dominant_emotion.lower():
+            cv2.putText(panel, label, (tx, ty), ef, es+0.08, (0, 255, 200), 2, cv2.LINE_AA)
+        else:
+            cv2.putText(panel, label, (tx, ty), ef, es, (190, 190, 255), et, cv2.LINE_AA)
+
+    # Neutral in center
+    nc = (0, 255, 200) if dominant_emotion == "neutral" else (160, 160, 160)
+    (tw, th), _ = cv2.getTextSize("Neutral", ef, es, et)
+    cv2.putText(panel, "Neutral", (cx-tw//2, cy+th//2), ef, es, nc, et, cv2.LINE_AA)
+
+    # Animated dot with glow
+    dot_x = int(cx + valence * radius * 0.88)
+    dot_y = int(cy - arousal * radius * 0.88)
+    cv2.circle(panel, (dot_x, dot_y), 15, (160, 120,  0), -1)
+    cv2.circle(panel, (dot_x, dot_y), 11, (220, 180,  0), -1)
+    cv2.circle(panel, (dot_x, dot_y),  7, (255, 230, 60), -1)
+
     return panel
 
-def draw_metric_bars(frame, x, y, confidence, clarity, stress):
-    """Renders horizontal performance bars."""
-    metrics = [("Confidence", confidence, (70, 180, 255)), ("Clarity", clarity, (100, 220, 150)), ("Stress", stress, (255, 120, 100))]
-    for i, (label, val, col) in enumerate(metrics):
-        curr_y = y + i * 40
-        cv2.putText(frame, label, (x, curr_y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,255,255), 1)
-        cv2.rectangle(frame, (x, curr_y), (x+200, curr_y+15), (50,50,50), -1) # BG
-        cv2.rectangle(frame, (x, curr_y), (x+int(val*2), curr_y+15), col, -1) # Fill
-    return frame
+BAR_CONFIGS = [
+    ("Confidence",   (70, 180, 255),   (30, 50, 100)),      # light blue
+    ("Clarity",  (100, 220, 150),   (25, 70, 50)),       # light cyan
+    ("Stress", (255, 120, 100),   (100, 40, 30)),        # light coral
+]
+
+def draw_metric_bars(panel,
+                    bars_x_start,
+                    bar_y_top,
+                    bar_height,
+                    bar_width,
+                    bar_gap,
+                    confidence,
+                    clarity,
+                    stress):
+    """
+    Draw horizontal metric bars with label above each bar
+    """
+
+    values = [confidence, clarity, stress]
+    labels_list = ["Confidence", "Clarity", "Stress"]
+
+    # Extra vertical space for labels
+    label_space = 20
+
+    for i, value in enumerate(values):
+
+        label, fill_color, bg_color = BAR_CONFIGS[i]
+
+        # Each bar block height = label + bar + gap
+        y = bar_y_top + i * (bar_height + label_space + bar_gap)
+
+        x_right = bars_x_start + bar_width
+
+        filled = int((value / 100) * bar_width)
+
+        # Draw label above bar
+        cv2.putText(
+            panel,
+            label,
+            (bars_x_start, y),
+            cv2.FONT_HERSHEY_DUPLEX,
+            0.6,
+            (230, 230, 230),
+            1,
+            cv2.LINE_AA
+        )
+
+        # Move bar slightly down to leave space for label
+        bar_y = y + 8
+
+        # Draw background bar
+        cv2.rectangle(
+            panel,
+            (bars_x_start, bar_y),
+            (x_right, bar_y + bar_height),
+            bg_color,
+            -1
+        )
+
+        # Draw filled portion
+        cv2.rectangle(
+            panel,
+            (bars_x_start, bar_y),
+            (bars_x_start + filled, bar_y + bar_height),
+            fill_color,
+            -1
+        )
+
+        # Draw percentage text
+        cv2.putText(
+            panel,
+            f"{int(value)}%",
+            (bars_x_start + 12, bar_y + bar_height - 6),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.6,
+            (255, 255, 255),
+            2,
+            cv2.LINE_AA
+        )
 
-# --- Core Video Processing ---
+    return panel
+
+##Integrated Video Processing (Analysis + Annotation)
 
-def process_video_segment(video_path, output_dir, q_id, audio_scores):
-    """Analyzes visual data and generates annotated video."""
+def process_video_segment(video_path, output_dir, segment_id, audio_scores_global=None):
     base_options = python.BaseOptions(model_asset_path=MODEL_PATH)
-    options = vision.FaceLandmarkerOptions(base_options=base_options, running_mode=vision.RunningMode.VIDEO)
-    
+    options = vision.FaceLandmarkerOptions(base_options=base_options, running_mode=vision.RunningMode.VIDEO, num_faces=1)
+
     cap = cv2.VideoCapture(video_path)
-    fps = cap.get(cv2.CAP_PROP_FPS)
-    w, h = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
-    
-    temp_v = os.path.join(output_dir, f"annotated_{q_id}.mp4")
-    out = cv2.VideoWriter(temp_v, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))
-    
-    frame_idx, face_accum, eye_accum = 0, [], []
-    s_v, s_a, dom_emo = 0.0, 0.0, "neutral"
+    fps, width, height = cap.get(cv2.CAP_PROP_FPS), int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    temp_video = os.path.join(output_dir, f"temp_annotated_{segment_id}.mp4")
+    # out = cv2.VideoWriter(temp_video, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
+    # Use 'avc1' or 'H264' for web compatibility
+    fourcc = cv2.VideoWriter_fourcc(*'XVID')
+    out = cv2.VideoWriter(temp_video, fourcc, fps, (width, height))
+
+    face_conf_accum, eye_accum, frame_idx = [], [], 0
+    smooth_v, smooth_a, dom_emo = 0.0, 0.0, "neutral"
     
+    # --- Optimization Variables ---
+    frame_stride = 3  # Process AI every 3 frames
+    last_results = None
+    last_emotions = None
+    last_eye_s = 0.5
+    last_lm = None
+    # ------------------------------
+
+    b_conf = audio_scores_global.get("confidence_audio", 50)
+    b_clar = audio_scores_global.get("clarity", 50)
+    b_stress = audio_scores_global.get("stress", 20)
+
     with vision.FaceLandmarker.create_from_options(options) as landmarker:
         while cap.isOpened():
             ret, frame = cap.read()
-            if not ret: break
-            
-            # Optimization: Run AI every 3 frames
-            if frame_idx % 3 == 0:
-                mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
-                res = landmarker.detect_for_video(mp_img, int((frame_idx/fps)*1000))
+            if not ret:
+                break
+
+            # 1. RUN HEAVY AI ONLY ON STRIDE FRAMES
+            if frame_idx % frame_stride == 0:
+                mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+                last_results = landmarker.detect_for_video(mp_image, int((frame_idx/fps)*1000))
                 
-                if res.face_landmarks:
-                    lm = res.face_landmarks[0]
-                    emotions = analyze_face_emotion(frame)
-                    eye_s = compute_eye_contact_ratio(frame, lm)
-                    
-                    # Statistical accumulation
-                    face_accum.append((emotions.get("neutral", 0) + emotions.get("happy", 0)) * 100)
-                    eye_accum.append(eye_s)
-                    
-                    dom_emo = max(emotions, key=emotions.get)
-                    v_t = sum(emotion_va[e][0]*s for e,s in emotions.items() if e in emotion_va)
-                    a_t = sum(emotion_va[e][1]*s for e,s in emotions.items() if e in emotion_va)
-                    
-                    # Moving average for smooth UI animation
-                    s_v += 0.2 * (v_t - s_v)
-                    s_a += 0.2 * (a_t - s_a)
-                    
-                    # Draw Bounding Box
-                    xs, ys = [l.x*w for l in lm], [l.y*h for l in lm]
-                    draw_face_ui(frame, int(min(xs)), int(min(ys)), int(max(xs)-min(xs)), int(max(ys)-min(ys)), dom_emo)
+                if last_results.face_landmarks:
+                    last_lm = last_results.face_landmarks[0]
+                    last_emotions = analyze_face_emotion(frame)
+                    last_eye_s = compute_eye_contact_ratio(frame, last_lm)
+
+            # 2. USE LAST KNOWN DATA FOR CALCULATIONS & DRAWING
+            d_conf, d_clar, d_stress = b_conf, b_clar, b_stress
             
-            # UI Overlays
-            draw_emotion_wheel(frame, (w-100, h-100), 70, s_v, s_a, dom_emo)
-            draw_metric_bars(frame, 30, h-120, audio_scores["confidence_audio"], audio_scores["clarity"], audio_scores["stress"])
+            if last_results and last_results.face_landmarks:
+                # Use current local variables from 'last' successful AI run
+                curr_f_conf = (last_emotions.get("neutral", 0) + last_emotions.get("happy", 0)) * 100
+                d_conf = (b_conf * 0.7) + (curr_f_conf * 0.3)
+                d_clar = (b_clar * 0.8) + (last_eye_s * 100 * 0.2)
+                d_stress = (b_stress * 0.7) + ((last_emotions.get("sad",0)+last_emotions.get("angry",0))*30)
+
+                # Update accumulators only on stride frames to keep averages accurate
+                if frame_idx % frame_stride == 0:
+                    face_conf_accum.append(curr_f_conf)
+                    eye_accum.append(last_eye_s)
+
+                dom_emo = max(last_emotions, key=last_emotions.get)
+                v_t = sum(emotion_va[e][0]*s for e,s in last_emotions.items() if e in emotion_va)
+                a_t = sum(emotion_va[e][1]*s for e,s in last_emotions.items() if e in emotion_va)
+                
+                # Keep smoothing every frame for fluid movement
+                smooth_v += 0.15 * (v_t - smooth_v)
+                smooth_a += 0.15 * (a_t - smooth_a)
+
+                # Draw face box using the last known landmarks
+                xs, ys = [l.x*width for l in last_lm], [l.y*height for l in last_lm]
+                draw_face_box(
+                    frame,
+                    int(min(xs)), int(min(ys)),
+                    int(max(xs) - min(xs)), int(max(ys) - min(ys)),
+                    dom_emo
+                )
+
+            # 3. ALWAYS DRAW UI (Wheel and Bars)
+            frame = draw_full_emotion_wheel(frame, (width-130, height-100), 90, smooth_v, smooth_a, dom_emo)
+            frame = draw_metric_bars(frame, 30, height-160, 28, 200, 6, d_conf, d_clar, d_stress)
             
             out.write(frame)
             frame_idx += 1
-            
+
     cap.release()
     out.release()
-    return temp_v, np.mean(face_accum) if face_accum else 50, np.mean(eye_accum)*100 if eye_accum else 50
-
-# --- Main Entry Point ---
+    return temp_video, np.mean(face_conf_accum) if face_conf_accum else 50, np.mean(eye_accum)*100 if eye_accum else 50
 
+##Main pipeline
 def run_intervision_pipeline(video_path, questions_config, output_dir):
-    """Main pipeline execution for all interview questions."""
+    if not os.path.exists(video_path):
+        return f"Error: Video file not found at {video_path}"
+
     os.makedirs(output_dir, exist_ok=True)
-    
-    # Baseline for fairness (first 10s)
+
+    # Establish baseline from first 10s
     try:
-        yb, srb = librosa.load(video_path, sr=16000, duration=10)
-        baseline = extract_audio_features(yb, srb)
-    except: baseline = None
+        y_b, sr_b = librosa.load(video_path, sr=16000, duration=10)
+        baseline = extract_audio_features(y_b, sr_b)
+    except Exception as e:
+        print(f"Baseline Load Warning: {e}. Using defaults.")
+        baseline = None
 
-    final_reports, video_segments = [], []
+    final_reports, segments = [], []
 
     for q in questions_config:
         q_id = q['question_id']
         raw_seg = os.path.join(output_dir, f"q{q_id}_raw.mp4")
-        
-        # Clip segment using FFmpeg
-        dur = q["end_time"] - q["start_time"]
-        subprocess.run(['ffmpeg', '-y', '-ss', str(q["start_time"]), '-t', str(dur), '-i', video_path, '-c', 'copy', raw_seg], quiet=True)
-        
-        # Audio Processing
-        y, sr = librosa.load(raw_seg, sr=16000)
+        wav_p = os.path.join(output_dir, f"q{q_id}.wav")
+
+        # Precise FFmpeg cutting with error handling
+        duration = q["end_time"] - q["start_time"]
+        try:
+            subprocess.run([
+                'ffmpeg', '-y', '-ss', str(q["start_time"]), '-t', str(duration),
+                '-i', video_path, '-c:v', 'libx264', '-c:a', 'aac', '-strict', 'experimental', raw_seg
+            ], check=True, capture_output=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Skipping Question {q_id}: Time range might be out of video bounds.")
+            continue
+
+        # Audio Extraction
+        try:
+            y, sr = librosa.load(raw_seg, sr=16000)
+            import soundfile as sf
+            sf.write(wav_p, y, sr)
+        except Exception as e:
+            print(f"Error extracting audio for Q{q_id}: {e}")
+            continue
+
+        # Audio Analysis
         a_scores = compute_audio_scores(extract_audio_features(y, sr), baseline)
-        
-        # NLP Analysis
-        transcription = asr(raw_seg)["text"].strip()
-        sim_score = round(float(cosine_similarity(semantic_model.encode([transcription, q["ideal_answer"]]))[0][0]*100), 2)
-        rel_score = round(float(1/(1+np.exp(-cross_encoder.predict([(q["question_text"], transcription)])[0]))*100), 2)
-        
+
+        # Whisper Transcription
+        try:
+            transcription_data = asr(wav_p, chunk_length_s=30, return_timestamps=True)
+            transcription = transcription_data["text"].strip()
+        except:
+            transcription = "[Transcription Error]"
+
+        similarity_score = compute_similarity_score(transcription, q["ideal_answer"])
+        relevance_score = compute_relevance_score(q["question_text"], transcription)
+
         # Visual Analysis
-        ann_v, f_conf, e_conf = process_video_segment(raw_seg, output_dir, q_id, a_scores)
-        final_v = os.path.join(output_dir, f"q{q_id}_final.mp4")
-        subprocess.run(['ffmpeg', '-y', '-i', ann_v, '-i', raw_seg, '-map', '0:v', '-map', '1:a', '-c:v', 'copy', '-c:a', 'aac', final_v], quiet=True)
-        
-        video_segments.append(final_v)
-        final_reports.append({
-            "questionId": q_id,
-            "userAnswerText": transcription,
-            "toneOfVoice": a_scores["tone_of_voice"], # Integer ID (0-3)
-            "clarity": a_scores["clarity"],
-            "stress": a_scores["stress"],
-            "confidence": round((a_scores["confidence_audio"] + f_conf + e_conf) / 3, 2),
-            "pauses": a_scores["pauses"],
-            "score": sim_score,
-            "relevance": rel_score
-        })
-        
-    # Final Concatenation
-    if video_segments:
-        list_p = os.path.join(output_dir, "list.txt")
-        with open(list_p, "w") as f:
-            for s in video_segments: f.write(f"file '{os.path.abspath(s)}'\n")
-        
-        final_out = os.path.join(output_dir, "Intervision_Final_Result.mp4")
-        os.system(f"ffmpeg -f concat -safe 0 -i {list_p} -c copy -y {final_out}")
-        
+        try:
+            ann_v, f_c, e_c = process_video_segment(raw_seg, output_dir, q_id, a_scores)
+
+            final_v = os.path.join(output_dir, f"q{q_id}_final.mp4")
+            subprocess.run([
+                'ffmpeg', '-y', '-i', ann_v, '-i', raw_seg, '-map', '0:v', '-map', '1:a',
+                '-c:v', 'copy', '-c:a', 'aac', final_v
+            ], check=True, capture_output=True)
+
+            segments.append(final_v)
+
+            final_reports.append({
+                "questionId": q_id,
+                "userAnswerText": transcription,
+                "toneOfVoice": a_scores["tone_of_voice"],
+                "clarity": a_scores["clarity"],
+                "stress": a_scores["stress"],
+                "confidence": round((a_scores["confidence_audio"] + f_c + e_c) / 3, 2),
+                "pauses": a_scores["pauses"],
+                "score": similarity_score,
+                "relevance": relevance_score
+            })
+        except Exception as e:
+            print(f"Visual analysis failed for Q{q_id}: {e}")
+
+        torch.cuda.empty_cache()
+
+    # Final concatenation
+    if segments:
+        list_path = os.path.join(output_dir, "list.txt")
+        with open(list_path, "w") as f:
+            for s in segments:
+                f.write(f"file '{os.path.abspath(s)}'\n")
+
+        final_output = os.path.join(output_dir, "Intervision_Final_Result.mp4")
+        os.system(f"ffmpeg -f concat -safe 0 -i {list_path} -c:v libx264 -preset superfast -crf 23 -c:a aac -y {final_output}")
+
         with open(os.path.join(output_dir, "report.json"), "w") as f:
             json.dump({"listOfAnswerReport": final_reports}, f, indent=4)
-            
-    return "Pipeline Finished Successfully"
+
+        return f"Successfully processed {len(segments)} questions."
+    else:
+        return "No segments were processed. Check your video time ranges."