Update pipeline.py

pipeline.py

@@ -480,32 +480,41 @@ def draw_metric_bars(panel,
 
 ##Integrated Video Processing (Analysis + Annotation)
 
-def process_video_segment(video_path, output_dir, segment_id, audio_scores_global=None):
-    base_options = python.BaseOptions(model_asset_path=MODEL_PATH)
-    options = vision.FaceLandmarkerOptions(base_options=base_options, running_mode=vision.RunningMode.VIDEO, num_faces=1)
-
+def process_full_video(video_path, output_dir, questions_config, audio_results_map):
+    """
+    Enhanced video processing with:
+    1. Real-time (Live) Audio Metric Bars using a sliding window.
+    2. Dynamic Emotion Wheel and Face Tracking.
+    3. Auto-wrapping Question Text that avoids UI overlap.
+    """
     cap = cv2.VideoCapture(video_path)
-    fps, width, height = cap.get(cv2.CAP_PROP_FPS), int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
-    temp_video = os.path.join(output_dir, f"temp_annotated_{segment_id}.mp4")
-    # out = cv2.VideoWriter(temp_video, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
-    # Use 'avc1' or 'H264' for web compatibility
-    fourcc = cv2.VideoWriter_fourcc(*'XVID')
-    out = cv2.VideoWriter(temp_video, fourcc, fps, (width, height))
-
-    face_conf_accum, eye_accum, frame_idx = [], [], 0
-    smooth_v, smooth_a, dom_emo = 0.0, 0.0, "neutral"
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+    # Load full audio for live segment analysis
+    full_audio, sr = librosa.load(video_path, sr=16000)
+
+    temp_output = os.path.join(output_dir, "annotated_full_raw.mp4")
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+    out = cv2.VideoWriter(temp_output, fourcc, fps, (width, height))
+
+    # Mediapipe Setup
+    base_options = python.BaseOptions(model_asset_path="face_landmarker.task")
+    options = vision.FaceLandmarkerOptions(
+        base_options=base_options,
+        running_mode=vision.RunningMode.VIDEO,
+        num_faces=1
+    )
+
+    frame_idx = 0
+    smooth_v, smooth_a = 0.0, 0.0
+    dom_emo = "neutral"
+    last_landmarks = None
     
-    # --- Optimization Variables ---
-    frame_stride = 3  # Process AI every 3 frames
-    last_results = None
-    last_emotions = None
-    last_eye_s = 0.5
-    last_lm = None
-    # ------------------------------
-
-    b_conf = audio_scores_global.get("confidence_audio", 50)
-    b_clar = audio_scores_global.get("clarity", 50)
-    b_stress = audio_scores_global.get("stress", 20)
+    # Live Scores Buffering
+    live_scores = {"confidence_audio": 0.0, "clarity": 0.0, "stress": 0.0}
+    smoothing_factor = 0.15 # Controls how "bouncy" the bars are
 
     with vision.FaceLandmarker.create_from_options(options) as landmarker:
         while cap.isOpened():
@@ -513,58 +522,109 @@ def process_video_segment(video_path, output_dir, segment_id, audio_scores_globa
             if not ret:
                 break
 
-            # 1. RUN HEAVY AI ONLY ON STRIDE FRAMES
-            if frame_idx % frame_stride == 0:
+            current_time = frame_idx / fps
+            active_answer = next((q for q in questions_config if q["start_time"] <= current_time <= q["end_time"]), None)
+            next_q = next((q for q in questions_config if current_time < q["start_time"]), None)
+            next_text = f"Q) {next_q['question_text']}" if next_q else "Preparing..."
+
+            # --- 1. LIVE AUDIO ANALYSIS (Every 10 frames) ---
+            if frame_idx % 10 == 0:
+                # Analyze the last 3 seconds of audio for "Live" feel
+                start_sample = max(0, int((current_time - 3) * sr))
+                end_sample = int(current_time * sr)
+                audio_segment = full_audio[start_sample:end_sample]
+
+                if len(audio_segment) > sr * 0.5: # At least 0.5s of audio to analyze
+                    feats = extract_audio_features(audio_segment, sr)
+                    # Use global baseline if available
+                    instant_scores = compute_audio_scores(feats, baseline=None) 
+                    
+                    # Apply smoothing to prevent jittery bars
+                    live_scores["confidence_audio"] += smoothing_factor * (instant_scores["confidence_audio"] - live_scores["confidence_audio"])
+                    live_scores["clarity"] += smoothing_factor * (instant_scores["clarity"] - live_scores["clarity"])
+                    live_scores["stress"] += smoothing_factor * (instant_scores["stress"] - live_scores["stress"])
+
+            # --- 2. VISUAL AI (Face & Emotion) ---
+            if frame_idx % 4 == 0:
                 mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
-                last_results = landmarker.detect_for_video(mp_image, int((frame_idx/fps)*1000))
+                results = landmarker.detect_for_video(mp_image, int(current_time * 1000))
                 
-                if last_results.face_landmarks:
-                    last_lm = last_results.face_landmarks[0]
-                    last_emotions = analyze_face_emotion(frame)
-                    last_eye_s = compute_eye_contact_ratio(frame, last_lm)
+                if results.face_landmarks:
+                    last_landmarks = results.face_landmarks[0]
+                    emo_probs = analyze_face_emotion(frame)
+                    dom_emo = max(emo_probs, key=emo_probs.get)
+                    v_target, a_target = compute_valence_arousal_from_probs(emo_probs)
+                    smooth_v += 0.15 * (v_target - smooth_v)
+                    smooth_a += 0.15 * (a_target - smooth_a)
+
+            # --- 3. RENDERING UI ELEMENTS ---
+            # Face Box
+            if last_landmarks:
+                xs = [lm.x * width for lm in last_landmarks]
+                ys = [lm.y * height for lm in last_landmarks]
+                draw_face_box(frame, int(min(xs)), int(min(ys)), int(max(xs)-min(xs)), int(max(ys)-min(ys)), dom_emo)
+
+            # Emotion Wheel
+            draw_full_emotion_wheel(frame, (width - 130, height - 100), 90, smooth_v, smooth_a, dom_emo)
+
+            # Live Metric Bars
+            draw_metric_bars(
+                frame, 30, height - 160, 28, 200, 6,
+                live_scores["confidence_audio"], live_scores["clarity"], live_scores["stress"]
+            )
+
+            # Question Overlay (Wrapped Text)
+            if not active_answer:
+                frame = draw_question_overlay(frame, next_text, width, height)
 
-            # 2. USE LAST KNOWN DATA FOR CALCULATIONS & DRAWING
-            d_conf, d_clar, d_stress = b_conf, b_clar, b_stress
-            
-            if last_results and last_results.face_landmarks:
-                # Use current local variables from 'last' successful AI run
-                curr_f_conf = (last_emotions.get("neutral", 0) + last_emotions.get("happy", 0)) * 100
-                d_conf = (b_conf * 0.7) + (curr_f_conf * 0.3)
-                d_clar = (b_clar * 0.8) + (last_eye_s * 100 * 0.2)
-                d_stress = (b_stress * 0.7) + ((last_emotions.get("sad",0)+last_emotions.get("angry",0))*30)
-
-                # Update accumulators only on stride frames to keep averages accurate
-                if frame_idx % frame_stride == 0:
-                    face_conf_accum.append(curr_f_conf)
-                    eye_accum.append(last_eye_s)
-
-                dom_emo = max(last_emotions, key=last_emotions.get)
-                v_t = sum(emotion_va[e][0]*s for e,s in last_emotions.items() if e in emotion_va)
-                a_t = sum(emotion_va[e][1]*s for e,s in last_emotions.items() if e in emotion_va)
-                
-                # Keep smoothing every frame for fluid movement
-                smooth_v += 0.15 * (v_t - smooth_v)
-                smooth_a += 0.15 * (a_t - smooth_a)
-
-                # Draw face box using the last known landmarks
-                xs, ys = [l.x*width for l in last_lm], [l.y*height for l in last_lm]
-                draw_face_box(
-                    frame,
-                    int(min(xs)), int(min(ys)),
-                    int(max(xs) - min(xs)), int(max(ys) - min(ys)),
-                    dom_emo
-                )
-
-            # 3. ALWAYS DRAW UI (Wheel and Bars)
-            frame = draw_full_emotion_wheel(frame, (width-130, height-100), 90, smooth_v, smooth_a, dom_emo)
-            frame = draw_metric_bars(frame, 30, height-160, 28, 200, 6, d_conf, d_clar, d_stress)
-            
             out.write(frame)
             frame_idx += 1
 
     cap.release()
     out.release()
-    return temp_video, np.mean(face_conf_accum) if face_conf_accum else 50, np.mean(eye_accum)*100 if eye_accum else 50
+    return temp_output
+
+def draw_question_overlay(frame, text, width, height):
+    """Draws a wrapped text box above the Wheel and Bars."""
+    font = cv2.FONT_HERSHEY_DUPLEX
+    font_scale = 0.65
+    thickness = 1
+    side_margin = 50
+    bottom_limit = height - 270 # Ensure it stays above the bars/wheel
+    line_height = 35
+
+    # Text Wrapping Logic
+    max_w = width - (2 * side_margin)
+    words = text.split(' ')
+    lines, current_line = [], ""
+    for word in words:
+        test = current_line + word + " "
+        (w, _), _ = cv2.getTextSize(test, font, font_scale, thickness)
+        if w < max_w: current_line = test
+        else:
+            lines.append(current_line)
+            current_line = word + " "
+    lines.append(current_line)
+
+    # Box dimensions
+    rect_h = (len(lines) * line_height) + 20
+    y2 = bottom_limit
+    y1 = y2 - rect_h
+
+    # Transparent Background
+    overlay = frame.copy()
+    cv2.rectangle(overlay, (side_margin - 10, y1), (width - side_margin + 10, y2), (20, 20, 20), -1)
+    cv2.addWeighted(overlay, 0.7, frame, 0.3, 0, frame)
+
+    # Draw Lines
+    for i, line in enumerate(lines):
+        (tw, th), _ = cv2.getTextSize(line.strip(), font, font_scale, thickness)
+        tx = (width - tw) // 2
+        ty = y1 + 25 + (i * line_height)
+        cv2.putText(frame, line.strip(), (tx, ty), font, font_scale, (255, 255, 255), thickness, cv2.LINE_AA)
+    
+    return frame
+
 
 ##Main pipeline
 def run_intervision_pipeline(video_path, questions_config, output_dir):
@@ -573,95 +633,79 @@ def run_intervision_pipeline(video_path, questions_config, output_dir):
 
     os.makedirs(output_dir, exist_ok=True)
 
-    # Establish baseline from first 10s
+    # 1. Establish Audio Baseline (First 10s)
     try:
         y_b, sr_b = librosa.load(video_path, sr=16000, duration=10)
         baseline = extract_audio_features(y_b, sr_b)
     except Exception as e:
-        print(f"Baseline Load Warning: {e}. Using defaults.")
+        print(f"Baseline Warning: {e}")
         baseline = None
 
-    final_reports, segments = [], []
+    final_reports = []
+    audio_results_map = {} # To pass scores to the visual function
 
+    # 2. Pre-process each question (Audio & Text Analysis only)
     for q in questions_config:
         q_id = q['question_id']
-        raw_seg = os.path.join(output_dir, f"q{q_id}_raw.mp4")
         wav_p = os.path.join(output_dir, f"q{q_id}.wav")
-
-        # Precise FFmpeg cutting with error handling
         duration = q["end_time"] - q["start_time"]
+
+        # Extract audio segment for analysis
         try:
             subprocess.run([
                 'ffmpeg', '-y', '-ss', str(q["start_time"]), '-t', str(duration),
-                '-i', video_path, '-c:v', 'libx264', '-c:a', 'aac', '-strict', 'experimental', raw_seg
+                '-i', video_path, '-vn', '-acodec', 'pcm_s16le', '-ar', '16000', wav_p
             ], check=True, capture_output=True)
-        except subprocess.CalledProcessError as e:
-            print(f"Skipping Question {q_id}: Time range might be out of video bounds.")
-            continue
-
-        # Audio Extraction
-        try:
-            y, sr = librosa.load(raw_seg, sr=16000)
-            import soundfile as sf
-            sf.write(wav_p, y, sr)
-        except Exception as e:
-            print(f"Error extracting audio for Q{q_id}: {e}")
-            continue
-
-        # Audio Analysis
-        a_scores = compute_audio_scores(extract_audio_features(y, sr), baseline)
-
-        # Whisper Transcription
-        try:
-            transcription_data = asr(wav_p, chunk_length_s=30, return_timestamps=True)
+            
+            y, sr = librosa.load(wav_p, sr=16000)
+            a_scores = compute_audio_scores(extract_audio_features(y, sr), baseline)
+            
+            # Whisper Transcription
+            transcription_data = asr(wav_p, chunk_length_s=30)
             transcription = transcription_data["text"].strip()
-        except:
-            transcription = "[Transcription Error]"
-
-        similarity_score = compute_similarity_score(transcription, q["ideal_answer"])
-        relevance_score = compute_relevance_score(q["question_text"], transcription)
-
-        # Visual Analysis
-        try:
-            ann_v, f_c, e_c = process_video_segment(raw_seg, output_dir, q_id, a_scores)
-
-            final_v = os.path.join(output_dir, f"q{q_id}_final.mp4")
-            subprocess.run([
-                'ffmpeg', '-y', '-i', ann_v, '-i', raw_seg, '-map', '0:v', '-map', '1:a',
-                '-c:v', 'copy', '-c:a', 'aac', final_v
-            ], check=True, capture_output=True)
-
-            segments.append(final_v)
+            
+            # Scores
+            similarity_score = compute_similarity_score(transcription, q["ideal_answer"])
+            relevance_score = compute_relevance_score(q["question_text"], transcription)
 
+            # Store results for the visual processing step
+            audio_results_map[q_id] = a_scores
+            
+            # Prepare the final JSON report entry
             final_reports.append({
                 "questionId": q_id,
                 "userAnswerText": transcription,
                 "toneOfVoice": a_scores["tone_of_voice"],
                 "clarity": a_scores["clarity"],
                 "stress": a_scores["stress"],
-                "confidence": round((a_scores["confidence_audio"] + f_c + e_c) / 3, 2),
+                "confidence": a_scores["confidence_audio"], # We'll refine this after visual if needed
                 "pauses": a_scores["pauses"],
                 "score": similarity_score,
                 "relevance": relevance_score
             })
+            
         except Exception as e:
-            print(f"Visual analysis failed for Q{q_id}: {e}")
-
-        torch.cuda.empty_cache()
-
-    # Final concatenation
-    if segments:
-        list_path = os.path.join(output_dir, "list.txt")
-        with open(list_path, "w") as f:
-            for s in segments:
-                f.write(f"file '{os.path.abspath(s)}'\n")
-
-        final_output = os.path.join(output_dir, "Intervision_Final_Result.mp4")
-        os.system(f"ffmpeg -f concat -safe 0 -i {list_path} -c:v libx264 -preset superfast -crf 23 -c:a aac -y {final_output}")
+            print(f"Error analyzing Question {q_id}: {e}")
+            continue
 
+    # 3. Process the entire video visually (Annotations + Overlays)
+    try:
+        # This calls the English function we wrote in the previous message
+        annotated_video_raw = process_full_video(video_path, output_dir, questions_config, audio_results_map)
+        
+        # 4. Final Merge: Put original audio back onto the annotated video
+        final_output = os.path.join(output_dir, "Intervision_Final_Report.mp4")
+        subprocess.run([
+            'ffmpeg', '-y', '-i', annotated_video_raw, '-i', video_path, 
+            '-map', '0:v:0', '-map', '1:a:0', '-c:v', 'libx264', '-preset', 'veryfast', 
+            '-crf', '22', '-c:a', 'aac', '-b:a', '192k', '-shortest', final_output
+        ], check=True)
+        
+        # Save the JSON report
         with open(os.path.join(output_dir, "report.json"), "w") as f:
             json.dump({"listOfAnswerReport": final_reports}, f, indent=4)
 
-        return f"Successfully processed {len(segments)} questions."
-    else:
-        return "No segments were processed. Check your video time ranges."
+        return f"Success! Full video generated at {final_output}"
+
+    except Exception as e:
+        return f"Visual processing or merging failed: {e}"