Upload 5 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+face_landmarker.task filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,21 @@
+# Use an official Python runtime with GPU support if needed, or standard CPU
+FROM python:3.9
+
+# Set working directory
+WORKDIR /code
+
+# Install system dependencies for OpenCV and MediaPipe
+RUN apt-get update && apt-get install -y \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install
+COPY ./requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+# Copy the rest of the application
+COPY . .
+
+# Command to run the application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,281 @@
+import os
+import json
+import requests
+import cloudinary
+import cloudinary.uploader
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from fastapi import FastAPI, Body, HTTPException, BackgroundTasks
+from dotenv import load_dotenv
+from datetime import datetime
+from pipeline import run_intervision_pipeline
+
+# --- Setup Retry Strategy ---
+retry_strategy = Retry(
+    total=3,
+    backoff_factor=1, # Wait 1s, 2s, 4s between retries
+    status_forcelist=[429, 500, 502, 503, 504],
+)
+adapter = HTTPAdapter(max_retries=retry_strategy)
+http = requests.Session()
+http.mount("https://", adapter)
+http.mount("http://", adapter)
+
+# Load environment variables from .env file
+load_dotenv()
+
+app = FastAPI(title="Intervision AI Engine")
+
+# Cloudinary Configuration
+cloudinary.config( 
+    cloud_name = os.getenv("CLOUDINARY_CLOUD_NAME"), 
+    api_key = os.getenv("CLOUDINARY_API_KEY"), 
+    api_secret = os.getenv("CLOUDINARY_API_SECRET") 
+)
+
+# Directory Setup
+RESULT_DIR = "temp_data/results"
+UPLOAD_DIR = "temp_data/uploads"
+os.makedirs(RESULT_DIR, exist_ok=True)
+os.makedirs(UPLOAD_DIR, exist_ok=True)
+
+def time_to_seconds(t_str: str) -> int:
+    """Converts HH:MM:SS timestamp format to total seconds."""
+    if not t_str: return 0
+    h, m, s = map(int, t_str.split(':'))
+    return h * 3600 + m * 60 + s
+
+def background_processing(session_data: dict):
+    """
+    Handles heavy AI processing: video download, pipeline execution, 
+    result upload, and backend notification (callback).
+    """
+    session_id = session_data.get('sessionId')
+    video_url = session_data.get('originalVideoUrl')
+    callback_url = session_data.get('callbackBaseUrl')
+    
+    print(f"[LOG] Processing started for session: {session_id}")
+    
+    # 1. Download the original video from the provided URL
+    local_input_path = os.path.join(UPLOAD_DIR, f"{session_id}_input.mp4")
+# 1. Download with increased timeout and Retry logic
+    try:
+        print(f"[LOG] Downloading video: {video_url}")
+        # Increased timeout to 300s (5 minutes) for large files
+        response = http.get(video_url, stream=True, timeout=300)
+        response.raise_for_status()
+        with open(local_input_path, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=1024*1024):
+                f.write(chunk)
+    except Exception as e:
+        print(f"[DOWNLOAD ERROR]: {e}")
+        # Notify backend that it failed due to download
+        return
+
+    # 2. Prepare question list for the AI Pipeline
+    final_questions = []
+    skipped_failed_reports = []
+    
+    for q in session_data.get('answers', []):
+        if q.get('isAnswered'):
+            final_questions.append({
+                "question_id": q['aiQuestionId'],
+                "question_text": q['questionText'],
+                "ideal_answer": q['expectedAnswer'],
+                "start_time": time_to_seconds(q['startedAt']),
+                "end_time": time_to_seconds(q['submittedAt'])
+            })
+        else:
+            # Handle questions that weren't answered during the session
+            skipped_failed_reports.append({
+                "questionId": q['aiQuestionId'],
+                "userAnswerText": "N/A",
+                "score": 0.0,                       
+                "relevance": 0.0,
+                "confidence": 0.0,
+                "stress": 0.0,
+                "clarity": 0.0,
+                "pauses": 0.0,
+                "toneOfVoice": "N/A",
+                "status": "skipped" if q.get('isSkipped') else "failed"
+            })
+
+    # 3. Execute AI Pipeline (Analysis & Visualization)
+    ai_results = []
+    if final_questions:
+        # run_intervision_pipeline generates Intervision_Final_Result.mp4
+        run_intervision_pipeline(local_input_path, final_questions, RESULT_DIR)
+        report_path = os.path.join(RESULT_DIR, "report.json")
+        if os.path.exists(report_path):
+            with open(report_path, "r") as f:
+                ai_results = json.load(f).get("listOfAnswerReport", [])
+
+    # 4. Upload the processed video to Cloudinary
+    final_video_path = os.path.join(RESULT_DIR, "Intervision_Final_Result.mp4")
+    final_video_url = None
+    if os.path.exists(final_video_path):
+        try:
+            upload_res = cloudinary.uploader.upload(
+                final_video_path, 
+                public_id=f"res_{session_id}", 
+                folder="intervision_results",
+                resource_type="video",
+                chunk_size=6000000
+            )
+            final_video_url = upload_res.get("secure_url")
+        except Exception as e:
+            print(f"[UPLOAD ERROR]: {e}")
+
+    # 5. Construct final payload and notify Backend via Callback
+    final_payload = {
+        "sessionId": session_id,
+        "finalVideoUrl": final_video_url,
+        "report": ai_results + skipped_failed_reports
+    }
+    
+    try:
+        # Notify backend that processing is complete
+        cb_response = requests.post(f"{callback_url}/api/ai-callback", json=final_payload, timeout=30)
+        print(f"[LOG] Callback sent to {callback_url}. Status: {cb_response.status_code}")
+        
+        # 6. Local Cleanup: Remove files to save disk space
+        if os.path.exists(local_input_path): os.remove(local_input_path)
+        if os.path.exists(final_video_path): os.remove(final_video_path)
+            
+    except Exception as e:
+        print(f"[CALLBACK ERROR]: {e}")
+
+@app.post("/process-interview/")
+async def process_interview(background_tasks: BackgroundTasks, data: dict = Body(...)):
+    """Entry point to start the AI analysis asynchronously."""
+    background_tasks.add_task(background_processing, data)
+    return {"message": "Processing started", "sessionId": data.get('sessionId')}
+
+@app.post("/delete-video-by-url/")
+async def delete_video_by_url(data: dict = Body(...)):
+    """
+    Deletes a video from Cloudinary based on its URL.
+    Input JSON: {"videoUrl": "https://..."}
+    """
+    video_url = data.get("videoUrl")
+    if not video_url:
+        raise HTTPException(status_code=400, detail="videoUrl is required")
+
+    try:
+        # Logic to extract the public_id from a Cloudinary URL
+        # Example: .../folder/public_id.mp4 -> folder/public_id
+        url_parts = video_url.split('/')
+        filename_with_ext = url_parts[-1]
+        filename = filename_with_ext.split('.')[0]
+        
+        # Check if the video is inside the results folder
+        folder = url_parts[-2] if "intervision_results" in url_parts[-2] else ""
+        public_id = f"{folder}/{filename}" if folder else filename
+
+        # Trigger deletion from Cloudinary
+        result = cloudinary.uploader.destroy(public_id, resource_type="video")
+
+        if result.get("result") == "ok":
+            return {"status": "success", "message": f"Deleted {public_id}"}
+        return {"status": "failed", "details": result}
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+    
+# @app.post("/process-interview-test/")
+# async def process_test(data: dict = Body(...)):
+#     try:
+#         print(f"--- [TEST LOG] Processing Session: {data['sessionId']} ---")
+        
+#         # 1. Path Check
+#         local_path = r"D:\FayzaAhmed\Graduation_project\models\MultiModal\deployment\interview_test.mp4" 
+#         if not os.path.exists(local_path):
+#             return {"error": f"Video file not found at {local_path}"}
+
+#         # 2. Prepare Data
+#         final_questions = []
+#         for q in data['answers']:
+#             if q.get('isAnswered'):
+#                 final_questions.append({
+#                     "question_id": q['aiQuestionId'],
+#                     "question_text": q['questionText'],
+#                     "ideal_answer": q['expectedAnswer'],
+#                     "start_time": time_to_seconds(q['startedAt']),
+#                     "end_time": time_to_seconds(q['submittedAt'])
+#                 })
+
+#         # 3. Run Pipeline
+#         print("[LOG] Running AI Pipeline...")
+#         run_intervision_pipeline(local_path, final_questions, RESULT_DIR)
+        
+#         # 4. Upload
+#         print("[LOG] Uploading to Cloudinary...")
+#         final_video_path = os.path.join(RESULT_DIR, "Intervision_Final_Result.mp4")
+#         upload_res = cloudinary.uploader.upload(
+#                     final_video_path, 
+#                     public_id=f"{data['sessionId']}_test",
+#                     folder="intervision_tests",
+#                     resource_type="video"  # This is the important part
+# )
+
+#         # 5. Load Report
+#         report_path = os.path.join(RESULT_DIR, "report.json")
+#         if not os.path.exists(report_path):
+#             return {"error": "Pipeline finished but report.json was not created."}
+
+#         with open(report_path, "r") as f:
+#             ai_results = json.load(f)["listOfAnswerReport"]
+
+#         return {
+#             "status": "Success",
+#             "videoUrl": upload_res.get("secure_url"),
+#             "report": ai_results
+#         }
+
+#     except Exception as e:
+#         print(f"[CRITICAL ERROR]: {str(e)}")
+#         return {"error": str(e), "traceback": "Check Terminal for details"}
+
+"""QUESTIONS_CONFIG = 
+[
+    {
+        "question_id": 1,
+        "question_text": "how do you describe yourself",
+        "ideal_answer": "Being different means you have to work at belonging...",
+        "start_time": 0,
+        "end_time": 15,
+    },
+    {
+        "question_id": 2,
+        "question_text": "Tell us about your biggest achievement",
+        "ideal_answer": "I am proud of accomplishing...",
+        "start_time": 15,
+        "end_time": 24,
+    }
+]
+"""
+
+"""
+{
+    "sessionId": "test-session-123",
+    "originalVideoUrl": "local_test_no_url",
+    "callbackBaseUrl": "http://localhost:8000",
+    "answers": [
+        {
+        "questionId": "q-1",
+        "aiQuestionId": 1,
+        "questionText": "How does the speaker encourage people to deal with their differences and uniqueness?",
+        "expectedAnswer": "When you're different, you have to work at the longing. Everybody wants to feel valued and accepted, and we think it should happen spontaneously, but it doesn't. Sometimes society tells us, and we tell ourselves, we don't fit the mold. Take a piece of paper and write down what makes you different. And I want you to celebrate it today and every day. Shout it from the rooftops. What makes me different is what has made me stand out and be successful. I also encourage you to be curious and ask, what is on other people's pieces of paper? What makes them different? Let's celebrate those imperfections that make us special. I hope that it teaches you that nobody has a claim on the word normal. We are all different. We are all quirky and unique, and that is what makes us wonderful.",
+        "isAnswered": true,
+        "isSkipped": false,
+        "isFailed": false,
+        "startedAt": "00:00:00",
+        "submittedAt": "00:00:55"
+        }
+    ]
+}
+"""

face_landmarker.task

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff
+size 3758596

pipeline.py

@@ -0,0 +1,659 @@
+import os
+import subprocess
+import cv2
+import json
+import math
+import torch
+import librosa
+import ffmpeg
+import numpy as np
+import soundfile as sf
+import mediapipe as mp
+from PIL import Image
+from transformers import AutoImageProcessor, AutoModelForImageClassification, pipeline
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from sklearn.metrics.pairwise import cosine_similarity
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+
+# Ignore unnecessary warnings
+import warnings
+warnings.filterwarnings("ignore", category=UserWarning)
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# 2. Download and Initialize Mediapipe once (Global)
+MODEL_PATH = "face_landmarker.task"
+if not os.path.exists(MODEL_PATH):
+    os.system(f"wget -O {MODEL_PATH} -q https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task")
+
+# 3. Initialize Models
+asr = pipeline("automatic-speech-recognition", model="openai/whisper-small", device=0 if torch.cuda.is_available() else -1)
+semantic_model = SentenceTransformer("all-MiniLM-L6-v2")
+cross_encoder = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
+
+FACE_MODEL_NAME = "dima806/facial_emotions_image_detection"
+face_processor = AutoImageProcessor.from_pretrained(FACE_MODEL_NAME)
+face_model = AutoModelForImageClassification.from_pretrained(FACE_MODEL_NAME).to(device).eval()
+
+# Emotion Mapping for Wheel
+emotion_va = {
+    "happy": (0.8, 0.2), "fear": (0.2, 0.8), "angry": (-0.7, 0.65),
+    "sad": (-0.65, -0.55), "surprise": (0.1, -0.75), "disgust": (0.6, -0.4), "neutral": (0.0, 0.0)
+}
+EMOTION_RING = [
+    ("Happy", 0, 0.84), ("Surprise", 45, 0.84), ("Fear", 100, 0.84),
+    ("Sad", 160, 0.84), ("Disgust", 215, 0.84), ("Angry", 270, 0.84)
+]
+
+##Utility functions
+
+def normalize(v, mn, mx):
+    return np.clip((v - mn) / (mx - mn), 0, 1) if mx - mn != 0 else 0.0
+
+def extract_audio(v_in, a_out):
+    ffmpeg.input(v_in).output(a_out, ac=1, ar=16000).overwrite_output().run(quiet=True)
+
+def merge_audio_video(v_in, a_in, v_out):
+    ffmpeg.output(ffmpeg.input(v_in).video, ffmpeg.input(a_in).audio, v_out, vcodec="libx264", acodec="aac").overwrite_output().run(quiet=True)
+
+def draw_face_box(frame, x, y, w, h, emotion_name=""):
+    color, th, cl = (0, 255, 100), 2, 20  # Green color
+    cv2.rectangle(frame, (x, y), (x+w, y+h), color, 1)
+
+    # Add emotion name above face box
+    if emotion_name:
+        cv2.putText(
+            frame,
+            emotion_name.upper(),
+            (x + 10, y - 15),
+            cv2.FONT_HERSHEY_DUPLEX,
+            0.7,
+            (0, 255, 100),
+            2,
+            cv2.LINE_AA
+        )
+
+    # Corners
+    for px, py, dx, dy in [(x,y,cl,0), (x,y,0,cl), (x+w,y,-cl,0), (x+w,y,0,cl), (x,y+h,cl,0), (x,y+h,0,-cl), (x+w,y+h,-cl,0), (x+w,y+h,0,-cl)]:
+        cv2.line(frame, (px, py), (px+dx, py+dy), color, 5)
+    return frame
+
+def compute_eye_contact_ratio(frame, landmarks):
+    h, w, _ = frame.shape
+    def ear(idx):
+        p = [np.array([landmarks[i].x * w, landmarks[i].y * h]) for i in idx]
+        return (np.linalg.norm(p[1]-p[5]) + np.linalg.norm(p[2]-p[4])) / (2.0 * np.linalg.norm(p[0]-p[3]))
+    avg_ear = (ear([33, 160, 158, 133, 153, 144]) + ear([362, 385, 387, 263, 373, 380])) / 2.0
+    return min(max(avg_ear * 3, 0), 1)
+
+def analyze_face_emotion(frame):
+    img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+    inputs = face_processor(images=img, return_tensors="pt").to(device)
+    with torch.no_grad():
+        outputs = face_model(**inputs)
+    probs = torch.nn.functional.softmax(outputs.logits, dim=-1)[0]
+    return {face_model.config.id2label[i].lower(): float(probs[i]) for i in range(len(probs))}
+
+##Audio analysis
+
+def extract_audio_features(y, sr):
+    duration = librosa.get_duration(y=y, sr=sr)
+    if duration == 0:
+        return {"pitch_std": 0, "jitter": 0, "energy_std": 0, "pause_ratio": 0, "speech_rate": 0}
+
+    # Pitch & Jitter
+    f0 = librosa.yin(y, fmin=75, fmax=300, sr=sr)
+    f0 = f0[~np.isnan(f0)]
+    pitch_std = np.std(f0) if len(f0) else 0
+    jitter = np.mean(np.abs(np.diff(f0)) / np.maximum(f0[:-1], 1e-6)) if len(f0) > 1 else 0
+
+    # Energy
+    rms = librosa.feature.rms(y=y)[0]
+    energy_std = np.std(rms)
+
+    intervals = librosa.effects.split(y, top_db=20)
+    speech_duration = sum((e - s) for s, e in intervals) / sr
+    pause_ratio = 1 - (speech_duration / duration) if duration > 0 else 0
+
+    # Speech Rate
+    oenv = librosa.onset.onset_strength(y=y, sr=sr)
+    onsets = librosa.onset.onset_detect(onset_envelope=oenv, sr=sr)
+    speech_rate = len(onsets) / duration if duration > 0 else 0
+
+    return {
+        "pitch_std": pitch_std,
+        "jitter": jitter,
+        "energy_std": energy_std,
+        "pause_ratio": pause_ratio,
+        "speech_rate": speech_rate
+    }
+
+
+def compute_audio_scores(features, baseline=None):
+    """
+    Fairness-aware audio scoring with personal baseline comparison
+    """
+    # Use standard defaults if no baseline provided
+    if baseline is None:
+        baseline = {"pitch_std": 30.0, "energy_std": 0.05, "jitter": 0.02, "pause_ratio": 0.2, "speech_rate": 4.0}
+
+    # Calculate Relative Ratios (Current / Baseline)
+    pitch_ratio = features["pitch_std"] / max(baseline["pitch_std"], 1e-6)
+    energy_ratio = features["energy_std"] / max(baseline["energy_std"], 1e-6)
+    rate_ratio = features["speech_rate"] / max(baseline["speech_rate"], 1e-6)
+
+    # Stress Score (Relative)
+    pitch_dev = abs(1 - pitch_ratio)
+    energy_dev = abs(1 - energy_ratio)
+    stress_val = (pitch_dev * 0.4 + energy_dev * 0.4 + features["jitter"] * 0.2) * 150
+    stress = np.clip(stress_val + 20, 0, 100)
+
+    # Clarity Score (Relative)
+    pause_dev = max(0, features["pause_ratio"] - baseline["pause_ratio"])
+    clarity = 100 - (pause_dev * 120 + features["jitter"] * 400)
+
+    # Confidence Score (Relative)
+    rate_dev = abs(1 - rate_ratio)
+    confidence_audio = 100 - (rate_dev * 40 + energy_dev * 30 + features["pause_ratio"] * 50)
+
+    # Tone classification based on relative shifts
+    tones = {
+        "Confident": confidence_audio,
+        "Hesitant": features["pause_ratio"] * 150,
+        "Excited": (energy_ratio - 1) * 100 if energy_ratio > 1 else 0,
+        "Unstable": stress,
+        "Natural": 100 - (pitch_dev * 60 + rate_dev * 40)
+    }
+
+    dominant_tone = max(tones, key=tones.get)
+
+    return {
+        "confidence_audio": round(float(np.clip(confidence_audio, 0, 100)), 2),
+        "clarity": round(float(np.clip(clarity, 0, 100)), 2),
+        "stress": round(float(np.clip(stress, 0, 100)), 2),
+        "pauses": round(float(features["pause_ratio"] * 100), 2),
+        "tone_of_voice": dominant_tone
+    }
+
+def analyze_audio_segment(audio_path, baseline=None):
+    """
+    Main entry point for audio segment analysis
+    """
+    y, sr = librosa.load(audio_path, sr=16000)
+    features = extract_audio_features(y, sr)
+    return compute_audio_scores(features, baseline)
+
+
+##Text analysis
+
+def get_user_answer(audio_path):
+    """Transcribe audio using Whisper"""
+    result = asr(audio_path, chunk_length_s=20)
+    return result["text"].strip()
+
+
+def compute_similarity_score(user_answer, ideal_answer):
+    emb = semantic_model.encode([user_answer, ideal_answer])
+    sim = cosine_similarity([emb[0]], [emb[1]])[0][0]
+    score = float(sim * 100)
+    return round(max(0, score), 2)
+
+def compute_relevance_score(question, user_answer):
+    raw_score = cross_encoder.predict([(question, user_answer)])[0]
+    prob = 1 / (1 + np.exp(-raw_score))
+    score = float(prob * 100)
+    return round(max(0, score), 2)
+
+##Video
+
+# Eye indices
+LEFT_EYE = [33, 160, 158, 133, 153, 144]
+RIGHT_EYE = [362, 385, 387, 263, 373, 380]
+
+# Eye Contact Function
+def compute_eye_contact_ratio(frame, landmarks):
+    """
+    Compute eye contact ratio from detected face landmarks
+    """
+
+    if not landmarks:
+        return 0.5
+
+    h, w, _ = frame.shape
+
+    def ear(indices):
+        points = [
+            np.array([
+                landmarks[i].x * w,
+                landmarks[i].y * h
+            ])
+            for i in indices
+        ]
+
+        v1 = np.linalg.norm(points[1] - points[5])
+        v2 = np.linalg.norm(points[2] - points[4])
+        h_dist = np.linalg.norm(points[0] - points[3])
+
+        return (v1 + v2) / (2.0 * h_dist)
+
+    ear_left = ear(LEFT_EYE)
+    ear_right = ear(RIGHT_EYE)
+
+    avg_ear = (ear_left + ear_right) / 2.0
+
+    eye_score = min(max(avg_ear * 3, 0), 1)
+
+    return eye_score
+
+def analyze_face_emotion(frame):
+    """
+    Predict facial emotion probabilities from single frame
+    """
+
+    # Convert BGR to RGB
+    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    image = Image.fromarray(rgb)
+
+    # Preprocess
+    inputs = face_processor(images=image, return_tensors="pt").to(device)
+
+    with torch.no_grad():
+        outputs = face_model(**inputs)
+
+    probs = torch.nn.functional.softmax(outputs.logits, dim=-1)[0]
+    labels = face_model.config.id2label
+
+    emotion_probs = {
+        labels[i].lower(): float(probs[i])
+        for i in range(len(probs))
+    }
+
+    return emotion_probs
+
+def draw_face_box(frame, x, y, w, h, emotion_label="Neutral"):
+    """
+    Draw face bounding box with emotion label above it
+    """
+
+    # Green color for face box
+    color = (0, 255, 0)
+
+    thickness = 2
+    corner_len = 22
+
+    # Main rectangle
+    cv2.rectangle(frame, (x, y), (x+w, y+h), color, thickness)
+
+    # Decorative corner lines
+    for (px, py, dx, dy) in [
+        (x, y, corner_len, 0), (x, y, 0, corner_len),
+        (x+w, y, -corner_len, 0), (x+w, y, 0, corner_len), 
+        (x, y+h, corner_len, 0), (x, y+h, 0, -corner_len),
+        (x+w, y+h, -corner_len, 0), (x+w, y+h, 0, -corner_len),
+    ]:
+        cv2.line(frame, (px, py), (px+dx, py+dy), color, 4)
+
+    # Draw emotion text above the face box
+    label_text = emotion_label.capitalize()
+
+    (tw, th), _ = cv2.getTextSize(
+        label_text,
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.7,
+        2
+    )
+
+    text_x = x + (w - tw) // 2
+    text_y = y - 10
+
+    cv2.putText(
+        frame,
+        label_text,
+        (text_x, text_y),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.7,
+        (0, 255, 0),
+        2,
+        cv2.LINE_AA
+    )
+
+    return frame
+
+def compute_valence_arousal_from_probs(emotion_probs):
+    """Computing Valence and Arousal from emotion probabilities"""
+    v, a, total = 0.0, 0.0, 0.0
+
+    for emo, score in emotion_probs.items():
+        emo = emo.lower()
+        if emo in emotion_va:
+            v += emotion_va[emo][0] * score
+            a += emotion_va[emo][1] * score
+            total += score
+
+    if total == 0:
+        return 0.0, 0.0
+
+    return v / total, a / total
+
+def draw_full_emotion_wheel(panel, center, radius, valence, arousal,
+                            dominant_emotion="neutral"):
+    cx, cy = center
+
+    # Circle background
+    cv2.circle(panel, center, radius + 5, (15, 15, 25), -1)
+    cv2.circle(panel, center, radius,     (60, 60, 85),  2)
+    for rf in [0.33, 0.66]:
+        cv2.circle(panel, center, int(radius * rf), (35, 35, 50), 1)
+
+    # Drawing dividing lines between emotions
+    for angle_deg in range(0, 360, 60):
+        rad = math.radians(angle_deg)
+        x1 = int(cx + radius * math.cos(rad))
+        y1 = int(cy - radius * math.sin(rad))
+        cv2.line(panel, (cx, cy), (x1, y1), (40, 40, 60), 1)
+
+    # Drawing emotion labels
+    ef, es, et = cv2.FONT_HERSHEY_SIMPLEX, 0.40, 1
+    for emotion_data in EMOTION_RING:
+        if emotion_data[1] is None:
+            continue
+
+        label, angle_deg, rf = emotion_data
+        rad = math.radians(angle_deg)
+        lx  = int(cx + rf * radius * math.cos(rad))
+        ly  = int(cy - rf * radius * math.sin(rad))
+        (tw, th), _ = cv2.getTextSize(label, ef, es, et)
+        tx, ty = lx - tw//2, ly + th//2
+
+        # Highlight active emotion
+        if label.lower() == dominant_emotion.lower():
+            cv2.putText(panel, label, (tx, ty), ef, es+0.08, (0, 255, 200), 2, cv2.LINE_AA)
+        else:
+            cv2.putText(panel, label, (tx, ty), ef, es, (190, 190, 255), et, cv2.LINE_AA)
+
+    # Neutral in center
+    nc = (0, 255, 200) if dominant_emotion == "neutral" else (160, 160, 160)
+    (tw, th), _ = cv2.getTextSize("Neutral", ef, es, et)
+    cv2.putText(panel, "Neutral", (cx-tw//2, cy+th//2), ef, es, nc, et, cv2.LINE_AA)
+
+    # Animated dot with glow
+    dot_x = int(cx + valence * radius * 0.88)
+    dot_y = int(cy - arousal * radius * 0.88)
+    cv2.circle(panel, (dot_x, dot_y), 15, (160, 120,  0), -1)
+    cv2.circle(panel, (dot_x, dot_y), 11, (220, 180,  0), -1)
+    cv2.circle(panel, (dot_x, dot_y),  7, (255, 230, 60), -1)
+
+    return panel
+
+BAR_CONFIGS = [
+    ("Confidence",   (70, 180, 255),   (30, 50, 100)),      # light blue
+    ("Clarity",  (100, 220, 150),   (25, 70, 50)),       # light cyan
+    ("Stress", (255, 120, 100),   (100, 40, 30)),        # light coral
+]
+
+def draw_metric_bars(panel,
+                    bars_x_start,
+                    bar_y_top,
+                    bar_height,
+                    bar_width,
+                    bar_gap,
+                    confidence,
+                    clarity,
+                    stress):
+    """
+    Draw horizontal metric bars with label above each bar
+    """
+
+    values = [confidence, clarity, stress]
+    labels_list = ["Confidence", "Clarity", "Stress"]
+
+    # Extra vertical space for labels
+    label_space = 20
+
+    for i, value in enumerate(values):
+
+        label, fill_color, bg_color = BAR_CONFIGS[i]
+
+        # Each bar block height = label + bar + gap
+        y = bar_y_top + i * (bar_height + label_space + bar_gap)
+
+        x_right = bars_x_start + bar_width
+
+        filled = int((value / 100) * bar_width)
+
+        # Draw label above bar
+        cv2.putText(
+            panel,
+            label,
+            (bars_x_start, y),
+            cv2.FONT_HERSHEY_DUPLEX,
+            0.6,
+            (230, 230, 230),
+            1,
+            cv2.LINE_AA
+        )
+
+        # Move bar slightly down to leave space for label
+        bar_y = y + 8
+
+        # Draw background bar
+        cv2.rectangle(
+            panel,
+            (bars_x_start, bar_y),
+            (x_right, bar_y + bar_height),
+            bg_color,
+            -1
+        )
+
+        # Draw filled portion
+        cv2.rectangle(
+            panel,
+            (bars_x_start, bar_y),
+            (bars_x_start + filled, bar_y + bar_height),
+            fill_color,
+            -1
+        )
+
+        # Draw percentage text
+        cv2.putText(
+            panel,
+            f"{int(value)}%",
+            (bars_x_start + 12, bar_y + bar_height - 6),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.6,
+            (255, 255, 255),
+            2,
+            cv2.LINE_AA
+        )
+
+    return panel
+
+##Integrated Video Processing (Analysis + Annotation)
+
+def process_video_segment(video_path, output_dir, segment_id, audio_scores_global=None):
+    base_options = python.BaseOptions(model_asset_path=MODEL_PATH)
+    options = vision.FaceLandmarkerOptions(base_options=base_options, running_mode=vision.RunningMode.VIDEO, num_faces=1)
+
+    cap = cv2.VideoCapture(video_path)
+    fps, width, height = cap.get(cv2.CAP_PROP_FPS), int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    temp_video = os.path.join(output_dir, f"temp_annotated_{segment_id}.mp4")
+    # out = cv2.VideoWriter(temp_video, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
+    # Use 'avc1' or 'H264' for web compatibility
+    fourcc = cv2.VideoWriter_fourcc(*'XVID')
+    out = cv2.VideoWriter(temp_video, fourcc, fps, (width, height))
+
+    face_conf_accum, eye_accum, frame_idx = [], [], 0
+    smooth_v, smooth_a, dom_emo = 0.0, 0.0, "neutral"
+    
+    # --- Optimization Variables ---
+    frame_stride = 3  # Process AI every 3 frames
+    last_results = None
+    last_emotions = None
+    last_eye_s = 0.5
+    last_lm = None
+    # ------------------------------
+
+    b_conf = audio_scores_global.get("confidence_audio", 50)
+    b_clar = audio_scores_global.get("clarity", 50)
+    b_stress = audio_scores_global.get("stress", 20)
+
+    with vision.FaceLandmarker.create_from_options(options) as landmarker:
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            # 1. RUN HEAVY AI ONLY ON STRIDE FRAMES
+            if frame_idx % frame_stride == 0:
+                mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+                last_results = landmarker.detect_for_video(mp_image, int((frame_idx/fps)*1000))
+                
+                if last_results.face_landmarks:
+                    last_lm = last_results.face_landmarks[0]
+                    last_emotions = analyze_face_emotion(frame)
+                    last_eye_s = compute_eye_contact_ratio(frame, last_lm)
+
+            # 2. USE LAST KNOWN DATA FOR CALCULATIONS & DRAWING
+            d_conf, d_clar, d_stress = b_conf, b_clar, b_stress
+            
+            if last_results and last_results.face_landmarks:
+                # Use current local variables from 'last' successful AI run
+                curr_f_conf = (last_emotions.get("neutral", 0) + last_emotions.get("happy", 0)) * 100
+                d_conf = (b_conf * 0.7) + (curr_f_conf * 0.3)
+                d_clar = (b_clar * 0.8) + (last_eye_s * 100 * 0.2)
+                d_stress = (b_stress * 0.7) + ((last_emotions.get("sad",0)+last_emotions.get("angry",0))*30)
+
+                # Update accumulators only on stride frames to keep averages accurate
+                if frame_idx % frame_stride == 0:
+                    face_conf_accum.append(curr_f_conf)
+                    eye_accum.append(last_eye_s)
+
+                dom_emo = max(last_emotions, key=last_emotions.get)
+                v_t = sum(emotion_va[e][0]*s for e,s in last_emotions.items() if e in emotion_va)
+                a_t = sum(emotion_va[e][1]*s for e,s in last_emotions.items() if e in emotion_va)
+                
+                # Keep smoothing every frame for fluid movement
+                smooth_v += 0.15 * (v_t - smooth_v)
+                smooth_a += 0.15 * (a_t - smooth_a)
+
+                # Draw face box using the last known landmarks
+                xs, ys = [l.x*width for l in last_lm], [l.y*height for l in last_lm]
+                draw_face_box(
+                    frame,
+                    int(min(xs)), int(min(ys)),
+                    int(max(xs) - min(xs)), int(max(ys) - min(ys)),
+                    dom_emo
+                )
+
+            # 3. ALWAYS DRAW UI (Wheel and Bars)
+            frame = draw_full_emotion_wheel(frame, (width-130, height-100), 90, smooth_v, smooth_a, dom_emo)
+            frame = draw_metric_bars(frame, 30, height-160, 28, 200, 6, d_conf, d_clar, d_stress)
+            
+            out.write(frame)
+            frame_idx += 1
+
+    cap.release()
+    out.release()
+    return temp_video, np.mean(face_conf_accum) if face_conf_accum else 50, np.mean(eye_accum)*100 if eye_accum else 50
+
+##Main pipeline
+def run_intervision_pipeline(video_path, questions_config, output_dir):
+    if not os.path.exists(video_path):
+        return f"Error: Video file not found at {video_path}"
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Establish baseline from first 10s
+    try:
+        y_b, sr_b = librosa.load(video_path, sr=16000, duration=10)
+        baseline = extract_audio_features(y_b, sr_b)
+    except Exception as e:
+        print(f"Baseline Load Warning: {e}. Using defaults.")
+        baseline = None
+
+    final_reports, segments = [], []
+
+    for q in questions_config:
+        q_id = q['question_id']
+        raw_seg = os.path.join(output_dir, f"q{q_id}_raw.mp4")
+        wav_p = os.path.join(output_dir, f"q{q_id}.wav")
+
+        # Precise FFmpeg cutting with error handling
+        duration = q["end_time"] - q["start_time"]
+        try:
+            subprocess.run([
+                'ffmpeg', '-y', '-ss', str(q["start_time"]), '-t', str(duration),
+                '-i', video_path, '-c:v', 'libx264', '-c:a', 'aac', '-strict', 'experimental', raw_seg
+            ], check=True, capture_output=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Skipping Question {q_id}: Time range might be out of video bounds.")
+            continue
+
+        # Audio Extraction
+        try:
+            y, sr = librosa.load(raw_seg, sr=16000)
+            import soundfile as sf
+            sf.write(wav_p, y, sr)
+        except Exception as e:
+            print(f"Error extracting audio for Q{q_id}: {e}")
+            continue
+
+        # Audio Analysis
+        a_scores = compute_audio_scores(extract_audio_features(y, sr), baseline)
+
+        # Whisper Transcription
+        try:
+            transcription_data = asr(wav_p, chunk_length_s=30, return_timestamps=True)
+            transcription = transcription_data["text"].strip()
+        except:
+            transcription = "[Transcription Error]"
+
+        similarity_score = compute_similarity_score(transcription, q["ideal_answer"])
+        relevance_score = compute_relevance_score(q["question_text"], transcription)
+
+        # Visual Analysis
+        try:
+            ann_v, f_c, e_c = process_video_segment(raw_seg, output_dir, q_id, a_scores)
+
+            final_v = os.path.join(output_dir, f"q{q_id}_final.mp4")
+            subprocess.run([
+                'ffmpeg', '-y', '-i', ann_v, '-i', raw_seg, '-map', '0:v', '-map', '1:a',
+                '-c:v', 'copy', '-c:a', 'aac', final_v
+            ], check=True, capture_output=True)
+
+            segments.append(final_v)
+
+            final_reports.append({
+                "questionId": q_id,
+                "userAnswerText": transcription,
+                "toneOfVoice": a_scores["tone_of_voice"],
+                "clarity": a_scores["clarity"],
+                "stress": a_scores["stress"],
+                "confidence": round((a_scores["confidence_audio"] + f_c + e_c) / 3, 2),
+                "pauses": a_scores["pauses"],
+                "score": similarity_score,
+                "relevance": relevance_score
+            })
+        except Exception as e:
+            print(f"Visual analysis failed for Q{q_id}: {e}")
+
+        torch.cuda.empty_cache()
+
+    # Final concatenation
+    if segments:
+        list_path = os.path.join(output_dir, "list.txt")
+        with open(list_path, "w") as f:
+            for s in segments:
+                f.write(f"file '{os.path.abspath(s)}'\n")
+
+        final_output = os.path.join(output_dir, "Intervision_Final_Result.mp4")
+        os.system(f"ffmpeg -f concat -safe 0 -i {list_path} -c:v libx264 -preset superfast -crf 23 -c:a aac -y {final_output}")
+
+        with open(os.path.join(output_dir, "report.json"), "w") as f:
+            json.dump({"listOfAnswerReport": final_reports}, f, indent=4)
+
+        return f"Successfully processed {len(segments)} questions."
+    else:
+        return "No segments were processed. Check your video time ranges."

requirements.txt

@@ -0,0 +1,23 @@
+fastapi
+uvicorn
+python-multipart
+ffmpeg-python
+transformers
+timm
+mediapipe
+deepface
+sentence-transformers
+librosa
+torchaudio
+opencv-python
+numpy
+soundfile
+pillow
+scikit-learn
+ffmpeg
+python-multipart
+cloudinary 
+requests
+python-dotenv
+opencv-python-headless
+openai-whisper