Spaces:
Sleeping
Sleeping
Update main.py
Browse files
main.py
CHANGED
|
@@ -1,58 +1,44 @@
|
|
| 1 |
-
# main.py β AI Interview Confidence Analyzer
|
| 2 |
-
# Combines: Face Landmarks + Expression + Eye Contact + Head Pose + STT + LLM + TTS
|
| 3 |
-
# Phase 1: Full interview loop β STT + LLM eval + follow-up questions
|
| 4 |
|
| 5 |
import cv2
|
| 6 |
import sys
|
| 7 |
import os
|
| 8 |
import time
|
| 9 |
-
import threading
|
| 10 |
import numpy as np
|
| 11 |
from collections import deque
|
|
|
|
|
|
|
| 12 |
|
| 13 |
-
# Add modules folder to path
|
| 14 |
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "modules"))
|
| 15 |
|
| 16 |
from modules.face_landmarks import FaceLandmarkExtractor
|
| 17 |
from modules.expression_detection import ExpressionDetector
|
| 18 |
from modules.eye_contact import EyeContactDetector
|
| 19 |
from modules.head_pose import HeadPoseEstimator
|
| 20 |
-
from modules.stt import transcribe
|
| 21 |
-
from modules.tts import speak
|
| 22 |
from modules.llm import (
|
| 23 |
generate_questions,
|
| 24 |
-
generate_followup,
|
| 25 |
evaluate_answer,
|
| 26 |
generate_final_summary,
|
| 27 |
-
FIRST_QUESTION,
|
| 28 |
-
LAST_QUESTION,
|
| 29 |
)
|
| 30 |
from modules.resume_parser import resume_to_profile, get_resume_context_for_llm
|
| 31 |
|
| 32 |
-
|
| 33 |
-
# CONFIDENCE SCORE WEIGHTS (must sum to 100)
|
| 34 |
-
|
| 35 |
-
|
| 36 |
WEIGHTS = {
|
| 37 |
"eye_contact": 30,
|
| 38 |
"expression": 25,
|
| 39 |
"head_stability": 25,
|
| 40 |
"nervousness": 20,
|
| 41 |
}
|
| 42 |
-
|
| 43 |
-
SCORE_HISTORY_LEN = 45 # ~1.5 seconds at 30fps
|
| 44 |
|
| 45 |
|
| 46 |
-
#
|
| 47 |
-
|
| 48 |
def compute_confidence_score(expr_result, eye_result, head_result):
|
| 49 |
eye_score = eye_result.get("score", 0)
|
| 50 |
expr_score = expr_result.get("expression_score", 0)
|
| 51 |
head_score = head_result.get("stability_score", 0)
|
| 52 |
nerv_score = expr_result.get("nervousness_score", 0)
|
| 53 |
-
|
| 54 |
nerv_contribution = max(0, 100 - nerv_score)
|
| 55 |
-
|
| 56 |
final = (
|
| 57 |
eye_score * WEIGHTS["eye_contact"] / 100 +
|
| 58 |
expr_score * WEIGHTS["expression"] / 100 +
|
|
@@ -63,586 +49,335 @@ def compute_confidence_score(expr_result, eye_result, head_result):
|
|
| 63 |
|
| 64 |
|
| 65 |
def confidence_label(score):
|
| 66 |
-
if score >= 80: return "High",
|
| 67 |
-
if score >= 60: return "Moderate",
|
| 68 |
-
if score >= 40: return "Low",
|
| 69 |
-
return "Very Low",
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
# DRAW DASHBOARD PANEL
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
def draw_dashboard(frame, expr_result, eye_result, head_result,
|
| 77 |
-
confidence, score_history, current_question="", status=""):
|
| 78 |
-
h, w = frame.shape[:2]
|
| 79 |
-
|
| 80 |
-
panel_x = w - 260
|
| 81 |
-
overlay = frame.copy()
|
| 82 |
-
cv2.rectangle(overlay, (panel_x - 10, 0), (w, h), (20, 20, 20), -1)
|
| 83 |
-
cv2.addWeighted(overlay, 0.55, frame, 0.45, 0, frame)
|
| 84 |
-
|
| 85 |
-
x = panel_x
|
| 86 |
-
y = 30
|
| 87 |
-
dy = 28
|
| 88 |
-
|
| 89 |
-
def put(text, color=(220, 220, 220), scale=0.55, bold=1):
|
| 90 |
-
nonlocal y
|
| 91 |
-
cv2.putText(frame, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, scale, color, bold)
|
| 92 |
-
y += dy
|
| 93 |
-
|
| 94 |
-
put("CONFIDENCE ANALYZER", (255, 255, 255), 0.55, 2)
|
| 95 |
-
put("-" * 28, (80, 80, 80))
|
| 96 |
-
|
| 97 |
-
label, color = confidence_label(confidence)
|
| 98 |
-
put(f"SCORE: {confidence}/100", color, 0.75, 2)
|
| 99 |
-
put(f"Level: {label}", color)
|
| 100 |
-
|
| 101 |
-
bar_w = int((confidence / 100) * 230)
|
| 102 |
-
cv2.rectangle(frame, (x, y), (x + 230, y + 12), (60, 60, 60), -1)
|
| 103 |
-
cv2.rectangle(frame, (x, y), (x + bar_w, y + 12), color, -1)
|
| 104 |
-
y += 22
|
| 105 |
-
put("-" * 28, (80, 80, 80))
|
| 106 |
-
|
| 107 |
-
expr = expr_result.get("expression", "N/A")
|
| 108 |
-
nerv = expr_result.get("nervousness_score", 0)
|
| 109 |
-
blink_r = expr_result.get("blink_rate", 0)
|
| 110 |
-
expr_col = (0, 220, 0) if expr == "Happy" else (0, 165, 255) if nerv > 40 else (220, 220, 220)
|
| 111 |
-
put("EXPRESSION", (180, 180, 255), 0.5, 1)
|
| 112 |
-
put(f" {expr}", expr_col)
|
| 113 |
-
put(f" Nerv: {nerv}/100 Blink:{blink_r}/m")
|
| 114 |
-
put("-" * 28, (80, 80, 80))
|
| 115 |
-
|
| 116 |
-
gaze = eye_result.get("gaze_direction", "N/A")
|
| 117 |
-
eye_pct = eye_result.get("eye_contact_pct", 0)
|
| 118 |
-
eye_col = (0, 220, 0) if gaze == "Center" else (0, 165, 255)
|
| 119 |
-
put("EYE CONTACT", (180, 180, 255), 0.5, 1)
|
| 120 |
-
put(f" Gaze: {gaze}", eye_col)
|
| 121 |
-
put(f" Contact: {eye_pct}%")
|
| 122 |
-
put("-" * 28, (80, 80, 80))
|
| 123 |
-
|
| 124 |
-
direction = head_result.get("direction", "N/A")
|
| 125 |
-
stability = head_result.get("stability_score", 0)
|
| 126 |
-
head_col = (0, 220, 0) if direction == "Forward" else (0, 165, 255)
|
| 127 |
-
put("HEAD POSE", (180, 180, 255), 0.5, 1)
|
| 128 |
-
put(f" Dir: {direction}", head_col)
|
| 129 |
-
put(f" Stability: {stability}/100")
|
| 130 |
-
pitch = head_result.get("pitch", 0)
|
| 131 |
-
yaw = head_result.get("yaw", 0)
|
| 132 |
-
put(f" P:{pitch:.1f} Y:{yaw:.1f}")
|
| 133 |
-
put("-" * 28, (80, 80, 80))
|
| 134 |
-
|
| 135 |
-
put("SCORE TREND", (180, 180, 255), 0.5, 1)
|
| 136 |
-
if len(score_history) > 1:
|
| 137 |
-
pts = list(score_history)
|
| 138 |
-
gx, gy, gw, gh = x, y, 230, 50
|
| 139 |
-
cv2.rectangle(frame, (gx, gy), (gx + gw, gy + gh), (40, 40, 40), -1)
|
| 140 |
-
for i in range(1, len(pts)):
|
| 141 |
-
x1 = gx + int((i - 1) / (SCORE_HISTORY_LEN - 1) * gw)
|
| 142 |
-
x2 = gx + int(i / (SCORE_HISTORY_LEN - 1) * gw)
|
| 143 |
-
y1 = gy + gh - int(pts[i - 1] / 100 * gh)
|
| 144 |
-
y2 = gy + gh - int(pts[i] / 100 * gh)
|
| 145 |
-
cv2.line(frame, (x1, y1), (x2, y2), (0, 200, 100), 1)
|
| 146 |
-
y += gh + 8
|
| 147 |
-
|
| 148 |
-
put("-" * 28, (80, 80, 80))
|
| 149 |
-
put("TIPS", (180, 180, 255), 0.5, 1)
|
| 150 |
-
if gaze != "Center":
|
| 151 |
-
put(" Look at the camera", (0, 200, 255), 0.45)
|
| 152 |
-
if nerv > 50:
|
| 153 |
-
put(" Breathe, slow down", (0, 200, 255), 0.45)
|
| 154 |
-
if direction != "Forward":
|
| 155 |
-
put(" Face forward", (0, 200, 255), 0.45)
|
| 156 |
-
if nerv <= 50 and gaze == "Center" and direction == "Forward":
|
| 157 |
-
put(" Great job! Keep it up", (0, 220, 0), 0.45)
|
| 158 |
-
|
| 159 |
-
# ββ Interview status overlay (bottom of frame) ββ
|
| 160 |
-
if current_question:
|
| 161 |
-
words = current_question.split()
|
| 162 |
-
max_chars = 55
|
| 163 |
-
line1 = ""
|
| 164 |
-
line2 = ""
|
| 165 |
-
for word in words:
|
| 166 |
-
if len(line1) + len(word) + 1 <= max_chars:
|
| 167 |
-
line1 += (" " if line1 else "") + word
|
| 168 |
-
else:
|
| 169 |
-
line2 += (" " if line2 else "") + word
|
| 170 |
-
|
| 171 |
-
cv2.rectangle(frame, (0, h - 75), (panel_x - 20, h), (20, 20, 20), -1)
|
| 172 |
-
cv2.putText(frame, "Q: " + line1, (10, h - 52),
|
| 173 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 220, 255), 1)
|
| 174 |
-
if line2:
|
| 175 |
-
cv2.putText(frame, " " + line2, (10, h - 30),
|
| 176 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 220, 255), 1)
|
| 177 |
-
|
| 178 |
-
if status:
|
| 179 |
-
cv2.putText(frame, status, (10, h - 10),
|
| 180 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 200, 0), 1)
|
| 181 |
-
|
| 182 |
-
return frame
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
# SESSION SUMMARY
|
| 186 |
-
|
| 187 |
-
def print_summary(score_history, expr_counts, total_frames, duration_sec,
|
| 188 |
-
interview_results=None):
|
| 189 |
-
if not score_history:
|
| 190 |
-
print("\n[INFO] No data recorded.")
|
| 191 |
-
return
|
| 192 |
-
|
| 193 |
-
avg_score = int(np.mean(score_history))
|
| 194 |
-
max_score = int(np.max(score_history))
|
| 195 |
-
min_score = int(np.min(score_history))
|
| 196 |
-
label, _ = confidence_label(avg_score)
|
| 197 |
-
|
| 198 |
-
print("\n" + "=" * 50)
|
| 199 |
-
print(" SESSION SUMMARY")
|
| 200 |
-
print("=" * 50)
|
| 201 |
-
print(f" Duration : {duration_sec:.1f} seconds")
|
| 202 |
-
print(f" Frames Analyzed : {total_frames}")
|
| 203 |
-
print(f" Avg Score : {avg_score}/100 [{label}]")
|
| 204 |
-
print(f" Peak Score : {max_score}/100")
|
| 205 |
-
print(f" Lowest Score : {min_score}/100")
|
| 206 |
-
print("\n Expression Breakdown:")
|
| 207 |
-
total_expr = sum(expr_counts.values()) or 1
|
| 208 |
-
for expr, count in sorted(expr_counts.items(), key=lambda x: -x[1]):
|
| 209 |
-
pct = int(count / total_expr * 100)
|
| 210 |
-
print(f" {expr:<16} {pct:>3}% {'#' * (pct // 5)}")
|
| 211 |
-
|
| 212 |
-
if interview_results:
|
| 213 |
-
print("\n" + "=" * 50)
|
| 214 |
-
print(" INTERVIEW Q&A SUMMARY")
|
| 215 |
-
print("=" * 50)
|
| 216 |
-
for i, r in enumerate(interview_results, 1):
|
| 217 |
-
fb = r.get('feedback', {})
|
| 218 |
-
if isinstance(fb, dict):
|
| 219 |
-
score_str = fb.get('score_str', '?')
|
| 220 |
-
strength = fb.get('strength', '')
|
| 221 |
-
improve = fb.get('improvement', '')
|
| 222 |
-
vis_conf = fb.get('visual_confidence', '?')
|
| 223 |
-
else:
|
| 224 |
-
score_str = str(fb)
|
| 225 |
-
strength = improve = vis_conf = ''
|
| 226 |
-
|
| 227 |
-
print(f"\n Q{i}: {r['question']}")
|
| 228 |
-
print(f" Answer : {r['answer'][:200]}")
|
| 229 |
-
if r.get('followup'):
|
| 230 |
-
print(f" Follow-up: {r['followup']}")
|
| 231 |
-
print(f" FU Answer: {r.get('followup_answer','')[:120]}")
|
| 232 |
-
print(f" AI Score : {score_str} | Visual Conf: {vis_conf}/100")
|
| 233 |
-
print(f" Strength : {strength}")
|
| 234 |
-
print(f" Improve : {improve}")
|
| 235 |
-
print(" " + "-" * 46)
|
| 236 |
-
|
| 237 |
-
print("\n OVERALL AI FEEDBACK:")
|
| 238 |
-
try:
|
| 239 |
-
last_role = interview_results[-1].get('job_role', 'the role')
|
| 240 |
-
summary = generate_final_summary(interview_results, last_role)
|
| 241 |
-
print(f" Overall Score : {summary.get('overall_score_str','')}")
|
| 242 |
-
print(f" Top Strength : {summary.get('top_strength','')}")
|
| 243 |
-
print(f" Top Improvement : {summary.get('top_area_to_improve','')}")
|
| 244 |
-
print(f" Weak Topics : {', '.join(summary.get('weak_topics',[]))}")
|
| 245 |
-
print(f" Final Tip : {summary.get('final_tip','')}")
|
| 246 |
-
except Exception as e:
|
| 247 |
-
print(f" [WARN] Summary generation failed: {e}")
|
| 248 |
-
|
| 249 |
-
print("\n Improvement Tips:")
|
| 250 |
-
if avg_score < 40:
|
| 251 |
-
print(" - Practice maintaining eye contact")
|
| 252 |
-
print(" - Work on reducing visible nervousness")
|
| 253 |
-
print(" - Keep your head stable and facing forward")
|
| 254 |
-
elif avg_score < 70:
|
| 255 |
-
print(" - Good effort β focus on eye contact consistency")
|
| 256 |
-
print(" - Try to appear more relaxed")
|
| 257 |
-
else:
|
| 258 |
-
print(" - Strong performance!")
|
| 259 |
-
print(" - Keep practicing to maintain consistency")
|
| 260 |
-
print("=" * 50)
|
| 261 |
|
| 262 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 263 |
|
| 264 |
-
|
| 265 |
|
|
|
|
|
|
|
|
|
|
| 266 |
|
| 267 |
-
|
| 268 |
-
""
|
| 269 |
-
Returns a full profile dict with keys:
|
| 270 |
-
name, job_role, experience, skills, projects, education, summary, resume_text
|
| 271 |
-
"""
|
| 272 |
-
print("\n" + "=" * 50)
|
| 273 |
-
print(" INTERVIEW SETUP")
|
| 274 |
-
print("=" * 50)
|
| 275 |
-
print(" [1] Upload Resume (PDF / DOCX / TXT / MD) β auto fill")
|
| 276 |
-
print(" [2] Enter manually")
|
| 277 |
-
print("=" * 50)
|
| 278 |
-
choice = input(" Choice: ").strip()
|
| 279 |
-
|
| 280 |
-
if choice == "1":
|
| 281 |
-
path = input(" Resume path: ").strip().strip('"')
|
| 282 |
-
print(" Parsing resume...")
|
| 283 |
-
try:
|
| 284 |
-
profile = resume_to_profile(path)
|
| 285 |
-
print(f"\n Parsed Profile:")
|
| 286 |
-
print(f" Name : {profile['name']}")
|
| 287 |
-
print(f" Role : {profile['job_role']}")
|
| 288 |
-
print(f" Experience : {profile['experience']}")
|
| 289 |
-
print(f" Skills : {profile['skills']}")
|
| 290 |
-
if profile.get('projects'):
|
| 291 |
-
print(f" Projects : {', '.join(profile['projects'][:3])}")
|
| 292 |
-
if profile.get('education'):
|
| 293 |
-
print(f" Education : {profile['education']}")
|
| 294 |
-
confirm = input("\n Looks good? (y/n): ").strip().lower()
|
| 295 |
-
if confirm == 'y':
|
| 296 |
-
return profile
|
| 297 |
-
except Exception as e:
|
| 298 |
-
print(f" [WARN] Resume parse failed: {e}. Falling back to manual entry.")
|
| 299 |
-
|
| 300 |
-
# manual fallback β build the same dict shape
|
| 301 |
-
name = input(" Your Name : ").strip()
|
| 302 |
-
job_role = input(" Job Role : ").strip()
|
| 303 |
-
experience = input(" Experience : ").strip()
|
| 304 |
-
skills = input(" Skills : ").strip()
|
| 305 |
-
print("\n Resume text (paste, press Enter twice when done):")
|
| 306 |
-
lines = []
|
| 307 |
-
while True:
|
| 308 |
-
line = input()
|
| 309 |
-
if line == "":
|
| 310 |
-
break
|
| 311 |
-
lines.append(line)
|
| 312 |
-
return {
|
| 313 |
-
'name': name,
|
| 314 |
-
'job_role': job_role,
|
| 315 |
-
'experience': experience,
|
| 316 |
-
'skills': skills,
|
| 317 |
-
'projects': [],
|
| 318 |
-
'education': '',
|
| 319 |
-
'summary': '',
|
| 320 |
-
'resume_text': '\n'.join(lines),
|
| 321 |
-
}
|
| 322 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
|
| 324 |
-
|
| 325 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 326 |
|
| 327 |
-
def run_live_session():
|
| 328 |
-
print("\n" + "=" * 50)
|
| 329 |
-
print(" AI INTERVIEW CONFIDENCE ANALYZER")
|
| 330 |
-
print("=" * 50)
|
| 331 |
-
print(" Press Q to quit and see summary")
|
| 332 |
-
print(" Press S to take a snapshot")
|
| 333 |
-
print("=" * 50 + "\n")
|
| 334 |
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
experience = profile['experience']
|
| 339 |
-
skills = profile['skills']
|
| 340 |
-
resume_ctx = get_resume_context_for_llm(profile) # rich context string
|
| 341 |
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
name, job_role, experience, skills,
|
| 345 |
-
resume_text=resume_ctx,
|
| 346 |
-
num_questions=2,
|
| 347 |
-
)
|
| 348 |
-
print(f"[INFO] {len(questions)} questions ready.\n")
|
| 349 |
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
print('[ERROR] FaceLandmarkExtractor not enabled. Check MediaPipe.')
|
| 353 |
-
return
|
| 354 |
-
expr_detector = ExpressionDetector(fps=30)
|
| 355 |
-
eye_detector = EyeContactDetector()
|
| 356 |
-
head_estimator = HeadPoseEstimator()
|
| 357 |
-
|
| 358 |
-
cap = cv2.VideoCapture(0)
|
| 359 |
-
if not cap.isOpened():
|
| 360 |
-
print("[ERROR] Cannot open webcam."); return
|
| 361 |
-
|
| 362 |
-
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
| 363 |
-
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
| 364 |
-
|
| 365 |
-
score_history = deque(maxlen=SCORE_HISTORY_LEN)
|
| 366 |
-
all_scores = []
|
| 367 |
-
expr_counts = {}
|
| 368 |
-
total_frames = 0
|
| 369 |
-
start_time = time.time()
|
| 370 |
-
snapshot_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "outputs")
|
| 371 |
-
os.makedirs(snapshot_dir, exist_ok=True)
|
| 372 |
-
|
| 373 |
-
empty_expr = {"expression": "N/A", "nervousness_score": 0,
|
| 374 |
-
"expression_score": 0, "blink_rate": 0}
|
| 375 |
-
empty_eye = {"gaze_direction": "N/A", "eye_contact_pct": 0, "score": 0}
|
| 376 |
-
empty_head = {"direction": "N/A", "stability_score": 0, "pitch": 0.0, "yaw": 0.0}
|
| 377 |
-
|
| 378 |
-
# ββ Interview state ββ
|
| 379 |
-
interview_results = []
|
| 380 |
-
current_q_idx = 0
|
| 381 |
-
current_question = questions[current_q_idx]
|
| 382 |
-
status_text = "Listening... (speak your answer)"
|
| 383 |
-
answer_scores = [] # confidence scores during this answer
|
| 384 |
-
answer_start_time = time.time()
|
| 385 |
-
ANSWER_DURATION = 45 # seconds per answer
|
| 386 |
-
|
| 387 |
-
# Speak first question in background thread
|
| 388 |
-
threading.Thread(
|
| 389 |
-
target=speak,
|
| 390 |
-
args=(f"Welcome {name}. Question 1. {current_question}",),
|
| 391 |
-
daemon=True
|
| 392 |
-
).start()
|
| 393 |
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
if not ret:
|
| 397 |
-
print("[ERROR] Failed to read webcam frame."); break
|
| 398 |
|
| 399 |
-
|
| 400 |
-
lm_result = landmark_extractor.extract(frame)
|
| 401 |
-
disp = lm_result["annotated_frame"].copy()
|
| 402 |
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 406 |
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 411 |
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
answer_scores.append(confidence)
|
| 415 |
|
| 416 |
-
|
| 417 |
-
expr_counts[expr] = expr_counts.get(expr, 0) + 1
|
| 418 |
|
| 419 |
-
disp = draw_dashboard(disp, expr_result, eye_result, head_result,
|
| 420 |
-
confidence, score_history,
|
| 421 |
-
current_question, status_text)
|
| 422 |
-
else:
|
| 423 |
-
cv2.putText(disp, "No face detected β position yourself in frame",
|
| 424 |
-
(10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 100, 255), 2)
|
| 425 |
-
disp = draw_dashboard(disp, empty_expr, empty_eye, empty_head,
|
| 426 |
-
0, score_history, current_question, status_text)
|
| 427 |
-
|
| 428 |
-
# ββ Timer per answer ββ
|
| 429 |
-
elapsed_answer = time.time() - answer_start_time
|
| 430 |
-
remaining = max(0, int(ANSWER_DURATION - elapsed_answer))
|
| 431 |
-
cv2.putText(disp, f"Answer time left: {remaining}s", (10, 50),
|
| 432 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.55, (200, 200, 0), 1)
|
| 433 |
-
|
| 434 |
-
# ββ Move to next question when time is up ββ
|
| 435 |
-
if elapsed_answer >= ANSWER_DURATION and current_q_idx < len(questions):
|
| 436 |
-
status_text = "Transcribing your answer..."
|
| 437 |
-
cv2.imshow("AI Interview Analyzer (Q=quit S=snapshot)", disp)
|
| 438 |
-
cv2.waitKey(1)
|
| 439 |
-
|
| 440 |
-
# ββ 1. Transcribe answer via STT ββββββββββββββββββββββββββββββββββ
|
| 441 |
-
avg_conf = int(np.mean(answer_scores)) if answer_scores else 0
|
| 442 |
-
try:
|
| 443 |
-
transcribed_answer = transcribe() # records mic and returns text
|
| 444 |
-
except Exception as e:
|
| 445 |
-
print(f"[WARN] STT failed: {e}")
|
| 446 |
-
transcribed_answer = f"[STT unavailable β visual confidence: {avg_conf}/100]"
|
| 447 |
-
|
| 448 |
-
status_text = "Evaluating with AI..."
|
| 449 |
-
cv2.imshow("AI Interview Analyzer (Q=quit S=snapshot)", disp)
|
| 450 |
-
cv2.waitKey(1)
|
| 451 |
-
|
| 452 |
-
# ββ 2. Check if a follow-up question is warranted βββββββββββββββββ
|
| 453 |
-
followup_q = None
|
| 454 |
-
followup_answer = ""
|
| 455 |
-
if transcribed_answer and not transcribed_answer.startswith('['):
|
| 456 |
-
try:
|
| 457 |
-
followup_q = generate_followup(current_question, transcribed_answer, job_role)
|
| 458 |
-
except Exception as e:
|
| 459 |
-
print(f"[WARN] Follow-up generation failed: {e}")
|
| 460 |
-
|
| 461 |
-
if followup_q:
|
| 462 |
-
status_text = f"Follow-up: {followup_q[:60]}..."
|
| 463 |
-
threading.Thread(
|
| 464 |
-
target=speak,
|
| 465 |
-
args=(f"Follow-up: {followup_q}",),
|
| 466 |
-
daemon=True
|
| 467 |
-
).start()
|
| 468 |
-
# Give candidate FOLLOW_UP_DURATION seconds to answer follow-up
|
| 469 |
-
FOLLOW_UP_DURATION = 30
|
| 470 |
-
fu_start = time.time()
|
| 471 |
-
while time.time() - fu_start < FOLLOW_UP_DURATION:
|
| 472 |
-
ret2, frame2 = cap.read()
|
| 473 |
-
if ret2:
|
| 474 |
-
fu_remaining = max(0, int(FOLLOW_UP_DURATION - (time.time() - fu_start)))
|
| 475 |
-
cv2.putText(frame2,
|
| 476 |
-
f"Follow-up time left: {fu_remaining}s",
|
| 477 |
-
(10, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (200, 200, 0), 1)
|
| 478 |
-
cv2.putText(frame2,
|
| 479 |
-
f"Follow-up: {followup_q[:70]}",
|
| 480 |
-
(10, frame2.shape[0] - 20),
|
| 481 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 220, 255), 1)
|
| 482 |
-
cv2.imshow("AI Interview Analyzer (Q=quit S=snapshot)", frame2)
|
| 483 |
-
cv2.waitKey(1)
|
| 484 |
-
try:
|
| 485 |
-
followup_answer = transcribe()
|
| 486 |
-
except Exception:
|
| 487 |
-
followup_answer = ""
|
| 488 |
-
|
| 489 |
-
# ββ 3. LLM evaluation ββββββββββββββββββββββββββββββββββββββββββββ
|
| 490 |
-
try:
|
| 491 |
-
feedback = evaluate_answer(
|
| 492 |
-
question=current_question,
|
| 493 |
-
answer=transcribed_answer,
|
| 494 |
-
job_role=job_role,
|
| 495 |
-
followup=followup_q or '',
|
| 496 |
-
followup_answer=followup_answer,
|
| 497 |
-
)
|
| 498 |
-
except Exception as e:
|
| 499 |
-
print(f"[WARN] Evaluation failed: {e}")
|
| 500 |
-
feedback = {
|
| 501 |
-
'score': avg_conf // 10,
|
| 502 |
-
'score_str': f"{avg_conf // 10}/10",
|
| 503 |
-
'strength': 'Answer recorded.',
|
| 504 |
-
'improvement': 'AI evaluation unavailable.',
|
| 505 |
-
'detail': '',
|
| 506 |
-
'raw': '',
|
| 507 |
-
}
|
| 508 |
-
|
| 509 |
-
# Append visual confidence as extra context
|
| 510 |
-
feedback['visual_confidence'] = avg_conf
|
| 511 |
-
|
| 512 |
-
interview_results.append({
|
| 513 |
-
'question': current_question,
|
| 514 |
-
'answer': transcribed_answer,
|
| 515 |
-
'followup': followup_q,
|
| 516 |
-
'followup_answer': followup_answer,
|
| 517 |
-
'feedback': feedback,
|
| 518 |
-
'job_role': job_role,
|
| 519 |
-
})
|
| 520 |
-
|
| 521 |
-
# Print quick feedback to terminal
|
| 522 |
-
print(f"\n[EVAL] Q: {current_question}")
|
| 523 |
-
print(f"[EVAL] A: {transcribed_answer[:120]}...")
|
| 524 |
-
print(f"[EVAL] Score: {feedback.get('score_str','?')} | "
|
| 525 |
-
f"{feedback.get('strength','')}")
|
| 526 |
-
|
| 527 |
-
answer_scores = []
|
| 528 |
-
current_q_idx += 1
|
| 529 |
-
answer_start_time = time.time()
|
| 530 |
-
|
| 531 |
-
if current_q_idx < len(questions):
|
| 532 |
-
current_question = questions[current_q_idx]
|
| 533 |
-
status_text = "Listening... (speak your answer)"
|
| 534 |
-
threading.Thread(
|
| 535 |
-
target=speak,
|
| 536 |
-
args=(f"Question {current_q_idx + 1}. {current_question}",),
|
| 537 |
-
daemon=True
|
| 538 |
-
).start()
|
| 539 |
-
else:
|
| 540 |
-
status_text = "Interview complete! Press Q to see summary."
|
| 541 |
-
threading.Thread(
|
| 542 |
-
target=speak,
|
| 543 |
-
args=("Interview complete. Great job! Press Q to see your summary.",),
|
| 544 |
-
daemon=True
|
| 545 |
-
).start()
|
| 546 |
-
|
| 547 |
-
# ββ Elapsed session timer ββ
|
| 548 |
-
elapsed = int(time.time() - start_time)
|
| 549 |
-
cv2.putText(disp, f"{elapsed//60:02d}:{elapsed%60:02d}", (10, 22),
|
| 550 |
-
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (180, 180, 180), 1)
|
| 551 |
-
|
| 552 |
-
cv2.imshow("AI Interview Analyzer (Q=quit S=snapshot)", disp)
|
| 553 |
-
|
| 554 |
-
key = cv2.waitKey(1) & 0xFF
|
| 555 |
-
if key == ord("q"):
|
| 556 |
-
break
|
| 557 |
-
elif key == ord("s"):
|
| 558 |
-
ts = time.strftime("%Y%m%d_%H%M%S")
|
| 559 |
-
path = os.path.join(snapshot_dir, f"snapshot_{ts}.jpg")
|
| 560 |
-
cv2.imwrite(path, disp)
|
| 561 |
-
print(f"[INFO] Snapshot saved: {path}")
|
| 562 |
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
|
|
|
| 566 |
|
| 567 |
-
|
| 568 |
-
|
| 569 |
|
|
|
|
|
|
|
|
|
|
| 570 |
|
|
|
|
|
|
|
| 571 |
|
| 572 |
-
# MAIN β IMAGE TEST (unchanged)
|
| 573 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 574 |
|
| 575 |
-
|
| 576 |
-
|
|
|
|
| 577 |
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
|
|
|
|
|
|
| 581 |
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 586 |
|
| 587 |
-
|
| 588 |
|
| 589 |
-
if not
|
| 590 |
-
|
| 591 |
|
| 592 |
-
|
| 593 |
-
|
|
|
|
|
|
|
| 594 |
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
confidence = compute_confidence_score(expr_result, eye_result, head_result)
|
| 599 |
-
label, _ = confidence_label(confidence)
|
| 600 |
-
|
| 601 |
-
print("\n" + "=" * 50)
|
| 602 |
-
print(" ANALYSIS RESULT")
|
| 603 |
-
print("=" * 50)
|
| 604 |
-
print(f" Confidence Score : {confidence}/100 [{label}]")
|
| 605 |
-
print(f" Expression : {expr_result['expression']}")
|
| 606 |
-
print(f" Nervousness : {expr_result['nervousness_score']}/100")
|
| 607 |
-
print(f" Gaze : {eye_result['gaze_direction']}")
|
| 608 |
-
print(f" Eye Contact : {eye_result['eye_contact_pct']}%")
|
| 609 |
-
print(f" Head Direction : {head_result['direction']}")
|
| 610 |
-
print(f" Head Stability : {head_result['stability_score']}/100")
|
| 611 |
-
print("=" * 50)
|
| 612 |
-
|
| 613 |
-
disp = draw_dashboard(
|
| 614 |
-
lm_result["annotated_frame"].copy(),
|
| 615 |
-
expr_result, eye_result, head_result,
|
| 616 |
-
confidence, deque([confidence])
|
| 617 |
)
|
| 618 |
-
cv2.imshow("AI Interview Analyzer - Image Test (any key to close)", disp)
|
| 619 |
-
cv2.waitKey(0)
|
| 620 |
-
cv2.destroyAllWindows()
|
| 621 |
-
landmark_extractor.release()
|
| 622 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 623 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 624 |
|
| 625 |
-
# ENTRY POINT
|
| 626 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 627 |
|
| 628 |
if __name__ == "__main__":
|
| 629 |
-
|
| 630 |
-
run_image_test(sys.argv[2]); sys.exit(0)
|
| 631 |
-
elif len(sys.argv) >= 2 and sys.argv[1] == "--live":
|
| 632 |
-
run_live_session(); sys.exit(0)
|
| 633 |
-
|
| 634 |
-
print("\n" + "=" * 50)
|
| 635 |
-
print(" AI INTERVIEW CONFIDENCE ANALYZER")
|
| 636 |
-
print("=" * 50)
|
| 637 |
-
print(" [1] Live interview session (with AI questions)")
|
| 638 |
-
print(" [2] Test on image")
|
| 639 |
-
print("=" * 50)
|
| 640 |
-
choice = input(" Choice (1 or 2): ").strip()
|
| 641 |
-
|
| 642 |
-
if choice == "1":
|
| 643 |
-
run_live_session()
|
| 644 |
-
elif choice == "2":
|
| 645 |
-
path = input(" Image path: ").strip().strip('"')
|
| 646 |
-
run_image_test(path)
|
| 647 |
-
else:
|
| 648 |
-
print(" Invalid choice.")
|
|
|
|
| 1 |
+
# main.py β AI Interview Confidence Analyzer (Gradio / HuggingFace Spaces)
|
|
|
|
|
|
|
| 2 |
|
| 3 |
import cv2
|
| 4 |
import sys
|
| 5 |
import os
|
| 6 |
import time
|
|
|
|
| 7 |
import numpy as np
|
| 8 |
from collections import deque
|
| 9 |
+
import gradio as gr
|
| 10 |
+
import tempfile
|
| 11 |
|
|
|
|
| 12 |
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "modules"))
|
| 13 |
|
| 14 |
from modules.face_landmarks import FaceLandmarkExtractor
|
| 15 |
from modules.expression_detection import ExpressionDetector
|
| 16 |
from modules.eye_contact import EyeContactDetector
|
| 17 |
from modules.head_pose import HeadPoseEstimator
|
|
|
|
|
|
|
| 18 |
from modules.llm import (
|
| 19 |
generate_questions,
|
|
|
|
| 20 |
evaluate_answer,
|
| 21 |
generate_final_summary,
|
|
|
|
|
|
|
| 22 |
)
|
| 23 |
from modules.resume_parser import resume_to_profile, get_resume_context_for_llm
|
| 24 |
|
| 25 |
+
# ββ Weights ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
|
|
|
|
|
|
| 26 |
WEIGHTS = {
|
| 27 |
"eye_contact": 30,
|
| 28 |
"expression": 25,
|
| 29 |
"head_stability": 25,
|
| 30 |
"nervousness": 20,
|
| 31 |
}
|
| 32 |
+
SCORE_HISTORY_LEN = 45
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
+
# ββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 36 |
def compute_confidence_score(expr_result, eye_result, head_result):
|
| 37 |
eye_score = eye_result.get("score", 0)
|
| 38 |
expr_score = expr_result.get("expression_score", 0)
|
| 39 |
head_score = head_result.get("stability_score", 0)
|
| 40 |
nerv_score = expr_result.get("nervousness_score", 0)
|
|
|
|
| 41 |
nerv_contribution = max(0, 100 - nerv_score)
|
|
|
|
| 42 |
final = (
|
| 43 |
eye_score * WEIGHTS["eye_contact"] / 100 +
|
| 44 |
expr_score * WEIGHTS["expression"] / 100 +
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def confidence_label(score):
|
| 52 |
+
if score >= 80: return "High", "#00dc00"
|
| 53 |
+
if score >= 60: return "Moderate", "#00c896"
|
| 54 |
+
if score >= 40: return "Low", "#00a5ff"
|
| 55 |
+
return "Very Low", "#003cff"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
+
def analyze_frame(frame):
|
| 59 |
+
"""Run all detectors on a single frame. Returns annotated frame + metrics dict."""
|
| 60 |
+
landmark_extractor = FaceLandmarkExtractor()
|
| 61 |
+
expr_detector = ExpressionDetector(fps=30)
|
| 62 |
+
eye_detector = EyeContactDetector()
|
| 63 |
+
head_estimator = HeadPoseEstimator()
|
| 64 |
|
| 65 |
+
lm_result = landmark_extractor.extract(frame)
|
| 66 |
|
| 67 |
+
if not lm_result["face_detected"]:
|
| 68 |
+
landmark_extractor.release()
|
| 69 |
+
return frame, None
|
| 70 |
|
| 71 |
+
kp = lm_result["key_points"]
|
| 72 |
+
landmarks = lm_result["landmarks"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
+
expr_result = expr_detector.detect(kp, frame.shape)
|
| 75 |
+
eye_result = eye_detector.detect(kp, frame.shape)
|
| 76 |
+
head_result = head_estimator.detect(landmarks, frame.shape)
|
| 77 |
+
confidence = compute_confidence_score(expr_result, eye_result, head_result)
|
| 78 |
|
| 79 |
+
landmark_extractor.release()
|
| 80 |
|
| 81 |
+
metrics = {
|
| 82 |
+
"confidence": confidence,
|
| 83 |
+
"expression": expr_result.get("expression", "N/A"),
|
| 84 |
+
"nervousness": expr_result.get("nervousness_score", 0),
|
| 85 |
+
"blink_rate": expr_result.get("blink_rate", 0),
|
| 86 |
+
"gaze": eye_result.get("gaze_direction", "N/A"),
|
| 87 |
+
"eye_pct": eye_result.get("eye_contact_pct", 0),
|
| 88 |
+
"direction": head_result.get("direction", "N/A"),
|
| 89 |
+
"stability": head_result.get("stability_score", 0),
|
| 90 |
+
"pitch": head_result.get("pitch", 0.0),
|
| 91 |
+
"yaw": head_result.get("yaw", 0.0),
|
| 92 |
+
}
|
| 93 |
+
return lm_result["annotated_frame"], metrics
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
|
| 96 |
+
def metrics_to_markdown(metrics, question=""):
|
| 97 |
+
if metrics is None:
|
| 98 |
+
return "## No face detected\nPosition yourself properly in the frame."
|
|
|
|
|
|
|
|
|
|
| 99 |
|
| 100 |
+
score = metrics["confidence"]
|
| 101 |
+
label, color = confidence_label(score)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
+
bar_filled = "β" * (score // 5)
|
| 104 |
+
bar_empty = "β" * (20 - score // 5)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
|
| 106 |
+
md = f"""
|
| 107 |
+
## Confidence Score: {score}/100 β {label}
|
|
|
|
|
|
|
| 108 |
|
| 109 |
+
`{bar_filled}{bar_empty}` {score}%
|
|
|
|
|
|
|
| 110 |
|
| 111 |
+
| Metric | Value |
|
| 112 |
+
|---|---|
|
| 113 |
+
| Expression | {metrics['expression']} |
|
| 114 |
+
| Nervousness | {metrics['nervousness']}/100 |
|
| 115 |
+
| Blink Rate | {metrics['blink_rate']}/min |
|
| 116 |
+
| Gaze | {metrics['gaze']} |
|
| 117 |
+
| Eye Contact | {metrics['eye_pct']}% |
|
| 118 |
+
| Head Direction | {metrics['direction']} |
|
| 119 |
+
| Head Stability | {metrics['stability']}/100 |
|
| 120 |
+
| Pitch / Yaw | {metrics['pitch']:.1f} / {metrics['yaw']:.1f} |
|
| 121 |
|
| 122 |
+
### Tips
|
| 123 |
+
"""
|
| 124 |
+
if metrics["gaze"] != "Center":
|
| 125 |
+
md += "- Look directly at the camera\n"
|
| 126 |
+
if metrics["nervousness"] > 50:
|
| 127 |
+
md += "- Breathe slowly, you got this\n"
|
| 128 |
+
if metrics["direction"] != "Forward":
|
| 129 |
+
md += "- Face forward\n"
|
| 130 |
+
if metrics["gaze"] == "Center" and metrics["nervousness"] <= 50 and metrics["direction"] == "Forward":
|
| 131 |
+
md += "- Great posture! Keep it up\n"
|
| 132 |
|
| 133 |
+
if question:
|
| 134 |
+
md += f"\n---\n**Current Question:** {question}"
|
|
|
|
| 135 |
|
| 136 |
+
return md
|
|
|
|
| 137 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
|
| 139 |
+
# ββ Tab 1: Image Analysis βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 140 |
+
def analyze_image(image):
|
| 141 |
+
if image is None:
|
| 142 |
+
return None, "Please upload an image."
|
| 143 |
|
| 144 |
+
frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
|
| 145 |
+
annotated, metrics = analyze_frame(frame)
|
| 146 |
|
| 147 |
+
if metrics is None:
|
| 148 |
+
out_img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 149 |
+
return out_img, "No face detected in image."
|
| 150 |
|
| 151 |
+
out_img = cv2.cvtColor(annotated, cv2.COLOR_BGR2RGB)
|
| 152 |
+
return out_img, metrics_to_markdown(metrics)
|
| 153 |
|
|
|
|
| 154 |
|
| 155 |
+
# ββ Tab 2: Video Analysis βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 156 |
+
def analyze_video(video_path):
|
| 157 |
+
if video_path is None:
|
| 158 |
+
return None, "Please upload a video."
|
| 159 |
|
| 160 |
+
cap = cv2.VideoCapture(video_path)
|
| 161 |
+
if not cap.isOpened():
|
| 162 |
+
return None, "Could not open video."
|
| 163 |
|
| 164 |
+
all_scores = []
|
| 165 |
+
expr_counts = {}
|
| 166 |
+
frame_count = 0
|
| 167 |
+
sample_every = 10 # analyze every 10th frame for speed
|
| 168 |
+
last_annotated = None
|
| 169 |
|
| 170 |
+
while True:
|
| 171 |
+
ret, frame = cap.read()
|
| 172 |
+
if not ret:
|
| 173 |
+
break
|
| 174 |
+
frame_count += 1
|
| 175 |
+
if frame_count % sample_every != 0:
|
| 176 |
+
continue
|
| 177 |
+
|
| 178 |
+
annotated, metrics = analyze_frame(frame)
|
| 179 |
+
if metrics:
|
| 180 |
+
all_scores.append(metrics["confidence"])
|
| 181 |
+
expr = metrics["expression"]
|
| 182 |
+
expr_counts[expr] = expr_counts.get(expr, 0) + 1
|
| 183 |
+
last_annotated = annotated
|
| 184 |
|
| 185 |
+
cap.release()
|
| 186 |
|
| 187 |
+
if not all_scores:
|
| 188 |
+
return None, "No face detected in video."
|
| 189 |
|
| 190 |
+
avg = int(np.mean(all_scores))
|
| 191 |
+
peak = int(np.max(all_scores))
|
| 192 |
+
low = int(np.min(all_scores))
|
| 193 |
+
label, _ = confidence_label(avg)
|
| 194 |
|
| 195 |
+
expr_breakdown = "\n".join(
|
| 196 |
+
f"- {e}: {int(c / sum(expr_counts.values()) * 100)}%"
|
| 197 |
+
for e, c in sorted(expr_counts.items(), key=lambda x: -x[1])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
|
| 200 |
+
summary_md = f"""
|
| 201 |
+
## Video Analysis Summary
|
| 202 |
+
|
| 203 |
+
| Metric | Value |
|
| 204 |
+
|---|---|
|
| 205 |
+
| Avg Confidence | {avg}/100 [{label}] |
|
| 206 |
+
| Peak Score | {peak}/100 |
|
| 207 |
+
| Lowest Score | {low}/100 |
|
| 208 |
+
| Frames Analyzed | {len(all_scores)} |
|
| 209 |
+
|
| 210 |
+
### Expression Breakdown
|
| 211 |
+
{expr_breakdown}
|
| 212 |
+
"""
|
| 213 |
+
out_img = cv2.cvtColor(last_annotated, cv2.COLOR_BGR2RGB) if last_annotated is not None else None
|
| 214 |
+
return out_img, summary_md
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
# ββ Tab 3: AI Mock Interview ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 218 |
+
def setup_interview(name, job_role, experience, skills, resume_file):
|
| 219 |
+
"""Generate questions from profile or resume."""
|
| 220 |
+
if not name or not job_role:
|
| 221 |
+
return "Please fill Name and Job Role.", gr.update(visible=False), []
|
| 222 |
+
|
| 223 |
+
profile = {
|
| 224 |
+
'name': name, 'job_role': job_role,
|
| 225 |
+
'experience': experience, 'skills': skills,
|
| 226 |
+
'projects': [], 'education': '', 'summary': '', 'resume_text': ''
|
| 227 |
+
}
|
| 228 |
|
| 229 |
+
if resume_file is not None:
|
| 230 |
+
try:
|
| 231 |
+
parsed = resume_to_profile(resume_file.name)
|
| 232 |
+
profile.update({k: v for k, v in parsed.items() if v})
|
| 233 |
+
except Exception as e:
|
| 234 |
+
print(f"[WARN] Resume parse failed: {e}")
|
| 235 |
+
|
| 236 |
+
resume_ctx = get_resume_context_for_llm(profile)
|
| 237 |
+
|
| 238 |
+
try:
|
| 239 |
+
questions = generate_questions(
|
| 240 |
+
profile['name'], profile['job_role'],
|
| 241 |
+
profile['experience'], profile['skills'],
|
| 242 |
+
resume_text=resume_ctx,
|
| 243 |
+
num_questions=3,
|
| 244 |
+
)
|
| 245 |
+
except Exception as e:
|
| 246 |
+
return f"Question generation failed: {e}", gr.update(visible=False), []
|
| 247 |
+
|
| 248 |
+
q_display = "\n\n".join([f"**Q{i+1}:** {q}" for i, q in enumerate(questions)])
|
| 249 |
+
return (
|
| 250 |
+
f"Interview ready! {len(questions)} questions generated.\n\n{q_display}",
|
| 251 |
+
gr.update(visible=True),
|
| 252 |
+
questions
|
| 253 |
+
)
|
| 254 |
|
|
|
|
| 255 |
|
| 256 |
+
def evaluate_single_answer(question, answer, job_role):
|
| 257 |
+
"""Evaluate one Q&A pair with LLM."""
|
| 258 |
+
if not question or not answer:
|
| 259 |
+
return "Please provide both question and answer."
|
| 260 |
+
try:
|
| 261 |
+
feedback = evaluate_answer(
|
| 262 |
+
question=question,
|
| 263 |
+
answer=answer,
|
| 264 |
+
job_role=job_role,
|
| 265 |
+
followup='',
|
| 266 |
+
followup_answer='',
|
| 267 |
+
)
|
| 268 |
+
score_str = feedback.get('score_str', '?')
|
| 269 |
+
strength = feedback.get('strength', '')
|
| 270 |
+
improve = feedback.get('improvement', '')
|
| 271 |
+
detail = feedback.get('detail', '')
|
| 272 |
+
return f"""
|
| 273 |
+
## AI Evaluation
|
| 274 |
+
|
| 275 |
+
**Score:** {score_str}
|
| 276 |
+
|
| 277 |
+
**Strength:** {strength}
|
| 278 |
+
|
| 279 |
+
**Improvement:** {improve}
|
| 280 |
+
|
| 281 |
+
**Details:** {detail}
|
| 282 |
+
"""
|
| 283 |
+
except Exception as e:
|
| 284 |
+
return f"Evaluation failed: {e}"
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
# ββ Build Gradio UI βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 288 |
+
with gr.Blocks(
|
| 289 |
+
title="AI Interview Confidence Analyzer",
|
| 290 |
+
theme=gr.themes.Base(primary_hue="purple"),
|
| 291 |
+
css="""
|
| 292 |
+
.score-box { font-size: 1.4em; font-weight: bold; }
|
| 293 |
+
footer { display: none !important; }
|
| 294 |
+
"""
|
| 295 |
+
) as demo:
|
| 296 |
+
|
| 297 |
+
gr.Markdown("""
|
| 298 |
+
# ποΈ AI Interview Confidence Analyzer
|
| 299 |
+
Multimodal confidence analysis β Face | Expression | Eye Contact | Head Pose | LLM Evaluation
|
| 300 |
+
""")
|
| 301 |
+
|
| 302 |
+
with gr.Tabs():
|
| 303 |
+
|
| 304 |
+
# ββ Tab 1: Image ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 305 |
+
with gr.Tab("πΈ Analyze Image"):
|
| 306 |
+
gr.Markdown("Upload a photo from your interview or practice session.")
|
| 307 |
+
with gr.Row():
|
| 308 |
+
img_input = gr.Image(label="Upload Image", type="pil")
|
| 309 |
+
img_output = gr.Image(label="Annotated Result")
|
| 310 |
+
img_metrics = gr.Markdown(label="Metrics")
|
| 311 |
+
img_btn = gr.Button("Analyze", variant="primary")
|
| 312 |
+
img_btn.click(
|
| 313 |
+
fn=analyze_image,
|
| 314 |
+
inputs=img_input,
|
| 315 |
+
outputs=[img_output, img_metrics]
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
# ββ Tab 2: Video ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 319 |
+
with gr.Tab("π¬ Analyze Video"):
|
| 320 |
+
gr.Markdown("Upload a recorded interview video for frame-by-frame analysis.")
|
| 321 |
+
with gr.Row():
|
| 322 |
+
vid_input = gr.Video(label="Upload Video")
|
| 323 |
+
vid_output = gr.Image(label="Last Analyzed Frame")
|
| 324 |
+
vid_metrics = gr.Markdown(label="Summary")
|
| 325 |
+
vid_btn = gr.Button("Analyze Video", variant="primary")
|
| 326 |
+
vid_btn.click(
|
| 327 |
+
fn=analyze_video,
|
| 328 |
+
inputs=vid_input,
|
| 329 |
+
outputs=[vid_output, vid_metrics]
|
| 330 |
+
)
|
| 331 |
+
|
| 332 |
+
# ββ Tab 3: AI Mock Interview ββββββββββββββββββββββββββββββββββββββββββ
|
| 333 |
+
with gr.Tab("π€ AI Mock Interview"):
|
| 334 |
+
gr.Markdown("""
|
| 335 |
+
Enter your profile, generate personalized interview questions,
|
| 336 |
+
then type your answers to get AI feedback.
|
| 337 |
+
""")
|
| 338 |
+
|
| 339 |
+
with gr.Row():
|
| 340 |
+
with gr.Column():
|
| 341 |
+
name_input = gr.Textbox(label="Your Name", placeholder="Sunaina")
|
| 342 |
+
role_input = gr.Textbox(label="Job Role", placeholder="AI/ML Engineer")
|
| 343 |
+
exp_input = gr.Textbox(label="Experience", placeholder="1 year intern, built RAG systems")
|
| 344 |
+
skill_input = gr.Textbox(label="Skills", placeholder="Python, LangChain, FastAPI, FAISS")
|
| 345 |
+
resume_file = gr.File(label="Resume (PDF/DOCX/TXT) β optional", file_types=[".pdf", ".docx", ".txt"])
|
| 346 |
+
setup_btn = gr.Button("Generate Questions", variant="primary")
|
| 347 |
+
|
| 348 |
+
with gr.Column():
|
| 349 |
+
setup_output = gr.Markdown(label="Questions")
|
| 350 |
+
|
| 351 |
+
questions_state = gr.State([])
|
| 352 |
+
|
| 353 |
+
setup_btn.click(
|
| 354 |
+
fn=setup_interview,
|
| 355 |
+
inputs=[name_input, role_input, exp_input, skill_input, resume_file],
|
| 356 |
+
outputs=[setup_output, gr.Column(visible=False), questions_state]
|
| 357 |
+
)
|
| 358 |
+
|
| 359 |
+
gr.Markdown("---")
|
| 360 |
+
gr.Markdown("### Evaluate Your Answer")
|
| 361 |
+
gr.Markdown("Copy a question from above, paste it, write your answer, and get AI feedback.")
|
| 362 |
+
|
| 363 |
+
with gr.Row():
|
| 364 |
+
eval_question = gr.Textbox(label="Question", lines=2, placeholder="Paste the question here")
|
| 365 |
+
eval_role = gr.Textbox(label="Job Role", placeholder="AI/ML Engineer")
|
| 366 |
+
eval_answer = gr.Textbox(label="Your Answer", lines=5, placeholder="Type your answer here...")
|
| 367 |
+
eval_btn = gr.Button("Get AI Feedback", variant="primary")
|
| 368 |
+
eval_output = gr.Markdown(label="AI Feedback")
|
| 369 |
+
|
| 370 |
+
eval_btn.click(
|
| 371 |
+
fn=evaluate_single_answer,
|
| 372 |
+
inputs=[eval_question, eval_answer, eval_role],
|
| 373 |
+
outputs=eval_output
|
| 374 |
+
)
|
| 375 |
+
|
| 376 |
+
gr.Markdown("""
|
| 377 |
+
---
|
| 378 |
+
Built by **Sunaina** | AI/ML Engineer Intern @ Indux Technology
|
| 379 |
+
| [GitHub](https://github.com/Sunaina792/AI-Interview-system)
|
| 380 |
+
""")
|
| 381 |
|
| 382 |
if __name__ == "__main__":
|
| 383 |
+
demo.launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|