Sunaina792 commited on
Commit
768c1da
Β·
verified Β·
1 Parent(s): b4d38c5

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +299 -564
main.py CHANGED
@@ -1,58 +1,44 @@
1
- # main.py β€” AI Interview Confidence Analyzer
2
- # Combines: Face Landmarks + Expression + Eye Contact + Head Pose + STT + LLM + TTS
3
- # Phase 1: Full interview loop β€” STT + LLM eval + follow-up questions
4
 
5
  import cv2
6
  import sys
7
  import os
8
  import time
9
- import threading
10
  import numpy as np
11
  from collections import deque
 
 
12
 
13
- # Add modules folder to path
14
  sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "modules"))
15
 
16
  from modules.face_landmarks import FaceLandmarkExtractor
17
  from modules.expression_detection import ExpressionDetector
18
  from modules.eye_contact import EyeContactDetector
19
  from modules.head_pose import HeadPoseEstimator
20
- from modules.stt import transcribe
21
- from modules.tts import speak
22
  from modules.llm import (
23
  generate_questions,
24
- generate_followup,
25
  evaluate_answer,
26
  generate_final_summary,
27
- FIRST_QUESTION,
28
- LAST_QUESTION,
29
  )
30
  from modules.resume_parser import resume_to_profile, get_resume_context_for_llm
31
 
32
-
33
- # CONFIDENCE SCORE WEIGHTS (must sum to 100)
34
-
35
-
36
  WEIGHTS = {
37
  "eye_contact": 30,
38
  "expression": 25,
39
  "head_stability": 25,
40
  "nervousness": 20,
41
  }
42
-
43
- SCORE_HISTORY_LEN = 45 # ~1.5 seconds at 30fps
44
 
45
 
46
- # CONFIDENCE CALCULATOR
47
-
48
  def compute_confidence_score(expr_result, eye_result, head_result):
49
  eye_score = eye_result.get("score", 0)
50
  expr_score = expr_result.get("expression_score", 0)
51
  head_score = head_result.get("stability_score", 0)
52
  nerv_score = expr_result.get("nervousness_score", 0)
53
-
54
  nerv_contribution = max(0, 100 - nerv_score)
55
-
56
  final = (
57
  eye_score * WEIGHTS["eye_contact"] / 100 +
58
  expr_score * WEIGHTS["expression"] / 100 +
@@ -63,586 +49,335 @@ def compute_confidence_score(expr_result, eye_result, head_result):
63
 
64
 
65
  def confidence_label(score):
66
- if score >= 80: return "High", (0, 220, 0)
67
- if score >= 60: return "Moderate", (0, 200, 150)
68
- if score >= 40: return "Low", (0, 165, 255)
69
- return "Very Low", (0, 60, 255)
70
-
71
-
72
- # DRAW DASHBOARD PANEL
73
-
74
-
75
-
76
- def draw_dashboard(frame, expr_result, eye_result, head_result,
77
- confidence, score_history, current_question="", status=""):
78
- h, w = frame.shape[:2]
79
-
80
- panel_x = w - 260
81
- overlay = frame.copy()
82
- cv2.rectangle(overlay, (panel_x - 10, 0), (w, h), (20, 20, 20), -1)
83
- cv2.addWeighted(overlay, 0.55, frame, 0.45, 0, frame)
84
-
85
- x = panel_x
86
- y = 30
87
- dy = 28
88
-
89
- def put(text, color=(220, 220, 220), scale=0.55, bold=1):
90
- nonlocal y
91
- cv2.putText(frame, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, scale, color, bold)
92
- y += dy
93
-
94
- put("CONFIDENCE ANALYZER", (255, 255, 255), 0.55, 2)
95
- put("-" * 28, (80, 80, 80))
96
-
97
- label, color = confidence_label(confidence)
98
- put(f"SCORE: {confidence}/100", color, 0.75, 2)
99
- put(f"Level: {label}", color)
100
-
101
- bar_w = int((confidence / 100) * 230)
102
- cv2.rectangle(frame, (x, y), (x + 230, y + 12), (60, 60, 60), -1)
103
- cv2.rectangle(frame, (x, y), (x + bar_w, y + 12), color, -1)
104
- y += 22
105
- put("-" * 28, (80, 80, 80))
106
-
107
- expr = expr_result.get("expression", "N/A")
108
- nerv = expr_result.get("nervousness_score", 0)
109
- blink_r = expr_result.get("blink_rate", 0)
110
- expr_col = (0, 220, 0) if expr == "Happy" else (0, 165, 255) if nerv > 40 else (220, 220, 220)
111
- put("EXPRESSION", (180, 180, 255), 0.5, 1)
112
- put(f" {expr}", expr_col)
113
- put(f" Nerv: {nerv}/100 Blink:{blink_r}/m")
114
- put("-" * 28, (80, 80, 80))
115
-
116
- gaze = eye_result.get("gaze_direction", "N/A")
117
- eye_pct = eye_result.get("eye_contact_pct", 0)
118
- eye_col = (0, 220, 0) if gaze == "Center" else (0, 165, 255)
119
- put("EYE CONTACT", (180, 180, 255), 0.5, 1)
120
- put(f" Gaze: {gaze}", eye_col)
121
- put(f" Contact: {eye_pct}%")
122
- put("-" * 28, (80, 80, 80))
123
-
124
- direction = head_result.get("direction", "N/A")
125
- stability = head_result.get("stability_score", 0)
126
- head_col = (0, 220, 0) if direction == "Forward" else (0, 165, 255)
127
- put("HEAD POSE", (180, 180, 255), 0.5, 1)
128
- put(f" Dir: {direction}", head_col)
129
- put(f" Stability: {stability}/100")
130
- pitch = head_result.get("pitch", 0)
131
- yaw = head_result.get("yaw", 0)
132
- put(f" P:{pitch:.1f} Y:{yaw:.1f}")
133
- put("-" * 28, (80, 80, 80))
134
-
135
- put("SCORE TREND", (180, 180, 255), 0.5, 1)
136
- if len(score_history) > 1:
137
- pts = list(score_history)
138
- gx, gy, gw, gh = x, y, 230, 50
139
- cv2.rectangle(frame, (gx, gy), (gx + gw, gy + gh), (40, 40, 40), -1)
140
- for i in range(1, len(pts)):
141
- x1 = gx + int((i - 1) / (SCORE_HISTORY_LEN - 1) * gw)
142
- x2 = gx + int(i / (SCORE_HISTORY_LEN - 1) * gw)
143
- y1 = gy + gh - int(pts[i - 1] / 100 * gh)
144
- y2 = gy + gh - int(pts[i] / 100 * gh)
145
- cv2.line(frame, (x1, y1), (x2, y2), (0, 200, 100), 1)
146
- y += gh + 8
147
-
148
- put("-" * 28, (80, 80, 80))
149
- put("TIPS", (180, 180, 255), 0.5, 1)
150
- if gaze != "Center":
151
- put(" Look at the camera", (0, 200, 255), 0.45)
152
- if nerv > 50:
153
- put(" Breathe, slow down", (0, 200, 255), 0.45)
154
- if direction != "Forward":
155
- put(" Face forward", (0, 200, 255), 0.45)
156
- if nerv <= 50 and gaze == "Center" and direction == "Forward":
157
- put(" Great job! Keep it up", (0, 220, 0), 0.45)
158
-
159
- # ── Interview status overlay (bottom of frame) ──
160
- if current_question:
161
- words = current_question.split()
162
- max_chars = 55
163
- line1 = ""
164
- line2 = ""
165
- for word in words:
166
- if len(line1) + len(word) + 1 <= max_chars:
167
- line1 += (" " if line1 else "") + word
168
- else:
169
- line2 += (" " if line2 else "") + word
170
-
171
- cv2.rectangle(frame, (0, h - 75), (panel_x - 20, h), (20, 20, 20), -1)
172
- cv2.putText(frame, "Q: " + line1, (10, h - 52),
173
- cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 220, 255), 1)
174
- if line2:
175
- cv2.putText(frame, " " + line2, (10, h - 30),
176
- cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 220, 255), 1)
177
-
178
- if status:
179
- cv2.putText(frame, status, (10, h - 10),
180
- cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 200, 0), 1)
181
-
182
- return frame
183
-
184
-
185
- # SESSION SUMMARY
186
-
187
- def print_summary(score_history, expr_counts, total_frames, duration_sec,
188
- interview_results=None):
189
- if not score_history:
190
- print("\n[INFO] No data recorded.")
191
- return
192
-
193
- avg_score = int(np.mean(score_history))
194
- max_score = int(np.max(score_history))
195
- min_score = int(np.min(score_history))
196
- label, _ = confidence_label(avg_score)
197
-
198
- print("\n" + "=" * 50)
199
- print(" SESSION SUMMARY")
200
- print("=" * 50)
201
- print(f" Duration : {duration_sec:.1f} seconds")
202
- print(f" Frames Analyzed : {total_frames}")
203
- print(f" Avg Score : {avg_score}/100 [{label}]")
204
- print(f" Peak Score : {max_score}/100")
205
- print(f" Lowest Score : {min_score}/100")
206
- print("\n Expression Breakdown:")
207
- total_expr = sum(expr_counts.values()) or 1
208
- for expr, count in sorted(expr_counts.items(), key=lambda x: -x[1]):
209
- pct = int(count / total_expr * 100)
210
- print(f" {expr:<16} {pct:>3}% {'#' * (pct // 5)}")
211
-
212
- if interview_results:
213
- print("\n" + "=" * 50)
214
- print(" INTERVIEW Q&A SUMMARY")
215
- print("=" * 50)
216
- for i, r in enumerate(interview_results, 1):
217
- fb = r.get('feedback', {})
218
- if isinstance(fb, dict):
219
- score_str = fb.get('score_str', '?')
220
- strength = fb.get('strength', '')
221
- improve = fb.get('improvement', '')
222
- vis_conf = fb.get('visual_confidence', '?')
223
- else:
224
- score_str = str(fb)
225
- strength = improve = vis_conf = ''
226
-
227
- print(f"\n Q{i}: {r['question']}")
228
- print(f" Answer : {r['answer'][:200]}")
229
- if r.get('followup'):
230
- print(f" Follow-up: {r['followup']}")
231
- print(f" FU Answer: {r.get('followup_answer','')[:120]}")
232
- print(f" AI Score : {score_str} | Visual Conf: {vis_conf}/100")
233
- print(f" Strength : {strength}")
234
- print(f" Improve : {improve}")
235
- print(" " + "-" * 46)
236
-
237
- print("\n OVERALL AI FEEDBACK:")
238
- try:
239
- last_role = interview_results[-1].get('job_role', 'the role')
240
- summary = generate_final_summary(interview_results, last_role)
241
- print(f" Overall Score : {summary.get('overall_score_str','')}")
242
- print(f" Top Strength : {summary.get('top_strength','')}")
243
- print(f" Top Improvement : {summary.get('top_area_to_improve','')}")
244
- print(f" Weak Topics : {', '.join(summary.get('weak_topics',[]))}")
245
- print(f" Final Tip : {summary.get('final_tip','')}")
246
- except Exception as e:
247
- print(f" [WARN] Summary generation failed: {e}")
248
-
249
- print("\n Improvement Tips:")
250
- if avg_score < 40:
251
- print(" - Practice maintaining eye contact")
252
- print(" - Work on reducing visible nervousness")
253
- print(" - Keep your head stable and facing forward")
254
- elif avg_score < 70:
255
- print(" - Good effort β€” focus on eye contact consistency")
256
- print(" - Try to appear more relaxed")
257
- else:
258
- print(" - Strong performance!")
259
- print(" - Keep practicing to maintain consistency")
260
- print("=" * 50)
261
 
262
 
 
 
 
 
 
 
263
 
264
- # COLLECT USER PROFILE
265
 
 
 
 
266
 
267
- def collect_profile() -> dict:
268
- """
269
- Returns a full profile dict with keys:
270
- name, job_role, experience, skills, projects, education, summary, resume_text
271
- """
272
- print("\n" + "=" * 50)
273
- print(" INTERVIEW SETUP")
274
- print("=" * 50)
275
- print(" [1] Upload Resume (PDF / DOCX / TXT / MD) β€” auto fill")
276
- print(" [2] Enter manually")
277
- print("=" * 50)
278
- choice = input(" Choice: ").strip()
279
-
280
- if choice == "1":
281
- path = input(" Resume path: ").strip().strip('"')
282
- print(" Parsing resume...")
283
- try:
284
- profile = resume_to_profile(path)
285
- print(f"\n Parsed Profile:")
286
- print(f" Name : {profile['name']}")
287
- print(f" Role : {profile['job_role']}")
288
- print(f" Experience : {profile['experience']}")
289
- print(f" Skills : {profile['skills']}")
290
- if profile.get('projects'):
291
- print(f" Projects : {', '.join(profile['projects'][:3])}")
292
- if profile.get('education'):
293
- print(f" Education : {profile['education']}")
294
- confirm = input("\n Looks good? (y/n): ").strip().lower()
295
- if confirm == 'y':
296
- return profile
297
- except Exception as e:
298
- print(f" [WARN] Resume parse failed: {e}. Falling back to manual entry.")
299
-
300
- # manual fallback β€” build the same dict shape
301
- name = input(" Your Name : ").strip()
302
- job_role = input(" Job Role : ").strip()
303
- experience = input(" Experience : ").strip()
304
- skills = input(" Skills : ").strip()
305
- print("\n Resume text (paste, press Enter twice when done):")
306
- lines = []
307
- while True:
308
- line = input()
309
- if line == "":
310
- break
311
- lines.append(line)
312
- return {
313
- 'name': name,
314
- 'job_role': job_role,
315
- 'experience': experience,
316
- 'skills': skills,
317
- 'projects': [],
318
- 'education': '',
319
- 'summary': '',
320
- 'resume_text': '\n'.join(lines),
321
- }
322
 
 
 
 
 
323
 
324
- # MAIN β€” INTERVIEW LIVE SESSION
325
 
 
 
 
 
 
 
 
 
 
 
 
 
 
326
 
327
- def run_live_session():
328
- print("\n" + "=" * 50)
329
- print(" AI INTERVIEW CONFIDENCE ANALYZER")
330
- print("=" * 50)
331
- print(" Press Q to quit and see summary")
332
- print(" Press S to take a snapshot")
333
- print("=" * 50 + "\n")
334
 
335
- profile = collect_profile()
336
- name = profile['name']
337
- job_role = profile['job_role']
338
- experience = profile['experience']
339
- skills = profile['skills']
340
- resume_ctx = get_resume_context_for_llm(profile) # rich context string
341
 
342
- print("\n[INFO] Generating interview questions...")
343
- questions = generate_questions(
344
- name, job_role, experience, skills,
345
- resume_text=resume_ctx,
346
- num_questions=2,
347
- )
348
- print(f"[INFO] {len(questions)} questions ready.\n")
349
 
350
- landmark_extractor = FaceLandmarkExtractor()
351
- if not getattr(landmark_extractor, 'enabled', False):
352
- print('[ERROR] FaceLandmarkExtractor not enabled. Check MediaPipe.')
353
- return
354
- expr_detector = ExpressionDetector(fps=30)
355
- eye_detector = EyeContactDetector()
356
- head_estimator = HeadPoseEstimator()
357
-
358
- cap = cv2.VideoCapture(0)
359
- if not cap.isOpened():
360
- print("[ERROR] Cannot open webcam."); return
361
-
362
- cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
363
- cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
364
-
365
- score_history = deque(maxlen=SCORE_HISTORY_LEN)
366
- all_scores = []
367
- expr_counts = {}
368
- total_frames = 0
369
- start_time = time.time()
370
- snapshot_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "outputs")
371
- os.makedirs(snapshot_dir, exist_ok=True)
372
-
373
- empty_expr = {"expression": "N/A", "nervousness_score": 0,
374
- "expression_score": 0, "blink_rate": 0}
375
- empty_eye = {"gaze_direction": "N/A", "eye_contact_pct": 0, "score": 0}
376
- empty_head = {"direction": "N/A", "stability_score": 0, "pitch": 0.0, "yaw": 0.0}
377
-
378
- # ── Interview state ──
379
- interview_results = []
380
- current_q_idx = 0
381
- current_question = questions[current_q_idx]
382
- status_text = "Listening... (speak your answer)"
383
- answer_scores = [] # confidence scores during this answer
384
- answer_start_time = time.time()
385
- ANSWER_DURATION = 45 # seconds per answer
386
-
387
- # Speak first question in background thread
388
- threading.Thread(
389
- target=speak,
390
- args=(f"Welcome {name}. Question 1. {current_question}",),
391
- daemon=True
392
- ).start()
393
 
394
- while True:
395
- ret, frame = cap.read()
396
- if not ret:
397
- print("[ERROR] Failed to read webcam frame."); break
398
 
399
- total_frames += 1
400
- lm_result = landmark_extractor.extract(frame)
401
- disp = lm_result["annotated_frame"].copy()
402
 
403
- if lm_result["face_detected"]:
404
- kp = lm_result["key_points"]
405
- landmarks = lm_result["landmarks"]
 
 
 
 
 
 
 
406
 
407
- expr_result = expr_detector.detect(kp, frame.shape)
408
- eye_result = eye_detector.detect(kp, frame.shape)
409
- head_result = head_estimator.detect(landmarks, frame.shape)
410
- confidence = compute_confidence_score(expr_result, eye_result, head_result)
 
 
 
 
 
 
411
 
412
- score_history.append(confidence)
413
- all_scores.append(confidence)
414
- answer_scores.append(confidence)
415
 
416
- expr = expr_result.get("expression", "N/A")
417
- expr_counts[expr] = expr_counts.get(expr, 0) + 1
418
 
419
- disp = draw_dashboard(disp, expr_result, eye_result, head_result,
420
- confidence, score_history,
421
- current_question, status_text)
422
- else:
423
- cv2.putText(disp, "No face detected β€” position yourself in frame",
424
- (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 100, 255), 2)
425
- disp = draw_dashboard(disp, empty_expr, empty_eye, empty_head,
426
- 0, score_history, current_question, status_text)
427
-
428
- # ── Timer per answer ──
429
- elapsed_answer = time.time() - answer_start_time
430
- remaining = max(0, int(ANSWER_DURATION - elapsed_answer))
431
- cv2.putText(disp, f"Answer time left: {remaining}s", (10, 50),
432
- cv2.FONT_HERSHEY_SIMPLEX, 0.55, (200, 200, 0), 1)
433
-
434
- # ── Move to next question when time is up ──
435
- if elapsed_answer >= ANSWER_DURATION and current_q_idx < len(questions):
436
- status_text = "Transcribing your answer..."
437
- cv2.imshow("AI Interview Analyzer (Q=quit S=snapshot)", disp)
438
- cv2.waitKey(1)
439
-
440
- # ── 1. Transcribe answer via STT ──────────────────────────────────
441
- avg_conf = int(np.mean(answer_scores)) if answer_scores else 0
442
- try:
443
- transcribed_answer = transcribe() # records mic and returns text
444
- except Exception as e:
445
- print(f"[WARN] STT failed: {e}")
446
- transcribed_answer = f"[STT unavailable β€” visual confidence: {avg_conf}/100]"
447
-
448
- status_text = "Evaluating with AI..."
449
- cv2.imshow("AI Interview Analyzer (Q=quit S=snapshot)", disp)
450
- cv2.waitKey(1)
451
-
452
- # ── 2. Check if a follow-up question is warranted ─────────────────
453
- followup_q = None
454
- followup_answer = ""
455
- if transcribed_answer and not transcribed_answer.startswith('['):
456
- try:
457
- followup_q = generate_followup(current_question, transcribed_answer, job_role)
458
- except Exception as e:
459
- print(f"[WARN] Follow-up generation failed: {e}")
460
-
461
- if followup_q:
462
- status_text = f"Follow-up: {followup_q[:60]}..."
463
- threading.Thread(
464
- target=speak,
465
- args=(f"Follow-up: {followup_q}",),
466
- daemon=True
467
- ).start()
468
- # Give candidate FOLLOW_UP_DURATION seconds to answer follow-up
469
- FOLLOW_UP_DURATION = 30
470
- fu_start = time.time()
471
- while time.time() - fu_start < FOLLOW_UP_DURATION:
472
- ret2, frame2 = cap.read()
473
- if ret2:
474
- fu_remaining = max(0, int(FOLLOW_UP_DURATION - (time.time() - fu_start)))
475
- cv2.putText(frame2,
476
- f"Follow-up time left: {fu_remaining}s",
477
- (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (200, 200, 0), 1)
478
- cv2.putText(frame2,
479
- f"Follow-up: {followup_q[:70]}",
480
- (10, frame2.shape[0] - 20),
481
- cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 220, 255), 1)
482
- cv2.imshow("AI Interview Analyzer (Q=quit S=snapshot)", frame2)
483
- cv2.waitKey(1)
484
- try:
485
- followup_answer = transcribe()
486
- except Exception:
487
- followup_answer = ""
488
-
489
- # ── 3. LLM evaluation ────────────────────────────────────────────
490
- try:
491
- feedback = evaluate_answer(
492
- question=current_question,
493
- answer=transcribed_answer,
494
- job_role=job_role,
495
- followup=followup_q or '',
496
- followup_answer=followup_answer,
497
- )
498
- except Exception as e:
499
- print(f"[WARN] Evaluation failed: {e}")
500
- feedback = {
501
- 'score': avg_conf // 10,
502
- 'score_str': f"{avg_conf // 10}/10",
503
- 'strength': 'Answer recorded.',
504
- 'improvement': 'AI evaluation unavailable.',
505
- 'detail': '',
506
- 'raw': '',
507
- }
508
-
509
- # Append visual confidence as extra context
510
- feedback['visual_confidence'] = avg_conf
511
-
512
- interview_results.append({
513
- 'question': current_question,
514
- 'answer': transcribed_answer,
515
- 'followup': followup_q,
516
- 'followup_answer': followup_answer,
517
- 'feedback': feedback,
518
- 'job_role': job_role,
519
- })
520
-
521
- # Print quick feedback to terminal
522
- print(f"\n[EVAL] Q: {current_question}")
523
- print(f"[EVAL] A: {transcribed_answer[:120]}...")
524
- print(f"[EVAL] Score: {feedback.get('score_str','?')} | "
525
- f"{feedback.get('strength','')}")
526
-
527
- answer_scores = []
528
- current_q_idx += 1
529
- answer_start_time = time.time()
530
-
531
- if current_q_idx < len(questions):
532
- current_question = questions[current_q_idx]
533
- status_text = "Listening... (speak your answer)"
534
- threading.Thread(
535
- target=speak,
536
- args=(f"Question {current_q_idx + 1}. {current_question}",),
537
- daemon=True
538
- ).start()
539
- else:
540
- status_text = "Interview complete! Press Q to see summary."
541
- threading.Thread(
542
- target=speak,
543
- args=("Interview complete. Great job! Press Q to see your summary.",),
544
- daemon=True
545
- ).start()
546
-
547
- # ── Elapsed session timer ──
548
- elapsed = int(time.time() - start_time)
549
- cv2.putText(disp, f"{elapsed//60:02d}:{elapsed%60:02d}", (10, 22),
550
- cv2.FONT_HERSHEY_SIMPLEX, 0.6, (180, 180, 180), 1)
551
-
552
- cv2.imshow("AI Interview Analyzer (Q=quit S=snapshot)", disp)
553
-
554
- key = cv2.waitKey(1) & 0xFF
555
- if key == ord("q"):
556
- break
557
- elif key == ord("s"):
558
- ts = time.strftime("%Y%m%d_%H%M%S")
559
- path = os.path.join(snapshot_dir, f"snapshot_{ts}.jpg")
560
- cv2.imwrite(path, disp)
561
- print(f"[INFO] Snapshot saved: {path}")
562
 
563
- cap.release()
564
- landmark_extractor.release()
565
- cv2.destroyAllWindows()
 
566
 
567
- duration = time.time() - start_time
568
- print_summary(all_scores, expr_counts, total_frames, duration, interview_results)
569
 
 
 
 
570
 
 
 
571
 
572
- # MAIN β€” IMAGE TEST (unchanged)
573
 
 
 
 
 
574
 
575
- def run_image_test(image_path):
576
- print(f"\n[INFO] Running on image: {image_path}")
 
577
 
578
- frame = cv2.imread(image_path)
579
- if frame is None:
580
- print(f"[ERROR] Cannot load: {image_path}"); return
 
 
581
 
582
- landmark_extractor = FaceLandmarkExtractor()
583
- expr_detector = ExpressionDetector(fps=30)
584
- eye_detector = EyeContactDetector()
585
- head_estimator = HeadPoseEstimator()
 
 
 
 
 
 
 
 
 
 
586
 
587
- lm_result = landmark_extractor.extract_image(frame)
588
 
589
- if not lm_result["face_detected"]:
590
- print("[ERROR] No face detected in image."); return
591
 
592
- kp = lm_result["key_points"]
593
- landmarks = lm_result["landmarks"]
 
 
594
 
595
- expr_result = expr_detector.detect(kp, frame.shape)
596
- eye_result = eye_detector.detect(kp, frame.shape)
597
- head_result = head_estimator.detect(landmarks, frame.shape)
598
- confidence = compute_confidence_score(expr_result, eye_result, head_result)
599
- label, _ = confidence_label(confidence)
600
-
601
- print("\n" + "=" * 50)
602
- print(" ANALYSIS RESULT")
603
- print("=" * 50)
604
- print(f" Confidence Score : {confidence}/100 [{label}]")
605
- print(f" Expression : {expr_result['expression']}")
606
- print(f" Nervousness : {expr_result['nervousness_score']}/100")
607
- print(f" Gaze : {eye_result['gaze_direction']}")
608
- print(f" Eye Contact : {eye_result['eye_contact_pct']}%")
609
- print(f" Head Direction : {head_result['direction']}")
610
- print(f" Head Stability : {head_result['stability_score']}/100")
611
- print("=" * 50)
612
-
613
- disp = draw_dashboard(
614
- lm_result["annotated_frame"].copy(),
615
- expr_result, eye_result, head_result,
616
- confidence, deque([confidence])
617
  )
618
- cv2.imshow("AI Interview Analyzer - Image Test (any key to close)", disp)
619
- cv2.waitKey(0)
620
- cv2.destroyAllWindows()
621
- landmark_extractor.release()
622
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
623
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
624
 
625
- # ENTRY POINT
626
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
627
 
628
  if __name__ == "__main__":
629
- if len(sys.argv) >= 3 and sys.argv[1] == "--image":
630
- run_image_test(sys.argv[2]); sys.exit(0)
631
- elif len(sys.argv) >= 2 and sys.argv[1] == "--live":
632
- run_live_session(); sys.exit(0)
633
-
634
- print("\n" + "=" * 50)
635
- print(" AI INTERVIEW CONFIDENCE ANALYZER")
636
- print("=" * 50)
637
- print(" [1] Live interview session (with AI questions)")
638
- print(" [2] Test on image")
639
- print("=" * 50)
640
- choice = input(" Choice (1 or 2): ").strip()
641
-
642
- if choice == "1":
643
- run_live_session()
644
- elif choice == "2":
645
- path = input(" Image path: ").strip().strip('"')
646
- run_image_test(path)
647
- else:
648
- print(" Invalid choice.")
 
1
+ # main.py β€” AI Interview Confidence Analyzer (Gradio / HuggingFace Spaces)
 
 
2
 
3
  import cv2
4
  import sys
5
  import os
6
  import time
 
7
  import numpy as np
8
  from collections import deque
9
+ import gradio as gr
10
+ import tempfile
11
 
 
12
  sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "modules"))
13
 
14
  from modules.face_landmarks import FaceLandmarkExtractor
15
  from modules.expression_detection import ExpressionDetector
16
  from modules.eye_contact import EyeContactDetector
17
  from modules.head_pose import HeadPoseEstimator
 
 
18
  from modules.llm import (
19
  generate_questions,
 
20
  evaluate_answer,
21
  generate_final_summary,
 
 
22
  )
23
  from modules.resume_parser import resume_to_profile, get_resume_context_for_llm
24
 
25
+ # ── Weights ──────────────────────────────────────────────────────────────────
 
 
 
26
  WEIGHTS = {
27
  "eye_contact": 30,
28
  "expression": 25,
29
  "head_stability": 25,
30
  "nervousness": 20,
31
  }
32
+ SCORE_HISTORY_LEN = 45
 
33
 
34
 
35
+ # ── Helpers ───────────────────────────────────────────────────────────────────
 
36
  def compute_confidence_score(expr_result, eye_result, head_result):
37
  eye_score = eye_result.get("score", 0)
38
  expr_score = expr_result.get("expression_score", 0)
39
  head_score = head_result.get("stability_score", 0)
40
  nerv_score = expr_result.get("nervousness_score", 0)
 
41
  nerv_contribution = max(0, 100 - nerv_score)
 
42
  final = (
43
  eye_score * WEIGHTS["eye_contact"] / 100 +
44
  expr_score * WEIGHTS["expression"] / 100 +
 
49
 
50
 
51
  def confidence_label(score):
52
+ if score >= 80: return "High", "#00dc00"
53
+ if score >= 60: return "Moderate", "#00c896"
54
+ if score >= 40: return "Low", "#00a5ff"
55
+ return "Very Low", "#003cff"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
 
57
 
58
+ def analyze_frame(frame):
59
+ """Run all detectors on a single frame. Returns annotated frame + metrics dict."""
60
+ landmark_extractor = FaceLandmarkExtractor()
61
+ expr_detector = ExpressionDetector(fps=30)
62
+ eye_detector = EyeContactDetector()
63
+ head_estimator = HeadPoseEstimator()
64
 
65
+ lm_result = landmark_extractor.extract(frame)
66
 
67
+ if not lm_result["face_detected"]:
68
+ landmark_extractor.release()
69
+ return frame, None
70
 
71
+ kp = lm_result["key_points"]
72
+ landmarks = lm_result["landmarks"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
 
74
+ expr_result = expr_detector.detect(kp, frame.shape)
75
+ eye_result = eye_detector.detect(kp, frame.shape)
76
+ head_result = head_estimator.detect(landmarks, frame.shape)
77
+ confidence = compute_confidence_score(expr_result, eye_result, head_result)
78
 
79
+ landmark_extractor.release()
80
 
81
+ metrics = {
82
+ "confidence": confidence,
83
+ "expression": expr_result.get("expression", "N/A"),
84
+ "nervousness": expr_result.get("nervousness_score", 0),
85
+ "blink_rate": expr_result.get("blink_rate", 0),
86
+ "gaze": eye_result.get("gaze_direction", "N/A"),
87
+ "eye_pct": eye_result.get("eye_contact_pct", 0),
88
+ "direction": head_result.get("direction", "N/A"),
89
+ "stability": head_result.get("stability_score", 0),
90
+ "pitch": head_result.get("pitch", 0.0),
91
+ "yaw": head_result.get("yaw", 0.0),
92
+ }
93
+ return lm_result["annotated_frame"], metrics
94
 
 
 
 
 
 
 
 
95
 
96
+ def metrics_to_markdown(metrics, question=""):
97
+ if metrics is None:
98
+ return "## No face detected\nPosition yourself properly in the frame."
 
 
 
99
 
100
+ score = metrics["confidence"]
101
+ label, color = confidence_label(score)
 
 
 
 
 
102
 
103
+ bar_filled = "β–ˆ" * (score // 5)
104
+ bar_empty = "β–‘" * (20 - score // 5)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
 
106
+ md = f"""
107
+ ## Confidence Score: {score}/100 β€” {label}
 
 
108
 
109
+ `{bar_filled}{bar_empty}` {score}%
 
 
110
 
111
+ | Metric | Value |
112
+ |---|---|
113
+ | Expression | {metrics['expression']} |
114
+ | Nervousness | {metrics['nervousness']}/100 |
115
+ | Blink Rate | {metrics['blink_rate']}/min |
116
+ | Gaze | {metrics['gaze']} |
117
+ | Eye Contact | {metrics['eye_pct']}% |
118
+ | Head Direction | {metrics['direction']} |
119
+ | Head Stability | {metrics['stability']}/100 |
120
+ | Pitch / Yaw | {metrics['pitch']:.1f} / {metrics['yaw']:.1f} |
121
 
122
+ ### Tips
123
+ """
124
+ if metrics["gaze"] != "Center":
125
+ md += "- Look directly at the camera\n"
126
+ if metrics["nervousness"] > 50:
127
+ md += "- Breathe slowly, you got this\n"
128
+ if metrics["direction"] != "Forward":
129
+ md += "- Face forward\n"
130
+ if metrics["gaze"] == "Center" and metrics["nervousness"] <= 50 and metrics["direction"] == "Forward":
131
+ md += "- Great posture! Keep it up\n"
132
 
133
+ if question:
134
+ md += f"\n---\n**Current Question:** {question}"
 
135
 
136
+ return md
 
137
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
138
 
139
+ # ── Tab 1: Image Analysis ─────────────────────────────────────────────────────
140
+ def analyze_image(image):
141
+ if image is None:
142
+ return None, "Please upload an image."
143
 
144
+ frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
145
+ annotated, metrics = analyze_frame(frame)
146
 
147
+ if metrics is None:
148
+ out_img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
149
+ return out_img, "No face detected in image."
150
 
151
+ out_img = cv2.cvtColor(annotated, cv2.COLOR_BGR2RGB)
152
+ return out_img, metrics_to_markdown(metrics)
153
 
 
154
 
155
+ # ── Tab 2: Video Analysis ─────────────────────────────────────────────────────
156
+ def analyze_video(video_path):
157
+ if video_path is None:
158
+ return None, "Please upload a video."
159
 
160
+ cap = cv2.VideoCapture(video_path)
161
+ if not cap.isOpened():
162
+ return None, "Could not open video."
163
 
164
+ all_scores = []
165
+ expr_counts = {}
166
+ frame_count = 0
167
+ sample_every = 10 # analyze every 10th frame for speed
168
+ last_annotated = None
169
 
170
+ while True:
171
+ ret, frame = cap.read()
172
+ if not ret:
173
+ break
174
+ frame_count += 1
175
+ if frame_count % sample_every != 0:
176
+ continue
177
+
178
+ annotated, metrics = analyze_frame(frame)
179
+ if metrics:
180
+ all_scores.append(metrics["confidence"])
181
+ expr = metrics["expression"]
182
+ expr_counts[expr] = expr_counts.get(expr, 0) + 1
183
+ last_annotated = annotated
184
 
185
+ cap.release()
186
 
187
+ if not all_scores:
188
+ return None, "No face detected in video."
189
 
190
+ avg = int(np.mean(all_scores))
191
+ peak = int(np.max(all_scores))
192
+ low = int(np.min(all_scores))
193
+ label, _ = confidence_label(avg)
194
 
195
+ expr_breakdown = "\n".join(
196
+ f"- {e}: {int(c / sum(expr_counts.values()) * 100)}%"
197
+ for e, c in sorted(expr_counts.items(), key=lambda x: -x[1])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
198
  )
 
 
 
 
199
 
200
+ summary_md = f"""
201
+ ## Video Analysis Summary
202
+
203
+ | Metric | Value |
204
+ |---|---|
205
+ | Avg Confidence | {avg}/100 [{label}] |
206
+ | Peak Score | {peak}/100 |
207
+ | Lowest Score | {low}/100 |
208
+ | Frames Analyzed | {len(all_scores)} |
209
+
210
+ ### Expression Breakdown
211
+ {expr_breakdown}
212
+ """
213
+ out_img = cv2.cvtColor(last_annotated, cv2.COLOR_BGR2RGB) if last_annotated is not None else None
214
+ return out_img, summary_md
215
+
216
+
217
+ # ── Tab 3: AI Mock Interview ──────────────────────────────────────────────────
218
+ def setup_interview(name, job_role, experience, skills, resume_file):
219
+ """Generate questions from profile or resume."""
220
+ if not name or not job_role:
221
+ return "Please fill Name and Job Role.", gr.update(visible=False), []
222
+
223
+ profile = {
224
+ 'name': name, 'job_role': job_role,
225
+ 'experience': experience, 'skills': skills,
226
+ 'projects': [], 'education': '', 'summary': '', 'resume_text': ''
227
+ }
228
 
229
+ if resume_file is not None:
230
+ try:
231
+ parsed = resume_to_profile(resume_file.name)
232
+ profile.update({k: v for k, v in parsed.items() if v})
233
+ except Exception as e:
234
+ print(f"[WARN] Resume parse failed: {e}")
235
+
236
+ resume_ctx = get_resume_context_for_llm(profile)
237
+
238
+ try:
239
+ questions = generate_questions(
240
+ profile['name'], profile['job_role'],
241
+ profile['experience'], profile['skills'],
242
+ resume_text=resume_ctx,
243
+ num_questions=3,
244
+ )
245
+ except Exception as e:
246
+ return f"Question generation failed: {e}", gr.update(visible=False), []
247
+
248
+ q_display = "\n\n".join([f"**Q{i+1}:** {q}" for i, q in enumerate(questions)])
249
+ return (
250
+ f"Interview ready! {len(questions)} questions generated.\n\n{q_display}",
251
+ gr.update(visible=True),
252
+ questions
253
+ )
254
 
 
255
 
256
+ def evaluate_single_answer(question, answer, job_role):
257
+ """Evaluate one Q&A pair with LLM."""
258
+ if not question or not answer:
259
+ return "Please provide both question and answer."
260
+ try:
261
+ feedback = evaluate_answer(
262
+ question=question,
263
+ answer=answer,
264
+ job_role=job_role,
265
+ followup='',
266
+ followup_answer='',
267
+ )
268
+ score_str = feedback.get('score_str', '?')
269
+ strength = feedback.get('strength', '')
270
+ improve = feedback.get('improvement', '')
271
+ detail = feedback.get('detail', '')
272
+ return f"""
273
+ ## AI Evaluation
274
+
275
+ **Score:** {score_str}
276
+
277
+ **Strength:** {strength}
278
+
279
+ **Improvement:** {improve}
280
+
281
+ **Details:** {detail}
282
+ """
283
+ except Exception as e:
284
+ return f"Evaluation failed: {e}"
285
+
286
+
287
+ # ── Build Gradio UI ───────────────────────────────────────────────────────────
288
+ with gr.Blocks(
289
+ title="AI Interview Confidence Analyzer",
290
+ theme=gr.themes.Base(primary_hue="purple"),
291
+ css="""
292
+ .score-box { font-size: 1.4em; font-weight: bold; }
293
+ footer { display: none !important; }
294
+ """
295
+ ) as demo:
296
+
297
+ gr.Markdown("""
298
+ # πŸŽ™οΈ AI Interview Confidence Analyzer
299
+ Multimodal confidence analysis β€” Face | Expression | Eye Contact | Head Pose | LLM Evaluation
300
+ """)
301
+
302
+ with gr.Tabs():
303
+
304
+ # ── Tab 1: Image ──────────────────────────────────────────────────────
305
+ with gr.Tab("πŸ“Έ Analyze Image"):
306
+ gr.Markdown("Upload a photo from your interview or practice session.")
307
+ with gr.Row():
308
+ img_input = gr.Image(label="Upload Image", type="pil")
309
+ img_output = gr.Image(label="Annotated Result")
310
+ img_metrics = gr.Markdown(label="Metrics")
311
+ img_btn = gr.Button("Analyze", variant="primary")
312
+ img_btn.click(
313
+ fn=analyze_image,
314
+ inputs=img_input,
315
+ outputs=[img_output, img_metrics]
316
+ )
317
+
318
+ # ── Tab 2: Video ──────────────────────────────────────────────────────
319
+ with gr.Tab("🎬 Analyze Video"):
320
+ gr.Markdown("Upload a recorded interview video for frame-by-frame analysis.")
321
+ with gr.Row():
322
+ vid_input = gr.Video(label="Upload Video")
323
+ vid_output = gr.Image(label="Last Analyzed Frame")
324
+ vid_metrics = gr.Markdown(label="Summary")
325
+ vid_btn = gr.Button("Analyze Video", variant="primary")
326
+ vid_btn.click(
327
+ fn=analyze_video,
328
+ inputs=vid_input,
329
+ outputs=[vid_output, vid_metrics]
330
+ )
331
+
332
+ # ── Tab 3: AI Mock Interview ──────────────────────────────────────────
333
+ with gr.Tab("πŸ€– AI Mock Interview"):
334
+ gr.Markdown("""
335
+ Enter your profile, generate personalized interview questions,
336
+ then type your answers to get AI feedback.
337
+ """)
338
+
339
+ with gr.Row():
340
+ with gr.Column():
341
+ name_input = gr.Textbox(label="Your Name", placeholder="Sunaina")
342
+ role_input = gr.Textbox(label="Job Role", placeholder="AI/ML Engineer")
343
+ exp_input = gr.Textbox(label="Experience", placeholder="1 year intern, built RAG systems")
344
+ skill_input = gr.Textbox(label="Skills", placeholder="Python, LangChain, FastAPI, FAISS")
345
+ resume_file = gr.File(label="Resume (PDF/DOCX/TXT) β€” optional", file_types=[".pdf", ".docx", ".txt"])
346
+ setup_btn = gr.Button("Generate Questions", variant="primary")
347
+
348
+ with gr.Column():
349
+ setup_output = gr.Markdown(label="Questions")
350
+
351
+ questions_state = gr.State([])
352
+
353
+ setup_btn.click(
354
+ fn=setup_interview,
355
+ inputs=[name_input, role_input, exp_input, skill_input, resume_file],
356
+ outputs=[setup_output, gr.Column(visible=False), questions_state]
357
+ )
358
+
359
+ gr.Markdown("---")
360
+ gr.Markdown("### Evaluate Your Answer")
361
+ gr.Markdown("Copy a question from above, paste it, write your answer, and get AI feedback.")
362
+
363
+ with gr.Row():
364
+ eval_question = gr.Textbox(label="Question", lines=2, placeholder="Paste the question here")
365
+ eval_role = gr.Textbox(label="Job Role", placeholder="AI/ML Engineer")
366
+ eval_answer = gr.Textbox(label="Your Answer", lines=5, placeholder="Type your answer here...")
367
+ eval_btn = gr.Button("Get AI Feedback", variant="primary")
368
+ eval_output = gr.Markdown(label="AI Feedback")
369
+
370
+ eval_btn.click(
371
+ fn=evaluate_single_answer,
372
+ inputs=[eval_question, eval_answer, eval_role],
373
+ outputs=eval_output
374
+ )
375
+
376
+ gr.Markdown("""
377
+ ---
378
+ Built by **Sunaina** | AI/ML Engineer Intern @ Indux Technology
379
+ | [GitHub](https://github.com/Sunaina792/AI-Interview-system)
380
+ """)
381
 
382
  if __name__ == "__main__":
383
+ demo.launch()