husseinelsaadi Claude Opus 4.8 commited on
Commit
c3ec385
·
1 Parent(s): 5f99f0f

Fix interview scoring so strong answers score well

Browse files

Scores were capped at ~50% even for perfect interviews because the final
score weighted skills-match (exact string overlap, almost always ~0) at
40%. Rebalance to answer-driven scoring and reduce conservative bias:

- interview_api.py / report_generator.py: weight QA answers 85% and skills
match 15% (was 40/60); bump category values (Good 0.8->0.85,
Medium 0.6->0.65). Mirrored in both so screen + PDF report agree.
- interview_engine.py: rewrite the evaluate_answer prompt with a clear
rubric, tell LUNA to use the full range and not default to "Medium",
and to judge substance not grammar (answers are speech-to-text). Output
format unchanged, so parsing is untouched.

A flawless interview now scores ~85% (more with matching skills); weak
answers still score low.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/routes/interview_api.py CHANGED
@@ -443,13 +443,13 @@ def interview_complete():
443
  if ('excellent' in score_text) or ('5' in score_text) or ('10' in score_text):
444
  qa_scores.append(1.0)
445
  elif ('good' in score_text) or ('4' in score_text) or ('8' in score_text) or ('9' in score_text):
446
- qa_scores.append(0.8)
447
  elif ('satisfactory' in score_text) or ('medium' in score_text) or ('3' in score_text) or ('6' in score_text) or ('7' in score_text):
448
- qa_scores.append(0.6)
449
  elif ('needs improvement' in score_text) or ('poor' in score_text) or ('2' in score_text):
450
  qa_scores.append(0.4)
451
  else:
452
- qa_scores.append(0.2)
453
  except Exception:
454
  qa_scores = []
455
 
@@ -459,10 +459,11 @@ def interview_complete():
459
  # data.
460
  qa_average = (sum(qa_scores) / len(qa_scores)) if qa_scores else 0.5
461
 
462
- # Weight skills match (40%) and QA average (60%) to derive
463
- # the final overall score. Convert to a percentage for
464
- # display.
465
- overall = (ratio * 0.4) + (qa_average * 0.6)
 
466
  percentage = overall * 100.0
467
 
468
  # Assign a descriptive label based on the overall score.
 
443
  if ('excellent' in score_text) or ('5' in score_text) or ('10' in score_text):
444
  qa_scores.append(1.0)
445
  elif ('good' in score_text) or ('4' in score_text) or ('8' in score_text) or ('9' in score_text):
446
+ qa_scores.append(0.85)
447
  elif ('satisfactory' in score_text) or ('medium' in score_text) or ('3' in score_text) or ('6' in score_text) or ('7' in score_text):
448
+ qa_scores.append(0.65)
449
  elif ('needs improvement' in score_text) or ('poor' in score_text) or ('2' in score_text):
450
  qa_scores.append(0.4)
451
  else:
452
+ qa_scores.append(0.25)
453
  except Exception:
454
  qa_scores = []
455
 
 
459
  # data.
460
  qa_average = (sum(qa_scores) / len(qa_scores)) if qa_scores else 0.5
461
 
462
+ # Weight the interview answers (85%) as the primary signal and the
463
+ # skills match (15%) as a minor bonus. The skills ratio relies on
464
+ # exact skill-string overlap, which is often near zero even for
465
+ # strong candidates, so it must not dominate the final score.
466
+ overall = (qa_average * 0.85) + (ratio * 0.15)
467
  percentage = overall * 100.0
468
 
469
  # Assign a descriptive label based on the overall score.
backend/services/interview_engine.py CHANGED
@@ -475,17 +475,24 @@ def evaluate_answer(question, answer, job_role="Software Developer", seniority="
475
  }
476
 
477
  prompt = f"""
478
- You are LUNA, an experienced recruiter evaluating a candidate's spoken answer for a
479
- {seniority} {job_role} position. Judge it at the level expected for that seniority.
480
 
481
  Question: {question}
482
  Candidate Answer: {answer}
483
 
484
- Evaluate on: technical correctness, relevance to the question, depth/specificity, and clarity.
485
- Calibrate to seniority be more demanding for senior roles, more forgiving for junior ones.
486
- Reward concrete, specific answers; penalise vague, off-topic, or empty ones.
487
-
488
- Rate the answer as exactly one of: Poor, Medium, Good, Excellent.
 
 
 
 
 
 
 
489
 
490
  Respond in this exact format and nothing else:
491
  Score: [Poor/Medium/Good/Excellent]
 
475
  }
476
 
477
  prompt = f"""
478
+ You are LUNA, an experienced technical recruiter evaluating a candidate's spoken answer
479
+ for a {seniority} {job_role} position.
480
 
481
  Question: {question}
482
  Candidate Answer: {answer}
483
 
484
+ Rate the answer using this rubric:
485
+ - Excellent: correct, relevant and specific; shows clear understanding appropriate for a
486
+ {seniority} {job_role}. Small imperfections are fine.
487
+ - Good: mostly correct and relevant with reasonable detail, but missing some depth or specifics.
488
+ - Medium: partially correct or relevant, but vague, generic or incomplete.
489
+ - Poor: incorrect, off-topic, or no real answer.
490
+
491
+ Important:
492
+ - Judge the SUBSTANCE of the answer, not its grammar or wording — it was transcribed from
493
+ speech, so ignore filler words, punctuation and minor phrasing issues.
494
+ - A clear, correct, on-topic answer should be rated "Good" or "Excellent". Do NOT default to
495
+ "Medium" — use the full range and reward strong answers.
496
 
497
  Respond in this exact format and nothing else:
498
  Score: [Poor/Medium/Good/Excellent]
backend/services/report_generator.py CHANGED
@@ -476,18 +476,19 @@ def _calculate_overall_score(report_data: Dict[str, Any]) -> Dict[str, Any]:
476
  if 'excellent' in score_text or '5' in score_text or '10' in score_text:
477
  qa_scores.append(1.0)
478
  elif 'good' in score_text or '4' in score_text or '8' in score_text or '9' in score_text:
479
- qa_scores.append(0.8)
480
  elif 'satisfactory' in score_text or 'medium' in score_text or '3' in score_text or '6' in score_text or '7' in score_text:
481
- qa_scores.append(0.6)
482
- elif 'needs improvement' in score_text or 'poor' in score_text or '2' in score_text or '4' in score_text or '5' in score_text:
483
  qa_scores.append(0.4)
484
  else:
485
- qa_scores.append(0.2)
486
-
487
  qa_average = sum(qa_scores) / len(qa_scores) if qa_scores else 0.5
488
-
489
- # Calculate weighted average
490
- overall = (skills_ratio * 0.4) + (qa_average * 0.6)
 
491
  percentage = overall * 100
492
 
493
  if overall >= 0.8:
 
476
  if 'excellent' in score_text or '5' in score_text or '10' in score_text:
477
  qa_scores.append(1.0)
478
  elif 'good' in score_text or '4' in score_text or '8' in score_text or '9' in score_text:
479
+ qa_scores.append(0.85)
480
  elif 'satisfactory' in score_text or 'medium' in score_text or '3' in score_text or '6' in score_text or '7' in score_text:
481
+ qa_scores.append(0.65)
482
+ elif 'needs improvement' in score_text or 'poor' in score_text or '2' in score_text:
483
  qa_scores.append(0.4)
484
  else:
485
+ qa_scores.append(0.25)
486
+
487
  qa_average = sum(qa_scores) / len(qa_scores) if qa_scores else 0.5
488
+
489
+ # Weight the interview answers (85%) as the primary signal and the skills
490
+ # match (15%) as a minor bonus, mirroring backend/routes/interview_api.py.
491
+ overall = (qa_average * 0.85) + (skills_ratio * 0.15)
492
  percentage = overall * 100
493
 
494
  if overall >= 0.8: