feat: implement Stage 2 reranker with BGE cross-encoder and RRF fusion

backend/src/matching/scorer.py

@@ -0,0 +1,37 @@
+from typing import Any
+
+
+DEFAULT_WEIGHTS = {
+    "semantic": 0.20,
+    "skill": 0.35,
+    "yoe": 0.15,
+    "company": 0.10,
+    "growth": 0.10,
+    "education": 0.10,
+}
+
+
+def normalize_weights(weights: dict[str, float]) -> dict[str, float]:
+    total = sum(weights.values())
+    if total == 0:
+        return DEFAULT_WEIGHTS.copy()
+    return {k: v / total for k, v in weights.items()}
+
+
+def rerank_with_weights(
+    match_results: list[dict[str, Any]],
+    weights: dict[str, float],
+) -> list[dict[str, Any]]:
+    w = normalize_weights({**DEFAULT_WEIGHTS, **weights})
+
+    reranked = []
+    for item in match_results:
+        components = item.get("component_scores") or {}
+        new_score = sum(w.get(k, 0) * v for k, v in components.items())
+        reranked.append({**item, "final_score": round(new_score, 4), "weights_used": w})
+
+    reranked.sort(key=lambda x: x["final_score"], reverse=True)
+    for i, item in enumerate(reranked):
+        item["rank"] = i + 1
+
+    return reranked

backend/src/matching/stage2.py

@@ -0,0 +1,96 @@
+from typing import Any
+from ..ml.reranker import rerank
+
+
+def _compute_gaps(jd: dict, candidate: dict) -> list[dict]:
+    gaps = []
+
+    jd_skills = {s.lower().strip() for s in (jd.get("required_skills") or [])}
+    cand_skills = set()
+    for lst in [
+        candidate.get("programming_languages") or [],
+        candidate.get("backend_frameworks") or [],
+        candidate.get("frontend_technologies") or [],
+    ]:
+        cand_skills.update(s.lower().strip() for s in lst if s)
+    if candidate.get("parsed_skills"):
+        cand_skills.update(s.strip().lower() for s in candidate["parsed_skills"].split(",") if s.strip())
+
+    missing_skills = jd_skills - cand_skills
+    for skill in sorted(missing_skills)[:8]:
+        gaps.append({"type": "missing_skill", "detail": skill})
+
+    min_yoe = jd.get("min_yoe")
+    cand_yoe = candidate.get("years_of_experience")
+    if min_yoe and cand_yoe is not None and float(cand_yoe) < float(min_yoe):
+        gaps.append({
+            "type": "yoe_gap",
+            "detail": f"Requires {min_yoe}+ years, candidate has {cand_yoe}",
+        })
+
+    jd_location = (jd.get("location") or "").lower()
+    cand_location = (candidate.get("open_to_working_at") or "").lower()
+    if jd_location and jd_location not in ("remote", "") and cand_location:
+        if jd_location not in cand_location and cand_location not in jd_location:
+            remote_allowed = jd.get("remote_allowed", False)
+            gaps.append({
+                "type": "location_mismatch",
+                "detail": f"JD is in {jd.get('location')}, candidate is open to {candidate.get('open_to_working_at')}",
+                "mitigated_by_remote": bool(remote_allowed),
+            })
+
+    jd_engineer_type = (jd.get("engineer_type") or "").lower()
+    cand_engineer_type = (candidate.get("engineer_type") or "").lower()
+    if jd_engineer_type and cand_engineer_type and jd_engineer_type not in cand_engineer_type and cand_engineer_type not in jd_engineer_type:
+        gaps.append({
+            "type": "engineer_type_mismatch",
+            "detail": f"JD needs {jd.get('engineer_type')}, candidate is {candidate.get('engineer_type')}",
+        })
+
+    return gaps
+
+
+def _reciprocal_rank_fusion(stage1_scored: list[dict], reranker_scores: list[float], k: int = 60) -> list[dict]:
+    stage1_ranks = {item["candidate_id"]: i + 1 for i, item in enumerate(stage1_scored)}
+    reranker_ranks = {}
+    reranker_order = sorted(range(len(reranker_scores)), key=lambda i: reranker_scores[i], reverse=True)
+    for rank, idx in enumerate(reranker_order):
+        cid = stage1_scored[idx]["candidate_id"]
+        reranker_ranks[cid] = rank + 1
+
+    results = []
+    for item in stage1_scored:
+        cid = item["candidate_id"]
+        rrf_score = 1.0 / (k + stage1_ranks.get(cid, k)) + 1.0 / (k + reranker_ranks.get(cid, k))
+        results.append({**item, "stage2_score": round(reranker_scores[stage1_scored.index(item)], 4), "final_score": round(rrf_score, 6)})
+
+    results.sort(key=lambda x: x["final_score"], reverse=True)
+    return results
+
+
+async def stage2_rerank(jd: dict, shortlist: list[dict]) -> list[dict]:
+    if not shortlist:
+        return []
+
+    jd_query = f"{jd.get('title', '')} {jd.get('raw_text', '')}"
+
+    passages = []
+    for cand in shortlist:
+        parts = []
+        if cand.get("parsed_summary"):
+            parts.append(cand["parsed_summary"])
+        if cand.get("parsed_skills"):
+            parts.append(f"Skills: {cand['parsed_skills']}")
+        langs = cand.get("programming_languages") or []
+        if langs:
+            parts.append(f"Languages: {', '.join(langs[:10])}")
+        passages.append(" ".join(parts) or "No profile text")
+
+    reranker_scores = rerank(jd_query, passages)
+
+    results = _reciprocal_rank_fusion(shortlist, reranker_scores)
+
+    for cand in results:
+        cand["gaps"] = _compute_gaps(jd, cand)
+
+    return results[:20]