fix: use real candidate count, remove stage2 cap, add full error logging

backend/src/matching/stage2.py

@@ -1,6 +1,9 @@
+import logging
 from typing import Any
 from ..ml.reranker import rerank
 
+logger = logging.getLogger(__name__)
+
 
 def _compute_gaps(jd: dict, candidate: dict) -> list[dict]:
     gaps = []
@@ -102,7 +105,13 @@ async def stage2_rerank(jd: dict, shortlist: list[dict]) -> list[dict]:
 
     from fastapi.concurrency import run_in_threadpool
     from .reranker import rerank
-    reranker_scores = await run_in_threadpool(rerank, jd_query, passages)
+    try:
+        logger.info(f"[Stage 2] Starting neural reranking of {len(passages)} candidates...")
+        reranker_scores = await run_in_threadpool(rerank, jd_query, passages)
+        logger.info(f"[Stage 2] Reranking complete. Got {len(reranker_scores)} scores.")
+    except Exception as exc:
+        logger.exception(f"[Stage 2] FATAL: Neural reranker crashed — {exc}")
+        raise
 
     weights = jd.get("custom_weights", {})
     results = _recompute_final_score(shortlist, reranker_scores, weights)
@@ -110,4 +119,5 @@ async def stage2_rerank(jd: dict, shortlist: list[dict]) -> list[dict]:
     for cand in results:
         cand["gaps"] = _compute_gaps(jd, cand)
 
-    return results[:250]
+    # Return ALL candidates — no arbitrary cap
+    return results

backend/src/routers/matching.py

@@ -1,11 +1,14 @@
 import uuid
 import json
+import logging
 import redis.asyncio as redis
 from datetime import datetime, timezone
 from fastapi import APIRouter, Depends, HTTPException, Request, Query
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import select, delete
 
+logger = logging.getLogger(__name__)
+
 from ..database import get_db
 from ..config import get_settings
 from ..models.jd import JobDescription
@@ -115,7 +118,8 @@ async def trigger_match(
             inserted_mrs.append(mr)
 
         await db.commit()
-    except Exception:
+    except Exception as exc:
+        logger.exception(f"[trigger_match] FATAL ERROR for JD {jd_id}: {exc}")
         await db.rollback()
         raise
 

frontend/src/app/pipeline/page.tsx

@@ -173,14 +173,24 @@ export default function PipelinePage() {
 
   const runMatches = async (jdIds: string[], sessionId: string, currentState: PipelineState) => {
     let pendingJds = [...jdIds];
-    
+
+    // Fetch the ACTUAL number of ingested candidates — never hardcode a cap
+    let candidateCount = 100; // safe fallback
+    try {
+      const countRes = await api.candidateCount(sessionId);
+      candidateCount = countRes.count;
+    } catch (_) {
+      // If count fetch fails, fall back to 100
+    }
+
     const pollMatches = async () => {
       try {
         const stillPending: string[] = [];
-        
+
         for (const jdId of pendingJds) {
           try {
-            await api.triggerMatch(jdId, sessionId, 500, 500); // Pass ALL candidates through both stages
+            // Pass EXACT candidate count as both Stage 1 and Stage 2 top_k
+            await api.triggerMatch(jdId, sessionId, candidateCount, candidateCount);
           } catch (e: any) {
             if (e.message === "202_ACCEPTED") {
               stillPending.push(jdId);