feat: slider max dynamically set from CSV row count (header excluded)

backend/main.py

@@ -11,9 +11,18 @@ from src.config import get_settings
 from src.models import JobDescription, Candidate, MatchResult, Session
 from src.routers import jds, candidates, matching, sessions, admin
 
+# Configure root logger so ALL module loggers (stage2, reranker, etc.)
+# emit to stdout — critical for error visibility on Hugging Face
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
 logger = logging.getLogger(__name__)
 settings = get_settings()
 
+
 _qdrant_client: QdrantClient | None = None
 _qdrant_ready: bool = False
 
@@ -58,17 +67,18 @@ async def lifespan(app: FastAPI):
 
     app.state.qdrant = _qdrant_client
     app.state.qdrant_ready = _qdrant_ready
-    
-    # -----------------------------------------------------
-    # CRITICAL: Pre-load the 2.3 GB Neural Cross-Encoder
-    # to entirely prevent HF Gateway 60-second 500 timeouts
-    # during user requests.
-    # -----------------------------------------------------
-    import asyncio
-    from src.ml.reranker import _get_reranker
-    logger.info(f"Preloading Neural Reranker `{settings.reranker_model}`. This may take ~60 seconds to cache...")
-    await asyncio.to_thread(_get_reranker)
-    logger.info("Neural Reranker fully loaded into memory!")
+
+    # Pre-load the lightweight CrossEncoder (~80MB) eagerly at startup so the
+    # first matching request doesn't pay the cold-start download cost.
+    try:
+        import asyncio
+        from src.ml.reranker import _get_reranker
+        logger.info("Warming up Neural CrossEncoder reranker...")
+        await asyncio.to_thread(_get_reranker)
+        logger.info("Neural CrossEncoder loaded and ready!")
+    except Exception as warm_exc:
+        # Log but don't crash — matching will attempt lazy-load on first request
+        logger.warning(f"Reranker warm-up failed (will retry on first request): {warm_exc}")
 
     yield
     _qdrant_client.close()

backend/src/ml/reranker.py

@@ -1,6 +1,9 @@
+import logging
 from FlagEmbedding import FlagReranker
 from ..config import get_settings
 
+logger = logging.getLogger(__name__)
+
 _reranker: FlagReranker | None = None
 
 
@@ -8,7 +11,9 @@ def _get_reranker() -> FlagReranker:
     global _reranker
     if _reranker is None:
         settings = get_settings()
+        logger.info(f"[Reranker] Loading model: {settings.reranker_model}")
         _reranker = FlagReranker(settings.reranker_model, use_fp16=False)
+        logger.info(f"[Reranker] Model loaded successfully: {settings.reranker_model}")
     return _reranker
 
 
@@ -17,7 +22,7 @@ def rerank(query: str, passages: list[str]) -> list[float]:
         return []
     reranker = _get_reranker()
     pairs = [[query, p] for p in passages]
-    # Enforce small batch_size to prevent OOM kills on 100+ candidates in cloud environments
+    # batch_size=8 limits peak RAM usage — critical for cloud CPU-only environments
     scores = reranker.compute_score(pairs, normalize=True, batch_size=8)
     if isinstance(scores, float):
         scores = [scores]

frontend/src/app/pipeline/page.tsx

@@ -38,6 +38,11 @@ export default function PipelinePage() {
   const [sessionName, setSessionName] = useState("");
   const [jds, setJds] = useState<JDInput[]>([{ title: "", desc: "" }]);
   const [file, setFile] = useState<File | null>(null);
+  // csvRowCount: actual number of data rows in the uploaded CSV (excludes header row)
+  const [csvRowCount, setCsvRowCount] = useState<number>(0);
+  // Ranking cap: max candidates to pass through Stage 1 + Stage 2
+  // Default 72 — safe upper bound for BAAI/bge-reranker-v2-m3 on HF free tier
+  const [rankingCap, setRankingCap] = useState(72);
 
   // Architecture state
   const [state, setState] = useState<PipelineState>(DEFAULT_STATE);
@@ -100,6 +105,27 @@ export default function PipelinePage() {
     });
   };
 
+  const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const selected = e.target.files?.[0] || null;
+    setFile(selected);
+    if (!selected) {
+      setCsvRowCount(0);
+      return;
+    }
+    const reader = new FileReader();
+    reader.onload = (ev) => {
+      const text = ev.target?.result as string;
+      if (!text) return;
+      // Count non-empty lines and subtract 1 for the header row
+      const lines = text.split(/\r?\n/).filter(l => l.trim().length > 0);
+      const dataRows = Math.max(0, lines.length - 1);
+      setCsvRowCount(dataRows);
+      // Clamp the current rankingCap to the new row count, keep it ≤ 72 initially
+      setRankingCap(prev => Math.min(prev, dataRows > 0 ? dataRows : 72));
+    };
+    reader.readAsText(selected);
+  };
+
   const addJd = () => setJds([...jds, { title: "", desc: "" }]);
   const removeJd = (idx: number) => {
     if (jds.length === 1) return;
@@ -174,23 +200,14 @@ export default function PipelinePage() {
   const runMatches = async (jdIds: string[], sessionId: string, currentState: PipelineState) => {
     let pendingJds = [...jdIds];
 
-    // Fetch the ACTUAL number of ingested candidates — never hardcode a cap
-    let candidateCount = 100; // safe fallback
-    try {
-      const countRes = await api.candidateCount(sessionId);
-      candidateCount = countRes.count;
-    } catch (_) {
-      // If count fetch fails, fall back to 100
-    }
-
     const pollMatches = async () => {
       try {
         const stillPending: string[] = [];
 
         for (const jdId of pendingJds) {
           try {
-            // Pass EXACT candidate count as both Stage 1 and Stage 2 top_k
-            await api.triggerMatch(jdId, sessionId, candidateCount, candidateCount);
+            // Use the user-configured ranking cap for both Stage 1 and Stage 2
+            await api.triggerMatch(jdId, sessionId, rankingCap, rankingCap);
           } catch (e: any) {
             if (e.message === "202_ACCEPTED") {
               stillPending.push(jdId);
@@ -294,7 +311,12 @@ export default function PipelinePage() {
              <label className="block text-xs font-medium text-[var(--color-muted)] mb-2">Candidates CSV (.csv, .json)</label>
              <input type="file" accept=".csv,.json,.jsonl" 
                className="w-full text-sm text-[var(--color-muted)] file:mr-4 file:py-2 file:px-4 file:rounded-xl file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-brand-dim)] file:text-[var(--color-brand-light)] hover:file:bg-[var(--color-brand)] hover:file:text-white transition-all cursor-pointer border border-[var(--color-border-strong)] rounded-xl p-2"
-               onChange={e => setFile(e.target.files?.[0] || null)} />
+               onChange={handleFileChange} />
+             {csvRowCount > 0 && (
+               <p className="mt-2 text-xs text-[var(--color-muted)]">
+                 📄 Detected <strong className="text-[var(--color-brand-light)]">{csvRowCount}</strong> candidate rows (excluding header)
+               </p>
+             )}
           </div>
 
           <div className="mb-6 border-t border-[var(--color-border-strong)] pt-6">
@@ -336,6 +358,44 @@ export default function PipelinePage() {
             </div>
           </div>
 
+          {/* RANKING CAP SLIDER */}
+          <div className="mb-6 border-t border-[var(--color-border-strong)] pt-6">
+            <div className="flex items-center justify-between mb-2">
+              <label className="text-sm font-bold text-[var(--color-text)]">Neural Ranking Cap</label>
+              <span className="text-lg font-mono font-bold text-[var(--color-brand-light)] bg-[var(--color-brand-dim)] px-3 py-1 rounded-lg border border-[var(--color-brand-glow)]">
+                {rankingCap}
+              </span>
+            </div>
+            <input
+              id="ranking-cap-slider"
+              type="range"
+              min={1}
+              max={csvRowCount > 0 ? csvRowCount : 200}
+              step={1}
+              value={rankingCap}
+              onChange={e => setRankingCap(Number(e.target.value))}
+              className="w-full h-2 rounded-lg appearance-none cursor-pointer"
+              style={{
+                background: `linear-gradient(to right, var(--color-brand) ${
+                  ((rankingCap / (csvRowCount > 0 ? csvRowCount : 200)) * 100).toFixed(1)
+                }%, var(--color-border-strong) ${
+                  ((rankingCap / (csvRowCount > 0 ? csvRowCount : 200)) * 100).toFixed(1)
+                }%)`
+              }}
+            />
+            <div className="flex justify-between text-[10px] text-[var(--color-muted)] mt-1">
+              <span>1</span>
+              <span>{csvRowCount > 0 ? csvRowCount : 200}</span>
+            </div>
+            {/* RAM Warning for BGE model */}
+            <div className="mt-3 flex items-start gap-2 bg-amber-500/10 border border-amber-500/25 rounded-xl px-4 py-3">
+              <span className="text-amber-400 text-sm mt-0.5">⚠️</span>
+              <p className="text-xs text-amber-300/90 leading-relaxed">
+                <strong>Hugging Face Free Tier Notice:</strong> We use <code className="font-mono bg-black/20 px-1 rounded">BAAI/bge-reranker-v2-m3</code> for neural reranking. On the free tier, this model exceeds available RAM above ~72 candidates and the backend will crash. <strong>Keep the cap at or below 72</strong> for stable results.
+              </p>
+            </div>
+          </div>
+
           <button onClick={startPipeline}
             className="w-full py-4 rounded-xl bg-[var(--color-brand)] text-white font-bold tracking-wide shadow-lg shadow-[var(--color-brand-glow)] hover:brightness-110 transition-all active:scale-[0.98]">
             START AUTOMATED PIPELINE