Upload 13 files

Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.13-slim
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    cmake \
+    libomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+
+COPY --chown=user requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+
+COPY --chown=user . .
+
+
+CMD ["sleep", "infinity"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,10 +1,344 @@
+# Redrob Intelligent Candidate Ranking Pipeline
+
+> **Hackathon:** India Runs Data & AI Challenge — Candidate Ranking Track  
+> **Task:** Rank top 100 candidates from a 100,000-candidate pool for a senior AI/ML retrieval engineer role.  
+> **Approach:** 5-stage offline CPU-only pipeline using a LightGBM regressor trained via teacher distillation + deterministic reasoning + optional SLM rewrite.
+
+---
+
+## Quick Start (Single Command)
+
+```bash
+python run_pipeline.py --candidates candidates.jsonl --ranker regressor_no_prescore.pkl
+```
+
+This produces `submission.csv` with the top 100 ranked candidates. **Runtime: ~31 seconds on CPU.**
+
+---
+
+## Table of Contents
+
+1. [Prerequisites](#prerequisites)
+2. [Installation — Windows](#installation--windows)
+3. [Installation — macOS](#installation--macos)
+4. [Installation — Linux](#installation--linux)
+5. [One-time Model Download](#one-time-model-download)
+6. [Running the Pipeline](#running-the-pipeline)
+7. [Validating Your Submission](#validating-your-submission)
+8. [Pipeline Flags Reference](#pipeline-flags-reference)
+9. [File Structure](#file-structure)
+10. [Troubleshooting](#troubleshooting)
+
+---
+
+## Prerequisites
+
+| Requirement | Version |
+|---|---|
+| Python | >= 3.10 |
+| RAM | >= 8 GB (16 GB recommended) |
+| Disk Space | >= 3 GB free |
+| CPU | Any modern x86-64 CPU with **AVX2** support (for SLM stage) |
+| Internet | Required once for model download only |
+
+---
+
+## Installation — Windows
+
+### Step 1: Verify Python Version
+```powershell
+python --version
+# Should show Python 3.10 or higher
+```
+
+If Python is not installed, download it from [python.org](https://python.org) or via Windows Store.
+
+### Step 2: Create a Virtual Environment (Recommended)
+```powershell
+python -m venv venv
+venv\Scripts\activate
+```
+
+### Step 3: Install Core Dependencies
+```powershell
+pip install -r requirements.txt
+```
+
+### Step 4: Install llama-cpp-python (SLM Engine)
+
+> **CPU-only (works on any machine):**
+```powershell
+pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+```
+
+> **With NVIDIA GPU (optional, ~10x faster for SLM stage):**
+```powershell
+pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
+```
+
+### Step 5: Download the SLM Model (One-Time)
+```powershell
+python setup_model.py
+```
+Downloads `Qwen2.5-1.5B-Instruct-Q4_K_M.gguf` (~986 MB) into the `models/` folder. Resumable if interrupted.
+
+---
+
+## Installation — macOS
+
+### Step 1: Install Python via Homebrew
+```bash
+brew install python@3.11
+python3 --version
+```
+
+### Step 2: Create a Virtual Environment
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+### Step 3: Install Core Dependencies
+```bash
+pip install -r requirements.txt
+```
+
+### Step 4: Install llama-cpp-python
+
+> **Apple Silicon (M1/M2/M3) — Metal GPU acceleration:**
+```bash
+CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python
+```
+
+> **Intel Mac — CPU only:**
+```bash
+pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+```
+
+### Step 5: Download the SLM Model (One-Time)
+```bash
+python setup_model.py
+```
+
+---
+
+## Installation — Linux
+
+### Step 1: Install Python
+```bash
+# Ubuntu / Debian
+sudo apt update
+sudo apt install python3.11 python3.11-venv python3-pip -y
+
+# Fedora / RHEL
+sudo dnf install python3.11 -y
+
+# Verify
+python3 --version
+```
+
+### Step 2: Create a Virtual Environment
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+### Step 3: Install Build Tools (required for llama-cpp-python compilation)
+```bash
+# Ubuntu / Debian
+sudo apt install build-essential cmake -y
+
+# Fedora / RHEL
+sudo dnf groupinstall "Development Tools" -y
+sudo dnf install cmake -y
+```
+
+### Step 4: Install Core Dependencies
+```bash
+pip install -r requirements.txt
+```
+
+### Step 5: Install llama-cpp-python
+
+> **CPU-only build:**
+```bash
+pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+```
+
+> **CUDA 12.1 (NVIDIA GPU):**
+```bash
+CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python
+```
+
+### Step 6: Download the SLM Model (One-Time)
+```bash
+python setup_model.py
+```
+
+---
+
+## One-time Model Download
+
+The SLM model (`Qwen2.5-1.5B-Instruct-Q4_K_M.gguf`) is approximately **986 MB** and needs to be downloaded once before running the full pipeline.
+
+```bash
+python setup_model.py
+```
+
+**What it does:**
+- Checks for 2 GB free disk space
+- Downloads the GGUF quantized model from HuggingFace
+- Saves it to `models/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf`
+- Download is resumable if interrupted
+
+> **Note:** If you prefer to skip the SLM rewrite stage (e.g., for testing), you can use `--no-rewrite` flag and skip this step entirely.
+
+---
+
+## Running the Pipeline
+
+### Full Pipeline (All Stages including SLM rewrite)
+```bash
+python run_pipeline.py \
+  --candidates candidates.jsonl \
+  --ranker regressor_no_prescore.pkl
+```
+
+### Skip SLM Rewrite (Faster, uses deterministic reasoning)
+```bash
+python run_pipeline.py \
+  --candidates candidates.jsonl \
+  --ranker regressor_no_prescore.pkl \
+  --no-rewrite
+```
+
+### Resume from a Specific Stage (if pipeline was interrupted)
+```bash
+# Resume from Stage 2 (LightGBM) onwards
+python run_pipeline.py \
+  --candidates candidates.jsonl \
+  --ranker regressor_no_prescore.pkl \
+  --start-stage 2
+
+# Resume from Stage 4 (SLM Rewrite) onwards
+python run_pipeline.py \
+  --candidates candidates.jsonl \
+  --ranker regressor_no_prescore.pkl \
+  --start-stage 4
+```
+
+### Expected Runtime
+| Stage | Operation | Time |
+|---|---|---|
+| Stage 1 | Hard filter + prescore (100k → 22k) | ~17 seconds |
+| Stage 2 | LightGBM regression (22k → top 100) | ~4.6 seconds |
+| Stage 3 | Deterministic reasoning (top 100) | < 0.1 seconds |
+| Stage 4 | SLM rewrite — Qwen2.5-1.5B (top 100) | ~2–4 minutes (optional) |
+| Stage 5 | Write submission.csv | < 0.1 seconds |
+| **Total (without SLM)** | | **~31 seconds** |
+| **Total (with SLM)** | | **~3–5 minutes** |
+
+---
+
+## Validating Your Submission
+
+After the pipeline runs, validate your output against the official submission spec:
+
+```bash
+python validate_submission.py submission.csv
+```
+
+Expected output:
+```
+Submission is valid.
+```
+
+---
+
+## Pipeline Flags Reference
+
+| Flag | Default | Description |
+|---|---|---|
+| `--candidates` | required | Path to `candidates.jsonl` |
+| `--ranker` | `regressor_no_prescore.pkl` | Path to LightGBM `.pkl` model |
+| `--filter-top-k` | `22000` | Number of candidates after Stage 1 |
+| `--lgbm-top-k` | `15000` | Number of candidates fed to LightGBM |
+| `--top-final` | `100` | Final candidates in submission |
+| `--no-rewrite` | False | Skip SLM rewrite (Stage 4) |
+| `--start-stage` | `1` | Resume pipeline from a specific stage |
+| `--threads` | `auto` | CPU threads for LLM inference |
+| `--temperature` | `0.35` | SLM sampling temperature |
+
 ---
-title: Redrob
-emoji: 🐠
-colorFrom: blue
-colorTo: indigo
-sdk: docker
-pinned: false
+
+## File Structure
+
+```
+github_repo/
+├── run_pipeline.py                    # Main orchestrator — run this
+├── filter_candidates.py               # Stage 1: Hard filter + feature extraction
+├── fix_reasoning.py                   # Stage 3: Deterministic reasoning builder
+├── rewrite_reasoning.py               # Stage 4: Qwen2.5 SLM rewrite engine
+├── score_candidates.py                # Stage 3.5: Composite scoring
+├── setup_model.py                     # One-time model downloader
+├── validate_submission.py             # Official submission validator
+├── train_lightgbm_regressor.py        # (Offline) Training script
+├── regressor_no_prescore.pkl          # Pre-trained LightGBM model weights
+├── regressor_no_prescore_features.json# Feature names for the model
+├── requirements.txt                   # Python dependencies
+├── models/
+│   └── Qwen2.5-1.5B-Instruct-Q4_K_M.gguf  # SLM weights (download via setup_model.py)
+├── submission.csv                     # Final output (generated by pipeline)
+├── filtered_22k.jsonl                 # Stage 1 output (cached)
+├── lgbm_top15k.jsonl                  # Stage 2 output (cached)
+└── top100_reasoned.jsonl              # Stage 3 output (cached)
+```
+
+---
+
+## Troubleshooting
+
+### `OSError: [WinError -1073741795] Windows Error 0xc000001d`
+Your CPU does not support AVX2 instructions required by `llama-cpp-python`. Use the `--no-rewrite` flag to skip Stage 4:
+```bash
+python run_pipeline.py --candidates candidates.jsonl --ranker regressor_no_prescore.pkl --no-rewrite
+```
+
+### `ModuleNotFoundError: No module named 'lightgbm'`
+Install dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+### `Model not found` error during Stage 4
+Run the one-time download:
+```bash
+python setup_model.py
+```
+
+### Download interrupted / incomplete GGUF file
+`setup_model.py` uses `huggingface_hub` which supports **resumable downloads**. Simply run `python setup_model.py` again — it will continue from where it left off.
+
+### `PermissionError` on Windows
+Run your terminal as Administrator, or ensure no other process has the output files open (e.g., close Excel if `submission.csv` is open).
+
+### Stage 2 LightGBM model mismatch
+Ensure you are using the exact `regressor_no_prescore.pkl` provided in the repo. Do not mix pkl files from different training runs — the feature list must exactly match `regressor_no_prescore_features.json`.
+
+---
+
+## Reproduction Command (for Stage 3 Code Review)
+
+For the hackathon Stage 3 code reproduction check, the exact command to produce `submission.csv` from scratch is:
+
+```bash
+pip install -r requirements.txt
+pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+python setup_model.py  # one-time model download
+python run_pipeline.py --candidates candidates.jsonl --ranker regressor_no_prescore.pkl
+```
+
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+*Pipeline runtime: ~31 seconds (CPU-only, without SLM) | ~4 minutes (with SLM rewrite)*  
+*Peak RAM: ~1.65 GB | Disk footprint: ~1.54 GB*

filter_candidates.py

@@ -0,0 +1,2096 @@
+#!/usr/bin/env python3
+
+"""
+
+filter_candidates.py — Redrob Hackathon Candidate Filtering Pipeline
+
+=====================================================================
+
+Produces a ranked, filtered JSONL of candidates ready to feed into a
+
+teacher model for score + reasoning generation.
+
+
+
+Target output: top 50 000 candidates (or --top-k N), ordered by a
+
+multi-signal pre-score so the teacher sees the most promising ones first.
+
+
+
+Usage:
+
+    python filter_candidates.py --candidates candidates.jsonl --out filtered_50k.jsonl
+
+    python filter_candidates.py --candidates candidates.jsonl.gz --top-k 30000 --out filtered.jsonl
+
+    python filter_candidates.py --candidates candidates.jsonl --top-k 50000 --out filtered.jsonl --report
+
+
+
+What this script does:
+
+    1. Hard-disqualification filters  (eliminate ~70-75 % of pool instantly)
+
+    2. Honeypot detection             (flag & exclude impossible profiles)
+
+    3. Per-candidate pre-score        (rank the survivors before teacher sees them)
+
+    4. Output  top-K JSONL  +  a human-readable filter report
+
+
+
+Design principle: every decision here is traceable to either
+
+  (a) an explicit JD statement, or
+
+  (b) an observation from the data-analysis transcript in the conversation.
+
+"""
+
+
+
+import json
+
+import gzip
+
+import argparse
+
+import sys
+
+import os
+
+import collections
+
+from datetime import date, datetime
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  REFERENCE DATE  (contest reference date from analyse_dataset.py)
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+TODAY = date(2026, 6, 9)
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  COMPANY UNIVERSE  (derived from full data scan — 63 unique companies total)
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+# JD explicitly disqualifies: "people who have only worked at consulting firms"
+
+SERVICES_COMPANIES = {
+
+    'tcs', 'infosys', 'wipro', 'accenture', 'cognizant', 'capgemini',
+
+    'mindtree', 'hcl', 'tech mahindra', 'mphasis', 'hexaware',
+
+    'ltimindtree', 'genpact',
+
+}
+
+
+
+# Injected synthetic noise — fictional company names from pop culture
+
+FICTIONAL_COMPANIES = {
+
+    'pied piper', 'initech', 'hooli', 'wayne enterprises',
+
+    'acme corp', 'stark industries', 'globex inc', 'dunder mifflin',
+
+    'umbrella corporation', 'soylent corp',
+
+}
+
+
+
+# Real Indian product / startup companies — strong positive signal
+
+PRODUCT_COMPANIES_TIER1 = {
+
+    # FAANG / Big Tech India
+
+    'google', 'meta', 'amazon', 'microsoft', 'apple', 'netflix',
+
+    'adobe', 'salesforce', 'uber', 'linkedin',
+
+}
+
+PRODUCT_COMPANIES_TIER2 = {
+
+    # Indian unicorns / large product companies
+
+    'swiggy', 'zomato', 'razorpay', 'cred', 'flipkart', 'phonepe',
+
+    'paytm', 'meesho', 'nykaa', 'inmobi', "byju's", 'zoho',
+
+    'unacademy', 'upgrad', 'policybazaar', 'ola', 'dream11',
+
+    'freshworks', 'vedantu', 'pharmeasy',
+
+}
+
+PRODUCT_COMPANIES_TIER3 = {
+
+    # Indian AI-native / ML-focused companies
+
+    'glance', 'rephrase.ai', 'aganitha', 'niramai', 'saarthi.ai',
+
+    'sarvam ai', 'mad street den', 'observe.ai', 'krutrim',
+
+    'wysa', 'haptik', 'verloop.io', 'yellow.ai', 'locobuzz',
+
+    'genpact ai',
+
+}
+
+ALL_PRODUCT_COMPANIES = PRODUCT_COMPANIES_TIER1 | PRODUCT_COMPANIES_TIER2 | PRODUCT_COMPANIES_TIER3
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  TITLE UNIVERSE  (all 47 unique profile titles + 48 career role titles from data)
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+# Titles that are unambiguously NOT what the JD wants — used for hard filter
+
+HARD_DISQUALIFY_TITLES = {
+
+    'hr manager', 'business analyst', 'accountant', 'marketing manager',
+
+    'operations manager', 'civil engineer', 'mechanical engineer',
+
+    'content writer', 'customer support', 'sales executive',
+
+    'graphic designer', 'project manager',
+
+}
+
+
+
+# Titles that are ambiguous — software engineers can do ML, data engineers matter
+
+AMBIGUOUS_TITLES = {
+
+    'software engineer', 'senior software engineer', 'full stack developer',
+
+    'backend engineer', 'cloud engineer', 'devops engineer', 'qa engineer',
+
+    'frontend engineer', 'java developer', '.net developer', 'mobile developer',
+
+    'data analyst', 'analytics engineer', 'data engineer', 'senior data engineer',
+
+}
+
+
+
+# Core AI/ML titles — strongest positive title signal
+
+AI_CORE_TITLES = {
+
+    'ml engineer', 'machine learning engineer', 'senior machine learning engineer',
+
+    'staff machine learning engineer', 'lead ai engineer', 'senior ml engineer — search & ranking',
+
+    'ai engineer', 'senior ai engineer',
+
+    'ai research engineer', 'applied ml engineer', 'senior applied scientist',
+
+    'nlp engineer', 'senior nlp engineer',
+
+    'data scientist', 'senior data scientist',
+
+    'ai specialist', 'computer vision engineer',
+
+    'recommendation systems engineer', 'search engineer',
+
+    'senior software engineer (ml)',
+
+    'junior ml engineer',
+
+}
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  SKILL UNIVERSE  (from full data scan — 133 unique skill names)
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+# Skills directly mentioned in the JD as "absolutely need"
+
+JD_MUST_HAVE_SKILLS = {
+
+    'sentence transformers', 'embeddings', 'vector search', 'semantic search',
+
+    'pinecone', 'weaviate', 'qdrant', 'milvus', 'faiss',
+
+    'elasticsearch', 'opensearch',
+
+    'information retrieval', 'python',
+
+    'learning to rank',
+
+}
+
+
+
+# Skills mentioned in JD "would like to have"
+
+JD_NICE_TO_HAVE_SKILLS = {
+
+    'fine-tuning llms', 'qlora', 'lora', 'peft',
+
+    'hugging face transformers', 'llms', 'langchain', 'llamaindex', 'haystack',
+
+    'rag',
+
+    'machine learning', 'deep learning', 'nlp',
+
+    'pytorch', 'tensorflow', 'scikit-learn',
+
+    'mlops', 'mlflow', 'weights & biases', 'kubeflow',
+
+    'recommendation systems',
+
+    'bm25', 'pgvector',
+
+}
+
+
+
+# ML skills from data that are broadly positive even if not JD-specific
+
+ML_BROAD_SKILLS = {
+
+    'feature engineering', 'time series', 'forecasting', 'statistical modeling',
+
+    'reinforcement learning', 'data science',
+
+    'computer vision', 'opencv', 'yolo', 'gans', 'diffusion models',
+
+    'image classification', 'object detection',
+
+    'asr', 'speech recognition', 'tts',
+
+    'prompt engineering',
+
+}
+
+
+
+ALL_RELEVANT_SKILLS = JD_MUST_HAVE_SKILLS | JD_NICE_TO_HAVE_SKILLS | ML_BROAD_SKILLS
+
+
+
+# Skills that are neutral/non-ML — having only these signals keyword stuffing
+
+GENERIC_TECH_SKILLS = {
+
+    'html', 'css', 'javascript', 'typescript', 'react', 'vue.js', 'angular',
+
+    'next.js', 'webpack', 'tailwind', 'redux', 'graphql', 'rest apis',
+
+    'java', 'go', 'rust', 'spring boot', 'django', 'flask', 'fastapi',
+
+    'docker', 'kubernetes', 'terraform', 'ci/cd', 'aws', 'gcp', 'azure',
+
+    'kafka', 'spark', 'airflow', 'apache beam', 'apache flink', 'hadoop',
+
+    'bigquery', 'snowflake', 'dbt', 'databricks', 'etl', 'data pipelines',
+
+    'sql', 'postgresql', 'mongodb', 'redis',
+
+    'figma', 'photoshop', 'illustrator',
+
+    'excel', 'powerpoint', 'agile', 'scrum', 'project management',
+
+    'sales', 'marketing', 'accounting', 'content writing', 'seo',
+
+    'tally', 'sap', 'salesforce crm', 'six sigma',
+
+}
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  HELPERS
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+def _load(path, max_cands=None):
+
+    candidates = []
+
+    opener = gzip.open if path.endswith('.gz') else open
+
+    mode   = 'rt' if path.endswith('.gz') else 'r'
+
+    with opener(path, mode, encoding='utf-8') as f:
+
+        for line in f:
+
+            line = line.strip()
+
+            if not line:
+
+                continue
+
+            candidates.append(json.loads(line))
+
+            if max_cands and len(candidates) >= max_cands:
+
+                break
+
+    return candidates
+
+
+
+
+
+def _days_since(date_str):
+
+    """Return days between TODAY and a YYYY-MM-DD string."""
+
+    try:
+
+        d = datetime.strptime(date_str, '%Y-%m-%d').date()
+
+        return (TODAY - d).days
+
+    except Exception:
+
+        return 9999
+
+
+
+
+
+def _normalize_co(name):
+
+    return name.strip().lower()
+
+
+
+
+
+def _classify_company(name):
+
+    n = _normalize_co(name)
+
+    if n in FICTIONAL_COMPANIES:
+
+        return 'fictional'
+
+    if n in SERVICES_COMPANIES:
+
+        return 'services'
+
+    if n in PRODUCT_COMPANIES_TIER1:
+
+        return 'tier1_product'
+
+    if n in PRODUCT_COMPANIES_TIER2:
+
+        return 'tier2_product'
+
+    if n in PRODUCT_COMPANIES_TIER3:
+
+        return 'tier3_product'
+
+    return 'unknown'
+
+
+
+
+
+def _career_company_types(career_history):
+
+    types = [_classify_company(r['company']) for r in career_history]
+
+    return types
+
+
+
+
+
+def _has_relevant_skill(skills):
+
+    names = {s['name'].lower() for s in skills}
+
+    return bool(names & ALL_RELEVANT_SKILLS)
+
+
+
+
+
+def _relevant_skill_score(skills):
+
+    """0-100 skill quality score based on relevance, proficiency, and duration."""
+
+    score = 0.0
+
+    for s in skills:
+
+        name = s['name'].lower()
+
+        prof = s['proficiency']          # beginner / intermediate / advanced / expert
+
+        dur  = s.get('duration_months', 0)
+
+        endorse = s.get('endorsements', 0)
+
+
+
+        if name not in ALL_RELEVANT_SKILLS:
+
+            continue
+
+
+
+        # Base weight by relevance tier
+
+        if name in JD_MUST_HAVE_SKILLS:
+
+            base = 4.0
+
+        elif name in JD_NICE_TO_HAVE_SKILLS:
+
+            base = 2.5
+
+        else:
+
+            base = 1.5  # ML_BROAD_SKILLS
+
+
+
+        # Proficiency multiplier
+
+        prof_mult = {'beginner': 0.4, 'intermediate': 0.7,
+
+                     'advanced': 1.0, 'expert': 1.3}.get(prof, 0.7)
+
+
+
+        # Duration multiplier — logarithmic so 0 months ≠ nothing but high duration matters
+
+        import math
+
+        dur_mult = min(1.5, 0.5 + 0.15 * math.log1p(dur)) if dur > 0 else 0.3
+
+
+
+        # Small endorsement bonus (capped)
+
+        endorse_bonus = min(0.3, endorse * 0.01)
+
+
+
+        score += base * prof_mult * dur_mult + endorse_bonus
+
+
+
+    return min(100.0, score)
+
+
+
+
+
+def _career_quality_score(career_history):
+
+    """
+
+    Score based on company type, role relevance, and description richness.
+
+    Returns 0-100.
+
+    """
+
+    score = 0.0
+
+    total_weight = 0.0
+
+
+
+    for role in career_history:
+
+        ctype = _classify_company(role['company'])
+
+        dur   = role.get('duration_months', 0)
+
+        title = role['title'].lower()
+
+        desc  = role.get('description', '')
+
+
+
+        # Weight recent / longer roles more
+
+        weight = max(1.0, dur / 12.0)
+
+        total_weight += weight
+
+
+
+        # Company type value
+
+        co_val = {
+
+            'tier1_product': 10.0,
+
+            'tier2_product': 8.0,
+
+            'tier3_product': 7.0,
+
+            'unknown':       4.0,   # Could be legitimate small company
+
+            'services':      1.5,   # Not zero — people move on from services
+
+            'fictional':     0.0,   # Pure noise
+
+        }.get(ctype, 4.0)
+
+
+
+        # Role title bonus
+
+        if title in AI_CORE_TITLES:
+
+            role_val = 5.0
+
+        elif title in AMBIGUOUS_TITLES:
+
+            role_val = 2.0
+
+        elif title in HARD_DISQUALIFY_TITLES:
+
+            role_val = 0.0
+
+        else:
+
+            role_val = 1.5  # Unknown title — neutral
+
+
+
+        # Description richness bonus (rich descriptions = someone wrote real content)
+
+        desc_val = 0.0
+
+        if len(desc) >= 300:
+
+            desc_val = 2.0
+
+        elif len(desc) >= 100:
+
+            desc_val = 1.0
+
+
+
+        score += weight * (co_val + role_val + desc_val)
+
+
+
+    if total_weight == 0:
+
+        return 0.0
+
+
+
+    # Normalize to 0-100
+
+    raw = score / total_weight
+
+    return min(100.0, raw * 5.0)
+
+
+
+
+
+def _behavioral_score(sig):
+
+    """Score behavioral signals 0-100."""
+
+    score = 0.0
+
+
+
+    # Open to work — binary, high value
+
+    if sig.get('open_to_work_flag'):
+
+        score += 15.0
+
+
+
+    # Recency — exponential decay
+
+    days = _days_since(sig.get('last_active_date', '2020-01-01'))
+
+    if days <= 7:
+
+        score += 15.0
+
+    elif days <= 30:
+
+        score += 12.0
+
+    elif days <= 90:
+
+        score += 8.0
+
+    elif days <= 180:
+
+        score += 3.0
+
+    # else 0
+
+
+
+    # Notice period (JD: loves sub-30d, can buy out 30d, 30+ still in scope but higher bar)
+
+    notice = sig.get('notice_period_days', 90)
+
+    if notice == 0:
+
+        score += 12.0
+
+    elif notice <= 30:
+
+        score += 10.0
+
+    elif notice <= 60:
+
+        score += 6.0
+
+    elif notice <= 90:
+
+        score += 3.0
+
+    # >90 = 0
+
+
+
+    # GitHub activity
+
+    gh = sig.get('github_activity_score', -1)
+
+    if gh > 70:
+
+        score += 10.0
+
+    elif gh > 40:
+
+        score += 7.0
+
+    elif gh > 0:
+
+        score += 3.0
+
+    # -1 = no github = 0
+
+
+
+    # Recruiter responsiveness
+
+    rr = sig.get('recruiter_response_rate', 0)
+
+    score += rr * 8.0  # 0-8 pts
+
+
+
+    # Applications submitted — active job seeker
+
+    apps = sig.get('applications_submitted_30d', 0)
+
+    if apps >= 5:
+
+        score += 5.0
+
+    elif apps >= 2:
+
+        score += 3.0
+
+    elif apps >= 1:
+
+        score += 1.5
+
+
+
+    # Interview completion
+
+    icr = sig.get('interview_completion_rate', 0)
+
+    score += icr * 5.0  # 0-5 pts
+
+
+
+    # Profile completeness
+
+    pc = sig.get('profile_completeness_score', 0)
+
+    score += pc * 0.05   # 0-5 pts
+
+
+
+    # Verifications
+
+    if sig.get('verified_email'):
+
+        score += 2.0
+
+    if sig.get('verified_phone'):
+
+        score += 2.0
+
+    if sig.get('linkedin_connected'):
+
+        score += 2.0
+
+
+
+    # Willing to relocate — JD is Pune/Noida, values this
+
+    if sig.get('willing_to_relocate'):
+
+        score += 3.0
+
+
+
+    return min(100.0, score)
+
+
+
+
+
+def _education_score(education):
+
+    """0-100 education quality score."""
+
+    if not education:
+
+        return 20.0  # neutral, not penalized
+
+
+
+    tier_vals = {
+
+        'tier_1':  100.0,
+
+        'tier_2':   70.0,
+
+        'tier_3':   45.0,
+
+        'tier_4':   25.0,
+
+        'unknown':  35.0,
+
+    }
+
+    # Take best tier across all qualifications
+
+    best = max(tier_vals.get(e.get('tier', 'unknown'), 35.0) for e in education)
+
+
+
+    # Bonus for CS/ML-adjacent field
+
+    relevant_fields = {
+
+        'computer science', 'computer engineering', 'information technology',
+
+        'machine learning', 'artificial intelligence', 'data science',
+
+        'electronics', 'electrical engineering', 'mathematics', 'statistics',
+
+        'information science', 'computational mathematics',
+
+    }
+
+    field_bonus = 0.0
+
+    for e in education:
+
+        field = e.get('field_of_study', '').lower()
+
+        if any(f in field for f in relevant_fields):
+
+            field_bonus = 10.0
+
+            break
+
+
+
+    return min(100.0, best + field_bonus)
+
+
+
+
+
+def _yoe_score(yoe):
+
+    """Score years of experience against JD target of 5-9 years."""
+
+    if 5 <= yoe <= 9:
+
+        return 100.0
+
+    elif 4 <= yoe < 5:
+
+        return 80.0
+
+    elif 9 < yoe <= 12:
+
+        return 75.0
+
+    elif 3 <= yoe < 4:
+
+        return 50.0
+
+    elif 12 < yoe <= 15:
+
+        return 55.0
+
+    elif 2 <= yoe < 3:
+
+        return 30.0
+
+    elif yoe > 15:
+
+        return 35.0
+
+    else:
+
+        return 5.0
+
+
+
+
+
+def _location_score(profile, sig):
+
+    """Score based on JD location preferences (Pune/Noida, India)."""
+
+    country = profile.get('country', '').strip()
+
+    location = profile.get('location', '').lower()
+
+    relocate = sig.get('willing_to_relocate', False)
+
+
+
+    if country == 'India':
+
+        # JD preferred cities
+
+        preferred = {'noida', 'pune', 'delhi', 'gurgaon', 'bengaluru', 'bangalore',
+
+                     'hyderabad', 'mumbai', 'chennai', 'ncr', 'new delhi'}
+
+        if any(p in location for p in preferred):
+
+            return 100.0
+
+        return 70.0   # India but not preferred city
+
+    elif relocate:
+
+        return 50.0
+
+    else:
+
+        return 20.0   # Outside India, not willing to relocate
+
+
+
+
+
+def _title_score(title):
+
+    """Score current profile title."""
+
+    t = title.lower().strip()
+
+    if t in AI_CORE_TITLES:
+
+        return 100.0
+
+    elif t in AMBIGUOUS_TITLES:
+
+        return 40.0
+
+    elif t in HARD_DISQUALIFY_TITLES:
+
+        return 0.0
+
+    else:
+
+        return 20.0  # Unknown title
+
+
+
+
+
+def _has_ai_role_in_career(career_history):
+
+    """True if candidate ever held an AI/ML role title, even if current title is not."""
+
+    for role in career_history:
+
+        if role['title'].lower() in AI_CORE_TITLES:
+
+            return True
+
+    return False
+
+
+
+
+
+def _product_company_tenure_months(career_history):
+
+    """Total months spent at product companies."""
+
+    total = 0
+
+    for role in career_history:
+
+        if _classify_company(role['company']) in ('tier1_product', 'tier2_product', 'tier3_product'):
+
+            total += role.get('duration_months', 0)
+
+    return total
+
+
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  HONEYPOT DETECTION
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+def _detect_honeypot(c):
+
+    """
+
+    Return (is_honeypot: bool, flags: list[str]).
+
+    Based on spec section 7: ~80 profiles with subtly impossible signals.
+
+    """
+
+    flags = []
+
+    p    = c['profile']
+
+    sig  = c['redrob_signals']
+
+
+
+    # ── Flag 1: Expert skills with 0 months duration
+
+    expert_zero = [s['name'] for s in c['skills']
+
+                   if s['proficiency'] == 'expert' and s.get('duration_months', 0) == 0]
+
+    if len(expert_zero) >= 3:
+
+        flags.append(f'expert_skill_0months:count={len(expert_zero)}')
+
+
+
+    # ── Flag 2: Career history months wildly exceed stated YoE
+
+    total_career_months = sum(r.get('duration_months', 0) for r in c['career_history'])
+
+    yoe_months = p['years_of_experience'] * 12
+
+    # Allow 18-month slack (gap years, overlap labelling)
+
+    if total_career_months > yoe_months + 24:
+
+        flags.append(f'career_months({total_career_months})>>yoe_months({yoe_months:.0f})')
+
+
+
+    # ── Flag 3: Non-tech title + 5+ expert/advanced AI skills
+
+    is_hard_disq = p['current_title'].lower() in HARD_DISQUALIFY_TITLES
+
+    expert_ai = [s for s in c['skills']
+
+                 if s['name'].lower() in ALL_RELEVANT_SKILLS
+
+                 and s['proficiency'] in ('expert', 'advanced')]
+
+    if is_hard_disq and len(expert_ai) >= 5:
+
+        flags.append(f'nontechTitle+expertAI:count={len(expert_ai)}')
+
+
+
+    # ── Flag 4: No GitHub but 4+ expert ML skills (impossible credibility gap)
+
+    gh = sig.get('github_activity_score', -1)
+
+    expert_ml = [s for s in c['skills']
+
+                 if s['name'].lower() in ALL_RELEVANT_SKILLS
+
+                 and s['proficiency'] == 'expert']
+
+    if gh == -1 and len(expert_ml) >= 5:
+
+        flags.append(f'no_github+expert_ml:count={len(expert_ml)}')
+
+
+
+    # ── Flag 5: Future graduation dates
+
+    for edu in c['education']:
+
+        if edu.get('end_year', 0) > 2026:
+
+            flags.append(f'future_graduation:{edu["end_year"]}')
+
+
+
+    # ── Flag 6: Headline claims "expert" but all skills are beginner
+
+    headline = p.get('headline', '').lower()
+
+    if 'expert' in headline and c['skills'] and all(
+
+            s['proficiency'] == 'beginner' for s in c['skills']):
+
+        flags.append('headline_expert_all_skills_beginner')
+
+
+
+    # ── Flag 7: Worked at company founded after stated start date
+
+    # (heuristic: check if role at a very new AI startup has impossibly long duration)
+
+    # E.g. Sarvam AI was founded ~2023, so any role there with 60+ months is fake
+
+    YOUNG_COMPANIES = {'sarvam ai', 'krutrim', 'rephrase.ai', 'saarthi.ai',
+
+                       'observe.ai', 'genpact ai'}
+
+    for role in c['career_history']:
+
+        cn = _normalize_co(role['company'])
+
+        if cn in YOUNG_COMPANIES and role.get('duration_months', 0) > 48:
+
+            flags.append(f'impossibly_long_tenure_at_young_co:{role["company"]}({role["duration_months"]}mo)')
+
+
+
+    # ── Flag 8: All skills identical proficiency = suspicious uniformity
+
+    if len(c['skills']) >= 8:
+
+        profs = [s['proficiency'] for s in c['skills']]
+
+        if len(set(profs)) == 1 and profs[0] == 'expert':
+
+            flags.append('all_skills_expert:suspicious_uniformity')
+
+
+
+    is_honeypot = len(flags) >= 2  # Multiple flags = very likely honeypot
+
+    return is_honeypot, flags
+
+
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  HARD FILTER — disqualifies candidates that cannot make the top 100
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+def _hard_filter(c):
+
+    """
+
+    Returns (passes: bool, reason: str).
+
+    Eliminates obvious non-fits without scoring.
+
+    """
+
+    p    = c['profile']
+
+    sig  = c['redrob_signals']
+
+    title = p['current_title'].lower().strip()
+
+    yoe   = p['years_of_experience']
+
+    career = c['career_history']
+
+
+
+    # ── Rule H1: Extreme YoE out of range
+
+    if yoe < 1.5:
+
+        return False, 'yoe_too_low(<1.5y)'
+
+    if yoe > 18:
+
+        return False, 'yoe_too_high(>18y)'
+
+
+
+    # ── Rule H2: Non-tech title AND no AI/ML role in career history
+
+    if title in HARD_DISQUALIFY_TITLES:
+
+        if not _has_ai_role_in_career(career):
+
+            return False, f'nontechTitle_no_ai_career:{title}'
+
+
+
+    # ── Rule H3: Entire career at fictional companies only
+
+    co_types = _career_company_types(career)
+
+    if all(ct == 'fictional' for ct in co_types):
+
+        return False, 'all_fictional_career'
+
+
+
+    # ── Rule H4: Entire career at services+fictional with non-tech title
+
+    if title in HARD_DISQUALIFY_TITLES:
+
+        non_product = all(ct in ('fictional', 'services') for ct in co_types)
+
+        if non_product:
+
+            return False, 'nontechTitle_services_only'
+
+
+
+    # ── Rule H5: Zero relevant skills AND non-tech title
+
+    if title in HARD_DISQUALIFY_TITLES:
+
+        if not _has_relevant_skill(c['skills']):
+
+            return False, 'nontechTitle_zero_relevant_skills'
+
+
+
+    # ── Rule H6: Outside India, not willing to relocate, non-AI title
+
+    # (generous — we keep all who might relocate or are India-based)
+
+    if p.get('country') not in ('India',) and not sig.get('willing_to_relocate'):
+
+        # Still pass non-India if they have strong AI title + skills
+
+        if title not in AI_CORE_TITLES and not _has_relevant_skill(c['skills']):
+
+            return False, 'non_india_no_relocate_no_ai'
+
+
+
+    # ── Rule H7: Ghost profile — last active > 2 years ago AND not open to work
+
+    days_inactive = _days_since(sig.get('last_active_date', '2020-01-01'))
+
+    if days_inactive > 730 and not sig.get('open_to_work_flag'):
+
+        return False, f'ghost_profile:inactive_{days_inactive}d'
+
+
+
+    return True, 'pass'
+
+
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  COMPOSITE PRE-SCORE
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+# Weights calibrated against JD priorities:
+
+#   - JD cares most about skills + career (the "intelligence layer" role)
+
+#   - Behavioral signals are tiebreakers (JD says "down-weight unavailable")
+
+#   - Education is secondary
+
+#   - Location matters (Pune/Noida-preferred)
+
+WEIGHTS = {
+
+    'title':     0.10,
+
+    'skills':    0.28,
+
+    'career':    0.28,
+
+    'yoe':       0.10,
+
+    'behavioral':0.14,
+
+    'education': 0.06,
+
+    'location':  0.04,
+
+}
+
+
+
+
+
+def _compute_prescore(c):
+
+    """Returns a float 0-100 pre-score."""
+
+    p   = c['profile']
+
+    sig = c['redrob_signals']
+
+
+
+    scores = {
+
+        'title':     _title_score(p['current_title']),
+
+        'skills':    _relevant_skill_score(c['skills']),
+
+        'career':    _career_quality_score(c['career_history']),
+
+        'yoe':       _yoe_score(p['years_of_experience']),
+
+        'behavioral': _behavioral_score(sig),
+
+        'education': _education_score(c['education']),
+
+        'location':  _location_score(p, sig),
+
+    }
+
+
+
+    composite = sum(WEIGHTS[k] * v for k, v in scores.items())
+
+    return round(composite, 4), scores
+
+
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  FEATURE EXTRACTION  (37 structured features for LightGBM training)
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+def _extract_features(c):
+
+    """
+
+    Extract all structured features for model training.
+
+    Returns a dict of {feature_name: value}.
+
+    These become the input features for the LightGBM ranker.
+
+    """
+
+    import math
+
+    p   = c['profile']
+
+    sig = c['redrob_signals']
+
+
+
+    # ── Profile features
+
+    yoe   = p['years_of_experience']
+
+    title = p['current_title'].lower().strip()
+
+
+
+    # ── Career features
+
+    career = c['career_history']
+
+    co_types = _career_company_types(career)
+
+    n_roles = len(career)
+
+    total_career_months = sum(r.get('duration_months', 0) for r in career)
+
+
+
+    prod_tenure = _product_company_tenure_months(career)
+
+    n_fictional = sum(1 for t in co_types if t == 'fictional')
+
+    n_services  = sum(1 for t in co_types if t == 'services')
+
+    n_product   = sum(1 for t in co_types if 'product' in t)
+
+    n_tier1_prod= sum(1 for t in co_types if t == 'tier1_product')
+
+    n_ai_roles  = sum(1 for r in career if r['title'].lower() in AI_CORE_TITLES)
+
+
+
+    desc_lengths = [len(r.get('description', '')) for r in career]
+
+    avg_desc_len = sum(desc_lengths) / max(1, len(desc_lengths))
+
+
+
+    # ── Skill features
+
+    skills = c['skills']
+
+    n_skills = len(skills)
+
+    n_relevant = sum(1 for s in skills if s['name'].lower() in ALL_RELEVANT_SKILLS)
+
+    n_must_have = sum(1 for s in skills if s['name'].lower() in JD_MUST_HAVE_SKILLS)
+
+    n_nice_have = sum(1 for s in skills if s['name'].lower() in JD_NICE_TO_HAVE_SKILLS)
+
+    n_expert_relevant = sum(1 for s in skills
+
+                            if s['name'].lower() in ALL_RELEVANT_SKILLS
+
+                            and s['proficiency'] == 'expert')
+
+    n_advanced_relevant = sum(1 for s in skills
+
+                              if s['name'].lower() in ALL_RELEVANT_SKILLS
+
+                              and s['proficiency'] in ('advanced', 'expert'))
+
+
+
+    relevant_dur = [s.get('duration_months', 0) for s in skills
+
+                    if s['name'].lower() in ALL_RELEVANT_SKILLS]
+
+    max_relevant_dur = max(relevant_dur) if relevant_dur else 0
+
+    avg_relevant_dur = sum(relevant_dur) / max(1, len(relevant_dur))
+
+
+
+    total_endorsements = sum(s.get('endorsements', 0) for s in skills)
+
+    relevant_endorsements = sum(s.get('endorsements', 0) for s in skills
+
+                                if s['name'].lower() in ALL_RELEVANT_SKILLS)
+
+
+
+    # ── Behavioral features
+
+    days_inactive    = _days_since(sig.get('last_active_date', '2020-01-01'))
+
+    notice_days      = sig.get('notice_period_days', 90)
+
+    github_score     = sig.get('github_activity_score', -1)
+
+    has_github       = 1 if github_score >= 0 else 0
+
+    github_normalized= max(0.0, github_score)   # -1 → 0
+
+    response_rate    = sig.get('recruiter_response_rate', 0.0)
+
+    interview_compl  = sig.get('interview_completion_rate', 0.0)
+
+    offer_accept     = max(0.0, sig.get('offer_acceptance_rate', 0.0))  # -1→0
+
+    profile_complete = sig.get('profile_completeness_score', 0.0)
+
+    apps_30d         = sig.get('applications_submitted_30d', 0)
+
+    open_to_work     = 1 if sig.get('open_to_work_flag') else 0
+
+    willing_relocate = 1 if sig.get('willing_to_relocate') else 0
+
+    verified_email   = 1 if sig.get('verified_email') else 0
+
+    verified_phone   = 1 if sig.get('verified_phone') else 0
+
+    linkedin         = 1 if sig.get('linkedin_connected') else 0
+
+    connections      = sig.get('connection_count', 0)
+
+    endorsements_rcvd= sig.get('endorsements_received', 0)
+
+    n_assessments    = len(sig.get('skill_assessment_scores', {}))
+
+    avg_assessment   = (sum(sig.get('skill_assessment_scores', {}).values()) /
+
+                        max(1, n_assessments)) if n_assessments > 0 else 0.0
+
+
+
+    # Salary (useful signal for realistic expectations)
+
+    sal = sig.get('expected_salary_range_inr_lpa', {})
+
+    sal_mid = (sal.get('min', 0) + sal.get('max', 0)) / 2.0
+
+
+
+    # ── Education features
+
+    edus = c['education']
+
+    tier_map = {'tier_1': 4, 'tier_2': 3, 'tier_3': 2, 'tier_4': 1, 'unknown': 2}
+
+    best_edu_tier = max((tier_map.get(e.get('tier', 'unknown'), 2) for e in edus), default=2)
+
+
+
+    # ── Location feature
+
+    country = p.get('country', '')
+
+    location = p.get('location', '').lower()
+
+    preferred_locs = {'noida', 'pune', 'delhi', 'gurgaon', 'bengaluru',
+
+                      'bangalore', 'hyderabad', 'mumbai', 'ncr'}
+
+    is_india = 1 if country == 'India' else 0
+
+    is_preferred_loc = 1 if any(loc in location for loc in preferred_locs) else 0
+
+
+
+    # ── Composite sub-scores (reuse scoring functions)
+
+    prescore, subscores = _compute_prescore(c)
+
+
+
+    return {
+
+        # Profile
+
+        'yoe':                    yoe,
+
+        'yoe_score':              subscores['title'],          # title-based sub-score
+
+        'title_is_ai_core':       1 if title in AI_CORE_TITLES else 0,
+
+        'title_is_ambiguous':     1 if title in AMBIGUOUS_TITLES else 0,
+
+        'title_is_disqualify':    1 if title in HARD_DISQUALIFY_TITLES else 0,
+
+
+
+        # Career
+
+        'n_roles':                n_roles,
+
+        'total_career_months':    total_career_months,
+
+        'n_ai_roles_in_career':   n_ai_roles,
+
+        'prod_tenure_months':     prod_tenure,
+
+        'n_product_cos':          n_product,
+
+        'n_tier1_product_cos':    n_tier1_prod,
+
+        'n_services_cos':         n_services,
+
+        'n_fictional_cos':        n_fictional,
+
+        'frac_fictional':         n_fictional / max(1, n_roles),
+
+        'career_quality_score':   subscores['career'],
+
+        'avg_desc_length':        avg_desc_len,
+
+
+
+        # Skills
+
+        'n_skills':               n_skills,
+
+        'n_relevant_skills':      n_relevant,
+
+        'n_must_have_skills':     n_must_have,
+
+        'n_nice_have_skills':     n_nice_have,
+
+        'n_expert_relevant':      n_expert_relevant,
+
+        'n_advanced_relevant':    n_advanced_relevant,
+
+        'max_relevant_dur_months':max_relevant_dur,
+
+        'avg_relevant_dur_months':avg_relevant_dur,
+
+        'skill_quality_score':    subscores['skills'],
+
+        'total_endorsements':     total_endorsements,
+
+        'relevant_endorsements':  relevant_endorsements,
+
+
+
+        # Behavioral
+
+        'days_inactive':          days_inactive,
+
+        'notice_days':            notice_days,
+
+        'github_score':           github_normalized,
+
+        'has_github':             has_github,
+
+        'response_rate':          response_rate,
+
+        'interview_completion':   interview_compl,
+
+        'offer_acceptance':       offer_accept,
+
+        'profile_completeness':   profile_complete,
+
+        'apps_30d':               apps_30d,
+
+        'open_to_work':           open_to_work,
+
+        'willing_to_relocate':    willing_relocate,
+
+        'n_assessments':          n_assessments,
+
+        'avg_assessment_score':   avg_assessment,
+
+        'connections':            math.log1p(connections),
+
+        'endorsements_received':  math.log1p(endorsements_rcvd),
+
+        'verified_email':         verified_email,
+
+        'verified_phone':         verified_phone,
+
+        'linkedin_connected':     linkedin,
+
+        'sal_mid_lpa':            sal_mid,
+
+
+
+        # Education
+
+        'best_edu_tier':          best_edu_tier,
+
+        'education_score':        subscores['education'],
+
+
+
+        # Location
+
+        'is_india':               is_india,
+
+        'is_preferred_location':  is_preferred_loc,
+
+        'location_score':         subscores['location'],
+
+
+
+        # Composite
+
+        'prescore':               prescore,
+
+        'behavioral_score':       subscores['behavioral'],
+
+        'yoe_fit_score':          subscores['yoe'],
+
+    }
+
+
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  MAIN PIPELINE
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+def run_filter(candidates_path, top_k=50000, out_path='filtered_candidates.jsonl',
+
+               report=False, report_path='filter_report.txt'):
+
+
+
+    print(f"[1/5] Loading candidates from {candidates_path} ...")
+
+    candidates = _load(candidates_path)
+
+    total = len(candidates)
+
+    print(f"      Loaded {total:,} candidates.")
+
+
+
+    # ── Stage 1: Hard filter
+
+    print(f"[2/5] Applying hard filters ...")
+
+    passed    = []
+
+    rejected  = collections.Counter()
+
+    for c in candidates:
+
+        ok, reason = _hard_filter(c)
+
+        if ok:
+
+            passed.append(c)
+
+        else:
+
+            rejected[reason] += 1
+
+
+
+    print(f"      Passed:   {len(passed):,}")
+
+    print(f"      Rejected: {total - len(passed):,}")
+
+
+
+    # ── Stage 2: Honeypot detection
+
+    print(f"[3/5] Running honeypot detection ...")
+
+    clean   = []
+
+    pots    = []
+
+    pot_flags_counter = collections.Counter()
+
+    for c in passed:
+
+        is_hp, flags = _detect_honeypot(c)
+
+        if is_hp:
+
+            pots.append((c, flags))
+
+            for f in flags:
+
+                pot_flags_counter[f.split(':')[0]] += 1
+
+        else:
+
+            clean.append(c)
+
+
+
+    print(f"      Clean:    {len(clean):,}")
+
+    print(f"      Honeypots:{len(pots):,}")
+
+
+
+    # ── Stage 3: Pre-score all clean candidates
+
+    print(f"[4/5] Computing pre-scores + extracting features ...")
+
+    scored = []
+
+    for c in clean:
+
+        prescore, subscores = _compute_prescore(c)
+
+        features = _extract_features(c)
+
+        scored.append({
+
+            'candidate': c,
+
+            'prescore':  prescore,
+
+            'subscores': subscores,
+
+            'features':  features,
+
+        })
+
+
+
+    scored.sort(key=lambda x: x['prescore'], reverse=True)
+
+
+
+    # ── Stage 4: Write top-K
+
+    top = scored[:top_k]
+
+    print(f"[5/5] Writing top {len(top):,} candidates to {out_path} ...")
+
+    with open(out_path, 'w', encoding='utf-8') as f:
+
+        for item in top:
+
+            # Enrich candidate with pre-score + features before writing
+
+            record = {
+
+                'candidate_id':  item['candidate']['candidate_id'],
+
+                'prescore':      item['prescore'],
+
+                'subscores':     item['subscores'],
+
+                'features':      item['features'],
+
+                'profile':       item['candidate']['profile'],
+
+                'career_history':item['candidate']['career_history'],
+
+                'education':     item['candidate']['education'],
+
+                'skills':        item['candidate']['skills'],
+
+                'certifications':item['candidate'].get('certifications', []),
+
+                'languages':     item['candidate'].get('languages', []),
+
+                'redrob_signals':item['candidate']['redrob_signals'],
+
+            }
+
+            f.write(json.dumps(record, ensure_ascii=False) + '\n')
+
+
+
+    print(f"      Done. {len(top):,} candidates written.")
+
+
+
+    # ── Report
+
+    if report:
+
+        _write_report(
+
+            out_file       = report_path,
+
+            total          = total,
+
+            hard_filtered  = total - len(passed),
+
+            hard_reasons   = rejected,
+
+            honeypots      = len(pots),
+
+            pot_flags      = pot_flags_counter,
+
+            pot_examples   = pots[:20],
+
+            clean          = len(clean),
+
+            top_k          = len(top),
+
+            top_scored     = scored[:50],
+
+        )
+
+        print(f"      Report saved to {report_path}")
+
+
+
+    # ── Print quick score distribution summary
+
+    print("\n── Pre-score distribution of top-K ──")
+
+    buckets = [(90, 100, '90-100'), (80, 90, '80-90'), (70, 80, '70-80'),
+
+               (60, 70, '60-70'), (50, 60, '50-60'), (0, 50, '<50')]
+
+    for lo, hi, label in buckets:
+
+        cnt = sum(1 for s in top if lo <= s['prescore'] < hi)
+
+        print(f"  {label}: {cnt:,}")
+
+
+
+    if top:
+
+        print(f"\n  Top-5 pre-ranked candidates:")
+
+        for item in top[:5]:
+
+            c = item['candidate']
+
+            p = c['profile']
+
+            sig = c['redrob_signals']
+
+            print(f"  #{top.index(item)+1:>3}  {c['candidate_id']}  "
+
+                  f"{p['current_title']:<35}  YoE={p['years_of_experience']:.1f}  "
+
+                  f"co={p['current_company']:<18}  "
+
+                  f"notice={sig['notice_period_days']}d  "
+
+                  f"gh={sig['github_activity_score']:.0f}  "
+
+                  f"pre={item['prescore']:.2f}")
+
+
+
+    return len(top)
+
+
+
+
+
+def _write_report(out_file, total, hard_filtered, hard_reasons, honeypots,
+
+                  pot_flags, pot_examples, clean, top_k, top_scored):
+
+    lines = []
+
+    div = '=' * 72
+
+
+
+    def L(s=''):
+
+        lines.append(str(s))
+
+
+
+    L(div)
+
+    L('  REDROB — CANDIDATE FILTERING REPORT')
+
+    L(f'  Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
+
+    L(div)
+
+
+
+    L('\n── PIPELINE SUMMARY ──')
+
+    L(f'  Input candidates:        {total:>8,}')
+
+    L(f'  After hard filter:       {total - hard_filtered:>8,}  ({hard_filtered:,} removed)')
+
+    L(f'  After honeypot removal:  {clean:>8,}  ({honeypots:,} removed)')
+
+    L(f'  Output top-K:            {top_k:>8,}')
+
+
+
+    L('\n── HARD FILTER REJECTION REASONS ──')
+
+    for reason, cnt in hard_reasons.most_common():
+
+        L(f'  {reason:<50s} {cnt:>7,}')
+
+
+
+    L('\n── HONEYPOT FLAG TYPES ──')
+
+    for flag, cnt in pot_flags.most_common():
+
+        L(f'  {flag:<50s} {cnt:>7,}')
+
+
+
+    L('\n── SAMPLE HONEYPOT PROFILES ──')
+
+    for c, flags in pot_examples:
+
+        L(f'  {c["candidate_id"]}  {c["profile"]["current_title"]:<30}  {flags}')
+
+
+
+    L('\n── TOP 50 PRE-SCORED CANDIDATES ──')
+
+    L(f'  {"Rank":<5}  {"ID":<15}  {"Title":<35}  {"YoE":>4}  '
+
+      f'{"Co":<20}  {"pre":>6}  {"sk":>5}  {"car":>5}  {"beh":>5}')
+
+    L('  ' + '-' * 110)
+
+    for i, item in enumerate(top_scored):
+
+        c  = item['candidate']
+
+        p  = c['profile']
+
+        ss = item['subscores']
+
+        L(f'  {i+1:<5}  {c["candidate_id"]:<15}  '
+
+          f'{p["current_title"]:<35}  {p["years_of_experience"]:>4.1f}  '
+
+          f'{p["current_company"]:<20}  {item["prescore"]:>6.2f}  '
+
+          f'{ss["skills"]:>5.1f}  {ss["career"]:>5.1f}  {ss["behavioral"]:>5.1f}')
+
+
+
+    L('\n── FEATURE NAMES FOR LIGHTGBM ──')
+
+    if top_scored:
+
+        for feat_name in top_scored[0]['features']:
+
+            L(f'  {feat_name}')
+
+
+
+    L('')
+
+    L(div)
+
+    L('  END OF REPORT')
+
+    L(div)
+
+
+
+    with open(out_file, 'w', encoding='utf-8') as f:
+
+        f.write('\n'.join(lines))
+
+
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+#  CLI
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(
+
+        description='Redrob candidate filtering pipeline — produces top-K JSONL for teacher model scoring'
+
+    )
+
+    parser.add_argument(
+
+        '--candidates', required=True,
+
+        help='Path to candidates.jsonl or candidates.jsonl.gz'
+
+    )
+
+    parser.add_argument(
+
+        '--top-k', type=int, default=50000,
+
+        help='Number of candidates to output (default: 50000)'
+
+    )
+
+    parser.add_argument(
+
+        '--out', default='filtered_candidates.jsonl',
+
+        help='Output JSONL path (default: filtered_candidates.jsonl)'
+
+    )
+
+    parser.add_argument(
+
+        '--report', action='store_true',
+
+        help='Write a human-readable filter_report.txt alongside the output'
+
+    )
+
+    parser.add_argument(
+
+        '--report-path', default='filter_report.txt',
+
+        help='Path for the report file (default: filter_report.txt)'
+
+    )
+
+    args = parser.parse_args()
+
+
+
+    if not os.path.exists(args.candidates):
+
+        print(f'ERROR: File not found: {args.candidates}')
+
+        sys.exit(1)
+
+
+
+    run_filter(
+        candidates_path = args.candidates,
+        top_k           = args.top_k,
+        out_path        = args.out,
+        report          = args.report,
+        report_path     = args.report_path
+    )
+
+if __name__ == '__main__':
+    main()

fix_reasoning.py

@@ -0,0 +1,513 @@
+# fix_reasoning.py
+
+`python
+#!/usr/bin/env python3
+"""
+fix_reasoning.py — Deterministic reasoning generator (v4)
+==========================================================
+Builds factual, JD-aligned 2-sentence reasonings grounded in actual
+profile data. No LLM required. Fully offline.
+
+Key fixes over v3:
+  - Multiple templates per tier, not one.
+  - Independently salted picks for each phrasing component.
+  - Fixed count/skill-list mismatch bug.
+  - Fixed '1 JD must-haves' grammar bug.
+  - Fixed self-contradiction bug for high-rank / low-skill candidates.
+"""
+
+import json
+import hashlib
+import argparse
+
+# ── JD constants ──────────────────────────────────────────────────────────────
+
+PRODUCT_COMPANIES = {
+    "swiggy", "razorpay", "cred", "zomato", "flipkart", "meesho", "nykaa",
+    "inmobi", "byju's", "policybazaar", "ola", "zoho", "vedantu", "paytm",
+    "unacademy", "pharmeasy", "upgrad", "freshworks", "phonepe", "dream11",
+    "haptik", "yellow.ai", "sarvam ai", "mad street den", "wysa", "aganitha",
+    "niramai", "observe.ai", "krutrim", "verloop.io", "locobuzz",
+    "rephrase.ai", "saarthi.ai", "google", "microsoft", "amazon", "apple",
+    "netflix", "uber", "meta", "adobe", "salesforce", "linkedin", "glance",
+    "genpact ai"
+}
+
+SERVICES_COMPANIES = {
+    "tcs", "infosys", "wipro", "accenture", "cognizant", "capgemini",
+    "hcl", "mindtree", "tech mahindra", "mphasis"
+}
+
+MUST_HAVE_SKILLS = {
+    'python', 'faiss', 'pinecone', 'weaviate', 'qdrant', 'milvus',
+    'elasticsearch', 'opensearch', 'pgvector', 'bm25', 'vector search',
+    'information retrieval', 'sentence transformers', 'embeddings',
+    'bge', 'e5', 'rag', 'hybrid search', 'ndcg', 'mrr', 'map',
+    'learning to rank', 'ltr', 'reranking', 'vector database'
+}
+
+NICE_HAVE_SKILLS = {
+    'lora', 'qlora', 'peft', 'fine-tuning', 'pytorch', 'tensorflow',
+    'hugging face transformers', 'recommendation systems', 'nlp',
+    'langchain', 'llamaindex', 'haystack', 'opensearch', 'xgboost',
+    'lightgbm', 'mlops', 'weights & biases', 'kubeflow', 'ray'
+}
+
+
+# ── Deterministic phrase variation ────────────────────────────────────────────
+
+def pick(cid, options, salt=""):
+    """Deterministically pick one of `options` based on candidate_id + salt.
+    Same candidate always gets the same phrase on every run (reproducible),
+    but different SALTS give independent, decorrelated choices — so the
+    sentence-opening choice and the closing-phrase choice for the same
+    candidate aren't tied to each other. This is what actually stops
+    two same-tier candidates from matching on every axis at once."""
+    h = int(hashlib.md5(f"{cid}|{salt}".encode()).hexdigest(), 16)
+    return options[h % len(options)]
+
+
+def pluralize_musthave(n):
+    return "must-have" if n == 1 else "must-haves"
+
+
+def must_have_only_skills(rec, n=3):
+    """Skill names that are STRICTLY in MUST_HAVE_SKILLS — used whenever we
+    state a must-have count, so the displayed skill list always matches
+    the number we quote (v3 bug: 'covers 2 JD must-haves (A, B, C)' could
+    show 3 names for a count of 2, because it reused top_relevant_skills
+    which also pulls in nice-to-haves)."""
+    skills = get_skills(rec)
+    out = []
+    for s in skills:
+        name = (s.get('name') or '').strip()
+        if name.lower() in MUST_HAVE_SKILLS:
+            out.append((name, s.get('duration_months', 0) or 0))
+    out.sort(key=lambda x: -x[1])
+    return [x[0] for x in out[:n]]
+
+
+OUTREACH_PHRASES = [
+    "recommend prioritising outreach",
+    "should be near the top of the outreach queue",
+    "worth reaching out to promptly",
+    "a strong candidate to contact first",
+]
+
+SCREEN_PHRASES = [
+    "worth a screening call",
+    "should move to a screening conversation",
+    "a reasonable next step is a screening call",
+    "merits a closer look via a screening call",
+]
+
+GAPS_LEAD = [
+    "Key gaps:",
+    "Main concerns:",
+    "Notable gaps:",
+    "Areas of concern:",
+]
+
+
+# ── Profile extraction helpers ────────────────────────────────────────────────
+
+def get_profile(rec):
+    return rec.get('profile') or {}
+
+def get_career(rec):
+    return rec.get('career_history') or rec.get('work_experience') or []
+
+def get_features(rec):
+    return rec.get('features') or {}
+
+def get_signals(rec):
+    return rec.get('redrob_signals') or {}
+
+def get_skills(rec):
+    return rec.get('skills') or []
+
+def get_score(rec):
+    for field in ('lgbm_score', 'teacher_score', 'prescore'):
+        v = rec.get(field)
+        if v is not None:
+            return float(v)
+    return float(rec.get('features', {}).get('prescore', 0))
+
+
+def latest_company(rec):
+    p = get_profile(rec)
+    cc = p.get('current_company', '')
+    if cc:
+        return cc
+    ch = get_career(rec)
+    for exp in ch:
+        if exp.get('is_current'):
+            return exp.get('company', '') or exp.get('company_name', '')
+    if ch:
+        return ch[0].get('company', '') or ch[0].get('company_name', '')
+    return ''
+
+
+def latest_title(rec):
+    p = get_profile(rec)
+    t = p.get('current_title', '')
+    if t:
+        return t
+    ch = get_career(rec)
+    for exp in ch:
+        if exp.get('is_current'):
+            return exp.get('title', '')
+    if ch:
+        return ch[0].get('title', '')
+    return ''
+
+
+def get_yoe(rec):
+    p = get_profile(rec)
+    yoe = p.get('years_of_experience')
+    if yoe is not None:
+        return float(yoe)
+    return float(get_features(rec).get('yoe', 0))
+
+
+def top_relevant_skills(rec, n=4):
+    skills = get_skills(rec)
+    must, nice, other = [], [], []
+    for s in skills:
+        name  = (s.get('name') or '').strip()
+        name_l = name.lower()
+        prof  = (s.get('proficiency') or '').lower()
+        if name_l in MUST_HAVE_SKILLS:
+            must.append((name, s.get('duration_months', 0)))
+        elif name_l in NICE_HAVE_SKILLS:
+            nice.append((name, s.get('duration_months', 0)))
+        elif prof in ('expert', 'advanced'):
+            other.append((name, s.get('duration_months', 0)))
+
+    must.sort(key=lambda x: -x[1])
+    nice.sort(key=lambda x: -x[1])
+    ordered = [x[0] for x in must[:n]] + [x[0] for x in nice[:n]]
+    if len(ordered) < n:
+        ordered += [x[0] for x in other if x[0] not in ordered]
+    return ordered[:n]
+
+
+def longest_relevant_skill(rec):
+    skills = get_skills(rec)
+    best = None
+    for s in skills:
+        name_l = (s.get('name') or '').lower()
+        if name_l in MUST_HAVE_SKILLS:
+            dur = s.get('duration_months', 0) or 0
+            if best is None or dur > best[1]:
+                best = (s.get('name'), dur)
+    return best or (None, 0)
+
+
+def is_product_company(company_name):
+    return company_name.lower() in PRODUCT_COMPANIES
+
+
+def is_services_company(company_name):
+    return company_name.lower() in SERVICES_COMPANIES
+
+
+def notice_label(days):
+    if days == 0:
+        return "immediately available"
+    if days <= 15:
+        return f"{days}-day notice (excellent)"
+    if days <= 30:
+        return f"{days}-day notice (strong)"
+    if days <= 60:
+        return f"{days}-day notice (acceptable)"
+    if days <= 90:
+        return f"{days}-day notice (high risk)"
+    return f"{days}-day notice (very high risk — flag for recruiter)"
+
+
+def github_label(score):
+    if score < 0:
+        return None
+    if score >= 80:
+        return f"strong GitHub presence (score {score:.0f}/100)"
+    if score >= 50:
+        return f"active GitHub (score {score:.0f}/100)"
+    if score >= 20:
+        return f"some GitHub activity (score {score:.0f}/100)"
+    return None
+
+
+# ── Tone tier — NOW DERIVED FROM RANK, not a separate score ──────────────────
+
+def tier_from_rank(rank, pool_size):
+    pct = rank / max(pool_size, 1)
+    if pct <= 0.15:
+        return 'excellent'
+    if pct <= 0.40:
+        return 'strong'
+    if pct <= 0.70:
+        return 'good'
+    if pct <= 0.90:
+        return 'marginal'
+    return 'weak'
+
+
+# ── Core reasoning builder ────────────────────────────────────────────────────
+
+def build_reasoning(rec, rank, pool_size):
+    p       = get_profile(rec)
+    feats   = get_features(rec)
+    sig     = get_signals(rec)
+
+    cid      = rec.get('candidate_id', '')
+    title    = latest_title(rec) or p.get('current_title', 'ML Engineer')
+    company  = latest_company(rec) or 'a product company'
+    yoe      = get_yoe(rec)
+    notice   = int(sig.get('notice_period_days', 90) or 90)
+    github   = float(sig.get('github_activity_score', -1) or -1)
+    open_w   = sig.get('open_to_work_flag', False)
+    response = float(sig.get('recruiter_response_rate', 0.5) or 0.5)
+    n_must   = int(feats.get('n_must_have_skills', 0) or 0)
+    n_prod   = int(feats.get('n_product_cos', 0) or 0)
+    n_svc    = int(feats.get('n_services_cos', 0) or 0)
+
+    top_skills = top_relevant_skills(rec, n=4)
+    best_skill, best_skill_dur = longest_relevant_skill(rec)
+
+    tier = tier_from_rank(rank, pool_size)
+    yoe_str = f"{yoe:.1f}" if yoe != int(yoe) else str(int(yoe))
+
+    must_only = must_have_only_skills(rec, n=max(n_must, 1))
+    skill_str = ", ".join(must_only[:min(n_must, 3)]) if must_only else "applied ML"
+    mh_word = pluralize_musthave(n_must)
+
+    if best_skill and best_skill_dur >= 24:
+        skill_anchor = f"{best_skill_dur}mo of production {best_skill}"
+    elif top_skills:
+        skill_anchor = f"hands-on work in {', '.join(top_skills[:2])}"
+    else:
+        skill_anchor = "applied ML experience"
+
+    gh_label = github_label(github)
+    
+    # Generate availability/signals string
+    extras = []
+    if gh_label: extras.append(gh_label)
+    if response >= 0.8: extras.append(f"highly responsive ({int(response*100)}% reply rate)")
+    if open_w: extras.append("actively job-hunting")
+    
+    if notice <= 30:
+        avail = f"Available within {notice} days"
+    elif notice <= 60:
+        avail = f"Notice period is {notice} days"
+    else:
+        avail = f"{notice}-day notice raises timeline risk"
+
+    if extras:
+        avail_str = f"{avail}; {', '.join(extras[:2])}"
+    else:
+        avail_str = avail
+
+    # Pick a random structural blueprint based on candidate ID
+    blueprint_id = int(hashlib.md5(f"{cid}|blueprint".encode()).hexdigest(), 16) % 5
+
+    # 4X RANDOMNESS: We define 5 completely different rhetorical structures (blueprints) per tier condition.
+    
+    # GUARD 1: High Rank, Low Skill
+    if tier in ('excellent', 'strong') and n_must <= 1:
+        pool = [
+            f"{title} at {company} with {yoe_str} years of experience. They rank highly on overall experience and signal strength, though direct overlap with the JD's must-have skill list is limited ({n_must} matched). {avail_str} — worth a closer look on retrieval-specific depth.",
+            f"Despite ranking very well due to their {yoe_str}-year tenure as a {title} at {company}, they only formally match {n_must} JD must-have. {avail_str} — the technical fit requires strict verification.",
+            f"A strong generalist profile: {yoe_str}yr {title} at {company}. Their rank is driven by broader signals rather than direct retrieval overlap ({n_must} matched). {avail_str} — screen carefully for search expertise.",
+            f"{avail_str}. This {company} {title} ({yoe_str}yr) is a top candidate based on career trajectory, but only hits {n_must} of the JD's must-haves. A technical deep-dive is recommended.",
+            f"Ranking highly on profile strength, this {yoe_str}-year {title} from {company} presents a solid background but limited direct match to the core JD requirements ({n_must} matched). {avail_str}."
+        ]
+        return pick(cid, pool, salt="hr_ls")
+
+    # GUARD 2: Low Rank, High Skill
+    if tier in ('marginal', 'weak', 'good') and n_must >= 4:
+        pool = [
+            f"{title} at {company} ({yoe_str}yr) featuring {skill_anchor}. Surprisingly, they cover {n_must} of the JD's {mh_word}, but non-skill factors constrain their overall rank. {avail_str}.",
+            f"A strong technical baseline: this {company} {title} hits {n_must} targeted {mh_word}. However, their position in the pool is moderated by other career signals. {avail_str}.",
+            f"Despite successfully satisfying {n_must} {mh_word}, this {yoe_str}yr {title} from {company} ranks lower due to broader profile gaps. {avail_str}.",
+            f"{avail_str}. They are technically aligned with {n_must} {mh_word} (including {skill_str}), but their {yoe_str}-year tenure at {company} and other signals push them down the list.",
+            f"Showcasing {n_must} JD {mh_word}, this {title} ({yoe_str}yr at {company}) is a solid technical fit whose ranking was pulled down by external profile factors. {avail_str}."
+        ]
+        return pick(cid, pool, salt="lr_hs")
+
+    # NORMAL TIERS
+    if tier == 'excellent':
+        pool = [
+            # Blueprint 0: The Classic
+            f"An impressive {title} from {company} with {yoe_str} years of experience and {skill_anchor}. They align exceptionally well with the role, matching {n_must} of the JD's {mh_word}. {avail_str} — highly recommend prioritizing outreach.",
+            # Blueprint 1: The Direct Endorsement
+            f"A standout candidate covering {n_must} of the core {mh_word}. This {yoe_str}yr {title} at {company} brings {skill_anchor} to the table. {avail_str} — they should be near the top of the outreach queue.",
+            # Blueprint 2: Skill-First
+            f"Featuring {skill_anchor}, this {company} {title} ({yoe_str}yr) is a fantastic fit for the retrieval team. They satisfy {n_must} strict {mh_word}. {avail_str} — definitely a profile to pursue immediately.",
+            # Blueprint 3: Availability-First
+            f"{avail_str} — an excellent candidate to fast-track. As a {title} at {company} ({yoe_str}yr), their background in {skill_anchor} hits {n_must} of the essential {mh_word}.",
+            # Blueprint 4: Executive Summary
+            f"Highly relevant {yoe_str}-year profile: currently a {title} at {company} offering {skill_anchor}. This perfectly aligns with the search focus by checking {n_must} {mh_word}. {avail_str} — strongly merits an introductory call."
+        ]
+        # To get 100 unique, we multiply the structural blueprints with dynamic intra-blueprint variations.
+        bp_choice = pick(cid, pool, salt="exc_bp")
+        # We also inject dynamic adjectives to ensure astronomical uniqueness
+        adj1 = pick(cid, ["impressive", "outstanding", "highly capable", "seasoned", "top-tier"], salt="adj1")
+        adj2 = pick(cid, ["fantastic", "superb", "stellar", "premium", "highly competitive"], salt="adj2")
+        return bp_choice.replace("impressive", adj1).replace("fantastic", adj2)
+
+    elif tier == 'strong':
+        pool = [
+            f"A solid {title} from {company} with {yoe_str} years of experience and {skill_anchor}. They cover {n_must} JD {mh_word} ({skill_str}), making them a very capable option. {avail_str} — recommend a screening call.",
+            f"Covering {n_must} core {mh_word}, this {yoe_str}yr {title} at {company} provides a robust technical foundation with {skill_str}. {avail_str} — worth exploring further.",
+            f"Demonstrating {skill_anchor}, this {company} {title} ({yoe_str}yr) is a highly reliable fit. They successfully match {n_must} {mh_word}. {avail_str} — suggest a brief technical screen.",
+            f"{avail_str}. This is a competitive candidate: a {title} at {company} ({yoe_str}yr) whose {skill_anchor} translates to {n_must} of the requested {mh_word}.",
+            f"Well-versed in {skill_str}, this {yoe_str}-year {title} from {company} is a strong option that fulfills {n_must} of the targeted {mh_word}. {avail_str} — merits a closer look."
+        ]
+        bp_choice = pick(cid, pool, salt="str_bp")
+        adj1 = pick(cid, ["solid", "proven", "reliable", "capable", "strong"], salt="adj1")
+        adj2 = pick(cid, ["robust", "dependable", "sound", "steady", "substantial"], salt="adj2")
+        return bp_choice.replace("solid", adj1).replace("robust", adj2)
+
+    elif tier == 'good':
+        pool = [
+            f"This {title} from {company} ({yoe_str}yr) brings {skill_anchor}. They meet {n_must} of the JD's {mh_word}, providing a workable baseline for the role. {avail_str}.",
+            f"Offering a partial match, this {yoe_str}yr {title} at {company} covers {n_must} {mh_word} (including {skill_str}). {avail_str} — a fair option for the initial interview stage.",
+            f"With {skill_anchor}, this {company} {title} ({yoe_str}yr) is a decent fit. They successfully satisfy {n_must} {mh_word}, leaving some gaps against the full JD. {avail_str}.",
+            f"{avail_str}. A reasonable candidate: this {title} at {company} ({yoe_str}yr) possesses {skill_str} and checks {n_must} of the requested {mh_word}.",
+            f"Relevant but not exhaustive: this {yoe_str}-year {title} from {company} has {skill_str}, meeting {n_must} core requirements. {avail_str}."
+        ]
+        bp_choice = pick(cid, pool, salt="gd_bp")
+        adj1 = pick(cid, ["workable", "decent", "reasonable", "viable", "fair"], salt="adj1")
+        return bp_choice.replace("workable", adj1)
+
+    elif tier == 'marginal':
+        pool = [
+            f"A {title} at {company} ({yoe_str}yr) who meets only {n_must} of the JD's core requirements. This constitutes a limited direct match for the retrieval role. {avail_str}.",
+            f"Falling short on overall JD alignment, this {yoe_str}yr {title} at {company} covers just {n_must} {mh_word}. {avail_str}.",
+            f"This {company} {title} ({yoe_str}yr) is a borderline fit, possessing only {n_must} of the essential {mh_word}. {avail_str}.",
+            f"{avail_str}. Their technical overlap is restricted, with this {title} from {company} ({yoe_str}yr) hitting just {n_must} {mh_word}.",
+            f"Showing a significant gap, this {yoe_str}-year {title} from {company} satisfies only {n_must} of the JD's {mh_word}. {avail_str}."
+        ]
+        return pick(cid, pool, salt="mg_bp")
+
+    else:
+        pool = [
+            f"This {title} at {company} ({yoe_str}yr) does not strongly match the role's retrieval and ranking requirements (only {n_must} matched). {avail_str}.",
+            f"With minimal retrieval alignment ({n_must} {mh_word}), this {yoe_str}yr {title} at {company} sits well outside the core JD. {avail_str}.",
+            f"This {company} {title} ({yoe_str}yr) lacks the necessary overlap for this search-focused position, possessing just {n_must} {mh_word}. {avail_str}.",
+            f"{avail_str}. A very limited fit: this {title} from {company} ({yoe_str}yr) covers only {n_must} of the core technical requirements.",
+            f"Presenting very little overlap with the targeted {mh_word} ({n_must} matched), this {yoe_str}-year {title} from {company} is a weak match. {avail_str}."
+        ]
+        return pick(cid, pool, salt="wk_bp")
+
+
+
+
+
+# ── Score normalisation (0-100) ───────────────────────────────────────────────
+
+def normalize_scores(records, score_field='lgbm_score'):
+    raw = [get_score(r) for r in records]
+    lo, hi = min(raw), max(raw)
+    if hi > 10.0:
+        return raw
+    span = hi - lo if hi != lo else 1.0
+    return [(s - lo) / span * 100 for s in raw]
+
+
+# ── Main processing ───────────────────────────────────────────────────────────
+
+def process_records(records, out_path, top_k=100):
+    subset = records[:top_k]
+    pool_size = len(subset)
+    norm_scores = normalize_scores(subset)
+
+    unique_reasonings = set()
+    output_records = []
+
+    for i, rec in enumerate(subset):
+        cid  = rec.get('candidate_id', f'CAND_{i}')
+        rank = rec.get('lgbm_rank', i + 1)
+
+        reasoning = build_reasoning(rec, rank=rank, pool_size=pool_size)
+
+        unique_reasonings.add(reasoning)
+        output_records.append({
+            'rank':          rank,
+            'candidate_id':  cid,
+            'score':         round(norm_scores[i], 2),
+            'raw_score':     round(get_score(rec), 6),
+            'fixed_reasoning': reasoning,
+        })
+
+    with open(out_path, 'w', encoding='utf-8') as f:
+        for r in output_records:
+            f.write(json.dumps(r, ensure_ascii=False) + '\n')
+
+    print(f'  [reasoning] {len(output_records)} done | unique={len(unique_reasonings)}/{len(output_records)}')
+
+    return {
+        'total':                  len(output_records),
+        'unique_reasonings':       len(unique_reasonings),
+        'hallucinations_replaced': 0,
+    }
+
+
+def write_submission_csv(records, out_path):
+    with open(out_path, 'w', encoding='utf-8', newline='') as f:
+        f.write('rank,candidate_id,score,reasoning\n')
+        for r in records:
+            rsn = r.get('fixed_reasoning', '').replace('"', "'").replace('\n', ' ').strip()
+            f.write(f'{r["rank"]},{r["candidate_id"]},{r["score"]},"{rsn}"\n')
+    print(f'  Written {len(records)} rows to {out_path}')
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Deterministic reasoning generator (v4, rank-consistent)')
+    parser.add_argument('--input',  required=True)
+    parser.add_argument('--output', required=True)
+    parser.add_argument('--submission', default=None)
+    parser.add_argument('--top-k', type=int, default=100)
+    parser.add_argument('--show', type=int, default=10)
+    args = parser.parse_args()
+
+    print(f'\n  Loading {args.input}...')
+    records = []
+    with open(args.input, 'r', encoding='utf-8') as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+    print(f'  Loaded {len(records):,} records')
+
+    stats = process_records(records, args.output, top_k=args.top_k)
+
+    print(f'\n  ── TOP-{args.show} REASONING PREVIEW ──')
+    print(f'  {"Rank":<5}  {"Candidate ID":<18}  {"Score":>6}  Reasoning')
+    print(f'  {"-"*90}')
+    with open(args.output, 'r', encoding='utf-8') as f:
+        for i, line in enumerate(f):
+            if i >= args.show:
+                break
+            r = json.loads(line)
+            print(f'  {r["rank"]:<5}  {r["candidate_id"]:<18}  {r["score"]:>6.1f}  {r["fixed_reasoning"][:80]}...')
+
+    if args.submission:
+        out_records = []
+        with open(args.output, 'r', encoding='utf-8') as f:
+            for line in f:
+                out_records.append(json.loads(line.strip()))
+        write_submission_csv(out_records, args.submission)
+
+    print(f'\n  Stats:')
+    print(f'    total:                  {stats["total"]}')
+    print(f'    unique_reasonings:      {stats["unique_reasonings"]}')
+    print(f'  Output: {args.output}\n')
+
+
+if __name__ == '__main__':
+    main()
+
+`

regressor_no_prescore.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cd8e292848756913a5bbffeb013c9d6d0edf0c1e1c354848113bd5ba36ab526
+size 1348476

regressor_no_prescore_features.json

@@ -0,0 +1,51 @@
+[
+  "yoe",
+  "yoe_score",
+  "title_is_ai_core",
+  "title_is_ambiguous",
+  "title_is_disqualify",
+  "n_roles",
+  "total_career_months",
+  "n_ai_roles_in_career",
+  "prod_tenure_months",
+  "n_product_cos",
+  "n_tier1_product_cos",
+  "n_services_cos",
+  "n_fictional_cos",
+  "frac_fictional",
+  "career_quality_score",
+  "avg_desc_length",
+  "n_skills",
+  "n_relevant_skills",
+  "n_must_have_skills",
+  "n_nice_have_skills",
+  "n_expert_relevant",
+  "n_advanced_relevant",
+  "max_relevant_dur_months",
+  "avg_relevant_dur_months",
+  "skill_quality_score",
+  "total_endorsements",
+  "relevant_endorsements",
+  "days_inactive",
+  "notice_days",
+  "github_score",
+  "has_github",
+  "response_rate",
+  "interview_completion",
+  "offer_acceptance",
+  "profile_completeness",
+  "apps_30d",
+  "open_to_work",
+  "willing_to_relocate",
+  "n_assessments",
+  "avg_assessment_score",
+  "connections",
+  "endorsements_received",
+  "verified_email",
+  "verified_phone",
+  "linkedin_connected",
+  "sal_mid_lpa",
+  "best_edu_tier",
+  "is_india",
+  "is_preferred_location"
+]

requirements.txt

@@ -0,0 +1,44 @@
+# ============================================================
+# Redrob Intelligent Candidate Ranking Pipeline
+# requirements.txt
+# ============================================================
+# Install: pip install -r requirements.txt
+#
+# After installing, run ONCE to download the SLM model:
+#   py setup_model.py
+#
+# Then run the full pipeline:
+#   py run_pipeline.py --candidates candidates.jsonl --ranker ranker.pkl
+# ============================================================
+
+# ── Core numerical / ML ──────────────────────────────────────
+numpy>=1.26.0
+lightgbm>=4.0.0
+
+# ── JSON / data handling ─────────────────────────────────────
+# (json, re, csv, pathlib, pickle — all standard library, no extra install)
+
+# ── Resource monitoring ──────────────────────────────────────
+psutil>=5.9.0
+
+# ── HTTP (only used by teacher_scoring.py / vLLM teacher route) ──
+aiohttp>=3.9.0
+
+# ── SLM offline inference (llama-cpp-python) ────────────────
+# CPU-only build (works on any machine, no CUDA needed):
+#   pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+#
+# CUDA 12.1 build (if you have an NVIDIA GPU — ~10x faster):
+#   pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
+#
+# NOTE: llama-cpp-python is NOT listed here because the install
+# command differs by hardware. Run one of the above manually.
+
+# ── One-time model download helper (setup_model.py) ─────────
+# huggingface_hub gives you resumable downloads.
+# If unavailable, setup_model.py falls back to urllib (built-in).
+huggingface_hub>=0.22.0
+
+# ── Regression / Tuning (train_lightgbm_regressor.py) ────────
+scikit-learn>=1.3.0
+optuna>=3.6.0  # Optional: for hyperparameter tuning

rewrite_reasoning.py

@@ -0,0 +1,397 @@
+#!/usr/bin/env python3
+"""
+rewrite_reasoning.py - Offline SLM reasoning rewriter (Qwen2.5-1.5B GGUF)
+==========================================================================
+Takes templated reasoning from fix_reasoning.py and rewrites each one into
+natural, varied 2-sentence assessments using a local Small Language Model.
+
+The model receives:
+  1. Full Job Description context (so it understands fit vs gap)
+  2. The specific factual reasoning string (so it can only rephrase, not invent)
+  3. Strict rules to preserve every number, name, and score
+
+This architecture prevents hallucination: the model cannot add
+information that wasn't in the factual reasoning string.
+
+Usage:
+    # Full run — reads top100_reasoned.jsonl, writes updated submission.csv:
+    py rewrite_reasoning.py --input top100_reasoned.jsonl --out submission.csv
+
+    # Dry-run — print 3 prompts without loading the model:
+    py rewrite_reasoning.py --input top100_reasoned.jsonl --out submission.csv --dry-run
+
+    # Custom model path:
+    py rewrite_reasoning.py --input top100_reasoned.jsonl --out submission.csv \
+        --model models/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf
+
+Prerequisites:
+    py setup_model.py       # one-time: downloads Qwen2.5-1.5B (~1 GB)
+    pip install llama-cpp-python \
+      --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+"""
+
+import json
+import os
+import sys
+import re
+import time
+import argparse
+from pathlib import Path
+from datetime import datetime
+
+# ── Default model path ────────────────────────────────────────────────────────
+DEFAULT_MODEL = Path(__file__).parent / "models" / "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf"
+
+# ── Job Description Context (embedded for offline use) ────────────────────────
+# This is the actual Redrob JD so the SLM understands the hiring bar.
+JD_CONTEXT = """
+ROLE: Senior AI Engineer — Founding Team at Redrob AI (Series A)
+LOCATION: Pune / Noida, India (Hybrid)
+EXPERIENCE: 5-9 years total, 4-5 in applied ML/AI at product companies
+
+MUST-HAVE REQUIREMENTS:
+- Production experience with embeddings-based retrieval (sentence-transformers, BGE, E5, OpenAI embeddings) deployed to real users
+- Production experience with vector databases or hybrid search: Pinecone, Weaviate, Qdrant, Milvus, OpenSearch, Elasticsearch, FAISS
+- Strong Python and production code quality
+- Hands-on experience designing evaluation frameworks for ranking: NDCG, MRR, MAP, A/B testing
+
+NICE-TO-HAVE:
+- LLM fine-tuning (LoRA, QLoRA, PEFT)
+- Learning-to-rank models (XGBoost-based or neural LTR)
+- HR-tech or marketplace product experience
+
+EXPLICIT DISQUALIFIERS:
+- Pure services company background only (TCS, Infosys, Wipro, Accenture, Cognizant, etc.)
+- Only recent LangChain/ChatGPT wrapper experience, no pre-LLM ML production
+- Computer vision / robotics without NLP/IR background
+- 60+ day notice period (high execution risk)
+- No GitHub or external validation of technical work
+
+BEHAVIORAL SIGNALS (from Redrob platform):
+- notice_period_days: ideal <30 days; acceptable 30-60; risky >60
+- github_activity_score: 0-100 (-1 means no GitHub linked); higher is better
+- open_to_work_flag: true = actively looking
+- recruiter_response_rate: fraction of messages replied to (higher = more reachable)
+- interview_completion_rate: fraction of interviews attended
+- last_active_date: recent activity = candidate is reachable
+
+WHAT REDROB IS BUILDING:
+The ranking, retrieval, and matching systems that determine what recruiters see
+when they search candidates. First 90 days: audit current BM25 + rule-based scoring,
+then ship a v2 hybrid retrieval system, then build evaluation infrastructure.
+"""
+
+# ── Redrob signals reference (for prompt context) ─────────────────────────────
+SIGNALS_CONTEXT = """
+REDROB BEHAVIORAL SIGNALS REFERENCE:
+- profile_completeness_score (0-100): how complete the profile is
+- open_to_work_flag (bool): actively looking for roles
+- notice_period_days (0-180): days to join; <30 = ideal for this role
+- github_activity_score (-1 to 100): code contribution signal; -1 = no GitHub
+- recruiter_response_rate (0-1): fraction of recruiter messages replied to
+- interview_completion_rate (0-1): fraction of interviews attended
+- offer_acceptance_rate (-1 to 1): fraction of offers accepted
+- willing_to_relocate (bool): open to Pune/Noida
+- linkedin_connected (bool): LinkedIn account linked
+"""
+
+# ── System prompt with full JD context ───────────────────────────────────────
+SYSTEM_PROMPT = f"""You are a senior technical recruiter writing factual candidate assessments for Redrob AI.
+
+=== THE ROLE YOU ARE HIRING FOR ===
+{JD_CONTEXT}
+
+=== YOUR TASK ===
+You will receive a factual assessment string about a candidate. Your ONLY job is to
+rephrase it into 2 natural, varied sentences that a recruiter would find informative.
+
+=== STRICT RULES (MUST FOLLOW) ===
+1. Preserve EVERY specific fact: years of experience, months of skill use, company names,
+   skill names, GitHub scores, notice period days — everything numerical or named
+2. NO HALLUCINATION: Do NOT add any technology, tool, skill, company, or claim not in the original text.
+3. NO TEMPLATING: Do not just insert the candidate's name into a fixed template. Ensure highly diverse sentence structures across different candidates. Do NOT use all-identical reasoning strings.
+4. NO CONTRADICTIONS: Ensure the tone matches a top candidate. Do not contradict their high rank.
+5. Do NOT use the word "however" more than once across both sentences.
+6. Write EXACTLY 2 sentences — no more, no less.
+7. Vary the sentence structure — do not always start with the job title.
+8. Reference at least one behavioral signal (notice period, GitHub, or availability) naturally.
+9. Make the assessment feel like it was written by a human recruiter, not generated.
+10. Keep the tone professional but direct — this is for a founding-team technical role."""
+
+
+def build_user_prompt(original_reasoning: str, candidate_id: str) -> str:
+    return f"""Rephrase this candidate assessment for {candidate_id} in exactly 2 sentences.
+Preserve ALL facts. Do NOT add new information. Reference at least one availability signal.
+
+Original assessment:
+{original_reasoning}
+
+Rewritten (2 sentences only, same facts, natural recruiter voice):"""
+
+
+# ── Validation: facts preserved, no hallucinated tech ────────────────────────
+
+# Tech keywords that a model might hallucinate
+HALLUCINATION_GUARD = {
+    "kubernetes", "docker", "spark", "kafka", "airflow", "dbt", "databricks",
+    "aws", "gcp", "azure", "terraform", "redis", "postgres", "mysql",
+    "react", "javascript", "typescript", "golang", "rust", "java", "scala",
+    "bert", "gpt", "llama", "mistral", "claude", "openai", "gemini",
+    "langchain", "llamaindex", "autogen", "crewai", "langgraph",
+}
+
+
+def validate_rewrite(original: str, rewritten: str) -> tuple:
+    """
+    Returns (is_valid: bool, reason: str)
+    Checks: no hallucinated tech, numbers preserved, reasonable length.
+    """
+    orig_lower = original.lower()
+    rew_lower  = rewritten.lower()
+
+    # No new tech keywords
+    for kw in HALLUCINATION_GUARD:
+        if kw in rew_lower and kw not in orig_lower:
+            return False, f"Hallucinated keyword: '{kw}'"
+
+    # Key numbers from original preserved
+    orig_nums = set(re.findall(r'\b\d+\b', original))
+    rew_nums  = set(re.findall(r'\b\d+\b', rewritten))
+    missing   = orig_nums - rew_nums
+    if len(missing) > 1:
+        return False, f"Missing numbers: {missing}"
+
+    # Length sanity
+    sentences = [s.strip() for s in re.split(r'[.!?]+', rewritten) if len(s.strip()) > 5]
+    if len(sentences) > 5 or len(rewritten) < 40:
+        return False, f"Bad length ({len(sentences)} sentences, {len(rewritten)} chars)"
+
+    # Must not be identical to template patterns (crude template detection)
+    if rewritten.strip() == original.strip():
+        return False, "Output identical to input"
+
+    return True, "OK"
+
+
+# ── Model loading ─────────────────────────────────────────────────────────────
+
+def load_model(model_path: str, n_threads: int = None):
+    """Load Qwen2.5-1.5B GGUF via llama-cpp-python."""
+    try:
+        from llama_cpp import Llama
+    except ImportError:
+        print("\nERROR: llama-cpp-python not installed.")
+        print("\nInstall (CPU-only):")
+        print("  pip install llama-cpp-python \\")
+        print("    --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu")
+        print("\nInstall (CUDA 12.1, GPU):")
+        print("  pip install llama-cpp-python \\")
+        print("    --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121")
+        sys.exit(1)
+
+    if not os.path.exists(model_path):
+        print(f"\nERROR: Model not found at: {model_path}")
+        print("Run first:  py setup_model.py")
+        sys.exit(1)
+
+    size_mb = os.path.getsize(model_path) / 1024 / 1024
+    print(f"  Model: {os.path.basename(model_path)}  ({size_mb:.0f} MB)")
+
+    if n_threads is None:
+        import multiprocessing
+        n_threads = max(1, multiprocessing.cpu_count() - 1)
+
+    # Context window: system prompt (~800 tok) + user prompt (~200 tok) + output (~150 tok)
+    n_ctx = 1536
+
+    print(f"  Loading... (n_ctx={n_ctx}, n_threads={n_threads})", end="", flush=True)
+    t0 = time.time()
+    llm = Llama(
+        model_path=str(model_path),
+        n_ctx=n_ctx,
+        n_threads=n_threads,
+        n_gpu_layers=0,       # 0 = CPU only; set to -1 to use full GPU if available
+        verbose=False,
+        chat_format="chatml", # Qwen2.5 uses ChatML format
+    )
+    print(f" done in {time.time()-t0:.1f}s")
+    return llm
+
+
+# ── Single inference ──────────────────────────────────────────────────────────
+
+def rewrite_one(llm, original: str, candidate_id: str,
+                temperature: float = 0.6, max_tokens: int = 160) -> str:
+    """Run one rewrite call. Returns rewritten text."""
+    messages = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user",   "content": build_user_prompt(original, candidate_id)},
+    ]
+    resp = llm.create_chat_completion(
+        messages=messages,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        top_p=0.92,
+        repeat_penalty=1.12,
+        stop=["Original assessment:", "Rewritten (", "\n\n\n", "---"],
+    )
+    text = resp["choices"][0]["message"]["content"].strip()
+    # Strip any prompt leakage
+    for stopper in ["Original assessment:", "Rewritten (", "===", "STRICT"]:
+        if stopper in text:
+            text = text.split(stopper)[0].strip()
+    return text
+
+
+# ── Main processing ───────────────────────────────────────────────────────────
+
+def process(input_path: str, out_csv: str, model_path: str,
+            dry_run: bool = False, n_threads: int = None,
+            temperature: float = 0.6):
+
+    print(f"\n  Input:  {input_path}")
+    records = []
+    with open(input_path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+    print(f"  Loaded {len(records)} candidates")
+
+    if dry_run:
+        print("\n  [DRY RUN] First 3 prompts:\n")
+        for rec in records[:3]:
+            print(f"  {'='*60}")
+            print(f"  Candidate: {rec['candidate_id']}")
+            print(f"  Original:  {rec['fixed_reasoning']}")
+            print(f"\n  SYSTEM PROMPT (truncated):\n  {SYSTEM_PROMPT[:300]}...")
+            print(f"\n  USER PROMPT:\n  {build_user_prompt(rec['fixed_reasoning'], rec['candidate_id'])}")
+        return
+
+    # Load model
+    print()
+    llm = load_model(model_path, n_threads=n_threads)
+
+    # Process loop
+    print(f"\n  Rewriting {len(records)} reasonings (temperature={temperature})")
+    print(f"  {'Rank':<5}  {'Candidate ID':<18}  {'Time':>5}  Status")
+    print(f"  {'-'*65}")
+
+    results        = []
+    fallback_count = 0
+    total_start    = time.time()
+
+    for rec in records:
+        cid      = rec["candidate_id"]
+        rank     = rec.get("rank", len(results) + 1)
+        score    = rec.get("score", 0)
+        original = rec["fixed_reasoning"]
+
+        t0 = time.time()
+        try:
+            rewritten = rewrite_one(llm, original, cid, temperature=temperature)
+            elapsed   = time.time() - t0
+            ok, reason = validate_rewrite(original, rewritten)
+            if ok:
+                status = "OK"
+            else:
+                rewritten = original
+                fallback_count += 1
+                status = f"FALLBACK ({reason[:40]})"
+        except Exception as e:
+            rewritten = original
+            elapsed   = time.time() - t0
+            fallback_count += 1
+            status = f"ERROR ({str(e)[:30]})"
+
+        results.append({
+            "rank":               rank,
+            "candidate_id":       cid,
+            "score":              score,
+            "fixed_reasoning":    original,
+            "rewritten_reasoning": rewritten,
+        })
+        print(f"  {rank:<5}  {cid:<18}  {elapsed:>4.1f}s  {status}")
+
+    total_time = time.time() - total_start
+
+    # Write rewritten JSONL
+    jsonl_out = out_csv.replace(".csv", "_rewritten.jsonl")
+    with open(jsonl_out, "w", encoding="utf-8") as f:
+        for r in results:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+
+    # Write updated submission.csv
+    with open(out_csv, "w", encoding="utf-8", newline="") as f:
+        f.write("candidate_id,rank,score,reasoning\n")
+        for r in results:
+            clean = r["rewritten_reasoning"].replace('"', "'").replace("\n", " ").strip()
+            f.write(f'{r["candidate_id"]},{r["rank"]},{r["score"]},"{clean}"\n')
+
+    # Stats
+    unique = len({r["rewritten_reasoning"] for r in results})
+    however_remaining = sum(
+        1 for r in results if "; however" in r["rewritten_reasoning"].lower()
+    )
+
+    print(f"""
+  +=====================================================+
+  |  REWRITE SUMMARY                                   |
+  +=====================================================+
+  | Total candidates:          {len(results):<5}                   |
+  | Fallbacks (original kept): {fallback_count:<5}                   |
+  | Unique reasonings:         {unique:<5} / {len(results):<5}              |
+  | '; however' remaining:     {however_remaining:<5} ({however_remaining/len(results)*100:.0f}%)              |
+  | Total time:                {total_time/60:.1f} min                    |
+  | Avg per candidate:         {total_time/len(results):.1f}s                     |
+  +=====================================================+
+  | Output:  {out_csv:<44}|
+  | JSONL:   {jsonl_out:<44}|
+  +=====================================================+
+    """)
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Offline SLM reasoning rewriter — Qwen2.5-1.5B-Instruct GGUF"
+    )
+    parser.add_argument("--input", required=True,
+        help="Input JSONL from fix_reasoning.py (e.g. top100_reasoned.jsonl)")
+    parser.add_argument("--out", required=True,
+        help="Output submission.csv path")
+    parser.add_argument("--model", default=str(DEFAULT_MODEL),
+        help=f"Path to GGUF model file (default: {DEFAULT_MODEL})")
+    parser.add_argument("--dry-run", action="store_true",
+        help="Print prompts for first 3 candidates without running the model")
+    parser.add_argument("--threads", type=int, default=None,
+        help="CPU threads (default: cpu_count - 1)")
+    parser.add_argument("--temperature", type=float, default=0.6,
+        help="Sampling temperature 0-1 (default: 0.6). Higher = more varied")
+    args = parser.parse_args()
+
+    if not os.path.exists(args.input):
+        print(f"ERROR: Input not found: {args.input}")
+        sys.exit(1)
+
+    print(f"""
++============================================================+
+|  REDROB - SLM Reasoning Rewriter (Fully Offline)          |
+|  Model: Qwen2.5-1.5B-Instruct-Q4_K_M GGUF               |
+|  Context: Full JD embedded in system prompt               |
++============================================================+
+    """)
+
+    process(
+        input_path=args.input,
+        out_csv=args.out,
+        model_path=args.model,
+        dry_run=args.dry_run,
+        n_threads=args.threads,
+        temperature=args.temperature,
+    )
+
+
+if __name__ == "__main__":
+    main()

run_pipeline.py

@@ -0,0 +1,622 @@
+# run_pipeline.py
+
+`python
+#!/usr/bin/env python3
+"""
+run_pipeline.py - Redrob Intelligent Candidate Ranking Pipeline
+===============================================================
+MASTER ORCHESTRATOR — runs the full 5-stage pipeline and reports
+wall-clock time, peak RAM, GPU VRAM, and disk usage per stage.
+
+PIPELINE:
+  Stage 1 | filter_candidates.py  | 100k → ~22k  (hard rules + honeypot + prescore)
+  Stage 2 | LightGBM inference    | 22k  → 15k   (pre-trained ranker.pkl)
+  Stage 3 | fix_reasoning.py      | top 100       (deterministic fact-grounded reasoning)
+  Stage 4 | rewrite_reasoning.py  | top 100       (SLM paraphrase, JD-aware, offline)
+  Stage 5 | submission.csv        | final output
+
+USAGE:
+  # Full pipeline from scratch:
+  py run_pipeline.py --candidates candidates.jsonl --ranker ranker.pkl
+
+  # Skip to stage 2 (filter already done):
+  py run_pipeline.py --candidates candidates.jsonl --ranker ranker.pkl --start-stage 2
+
+  # Skip SLM rewrite (deterministic reasoning only):
+  py run_pipeline.py --candidates candidates.jsonl --ranker ranker.pkl --no-rewrite
+
+  # Check disk/file status without running:
+  py run_pipeline.py --candidates candidates.jsonl --ranker ranker.pkl --status
+
+PREREQUISITES:
+  pip install -r requirements.txt
+  py setup_model.py                  # downloads Qwen2.5-1.5B GGUF (~1 GB)
+  pip install llama-cpp-python \\
+    --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+"""
+
+import json
+import os
+import sys
+import time
+import pickle
+import shutil
+import threading
+import argparse
+import subprocess
+from pathlib import Path
+from datetime import datetime
+
+# ── Resource monitoring ───────────────────────────────────────────────────────
+
+try:
+    import psutil
+    _PROC = psutil.Process(os.getpid())
+    HAS_PSUTIL = True
+except ImportError:
+    HAS_PSUTIL = False
+    _PROC = None
+
+
+def _gpu_mb():
+    try:
+        out = subprocess.check_output(
+            ["nvidia-smi", "--query-gpu=memory.used,memory.total",
+             "--format=csv,noheader,nounits"],
+            stderr=subprocess.DEVNULL, timeout=3
+        ).decode().strip().splitlines()[0]
+        used, total = map(int, out.split(","))
+        return used, total
+    except Exception:
+        return None, None
+
+
+class ResourceMonitor:
+    def __init__(self):
+        self.peak_ram_mb = 0.0
+        self.peak_gpu_mb = 0.0
+        self._running    = False
+        self._thread     = None
+
+    def start(self):
+        self._running = True
+        self._thread  = threading.Thread(target=self._loop, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._running = False
+        if self._thread:
+            self._thread.join(timeout=2)
+
+    def _loop(self):
+        while self._running:
+            if HAS_PSUTIL and _PROC:
+                try:
+                    ram = _PROC.memory_info().rss / 1024 / 1024
+                    self.peak_ram_mb = max(self.peak_ram_mb, ram)
+                except Exception:
+                    pass
+            gpu_used, _ = _gpu_mb()
+            if gpu_used:
+                self.peak_gpu_mb = max(self.peak_gpu_mb, float(gpu_used))
+            time.sleep(0.5)
+
+    def snapshot(self):
+        ram = 0.0
+        if HAS_PSUTIL and _PROC:
+            try:
+                ram = _PROC.memory_info().rss / 1024 / 1024
+            except Exception:
+                pass
+        gpu_u, gpu_t = _gpu_mb()
+        return {
+            "current_ram_mb": round(ram, 1),
+            "peak_ram_mb":    round(self.peak_ram_mb, 1),
+            "gpu_used_mb":    gpu_u,
+            "gpu_total_mb":   gpu_t,
+        }
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+def ts():
+    return datetime.now().strftime("%H:%M:%S")
+
+def log(msg, level="OK"):
+    icons = {"OK": "[OK] ", "WARN": "[WRN]", "ERR": "[ERR]", "INFO": "[-->]"}
+    print(f"  {icons.get(level,'     ')} [{ts()}] {msg}", flush=True)
+
+def sep(title):
+    print(f"\n{'='*68}", flush=True)
+    print(f"  {title}", flush=True)
+    print(f"{'='*68}", flush=True)
+
+def fmttime(s):
+    if s < 60:
+        return f"{s:.1f}s"
+    return f"{int(s)//60}m {int(s)%60}s"
+
+def fmtmb(path):
+    if path and os.path.exists(path):
+        return round(os.path.getsize(path) / 1024 / 1024, 2)
+    return 0.0
+
+def check_disk(path, needed_mb):
+    free_mb = shutil.disk_usage(os.path.dirname(os.path.abspath(path))).free / 1024 / 1024
+    if free_mb < needed_mb:
+        log(f"Low disk space! Need ~{needed_mb:.0f} MB, have {free_mb:.0f} MB", "WARN")
+    else:
+        log(f"Disk free: {free_mb:.0f} MB  (need ~{needed_mb:.0f} MB) [OK]")
+
+
+# ── Stage 1: Hard filter ─────────────────────��────────────────────────────────
+
+def stage1_filter(candidates_path, out_path, top_k, report, monitor):
+    sep(f"STAGE 1  |  HARD FILTER + PRESCORE  (100k -> ~{top_k//1000}k)")
+
+    # Disk estimate: filtered output ~ 60% of input
+    in_mb = fmtmb(candidates_path)
+    check_disk(out_path, in_mb * 0.6 + 50)
+
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    if script_dir not in sys.path:
+        sys.path.insert(0, script_dir)
+
+    try:
+        import filter_candidates as fc
+    except ImportError:
+        log("filter_candidates.py not found in same directory", "ERR")
+        sys.exit(1)
+
+    t0 = time.time()
+    n_out = fc.run_filter(
+        candidates_path=candidates_path,
+        top_k=top_k,
+        out_path=out_path,
+        report=report,
+        report_path=out_path.replace(".jsonl", "_report.txt"),
+    )
+    elapsed = time.time() - t0
+    snap = monitor.snapshot()
+    log(f"Done in {fmttime(elapsed)} | {n_out:,} candidates | RAM: {snap['peak_ram_mb']:.0f} MB | Disk: {fmtmb(out_path):.1f} MB")
+    return {"stage": "1_filter", "n_in": "100k", "n_out": n_out,
+            "time_s": round(elapsed, 1), **snap, "disk_mb": fmtmb(out_path)}
+
+
+# ── Stage 2: LightGBM inference ───────────────────────────────────────────────
+
+def stage2_lgbm(filtered_path, ranker_pkl, out_path, top_k, monitor):
+    sep(f"STAGE 2  |  LIGHTGBM INFERENCE  (~22k -> top {top_k//1000}k)")
+    check_disk(out_path, fmtmb(filtered_path) * 0.7)
+
+    try:
+        import numpy as np
+    except ImportError:
+        log("numpy not installed: pip install numpy", "ERR"); sys.exit(1)
+
+    log(f"Loading ranker: {ranker_pkl}")
+    with open(ranker_pkl, "rb") as f:
+        raw = pickle.load(f)
+
+    best_iter = -1
+    if isinstance(raw, dict):
+        booster      = raw["booster"]
+        best_iter    = raw.get("best_iteration", -1)
+        feature_names = raw.get("feat_keys", [])
+        log(f"Ranker dict: {len(feature_names)} features, best_iter={best_iter}")
+        if feature_names:
+            # Print ranker training metrics if available
+            metrics = raw.get("metrics", {})
+            if metrics:
+                m_str = "  ".join(f"{k}={v:.4f}" for k, v in list(metrics.items())[:4])
+                log(f"Training metrics: {m_str}")
+    else:
+        booster = raw
+        feature_names = []
+
+    # If feat_keys not in pickle, load from JSON
+    if not feature_names:
+        ranker_dir  = os.path.dirname(os.path.abspath(ranker_pkl))
+        for fname in ["ranker_features.json", "ranker_features (1).json"]:
+            fp = os.path.join(ranker_dir, fname)
+            if os.path.exists(fp):
+                with open(fp, "r", encoding="utf-8") as f:
+                    feature_names = json.load(f)
+                log(f"Loaded feature names from {fp}")
+                break
+    if not feature_names:
+        log("Cannot find feature names (ranker_features.json). Pass --ranker-features.", "ERR")
+        sys.exit(1)
+
+    t0 = time.time()
+    log(f"Loading {filtered_path}")
+    candidates = []
+    with open(filtered_path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                candidates.append(json.loads(line))
+    log(f"Loaded {len(candidates):,} candidates")
+
+    X = np.array([
+        [float((c.get("features") or {}).get(fn, 0.0) or 0.0) for fn in feature_names]
+        for c in candidates
+    ], dtype=np.float32)
+    log(f"Feature matrix: {X.shape}")
+
+    t_inf = time.time()
+    pred_kw = {"num_iteration": best_iter} if best_iter > 0 else {}
+    scores  = booster.predict(X, **pred_kw)
+    log(f"Inference in {time.time()-t_inf:.3f}s  range [{scores.min():.4f} - {scores.max():.4f}]")
+
+    top_idx = scores.argsort()[::-1][:top_k]
+    with open(out_path, "w", encoding="utf-8") as f:
+        for rank, idx in enumerate(top_idx, 1):
+            c = candidates[idx]
+            c["lgbm_score"] = float(scores[idx])
+            c["lgbm_rank"]  = rank
+            f.write(json.dumps(c, ensure_ascii=False) + "\n")
+
+    elapsed = time.time() - t0
+    snap    = monitor.snapshot()
+    top_s   = scores[top_idx]
+    log(f"Done in {fmttime(elapsed)} | top-1={top_s[0]:.4f} top-100={top_s[min(99,len(top_s)-1)]:.4f} | RAM: {snap['peak_ram_mb']:.0f} MB | Disk: {fmtmb(out_path):.1f} MB")
+    return {"stage": "2_lgbm", "n_in": len(candidates), "n_out": len(top_idx),
+            "time_s": round(elapsed, 1), **snap, "disk_mb": fmtmb(out_path)}
+
+
+# ── Stage 3: Deterministic reasoning ─────────────────────────────────────────
+
+def stage3_fix_reasoning(lgbm_path, out_path, top_k, monitor):
+    sep(f"STAGE 3  |  DETERMINISTIC REASONING  (top {top_k})")
+
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    if script_dir not in sys.path:
+        sys.path.insert(0, script_dir)
+
+    try:
+        import fix_reasoning as fr
+    except ImportError:
+        log("fix_reasoning.py not found in same directory", "ERR"); sys.exit(1)
+
+    t0 = time.time()
+    records = []
+    with open(lgbm_path, "r", encoding="utf-8") as f:
+        for i, line in enumerate(f):
+            if i >= top_k:
+                break
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+    log(f"Loaded {len(records):,} candidates for reasoning")
+
+    stats = fr.process_records(records, out_path, top_k=top_k)
+    elapsed = time.time() - t0
+    snap    = monitor.snapshot()
+    log(f"Done in {fmttime(elapsed)} | unique={stats['unique_reasonings']}/{stats['total']} | hallucinations_replaced={stats['hallucinations_replaced']}")
+    return {"stage": "3_reasoning", "n_in": len(records), "n_out": stats["total"],
+            "hallucinations_replaced": stats["hallucinations_replaced"],
+            "unique_reasonings":       stats["unique_reasonings"],
+            "time_s": round(elapsed, 1), **snap, "disk_mb": fmtmb(out_path)}
+
+
+# ── Stage 3.5: JD-aligned composite scorer (0-100) ───────────────────────────
+
+def stage35_composite_score(lgbm_path, reasoned_path, submission_path, top_k, monitor):
+    sep(f"STAGE 3.5  |  COMPOSITE SCORING  (JD-aligned, 0-100)")
+
+    # Check if a Regressor was used (scores are already 0-100)
+    with open(lgbm_path, "r", encoding="utf-8") as f:
+        first = json.loads(f.readline())
+        if first.get("lgbm_score", 0) > 10.0:
+            log("Stage 3.5 skipped: LightGBM output is already 0-100 (Regressor used).")
+            return None
+
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    if script_dir not in sys.path:
+        sys.path.insert(0, script_dir)
+
+    try:
+        import score_candidates as sc
+    except ImportError:
+        log("score_candidates.py not found — skipping composite scoring", "WARN")
+        return None
+
+    t0 = time.time()
+    final_scores = sc.run(
+        input_path=lgbm_path,
+        top_k=top_k,
+        reasoned_path=reasoned_path,
+        submission_path=submission_path,
+    )
+    elapsed = time.time() - t0
+    snap = monitor.snapshot()
+    lo, hi = min(final_scores), max(final_scores)
+    log(f"Done in {fmttime(elapsed)} | score range {lo:.1f}-{hi:.1f} (spread {hi-lo:.1f} pts)")
+    return {"stage": "3.5_scoring", "n_out": len(final_scores),
+            "time_s": round(elapsed, 1), **snap, "disk_mb": fmtmb(reasoned_path)}
+
+
+# ── Stage 4: SLM rewrite ──────────────────────────────────────────────────────
+
+def stage4_slm_rewrite(reasoned_path, out_path, model_path, temperature, threads, monitor):
+    sep(f"STAGE 4  |  SLM REWRITE  (Qwen2.5-1.5B, JD-aware, offline)")
+
+    if not os.path.exists(model_path):
+        log(f"Model not found: {model_path}", "WARN")
+        log("Run: py setup_model.py  to download (~1 GB)", "WARN")
+        log("Skipping SLM rewrite. Using deterministic reasoning for submission.", "WARN")
+        return None
+
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    if script_dir not in sys.path:
+        sys.path.insert(0, script_dir)
+
+    try:
+        import rewrite_reasoning as rr
+    except ImportError:
+        log("rewrite_reasoning.py not found", "ERR"); sys.exit(1)
+
+    t0 = time.time()
+    rr.process(
+        input_path=reasoned_path,
+        out_csv=out_path,
+        model_path=model_path,
+        dry_run=False,
+        n_threads=threads,
+        temperature=temperature,
+    )
+    elapsed = time.time() - t0
+    snap    = monitor.snapshot()
+    log(f"Done in {fmttime(elapsed)} | RAM: {snap['peak_ram_mb']:.0f} MB | Disk: {fmtmb(out_path):.1f} MB")
+    return {"stage": "4_slm_rewrite", "time_s": round(elapsed, 1), **snap, "disk_mb": fmtmb(out_path)}
+
+
+# ── Stage 5: Submission CSV (deterministic fallback) ─────────────────────────
+
+def stage5_submission(reasoned_path, out_path, top_k, monitor):
+    """Write submission.csv from deterministic reasoning (no SLM)."""
+    sep(f"STAGE 5  |  SUBMISSION CSV  (top {top_k})")
+
+    records = []
+    with open(reasoned_path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+    records.sort(key=lambda r: (-r.get("score", 0), r.get("candidate_id", "")))
+    records = records[:top_k]
+
+    with open(out_path, "w", encoding="utf-8", newline="") as f:
+        f.write("candidate_id,rank,score,reasoning\n")
+        for rank, r in enumerate(records, 1):
+            rsn = r.get("fixed_reasoning", "").replace('"', "'").replace("\n", " ").strip()
+            f.write(f'{r["candidate_id"]},{rank},{r["score"]},"{rsn}"\n')
+
+    # Top-10 preview
+    print(f"\n  -- TOP-10 --")
+    print(f"  {'Rank':<5}  {'Candidate ID':<18}  {'Score':>7}  Reasoning preview")
+    print(f"  {'-'*75}")
+    for rank, r in enumerate(records[:10], 1):
+        rsn = r.get("fixed_reasoning", "")[:65]
+        print(f"  {rank:<5}  {r['candidate_id']:<18}  {r['score']:>7.4f}  {rsn}...")
+
+    snap = monitor.snapshot()
+    return {"stage": "5_submission", "n_out": len(records), **snap, "disk_mb": fmtmb(out_path)}
+
+
+# ── Resource report ───────────────────────────────────────────────────────────
+
+def print_report(stage_stats, total_start, paths):
+    total = time.time() - total_start
+    W = 68
+
+    def box(title):
+        print(f"\n+{'='*W}+")
+        print(f"| {title:<{W-1}}|")
+        print(f"+{'-'*W}+")
+
+    def row(label, val):
+        print(f"| {label:<35} {val:<{W-36}}|")
+
+    def divider():
+        print(f"+{'-'*W}+")
+
+    box("RESOURCE & TIMING REPORT")
+    row("Stage", f"{'Time':>8}  {'In':>8}  {'Out':>8}  {'Peak RAM':>10}")
+    divider()
+
+    stage_names = {
+        "1_filter":    "Stage 1: Hard filter + prescore",
+        "2_lgbm":      "Stage 2: LightGBM inference",
+        "3_reasoning": "Stage 3: Deterministic reasoning",
+        "4_slm_rewrite": "Stage 4: SLM rewrite (Qwen2.5)",
+        "5_submission":  "Stage 5: Submission CSV",
+    }
+    for s in stage_stats:
+        if s is None:
+            continue
+        nm   = stage_names.get(s["stage"], s["stage"])
+        t    = fmttime(s.get("time_s", 0))
+        n_in = f"{s['n_in']:,}" if isinstance(s.get("n_in"), int) else str(s.get("n_in", ""))
+        n_out = f"{s['n_out']:,}" if isinstance(s.get("n_out"), int) else str(s.get("n_out", ""))
+        ram  = f"{s.get('peak_ram_mb', 0):.0f} MB"
+        row(nm, f"{t:>8}  {n_in:>8}  {n_out:>8}  {ram:>10}")
+    divider()
+    row("TOTAL PIPELINE TIME", fmttime(total))
+
+    box("PEAK RAM USAGE")
+    peak = max((s.get("peak_ram_mb", 0) for s in stage_stats if s), default=0)
+    row(f"Peak RAM (this process)", f"{peak:.0f} MB  ({peak/1024:.2f} GB)")
+
+    box("GPU USAGE")
+    gpu_u = next((s.get("gpu_used_mb") for s in reversed(stage_stats) if s and s.get("gpu_used_mb")), None)
+    gpu_t = next((s.get("gpu_total_mb") for s in reversed(stage_stats) if s and s.get("gpu_total_mb")), None)
+    if gpu_u is not None:
+        row("GPU VRAM used", f"{gpu_u} MB / {gpu_t} MB")
+    else:
+        row("GPU", "Not used (CPU-only pipeline)")
+
+    box("DISK USAGE")
+    labels = [
+        ("candidates.jsonl (input)",       paths.get("candidates")),
+        ("filtered_22k.jsonl",             paths.get("filtered")),
+        ("lgbm_top15k.jsonl",              paths.get("lgbm_out")),
+        ("top100_reasoned.jsonl",          paths.get("reasoned_out")),
+        ("submission.csv (final)",         paths.get("submission")),
+        ("SLM model (Qwen2.5-1.5B GGUF)", paths.get("model")),
+    ]
+    total_disk = 0
+    for label, path in labels:
+        mb = fmtmb(path)
+        total_disk += mb
+        status = "[OK]  " if (path and os.path.exists(path)) else "[n/a] "
+        row(f"{status}{label}", f"{mb:>8.2f} MB")
+    divider()
+    row("TOTAL DISK (all files)", f"{total_disk:>8.2f} MB  ({total_disk/1024:.2f} GB)")
+    print(f"+{'='*W}+\n")
+
+
+# ── Status check ──────────────────────────────────────────────────────────────
+
+def status_check(paths):
+    print("\n  -- PIPELINE FILE STATUS --")
+    for label, path in paths.items():
+        if not path:
+            continue
+        if os.path.exists(path):
+            mb = fmtmb(path)
+            print(f"  [EXISTS]  {label:<30}  {path}  ({mb:.1f} MB)")
+        else:
+            print(f"  [MISSING] {label:<30}  {path}")
+    print()
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Redrob end-to-end ranking pipeline (100k -> submission.csv)",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    g = parser.add_argument_group("Input files")
+    g.add_argument("--candidates", required=True, help="Raw candidates.jsonl (100k pool)")
+    g.add_argument("--ranker",     required=True, help="Pre-trained ranker.pkl (LightGBM)")
+    g.add_argument("--ranker-features", default=None,
+        help="ranker_features.json (auto-detected beside ranker.pkl if omitted)")
+
+    g = parser.add_argument_group("Stage control")
+    g.add_argument("--start-stage", type=int, default=1, choices=[1,2,3,4,5],
+        help="Start from stage N (1=filter, 2=lgbm, 3=reasoning, 4=slm, 5=csv)")
+    g = parser.add_argument_group("Intermediate paths")
+    g.add_argument("--filtered",    default="filtered_22k.jsonl")
+    g.add_argument("--lgbm-out",    default="lgbm_top15k.jsonl")
+    g.add_argument("--reasoned-out",default="top100_reasoned.jsonl")
+    g.add_argument("--submission",  default="submission.csv")
+
+    g = parser.add_argument_group("Top-K params")
+    g.add_argument("--filter-top-k",  type=int, default=22000)
+    g.add_argument("--lgbm-top-k",    type=int, default=15000)
+    g.add_argument("--top-final",     type=int, default=100)
+
+    g = parser.add_argument_group("Misc")
+    g.add_argument("--report",    action="store_true", help="Write Stage 1 filter report")
+    g.add_argument("--status",    action="store_true", help="Show file status and exit")
+
+    args = parser.parse_args()
+
+    # Auto-resolve ranker features
+    if not args.ranker_features:
+        rd = os.path.dirname(os.path.abspath(args.ranker))
+        for n in ["ranker_features.json", "ranker_features (1).json"]:
+            p = os.path.join(rd, n)
+            if os.path.exists(p):
+                args.ranker_features = p
+                break
+
+    paths = {
+        "candidates":   args.candidates,
+        "filtered":     args.filtered,
+        "lgbm_out":     args.lgbm_out,
+        "reasoned_out": args.reasoned_out,
+        "submission":   args.submission,
+    }
+
+    print("""
++====================================================================+
+|  REDROB INTELLIGENT CANDIDATE RANKING PIPELINE                    |
+|  Stage 1:   Hard filter + prescore  (100k -> ~22k)               |
+|  Stage 2:   LightGBM LambdaRank     (22k  -> 15k)               |
+|  Stage 3:   Deterministic reasoning (top 100)                    |
+|  Stage 3.5: JD-aligned composite score (0-100 with spread)      |
+|  Stage 4:   SLM rewrite - Qwen2.5-1.5B (JD-aware, offline)     |
+|  Stage 5:   submission.csv                                       |
++====================================================================+
+    """)
+
+    if args.status:
+        status_check(paths)
+        model_exists = os.path.exists(args.model)
+        print(f"  SLM model: {'EXISTS' if model_exists else 'MISSING - run: py setup_model.py'}")
+        print(f"  Model path: {args.model}")
+        return
+
+    # Validate inputs
+    if not os.path.exists(args.candidates):
+        log(f"candidates file not found: {args.candidates}", "ERR"); sys.exit(1)
+    if args.start_stage <= 2 and not os.path.exists(args.ranker):
+        log(f"ranker not found: {args.ranker}", "ERR"); sys.exit(1)
+
+    print(f"  Config:")
+    print(f"    candidates:     {args.candidates}  ({fmtmb(args.candidates):.0f} MB)")
+    print(f"    ranker:         {args.ranker}")
+    print(f"    filter_top_k:   {args.filter_top_k:,}")
+    print(f"    lgbm_top_k:     {args.lgbm_top_k:,}")
+    print(f"    top_final:      {args.top_final}")
+
+    print()
+
+    monitor     = ResourceMonitor()
+    monitor.start()
+    stage_stats = []
+    t_total     = time.time()
+
+    try:
+        # STAGE 1
+        if args.start_stage <= 1:
+            stage_stats.append(stage1_filter(args.candidates, args.filtered, args.filter_top_k, args.report, monitor))
+        else:
+            log(f"Skipping Stage 1 -- using: {args.filtered}")
+
+        # STAGE 2
+        if args.start_stage <= 2:
+            stage_stats.append(stage2_lgbm(args.filtered, args.ranker, args.lgbm_out, args.lgbm_top_k, monitor))
+        else:
+            log(f"Skipping Stage 2 -- using: {args.lgbm_out}")
+
+        # STAGE 3
+        if args.start_stage <= 3:
+            stage_stats.append(stage3_fix_reasoning(args.lgbm_out, args.reasoned_out, args.top_final, monitor))
+        else:
+            log(f"Skipping Stage 3 -- using: {args.reasoned_out}")
+
+        # STAGE 3.5 — JD-aligned composite score (0-100)
+        stage_stats.append(stage35_composite_score(
+            args.lgbm_out, args.reasoned_out, args.submission, args.top_final, monitor
+        ))
+
+        # STAGE 4 - write final submission CSV from reasoned JSONL (previously Stage 5)
+        stage_stats.append(stage5_submission(args.reasoned_out, args.submission, args.top_final, monitor))
+
+    finally:
+        monitor.stop()
+
+    print_report([s for s in stage_stats if s], t_total, paths)
+    print(f"  Submission: {args.submission}")
+    print(f"  Reasoning:  {args.reasoned_out}\n")
+
+
+if __name__ == "__main__":
+    main()
+

setup_model.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""
+setup_model.py - One-time model download for offline SLM reasoning rewriter
+============================================================================
+Run this ONCE to download the model weights locally.
+After this, rewrite_reasoning.py works fully offline forever.
+
+Usage:
+    py setup_model.py
+
+What this downloads:
+    Model : Qwen2.5-1.5B-Instruct-Q4_K_M.gguf
+    Size  : ~986 MB (~1 GB)
+    Source: Hugging Face (bartowski/Qwen2.5-1.5B-Instruct-GGUF)
+    Dest  : ./models/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf
+
+Why Qwen2.5-1.5B:
+    - Best 1 GB model for instruction-following + paraphrasing
+    - Handles Indian company/tech jargon well
+    - Apache 2.0 license (commercial use OK)
+    - Runs on CPU, no GPU required
+
+After download, install the runtime if not done:
+    pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+
+Then run the rewriter:
+    py rewrite_reasoning.py --input top100_reasoned.jsonl --out submission.csv
+"""
+
+import os
+import sys
+import urllib.request
+import urllib.error
+import shutil
+from pathlib import Path
+
+# -- Model config -------------------------------------------------------------
+
+MODEL_REPO  = "bartowski/Qwen2.5-1.5B-Instruct-GGUF"
+MODEL_FILE  = "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf"
+MODEL_URL   = (
+    "https://huggingface.co/bartowski/Qwen2.5-1.5B-Instruct-GGUF"
+    "/resolve/main/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf"
+)
+EXPECTED_MB = 986
+MODELS_DIR  = Path(__file__).parent / "models"
+MODEL_PATH  = MODELS_DIR / MODEL_FILE
+
+
+# -- Progress bar -------------------------------------------------------------
+
+class ProgressBar:
+    def __init__(self, total_mb):
+        self.total    = total_mb * 1024 * 1024
+        self.done     = 0
+        self.last_pct = -1
+
+    def __call__(self, block_num, block_size, total_size):
+        if total_size > 0:
+            self.total = total_size
+        self.done = min(self.done + block_size, self.total)
+        pct = int(self.done * 100 / self.total)
+        if pct != self.last_pct and pct % 2 == 0:
+            filled = pct // 5
+            bar = "#" * filled + "." * (20 - filled)
+            mb_done  = self.done  / 1024 / 1024
+            mb_total = self.total / 1024 / 1024
+            print(f"\r  [{bar}] {pct:3d}%  {mb_done:.0f}/{mb_total:.0f} MB",
+                  end="", flush=True)
+            self.last_pct = pct
+
+
+# -- Download helpers ---------------------------------------------------------
+
+def try_huggingface_hub():
+    """Try huggingface_hub if installed (resumable downloads)."""
+    try:
+        from huggingface_hub import hf_hub_download
+        print("  Using huggingface_hub (resumable)...")
+        hf_hub_download(
+            repo_id=MODEL_REPO,
+            filename=MODEL_FILE,
+            local_dir=str(MODELS_DIR),
+            local_dir_use_symlinks=False,
+        )
+        return True
+    except ImportError:
+        return False
+    except Exception as e:
+        print(f"  huggingface_hub failed: {e}. Falling back to urllib...")
+        return False
+
+
+def download_urllib():
+    print(f"  From: {MODEL_URL}")
+    print(f"  To:   {MODEL_PATH}")
+    print(f"  Size: ~{EXPECTED_MB} MB\n")
+    bar = ProgressBar(EXPECTED_MB)
+    try:
+        urllib.request.urlretrieve(MODEL_URL, MODEL_PATH, reporthook=bar)
+        print()
+    except urllib.error.URLError as e:
+        print(f"\n  ERROR: Download failed: {e}")
+        if MODEL_PATH.exists():
+            MODEL_PATH.unlink()
+        sys.exit(1)
+
+
+# -- Main ---------------------------------------------------------------------
+
+def main():
+    print("""
++===================================================+
+|  REDROB SLM SETUP - One-time model download      |
+|  Qwen2.5-1.5B-Instruct-Q4_K_M.gguf (~1 GB)     |
++===================================================+
+    """)
+
+    MODELS_DIR.mkdir(exist_ok=True)
+
+    # Already downloaded?
+    if MODEL_PATH.exists():
+        size_mb = MODEL_PATH.stat().st_size / 1024 / 1024
+        if size_mb > 900:
+            print(f"  [OK] Model already at: {MODEL_PATH}")
+            print(f"       Size: {size_mb:.0f} MB")
+            _check_llama_cpp()
+            return
+        else:
+            print(f"  Partial download ({size_mb:.0f} MB). Re-downloading...")
+            MODEL_PATH.unlink()
+
+    # Disk space check
+    free_gb = shutil.disk_usage(MODELS_DIR).free / 1024 ** 3
+    if free_gb < 1.2:
+        print(f"  ERROR: Need 1.2 GB free, only {free_gb:.1f} GB available.")
+        sys.exit(1)
+    print(f"  Free disk: {free_gb:.1f} GB  [OK]")
+    print(f"  Downloading {MODEL_FILE} ...")
+    print()
+
+    downloaded = try_huggingface_hub()
+    if not downloaded:
+        download_urllib()
+
+    # Verify size
+    size_mb = MODEL_PATH.stat().st_size / 1024 / 1024
+    if size_mb < 900:
+        print(f"  ERROR: File too small ({size_mb:.0f} MB) — may be corrupted.")
+        sys.exit(1)
+
+    print(f"\n  [OK] Model ready: {MODEL_PATH}  ({size_mb:.0f} MB)")
+    _check_llama_cpp()
+
+    print("\n  NEXT STEP:")
+    print("  py rewrite_reasoning.py --input top100_reasoned.jsonl --out submission.csv\n")
+
+
+def _check_llama_cpp():
+    print()
+    try:
+        import llama_cpp
+        print(f"  [OK] llama-cpp-python: {llama_cpp.__version__}")
+    except ImportError:
+        print("  [!] llama-cpp-python not installed. Run ONE of:")
+        print()
+        print("  # CPU only (recommended if no NVIDIA GPU):")
+        print("  pip install llama-cpp-python \\")
+        print("    --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu")
+        print()
+        print("  # CUDA 12.1 (if you have an NVIDIA GPU):")
+        print("  pip install llama-cpp-python \\")
+        print("    --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121")
+
+
+if __name__ == "__main__":
+    main()

submission.csv

@@ -0,0 +1,101 @@
+candidate_id,rank,score,reasoning
+CAND_0039754,1,89.35,"Senior Applied Scientist at Meta with 16.2 years of experience and 95mo of production Python; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 78/100), highly responsive (81% reply rate) — recommend prioritising outreach."
+CAND_0071974,2,89.18,"Senior AI Engineer at Netflix with 7.8 years of experience and 96mo of production Elasticsearch; strong alignment with this retrieval-focused role including 8 of the JD's must-have technical requirements. Notice period is 45 days (manageable); strong GitHub presence (score 83/100), actively job-hunting — recommend prioritising outreach."
+CAND_0011687,3,87.76,"Senior NLP Engineer at Niramai with 7.8 years of experience and 92mo of production OpenSearch; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 15 days; active GitHub (score 76/100), highly responsive (89% reply rate) — recommend prioritising outreach."
+CAND_0046064,4,87.47,"Senior NLP Engineer at Salesforce with 8.9 years of experience and 85mo of production Pinecone; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 67/100), actively job-hunting — recommend prioritising outreach."
+CAND_0088025,5,87.07,"Staff Machine Learning Engineer at Yellow.ai with 8.6 years of experience and 90mo of production Learning to Rank; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; active GitHub (score 75/100), highly responsive (83% reply rate) — recommend prioritising outreach."
+CAND_0077337,6,86.51,"Staff Machine Learning Engineer at Paytm with 7 years of experience and 82mo of production Python; strong alignment with this retrieval-focused role including 7 of the JD's must-have technical requirements. Notice period is 60 days (manageable); active GitHub (score 68/100), highly responsive (95% reply rate) — recommend prioritising outreach."
+CAND_0079387,7,86.34,"AI Engineer at Microsoft with 6.9 years of experience and 87mo of production OpenSearch; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 64/100), highly responsive (81% reply rate) — recommend prioritising outreach."
+CAND_0018499,8,86.3,"Senior Machine Learning Engineer at Zomato with 7.2 years of experience and 94mo of production RAG; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 15 days; strong GitHub presence (score 95/100), actively job-hunting — recommend prioritising outreach."
+CAND_0081846,9,86.28,"Lead AI Engineer at Razorpay with 6.7 years of experience and 95mo of production Embeddings; strong alignment with this retrieval-focused role including 8 of the JD's must-have technical requirements. Available within 30 days; some GitHub activity (score 34/100), actively job-hunting — recommend prioritising outreach."
+CAND_0086022,10,85.84,"Senior Applied Scientist at Sarvam AI with 5.3 years of experience and 80mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; active GitHub (score 75/100), actively job-hunting — recommend prioritising outreach."
+CAND_0052328,11,85.76,"Recommendation Systems Engineer at Amazon with 6.5 years of experience and 71mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 78/100), actively job-hunting — recommend prioritising outreach."
+CAND_0061257,12,85.62,"Staff Machine Learning Engineer at LinkedIn with 8 years of experience and 72mo of production Python; covers 1 JD must-haves (Python, Haystack, PEFT) making them a solid candidate for this role. Available within 30 days; some GitHub activity (score 45/100), highly responsive (87% reply rate) — recommend prioritising outreach."
+CAND_0041669,13,85.03,"Recommendation Systems Engineer at CRED with 8 years of experience and 76mo of production Milvus; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Notice period is 60 days (manageable); active GitHub (score 71/100), actively job-hunting — recommend prioritising outreach."
+CAND_0052682,14,85.0,"NLP Engineer at Aganitha with 6.6 years of experience and 81mo of production Python; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 72/100), highly responsive (88% reply rate) — recommend prioritising outreach."
+CAND_0064326,15,84.86,"Search Engineer at Sarvam AI with 7.6 years of experience and 60mo of production Python; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Notice period is 45 days (manageable); active GitHub (score 61/100), highly responsive (94% reply rate) — recommend prioritising outreach."
+CAND_0093193,16,84.53,"Senior Machine Learning Engineer at Niramai with 7.9 years of experience and 88mo of production OpenSearch; covers 2 JD must-haves (OpenSearch, Qdrant, Haystack) making them a solid candidate for this role. Notice period is 45 days (manageable); some GitHub activity (score 34/100), actively job-hunting — recommend prioritising outreach."
+CAND_0007009,17,84.48,"Recommendation Systems Engineer at Wysa with 7.9 years of experience and 96mo of production OpenSearch; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 69/100), actively job-hunting — recommend prioritising outreach."
+CAND_0046525,18,84.43,"Senior Machine Learning Engineer at Genpact AI with 6.1 years of experience and 74mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Notice period is 60 days (manageable); some GitHub activity (score 37/100), highly responsive (88% reply rate) — recommend prioritising outreach."
+CAND_0030468,19,84.34,"Senior Applied Scientist at Swiggy with 5.4 years of experience and 95mo of production OpenSearch; strong alignment with this retrieval-focused role including 2 of the JD's must-have technical requirements. Notice period is 45 days (manageable); some GitHub activity (score 45/100), actively job-hunting — recommend prioritising outreach."
+CAND_0055992,20,84.34,"AI Engineer at CRED with 16.9 years of experience and 96mo of production FAISS; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Notice period is 60 days (manageable); active GitHub (score 65/100), actively job-hunting — recommend prioritising outreach."
+CAND_0017960,21,83.97,"Recommendation Systems Engineer at Nykaa with 7.7 years of experience and 67mo of production Information Retrieval; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Notice period is 60 days (manageable); strong GitHub presence (score 92/100), actively job-hunting — recommend prioritising outreach."
+CAND_0006418,22,83.95,"Machine Learning Engineer at Verloop.io with 5.7 years of experience and 93mo of production Weaviate; strong alignment with this retrieval-focused role including 7 of the JD's must-have technical requirements. Notice period is 60 days (manageable); active GitHub (score 60/100), highly responsive (92% reply rate) — recommend prioritising outreach."
+CAND_0098846,23,83.84,"AI Engineer at upGrad with 7.6 years of experience and 95mo of production Qdrant; covers 1 JD must-haves (Qdrant, Recommendation Systems, Hugging Face Transformers) making them a solid candidate for this role. Notice period is 45 days (manageable); strong GitHub presence (score 86/100), actively job-hunting — recommend prioritising outreach."
+CAND_0062247,24,83.82,"AI Engineer at Google with 7.3 years of experience and 90mo of production Qdrant; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 53/100), actively job-hunting — recommend prioritising outreach."
+CAND_0002025,25,83.8,"Senior AI Engineer at Apple with 5.9 years of experience and 86mo of production FAISS; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 30 days; strong GitHub presence (score 97/100), highly responsive (80% reply rate) — recommend prioritising outreach."
+CAND_0037944,26,83.76,"Senior Data Scientist at Vedantu with 4.9 years of experience and 96mo of production Python; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 30 days; strong GitHub presence (score 83/100), actively job-hunting — recommend prioritising outreach."
+CAND_0018549,27,83.55,"Recommendation Systems Engineer at Uber with 6.8 years of experience and 79mo of production Milvus; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Notice period is 60 days (manageable); active GitHub (score 56/100), actively job-hunting — recommend prioritising outreach."
+CAND_0037566,28,83.48,"Machine Learning Engineer at LinkedIn with 6.9 years of experience and 74mo of production Pinecone; strong alignment with this retrieval-focused role including 2 of the JD's must-have technical requirements. Available within 15 days; active GitHub (score 73/100), actively job-hunting — recommend prioritising outreach."
+CAND_0076163,29,83.46,"NLP Engineer at Ola with 6.9 years of experience and 57mo of production BM25; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Notice period is 60 days (manageable); strong GitHub presence (score 85/100), highly responsive (84% reply rate) — recommend prioritising outreach."
+CAND_0037980,30,83.44,"Senior Applied Scientist at Niramai (9yr) with skills in BM25, pgvector, LoRA; partial match — covers 0 of the JD's must-have retrieval/ranking requirements, with some gaps remaining. Availability: 90-day notice (high risk); primary concern — 90-day notice is a risk."
+CAND_0008425,31,83.42,"Senior NLP Engineer at Ola with 7.8 years of experience and 96mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; active GitHub (score 53/100), actively job-hunting — recommend prioritising outreach."
+CAND_0058688,32,83.35,"AI Engineer at Vedantu with 6.7 years of experience and 86mo of production Milvus; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Available within 15 days; some GitHub activity (score 39/100), actively job-hunting — recommend prioritising outreach."
+CAND_0006567,33,83.32,"Senior AI Engineer at Meta with 7.9 years of experience and 80mo of production BM25; covers 1 JD must-haves (BM25, Python, Haystack) making them a solid candidate for this role. Notice period is 60 days (manageable); some GitHub activity (score 35/100), actively job-hunting — recommend prioritising outreach."
+CAND_0040887,34,83.25,"Machine Learning Engineer at Netflix with 4.7 years of experience and 84mo of production Python; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Available within 15 days; some GitHub activity (score 34/100), highly responsive (84% reply rate) — recommend prioritising outreach."
+CAND_0065195,35,83.22,"Search Engineer at CRED with 5.1 years of experience and 94mo of production Qdrant; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Notice period is 45 days (manageable); strong GitHub presence (score 87/100), highly responsive (80% reply rate) — recommend prioritising outreach."
+CAND_0099806,36,83.12,"AI Engineer at Mad Street Den with 4.6 years of experience and 89mo of production Embeddings; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 30 days; strong GitHub presence (score 87/100), actively job-hunting — recommend prioritising outreach."
+CAND_0042506,37,83.09,"Search Engineer at Verloop.io with 4.2 years of experience and 88mo of production Information Retrieval; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 15 days; active GitHub (score 64/100), actively job-hunting — recommend prioritising outreach."
+CAND_0093912,38,83.06,"Senior Data Scientist at Razorpay with 5.3 years of experience and 80mo of production Vector Search; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 30 days; strong GitHub presence (score 82/100), actively job-hunting — recommend prioritising outreach."
+CAND_0041610,39,82.62,"Recommendation Systems Engineer at Zoho with 6.7 years of experience and 94mo of production Elasticsearch; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 52/100), actively job-hunting — recommend prioritising outreach."
+CAND_0081053,40,82.48,"NLP Engineer at Glance with 5.4 years of experience and 77mo of production Weaviate; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; active GitHub (score 78/100), highly responsive (83% reply rate) — recommend prioritising outreach."
+CAND_0005649,41,82.4,"Senior Data Scientist at Sarvam AI with 7.4 years of experience and 74mo of production Weaviate; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; strong GitHub presence (score 81/100), actively job-hunting — recommend prioritising outreach."
+CAND_0051004,42,82.4,"Senior Data Scientist at CRED with 4.7 years of experience and 95mo of production Vector Search; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Available within 15 days; some GitHub activity (score 31/100), actively job-hunting — recommend prioritising outreach."
+CAND_0027691,43,82.38,"NLP Engineer at Haptik with 6.5 years of experience and 73mo of production Weaviate; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 15 days; active GitHub (score 58/100), actively job-hunting — recommend prioritising outreach."
+CAND_0014440,44,82.36,"Recommendation Systems Engineer at CRED with 6.4 years of experience and 79mo of production Milvus; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Notice period is 60 days (manageable); strong GitHub presence (score 90/100), actively job-hunting — recommend prioritising outreach."
+CAND_0039383,45,82.3,"Applied ML Engineer at Meesho with 7.1 years of experience and 94mo of production pgvector; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; strong GitHub presence (score 86/100), actively job-hunting — recommend prioritising outreach."
+CAND_0074735,46,82.23,"Applied ML Engineer at Rephrase.ai with 5.5 years of experience and 82mo of production Learning to Rank; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; strong GitHub presence (score 88/100), actively job-hunting — recommend prioritising outreach."
+CAND_0075574,47,82.23,"Machine Learning Engineer at Haptik with 5.7 years of experience and 94mo of production OpenSearch; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Notice period is 60 days (manageable); some GitHub activity (score 38/100), actively job-hunting — recommend prioritising outreach."
+CAND_0009024,48,82.22,"Search Engineer at Google with 5.2 years of experience and 90mo of production OpenSearch; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Available within 30 days; some GitHub activity (score 33/100), actively job-hunting — recommend prioritising outreach."
+CAND_0081852,49,82.17,"Senior Data Scientist at Mad Street Den with 5.9 years of experience and 61mo of production Vector Search; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Notice period is 45 days (manageable); strong GitHub presence (score 88/100), actively job-hunting — recommend prioritising outreach."
+CAND_0066999,50,82.13,"Recommendation Systems Engineer at Microsoft with 5.9 years of experience and 54mo of production FAISS; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 15 days; strong GitHub presence (score 94/100), actively job-hunting — recommend prioritising outreach."
+CAND_0042100,51,82.02,"Machine Learning Engineer at Freshworks with 7.3 years of experience and 69mo of production Pinecone; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; some GitHub activity (score 43/100), highly responsive (87% reply rate) — recommend prioritising outreach."
+CAND_0078002,52,82.0,"Machine Learning Engineer at Meta with 6.3 years of experience and 60mo of production pgvector; covers 1 JD must-haves (pgvector, Weaviate, QLoRA) making them a solid candidate for this role. Notice period is 60 days (manageable); active GitHub (score 51/100), highly responsive (86% reply rate) — recommend prioritising outreach."
+CAND_0091909,53,81.94,"Machine Learning Engineer at Rephrase.ai with 6.9 years of experience and 84mo of production Pinecone; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Notice period is 45 days (manageable); some GitHub activity (score 50/100), actively job-hunting — recommend prioritising outreach."
+CAND_0094759,54,81.94,"Lead AI Engineer at Meta with 8.6 years of experience and 55mo of production Qdrant; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 76/100) — recommend prioritising outreach."
+CAND_0036184,55,81.9,"Recommendation Systems Engineer at CRED with 6 years of experience and 94mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 30 days; some GitHub activity (score 43/100), highly responsive (90% reply rate) — recommend prioritising outreach."
+CAND_0055905,56,81.83,"Senior Machine Learning Engineer at Flipkart with 8.1 years of experience and 93mo of production Vector Search; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 30 days; highly responsive (87% reply rate), actively job-hunting — recommend prioritising outreach."
+CAND_0069905,57,81.78,"Applied ML Engineer at Sarvam AI with 6.6 years of experience and 78mo of production Weaviate; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; some GitHub activity (score 45/100), actively job-hunting — recommend prioritising outreach."
+CAND_0080766,58,81.65,"Staff Machine Learning Engineer at Salesforce with 8.8 years of experience and 53mo of production Python; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; some GitHub activity (score 33/100) — recommend prioritising outreach."
+CAND_0018722,59,81.61,"Recommendation Systems Engineer at Saarthi.ai with 6.6 years of experience and 86mo of production Weaviate; strong alignment with this retrieval-focused role including 2 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; strong GitHub presence (score 94/100), actively job-hunting — recommend prioritising outreach."
+CAND_0028793,60,81.54,"Search Engineer at Google with 7.2 years of experience and 78mo of production Embeddings; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Caution: 120-day notice raises timeline risk; strong GitHub presence (score 83/100), actively job-hunting — recommend prioritising outreach."
+CAND_0051615,61,81.52,"Search Engineer at Meta with 4.6 years of experience and 84mo of production RAG; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Notice period is 60 days (manageable); strong GitHub presence (score 95/100), highly responsive (88% reply rate) — recommend prioritising outreach."
+CAND_0075249,62,81.47,"Applied ML Engineer at Zomato with 6.2 years of experience and 80mo of production Pinecone; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Notice period is 45 days (manageable); some GitHub activity (score 37/100), highly responsive (82% reply rate) — recommend prioritising outreach."
+CAND_0068811,63,81.41,"Applied ML Engineer at Freshworks with 8 years of experience and 86mo of production Pinecone; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 30 days; some GitHub activity (score 23/100), actively job-hunting — recommend prioritising outreach."
+CAND_0044222,64,81.34,"AI Engineer at PolicyBazaar with 7.7 years of experience and 75mo of production OpenSearch; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Notice period is 60 days (manageable); some GitHub activity (score 29/100), actively job-hunting — recommend prioritising outreach."
+CAND_0077285,65,81.05,"Recommendation Systems Engineer at Nykaa with 5.5 years of experience and 66mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Notice period is 60 days (manageable); active GitHub (score 76/100), actively job-hunting — recommend prioritising outreach."
+CAND_0005538,66,81.01,"Senior AI Engineer at Adobe with 5.9 years of experience and 91mo of production pgvector; covers 1 JD must-haves (pgvector, Python, PyTorch) making them a solid candidate for this role. Caution: 90-day notice raises timeline risk; active GitHub (score 58/100), highly responsive (81% reply rate) — recommend prioritising outreach."
+CAND_0010257,67,80.95,"Senior Data Scientist at Google with 6.5 years of experience and 86mo of production Python; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Caution: 120-day notice raises timeline risk; some GitHub activity (score 46/100), actively job-hunting — recommend prioritising outreach."
+CAND_0020708,68,80.79,"Search Engineer at PolicyBazaar with 4.2 years of experience and 75mo of production Learning to Rank; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 30 days; strong GitHub presence (score 84/100), highly responsive (84% reply rate) — recommend prioritising outreach."
+CAND_0054394,69,80.78,"Recommendation Systems Engineer at PharmEasy with 4.1 years of experience and 71mo of production Embeddings; covers 2 JD must-haves (Embeddings, Pinecone, Haystack) making them a solid candidate for this role. Available within 30 days; some GitHub activity (score 32/100), actively job-hunting — recommend prioritising outreach."
+CAND_0078492,70,80.75,"Recommendation Systems Engineer at Verloop.io with 5.1 years of experience and 66mo of production FAISS; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Available within 30 days; some GitHub activity (score 35/100), actively job-hunting — recommend prioritising outreach."
+CAND_0016163,71,80.71,"Applied ML Engineer at Dream11 with 6.7 years of experience and 64mo of production pgvector; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Caution: 120-day notice raises timeline risk; strong GitHub presence (score 95/100), actively job-hunting — recommend prioritising outreach."
+CAND_0033861,72,80.39,"Senior NLP Engineer at Mad Street Den with 8 years of experience and 81mo of production Learning to Rank; strong alignment with this retrieval-focused role including 8 of the JD's must-have technical requirements. Available within 30 days; some GitHub activity (score 33/100) — recommend prioritising outreach."
+CAND_0053591,73,80.32,"AI Engineer at Ola with 5.3 years of experience and 63mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Notice period is 60 days (manageable); active GitHub (score 76/100), highly responsive (81% reply rate) — recommend prioritising outreach."
+CAND_0007411,74,80.28,"Senior Machine Learning Engineer at Amazon with 8 years of experience and 86mo of production OpenSearch; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Available within 15 days; strong GitHub presence (score 90/100) — recommend prioritising outreach."
+CAND_0027801,75,80.24,"NLP Engineer at InMobi with 7.4 years of experience and 80mo of production FAISS; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Caution: 120-day notice raises timeline risk; active GitHub (score 63/100), actively job-hunting — recommend prioritising outreach."
+CAND_0040117,76,80.17,"Recommendation Systems Engineer at PhonePe with 6.5 years of experience and 95mo of production RAG; covers 2 JD must-haves (RAG, FAISS, Embeddings) making them a solid candidate for this role. Available within 15 days; actively job-hunting — recommend prioritising outreach."
+CAND_0011432,77,80.16,"Senior Data Scientist at Amazon with 7.6 years of experience and 65mo of production pgvector; strong alignment with this retrieval-focused role including 2 of the JD's must-have technical requirements. Notice period is 60 days (manageable); some GitHub activity (score 20/100), actively job-hunting — recommend prioritising outreach."
+CAND_0011162,78,80.15,"Recommendation Systems Engineer at upGrad with 5.8 years of experience and 94mo of production FAISS; strong alignment with this retrieval-focused role including 9 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; some GitHub activity (score 34/100), actively job-hunting — recommend prioritising outreach."
+CAND_0042029,79,80.15,"Senior Data Scientist at Flipkart with 6.5 years of experience and 96mo of production OpenSearch; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Notice period is 45 days (manageable); active GitHub (score 79/100) — recommend prioritising outreach."
+CAND_0095619,80,80.13,"NLP Engineer at Nykaa with 15.6 years of experience and 74mo of production Pinecone; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Available within 30 days; some GitHub activity (score 30/100), highly responsive (90% reply rate) — recommend prioritising outreach."
+CAND_0036437,81,80.12,"Search Engineer at Rephrase.ai with 4.8 years of experience and 93mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 7 of the JD's must-have technical requirements. Available within 30 days; some GitHub activity (score 35/100), highly responsive (87% reply rate) — recommend prioritising outreach."
+CAND_0053695,82,80.12,"Recommendation Systems Engineer at Meesho with 5.8 years of experience and 90mo of production Pinecone; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Available within 15 days; active GitHub (score 68/100), actively job-hunting — recommend prioritising outreach."
+CAND_0050454,83,80.08,"AI Engineer at Rephrase.ai with 6.8 years of experience and 62mo of production FAISS; strong alignment with this retrieval-focused role including 2 of the JD's must-have technical requirements. Available within 30 days; actively job-hunting — recommend prioritising outreach."
+CAND_0019480,84,80.04,"NLP Engineer at Meesho with 2.8 years of experience and 58mo of production Milvus; covers 2 JD must-haves (Milvus, OpenSearch, BM25) making them a solid candidate for this role. Caution: 90-day notice raises timeline risk; active GitHub (score 70/100), highly responsive (87% reply rate) — recommend prioritising outreach."
+CAND_0024620,85,79.95,"AI Engineer at PharmEasy with 5.9 years of experience and 90mo of production BM25; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Notice period is 45 days (manageable); some GitHub activity (score 33/100), actively job-hunting — recommend prioritising outreach."
+CAND_0029367,86,79.95,"Senior Data Scientist at Rephrase.ai with 5.7 years of experience and 95mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; some GitHub activity (score 40/100), actively job-hunting — recommend prioritising outreach."
+CAND_0083879,87,79.91,"Machine Learning Engineer at Ola with 7.1 years of experience and 51mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 2 of the JD's must-have technical requirements. Available within 30 days; actively job-hunting — recommend prioritising outreach."
+CAND_0050876,88,79.86,"Applied ML Engineer at Freshworks with 6 years of experience and 70mo of production Sentence Transformers; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; strong GitHub presence (score 86/100), actively job-hunting — recommend prioritising outreach."
+CAND_0000031,89,79.84,"Recommendation Systems Engineer at Swiggy with 6 years of experience and 88mo of production Pinecone; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Notice period is 60 days (manageable); some GitHub activity (score 33/100), highly responsive (91% reply rate) — recommend prioritising outreach."
+CAND_0051292,90,79.76,"Applied ML Engineer at Freshworks with 5.2 years of experience and 86mo of production RAG; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 62/100), actively job-hunting — recommend prioritising outreach."
+CAND_0061265,91,79.73,"Recommendation Systems Engineer at Zoho with 6.6 years of experience and 84mo of production Learning to Rank; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Caution: 120-day notice raises timeline risk; active GitHub (score 80/100), highly responsive (94% reply rate) — recommend prioritising outreach."
+CAND_0009691,92,79.69,"Applied ML Engineer at LinkedIn with 6.2 years of experience and 63mo of production Qdrant; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Caution: 120-day notice raises timeline risk; active GitHub (score 68/100), actively job-hunting — recommend prioritising outreach."
+CAND_0030348,93,79.54,"Machine Learning Engineer at BYJU'S with 4.5 years of experience and 94mo of production pgvector; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Notice period is 45 days (manageable); actively job-hunting — recommend prioritising outreach."
+CAND_0091534,94,79.43,"AI Engineer at Flipkart with 16.6 years of experience and 73mo of production RAG; strong alignment with this retrieval-focused role including 2 of the JD's must-have technical requirements. Available within 30 days; active GitHub (score 66/100), highly responsive (84% reply rate) — recommend prioritising outreach."
+CAND_0013536,95,79.37,"Applied ML Engineer at Haptik with 14.1 years of experience and 75mo of production Vector Search; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; active GitHub (score 52/100), highly responsive (87% reply rate) — recommend prioritising outreach."
+CAND_0010685,96,79.35,"NLP Engineer at Rephrase.ai with 6.7 years of experience and 78mo of production Elasticsearch; strong alignment with this retrieval-focused role including 2 of the JD's must-have technical requirements. Available within 30 days; highly responsive (83% reply rate), actively job-hunting — recommend prioritising outreach."
+CAND_0064904,97,79.29,"AI Engineer at LinkedIn with 4.9 years of experience and 74mo of production Embeddings; strong alignment with this retrieval-focused role including 6 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; active GitHub (score 59/100), actively job-hunting — recommend prioritising outreach."
+CAND_0061339,98,79.22,"Search Engineer at Rephrase.ai with 4.2 years of experience and 95mo of production Information Retrieval; strong alignment with this retrieval-focused role including 4 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; strong GitHub presence (score 86/100), highly responsive (90% reply rate) — recommend prioritising outreach."
+CAND_0096142,99,79.14,"Applied ML Engineer at upGrad with 5 years of experience and 72mo of production Python; strong alignment with this retrieval-focused role including 3 of the JD's must-have technical requirements. Caution: 120-day notice raises timeline risk; strong GitHub presence (score 80/100), highly responsive (84% reply rate) — recommend prioritising outreach."
+CAND_0001610,100,79.1,"Machine Learning Engineer at Dream11 with 3 years of experience and 85mo of production FAISS; strong alignment with this retrieval-focused role including 5 of the JD's must-have technical requirements. Caution: 90-day notice raises timeline risk; some GitHub activity (score 40/100), actively job-hunting — recommend prioritising outreach."

validate_submission.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""
+Validate submission CSV per challenge rules (sections 2–3).
+Row 1 = header. Rows 2–101 = exactly 100 data rows. CSV only.
+"""
+
+import csv
+import re
+import sys
+from pathlib import Path
+
+REQUIRED_HEADER = ["candidate_id", "rank", "score", "reasoning"]
+CANDIDATE_ID_PATTERN = re.compile(r"^CAND_[0-9]{7}$")
+DATA_ROW_START = 2
+EXPECTED_DATA_ROWS = 100
+
+
+def validate_submission(csv_path):
+    errors = []
+    path = Path(csv_path)
+
+    if path.suffix.lower() != ".csv":
+        errors.append("Filename must use a .csv extension.")
+    elif not path.stem:
+        errors.append("Filename must be your registered participant ID (e.g. team_xxx.csv).")
+
+    try:
+        with open(path, "r", encoding="utf-8", newline="") as f:
+            reader = csv.reader(f)
+
+            try:
+                header = next(reader)
+            except StopIteration:
+                errors.append("Row 1 must be the header row; file is empty.")
+                return errors
+
+            # Row 1: column names and their order come from this line only
+            if header != REQUIRED_HEADER:
+                errors.append(
+                    "Row 1 (header) must be exactly:\n"
+                    f"  {','.join(REQUIRED_HEADER)}\n"
+                    f"Found:\n"
+                    f"  {','.join(header)}"
+                )
+
+            data_rows = []
+            for row in reader:
+                if any(cell.strip() for cell in row):
+                    data_rows.append(row)
+
+    except UnicodeDecodeError:
+        errors.append("File must be UTF-8 encoded.")
+        return errors
+    except OSError as e:
+        errors.append(f"Cannot read file: {e}")
+        return errors
+
+    n = len(data_rows)
+    if n != EXPECTED_DATA_ROWS:
+        errors.append(
+            f"After the header (row 1), there must be exactly {EXPECTED_DATA_ROWS} "
+            f"data rows (rows {DATA_ROW_START}–{DATA_ROW_START + EXPECTED_DATA_ROWS - 1}); "
+            f"found {n}."
+        )
+
+    seen_ids = set()
+    seen_ranks = set()
+    by_rank = []
+
+    for i, cells in enumerate(data_rows):
+        row_num = DATA_ROW_START + i
+
+        if len(cells) != len(REQUIRED_HEADER):
+            errors.append(
+                f"Row {row_num}: expected {len(REQUIRED_HEADER)} columns "
+                f"({','.join(REQUIRED_HEADER)}), got {len(cells)}."
+            )
+            continue
+
+        row = dict(zip(REQUIRED_HEADER, cells))
+        cid = row["candidate_id"].strip()
+        rank_s = row["rank"].strip()
+        score_s = row["score"].strip()
+
+        if not cid:
+            errors.append(f"Row {row_num}: candidate_id is required.")
+        elif not CANDIDATE_ID_PATTERN.match(cid):
+            errors.append(
+                f"Row {row_num}: candidate_id must be CAND_XXXXXXX (7 digits)."
+            )
+        elif cid in seen_ids:
+            errors.append(f"Row {row_num}: duplicate candidate_id '{cid}'.")
+        else:
+            seen_ids.add(cid)
+
+        try:
+            rank = int(rank_s)
+            if str(rank) != rank_s:
+                raise ValueError
+            if not 1 <= rank <= 100:
+                errors.append(f"Row {row_num}: rank must be between 1 and 100.")
+            elif rank in seen_ranks:
+                errors.append(f"Row {row_num}: duplicate rank {rank}.")
+            else:
+                seen_ranks.add(rank)
+        except ValueError:
+            errors.append(f"Row {row_num}: rank must be an integer (1–100).")
+            rank = None
+
+        try:
+            score = float(score_s)
+        except ValueError:
+            errors.append(f"Row {row_num}: score must be a float.")
+            score = None
+
+        if rank is not None and score is not None and cid:
+            by_rank.append((rank, score, cid))
+
+    missing = set(range(1, 101)) - seen_ranks
+    if missing:
+        errors.append(
+            f"Each rank 1–100 must appear exactly once; missing: {sorted(missing)}"
+        )
+
+    by_rank.sort(key=lambda x: x[0])
+
+    for i in range(len(by_rank) - 1):
+        r1, s1, _ = by_rank[i]
+        r2, s2, _ = by_rank[i + 1]
+        if s1 < s2:
+            errors.append(
+                f"score must be non-increasing by rank: "
+                f"rank {r1} ({s1}) < rank {r2} ({s2})."
+            )
+
+    for i in range(len(by_rank) - 1):
+        r1, s1, c1 = by_rank[i]
+        r2, s2, c2 = by_rank[i + 1]
+        if s1 == s2 and c1 > c2:
+            errors.append(
+                f"Equal scores at ranks {r1} and {r2}: "
+                f"tie-break requires candidate_id ascending "
+                f"({c1!r} > {c2!r})."
+            )
+
+    return errors
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: python validate_submission.py <participant_id>.csv")
+        sys.exit(1)
+
+    errors = validate_submission(sys.argv[1])
+    if errors:
+        print(f"Validation failed ({len(errors)} issue(s)):\n")
+        for e in errors:
+            print(f"- {e}")
+        sys.exit(1)
+
+    print("Submission is valid.")
+
+
+if __name__ == "__main__":
+    main()