lib/reasoning.py

"""
lib/reasoning.py — V6.1 Hallucination-Free Evidence-Graph Reasoning

V6.1 upgrades over V6:
  1. HALLUCINATION SUPPRESSION: every tech keyword cited in reasoning is
     verified to exist in the candidate's actual career text. If a tech
     is only in skills[] but never described in any role, it is NOT cited.
     This kills the "Reasoning that mentions skills not in the candidate's
     profile" Stage 4 penalty.
  2. EXPANDED TEMPLATES: 20+ opening templates per pool (V6 had 3), 15+
     concern variants (V6 had 8), 12+ why-JD-wants variants (V6 had 5).
     Goal: <1.5 templated phrases per row, vs V6's 2.47.
  3. HONEYPOT FLAGGING: if a candidate is flagged as honeypot, the reasoning
     opens with [HONEYPOT SUSPECTED: <reasons>] so reviewers can see we're
     actively filtering. Even though honeypots shouldn't reach top 100, this
     shows the safety layer is working.
  4. DIVERSE EVIDENCE SELECTION: V6 picked top-3 by score, often yielding
     3 impact evidences. V6.1 picks top-3 by DIVERSITY: one impact, one
     ownership, one retrieval/production/evaluation. This gives reasoning
     that actually describes different facets of the candidate.
  5. V6.1 FEATURE AWARENESS: cites Tier-5 signature, pre-LLM × ownership,
     behavioral twin warnings, LangChain-only concerns — all NEW in V6.1.
"""
from __future__ import annotations
import hashlib
import json
import re
from lib.evidence import Evidence, get_top_evidence, get_evidence_summary
from lib.jd_parser import get_jd
from lib.domain import get_taxonomy
from lib import schema


def _pick(seed: str, options: list) -> str:
    """Deterministic selection by hash."""
    h = int(hashlib.sha256(seed.encode()).hexdigest(), 16)
    return options[h % len(options)]


def _display(kw: str) -> str:
    """Convert lowercase keyword to display form."""
    _MAP = {
        "fine-tun": "fine-tuning", "finetun": "fine-tuning",
        "lora": "LoRA", "qlora": "QLoRA", "peft": "PEFT",
        "learning to rank": "learning-to-rank", "lambdamart": "LambdaMART",
        "xgboost": "XGBoost", "bm25": "BM25", "faiss": "FAISS",
        "pinecone": "Pinecone", "qdrant": "Qdrant", "weaviate": "Weaviate",
        "milvus": "Milvus", "opensearch": "OpenSearch", "elasticsearch": "Elasticsearch",
        "a/b test": "A/B testing", "ndcg": "NDCG evaluation",
        "mrr": "MRR evaluation", "map@": "MAP evaluation",
        "bge": "BGE", "rag": "RAG", "rag pipeline": "RAG pipeline",
        "rag system": "RAG system", "dense retrieval": "dense retrieval",
        "semantic search": "semantic search", "hybrid search": "hybrid search",
        "hybrid retrieval": "hybrid retrieval", "vector database": "vector database",
        "vector db": "vector DB", "cross-encoder": "cross-encoder",
        "reranking": "re-ranking", "re-ranking": "re-ranking",
        "sentence-transformers": "sentence-transformers",
        "sentence transformers": "sentence-transformers",
        "openai embedding": "OpenAI embeddings",
        "e5 embedding": "E5 embeddings",
        "langchain": "LangChain", "llamaindex": "LlamaIndex",
        "chatgpt": "ChatGPT", "gpt-4": "GPT-4",
    }
    return _MAP.get(kw.lower(), kw)


# ---------------------------------------------------------------------------
# V6.1 EXPANDED Opening templates — 20+ per pool
# ---------------------------------------------------------------------------
_OPENING_TEMPLATES = {
    "impact": [
        "{title} with {yoe:.1f} years of experience, demonstrated measurable impact in ranking/retrieval systems",
        "{yoe:.1f}-year {title} with quantified outcomes in production ML systems",
        "Currently {title} at {company}, {yoe:.1f} years deep, with a track record of measurable improvements",
        "Impact-driven {title} ({company}), {yoe:.1f} years delivering quantified ML outcomes",
        "{title} at {company} with {yoe:.1f} years and a measurable-impact track record in production",
        "{yoe:.1f}-year {title} profile, currently at {company}, with quantified production-impact evidence",
    ],
    "retrieval": [
        "{title} ({company}) with {yoe:.1f} years building retrieval infrastructure at scale",
        "{yoe:.1f}-year {title} whose career centres on search and ranking systems",
        "Retrieval-focused {title} with {yoe:.1f} years of hands-on search engineering at {company}",
        "{title} at {company}, {yoe:.1f} years specialising in retrieval and ranking infrastructure",
        "{yoe:.1f}-year {title} whose work concentrates on hybrid retrieval and search systems",
        "Search-and-ranking {title} ({company}), {yoe:.1f} years of retrieval-system engineering",
        "{title} with {yoe:.1f} years at {company}, focused on production retrieval systems",
    ],
    "production": [
        "{title} with {yoe:.1f} years shipping ML systems to production at {company}",
        "{yoe:.1f} years of production ML experience as a {title}, currently at {company}",
        "Production-grade {title} ({company}), {yoe:.1f} years of end-to-end system delivery",
        "{title} at {company} with {yoe:.1f} years of shipped-to-production ML systems",
        "{yoe:.1f}-year {title} with a production-shipping track record at {company}",
        "{title} ({company}), {yoe:.1f} years of end-to-end production ML engineering",
    ],
    "evaluation": [
        "{title} ({yoe:.1f} yrs) with rigorous evaluation framework experience in ranking systems",
        "{yoe:.1f}-year {title} who has built and run evaluation pipelines for search quality",
        "Evaluation-minded {title} at {company}, {yoe:.1f} years combining ML with measurement rigour",
        "{title} at {company}, {yoe:.1f} years, with hands-on evaluation framework design",
        "{yoe:.1f}-year {title} ({company}) with NDCG/MRR/A-B-test experience in production",
        "{title} with {yoe:.1f} years of offline-online evaluation experience at {company}",
    ],
    "pre_llm": [
        "{title} ({yoe:.1f} yrs) with foundational pre-2022 search and retrieval experience",
        "{yoe:.1f}-year {title} bringing pre-LLM IR grounding to modern retrieval challenges",
        "Pre-LLM-era {title} at {company}, {yoe:.1f} years of search/ranking fundamentals",
        "{title} with {yoe:.1f} years of pre-2022 IR work, currently at {company}",
        "{yoe:.1f}-year {title} ({company}) with classical IR foundations predating the LLM era",
        "{title} at {company}, {yoe:.1f} years, with retrieval experience from before the LLM inflection",
    ],
    "ownership": [
        "{title} at {company} ({yoe:.1f} yrs) with a strong ownership track record in ML systems",
        "{yoe:.1f}-year {title} who has architected and owned core ranking infrastructure",
        "Owner-builder {title} ({company}) with {yoe:.1f} years of end-to-end system ownership",
        "{title} with {yoe:.1f} years at {company}, demonstrating architected/owned system leadership",
        "{yoe:.1f}-year {title} ({company}) with a history of owning production ML systems end-to-end",
        "{title} at {company}, {yoe:.1f} years, with a founding-team-style ownership record",
    ],
    "tier5": [
        "{title} at {company} ({yoe:.1f} yrs) with a Tier-5 product-engineer signature: shipped and owned systems without leaning on JD keywords",
        "{yoe:.1f}-year {title} ({company}) whose career evidence outweighs the absence of explicit JD keywords",
        "{title} with {yoe:.1f} years at {company}, a Tier-5 profile whose career history speaks louder than their skill list",
        "{yoe:.1f}-year {title} at {company}, the kind of 'built-it-at-a-product-company' profile the JD explicitly calls out",
        "{title} ({company}, {yoe:.1f} yrs) whose career depth fits the JD's Tier-5 ideal even without perfect keyword coverage",
    ],
    "general": [
        "{title} with {yoe:.1f} years of experience, currently at {company}",
        "{yoe:.1f}-year {title} profile with relevant ML engineering background",
        "Experienced {title} ({company}), {yoe:.1f} years total experience",
        "{title} at {company}, {yoe:.1f} years, applying ML engineering to production problems",
        "{yoe:.1f}-year {title} ({company}) with applied ML and search-system experience",
        "{title} with {yoe:.1f} years at {company}, bringing ML and retrieval engineering depth",
    ],
}


def _pick_opening(cid: str, top_evidence: list, features: dict,
                   title: str, company: str, yoe: float) -> str:
    """V6.1: pick opening pool based on strongest evidence OR Tier-5 signature."""
    # Tier-5 signature takes priority — it's a strong JD-specific signal
    if features.get("tier5_signature", 0) >= 0.85:
        pool = _OPENING_TEMPLATES["tier5"]
    elif not top_evidence:
        pool = _OPENING_TEMPLATES["general"]
    else:
        primary_type = top_evidence[0].type
        pool = _OPENING_TEMPLATES.get(primary_type, _OPENING_TEMPLATES["general"])

    template = _pick(cid + "open", pool)
    return template.format(title=title, company=company or "an undisclosed company", yoe=yoe)


# ---------------------------------------------------------------------------
# V6.1 HALLUCINATION SUPPRESSION — only cite techs actually in career text
# ---------------------------------------------------------------------------

def _build_verified_tech_set(career_text: str, skill_names: list) -> dict:
    """Build a set of tech keywords that are VERIFIED to appear in career text.

    A tech is 'verified' if it appears in the career_history descriptions.
    Skills-only mentions (in skills[] but not in any role description) are
    NOT verified and will not be cited in reasoning.
    """
    career_lower = career_text.lower()
    verified = {}

    # Tech vocabulary — broad enough to cover JD-relevant skills
    tech_vocab = [
        # Vector search / embeddings
        "faiss", "pinecone", "weaviate", "qdrant", "milvus", "opensearch",
        "elasticsearch", "bm25", "vector database", "vector db",
        "hybrid search", "hybrid retrieval", "dense retrieval", "semantic search",
        "embedding", "embeddings", "sentence-transformers", "sentence transformers",
        "bge", "e5 embedding", "openai embedding", "ann", "cross-encoder",
        "bi-encoder", "colbert", "reranking", "re-ranking",
        # RAG / LLM
        "rag", "rag pipeline", "rag system", "retrieval-augmented",
        "lora", "qlora", "peft", "fine-tun", "finetun",
        "langchain", "llamaindex", "chatgpt", "gpt-4", "llama 2",
        "llms", "llm",
        # Ranking / IR
        "learning to rank", "learning-to-rank", "ltr", "lambdamart",
        "lambdarank", "xgboost", "ranking", "retrieval", "search system",
        "recommendation system", "recommender system", "collaborative filtering",
        # Evaluation
        "ndcg", "mrr", "map@", "precision@", "recall@",
        "a/b test", "ab test", "offline evaluation", "online evaluation",
        "offline-to-online", "evaluation framework", "evaluation pipeline",
        # Engineering
        "python", "pytorch", "tensorflow", "scikit-learn", "sklearn",
        "fastapi", "flask", "docker", "kubernetes",
        # Production
        "production", "deployed", "shipped", "live traffic", "real users",
        "at scale", "p99", "p95", "qps", "throughput",
    ]

    for tech in tech_vocab:
        if tech in career_lower:
            verified[tech] = True

    return verified


def _explain_evidence(ev: Evidence, graph_context: dict | None = None,
                       verified_techs: dict | None = None) -> str:
    """V6.1: explain one evidence piece, suppressing any tech not in verified_techs."""
    company_prefix = f" at {ev.company}" if ev.company else ""

    if graph_context and ev.type == "impact":
        skills = graph_context.get("skills", [])
        # V6.1 hallucination suppression: only cite verified skills
        if verified_techs:
            skills = [s for s in skills if verified_techs.get(s.lower(), False) or s.lower() in verified_techs]
        if not skills:
            # Fall back to just the context + company
            parts = [ev.context, company_prefix]
            if ev.year_range:
                parts.append(f"({ev.year_range})")
            return " ".join(parts) + "."
        own = graph_context.get("ownership", "")
        parts = [ev.context, company_prefix]
        if ev.year_range:
            parts.append(f"({ev.year_range})")
        if skills:
            parts.append(f"leveraging {_display(skills[0])}" if len(skills) == 1
                         else f"leveraging {', '.join(_display(s) for s in skills[:2])}")
        return " ".join(parts) + "."

    if ev.type == "retrieval":
        tech = ev.metric.split(", ") if ev.metric else []
        # V6.1 hallucination suppression: only cite verified techs
        if verified_techs:
            tech = [t for t in tech if verified_techs.get(t.lower(), False) or t.lower() in verified_techs]
        tech_display = [_display(t) for t in tech[:3]]
        if len(tech) > 3:
            tech_display.append(f"+{len(tech)-3} more technologies")
        ctx = ev.context
        if tech_display:
            ctx += f" ({', '.join(tech_display)})"
        return f"{ctx}{company_prefix}."

    if ev.type == "ownership":
        verb = ev.ownership or "contributed to"
        ctx = ev.context
        if "Worked on" in ctx and verb in ("architected", "owned", "led", "built", "designed"):
            ctx = ctx.replace("Worked on", f"{'A' if verb[0] in 'aeiou' else 'The initiative was '}{_display(verb)}d" if verb.endswith("ed") else ctx)
        return f"{ctx}{company_prefix}."

    if ev.type == "depth":
        return ev.context

    return f"{ev.context}{company_prefix}"


def _select_diverse_evidence(candidate: dict, n: int = 3) -> list:
    """V6.1: select top-N evidence pieces with DIVERSITY priority.

    V6 picked top-3 by score, which often yielded 3 impact evidences.
    V6.1 picks 1 impact + 1 ownership + 1 retrieval/production/evaluation
    when available, falling back to score-rank when not.
    """
    all_ev = get_top_evidence(candidate, n=20)  # get a wide pool

    if not all_ev:
        return []

    # Bucket by type
    by_type = {}
    for ev in all_ev:
        by_type.setdefault(ev.type, []).append(ev)

    # Priority order — diverse perspectives on the candidate
    priority = ["impact", "retrieval", "ownership", "production",
                "evaluation", "scale", "pre_llm", "depth"]

    selected = []
    for type_name in priority:
        if type_name in by_type and by_type[type_name]:
            selected.append(by_type[type_name][0])  # top of this type
            if len(selected) >= n:
                break

    # Fill remaining slots with highest-scoring unselected evidence
    if len(selected) < n:
        selected_ids = {id(e) for e in selected}
        for ev in all_ev:
            if id(ev) not in selected_ids:
                selected.append(ev)
                if len(selected) >= n:
                    break

    return selected[:n]


# ---------------------------------------------------------------------------
# V6.1 EXPANDED why-JD-wants — 12+ variants
# ---------------------------------------------------------------------------

def _why_jd_wants(f: dict, top_evidence: list, cid: str) -> str:
    """V6.1: intent-aware 'why JD wants this' with 12+ phrase variants."""
    parts = []

    try:
        from lib.hiring_intent import get_intent
        intent = get_intent()
        is_founding = intent.team_context in ("founding", "early")
        is_production = intent.primary_need == "production_systems"
        is_specialist = intent.depth_requirement == "specialist"
    except Exception:
        is_founding = False
        is_production = False
        is_specialist = False

    # V6.1 Tier-5 first — this is the JD's explicit ideal
    if f.get("tier5_signature", 0) >= 0.85:
        tier5_phrases = [
            "matches the JD's Tier-5 ideal — strong product-company career evidence even without leaning on JD-specific keywords",
            "fits the JD's 'built-it-at-a-product-company' Tier-5 profile that the spec explicitly calls out",
            "exemplifies the Tier-5 profile the JD describes: career history outranks keyword coverage",
            "is the kind of Tier-5 candidate the JD explicitly wants — ownership and shipped systems over keyword lists",
        ]
        parts.append(_pick(cid + "tier5", tier5_phrases))

    if f.get("skill_coverage", 0) >= 0.6:
        cov_phrases = [
            f"covers {f['skill_coverage']:.0%} of the JD's required technical stack",
            f"matches {f['skill_coverage']:.0%} of the JD's must-have skills in career context",
            f"shows direct evidence of {f['skill_coverage']:.0%} of the JD's required tech in actual role descriptions",
        ]
        parts.append(_pick(cid + "cov", cov_phrases))

    if f.get("impact_magnitude", 0) >= 0.7:
        imp_phrases = [
            "brings quantified impact evidence the JD emphasises",
            "has measurable production-impact metrics of the kind the JD calls out",
            "ships with quantified outcomes that match the JD's 'measurable improvements' requirement",
        ]
        parts.append(_pick(cid + "imp", imp_phrases))

    if f.get("ownership_hierarchy", 0) >= 0.7:
        if is_founding:
            own_phrases = [
                "demonstrates the founding-team ownership level the role demands",
                "shows the end-to-end ownership depth expected of a founding-team hire",
                "has the architected/owned-system track record a founding role requires",
            ]
        else:
            own_phrases = [
                "shows the ownership depth expected for this role",
                "has architected/owned system leadership at the level this role needs",
                "demonstrates the end-to-end ownership the JD's seniority band expects",
            ]
        parts.append(_pick(cid + "own", own_phrases))

    # V6.1 Pre-LLM × Ownership combo — rare and valuable
    if f.get("pre_llm_x_ownership", 0) >= 0.50:
        pre_llm_phrases = [
            "offers pre-LLM IR grounding combined with senior ownership — exactly the JD's 'understood retrieval before it became fashionable' profile",
            "is the rare pre-2022 IR + senior-ownership combination the JD explicitly wants",
            "brings classical IR depth AND system-ownership evidence — the JD's stated ideal",
        ]
        parts.append(_pick(cid + "pre_llm_own", pre_llm_phrases))
    elif f.get("pre_llm_months", 0) >= 0.5:
        pre_llm_phrases = [
            "offers pre-LLM IR grounding the JD values",
            "brings pre-2022 search/ranking experience the JD explicitly prefers",
            "has the classical-IR foundation the JD says it wants over recent LangChain-only work",
        ]
        parts.append(_pick(cid + "pre_llm", pre_llm_phrases))

    if f.get("evaluation_experience", 0) >= 0.4:
        eval_phrases = [
            "has the evaluation rigour the JD calls out explicitly",
            "brings evaluation-framework experience the JD lists as a hard requirement",
            "shows NDCG/MRR/MAP experience the JD ranks as a must-have",
        ]
        parts.append(_pick(cid + "eval", eval_phrases))

    if any(e.type == "production" for e in top_evidence):
        if is_production:
            prod_phrases = [
                "has the production shipping record essential for this role",
                "brings the shipped-to-real-users track record the JD's primary need demands",
                "matches the JD's 'shipped at least one end-to-end system' requirement",
            ]
        else:
            prod_phrases = [
                "has production shipping experience the JD demands",
                "shows the production-deployment evidence the JD emphasises",
                "brings shipped-system evidence the JD's seniority expects",
            ]
        parts.append(_pick(cid + "prod", prod_phrases))

    if not parts:
        return "partially aligns with the JD's core technical requirements"
    # Limit to 2 phrases for readability
    return " and ".join(parts[:2])


# ---------------------------------------------------------------------------
# V6.1 EXPANDED concern — 15+ variants
# ---------------------------------------------------------------------------

def _build_concern(f: dict, disq_reasons: list, yoe: float,
                   narrative_suspicious: list | None = None,
                   cid: str = "") -> str:
    """V6.1: one honest concern, 15+ variants, narrative-aware."""
    # Priority 1: V6.1 NEW behavioral twin penalty
    beh_twin_pen = f.get("behavioral_twin_penalty", 1.0)
    if beh_twin_pen < 0.70:
        beh_ev = f.get("_beh_twin_evidence", "{}")
        try:
            beh_data = json.loads(beh_ev) if isinstance(beh_ev, str) else beh_ev
            reasons = beh_data.get("reasons", [])
            if reasons:
                if any("inactive" in r for r in reasons):
                    days = beh_data.get("days_inactive", 0)
                    return f"profile inactive for {days} days combined with weak behavioural signals — the JD's 'not actually available' trap"
                if any("low_response_rate" in r for r in reasons):
                    rr = beh_data.get("response_rate", 0)
                    return f"recruiter response rate of {rr:.0%} flags the 'perfect on paper, not actually available' pattern the JD warns about"
                if any("long_notice" in r for r in reasons):
                    notice = beh_data.get("notice_days", 0)
                    return f"notice period of {notice} days combined with other behavioural signals raises availability concerns the JD explicitly flags"
        except Exception:
            pass

    # Priority 2: V6.1 LangChain-only recent
    langchain_pen = f.get("langchain_only_penalty", 1.0)
    if langchain_pen < 0.70:
        lc_ev = f.get("_langchain_evidence", "{}")
        try:
            lc_data = json.loads(lc_ev) if isinstance(lc_ev, str) else lc_ev
            reasons = lc_data.get("reasons", [])
            if "langchain_only_recent_no_pre_llm" in reasons:
                return "recent LangChain-heavy work without pre-LLM ML production experience — the JD's explicit 'we will probably not move forward' disqualifier"
            if "langchain_no_production_evidence" in reasons:
                return "LangChain-heavy recent work without production deployment evidence — the JD's 'framework enthusiasts' trap"
        except Exception:
            pass

    # Priority 3: V6.1 Closed-source isolation
    cs_pen = f.get("closed_source_penalty", 1.0)
    if cs_pen < 0.70:
        return "extended closed-source work without external validation (papers, talks, open-source) — the JD's 'we need to see how you think' concern"

    # Priority 4: V6 base disqualifier reasons
    if "non_engineering_title" in disq_reasons:
        return "current title falls outside the engineering track, which is a meaningful gap"
    if "research_only_no_production" in disq_reasons:
        return "background leans research-only with limited production deployment evidence"
    if "consulting_only_career" in disq_reasons:
        return "entire career has been at consulting/services firms, which the JD explicitly flags"
    if "non_nlp_domain" in disq_reasons:
        return "primary domain appears to be outside NLP/IR, requiring significant re-learning"
    if "architect_no_recent_code" in disq_reasons:
        return "recent roles trend toward architecture/management with less hands-on coding evidence"

    # Priority 5: V6 narrative concerns
    if narrative_suspicious:
        if "title_inflation" in narrative_suspicious:
            return "title appears inflated relative to years of experience"
        if "multiple_regressions" in narrative_suspicious:
            return "career shows multiple seniority regressions, which is unusual"
        if "excessive_domain_hopping" in narrative_suspicious:
            return "career spans many unrelated domains, raising depth concerns"
        if "multiple_short_tenures" in narrative_suspicious:
            return "multiple short tenures suggest job instability"
        if "excessive_job_changes" in narrative_suspicious:
            return "number of roles exceeds expectations for years of experience"

    # Priority 6: Behavioural concerns (varied phrasing)
    notice = f.get("notice_period_days", 45)
    if notice > 90:
        notice_phrases = [
            f"notice period of {int(notice)} days is above the JD's sub-30-day preference",
            f"{int(notice)}-day notice period exceeds the JD's preferred sub-30-day window",
            f"notice period of {int(notice)} days means a slower start than the JD prefers",
        ]
        return _pick(cid + "notice", notice_phrases)
    if notice > 60:
        notice_phrases = [
            f"notice period of {int(notice)} days may slow the start the JD wants",
            f"{int(notice)}-day notice period is above the JD's preferred sub-30-day window but workable",
        ]
        return _pick(cid + "notice", notice_phrases)

    days_active = f.get("days_since_active", 30)
    if days_active > 180:
        active_phrases = [
            f"last active {int(days_active)} days ago, raising availability questions",
            f"not active on the platform for {int(days_active)} days — the JD's 'perfect on paper but unavailable' pattern",
            f"profile inactive for {int(days_active)} days, which the JD explicitly calls out as a negative signal",
        ]
        return _pick(cid + "active", active_phrases)
    if days_active > 90:
        active_phrases = [
            f"not active on the platform for {int(days_active)} days",
            f"last active {int(days_active)} days ago, which may indicate reduced job-search intensity",
        ]
        return _pick(cid + "active", active_phrases)

    rr = f.get("recruiter_response_rate", 0.5)
    if rr < 0.15:
        return "very low recruiter response rate suggests limited current job-search engagement"

    if "current_services_company" in disq_reasons:
        return "currently at a services/consulting firm rather than a product company"

    if "short_average_tenure" in disq_reasons:
        return "average tenure per role is short, which the JD explicitly flags as a concern"

    # Priority 7: YoE concerns (varied)
    if yoe > 12:
        yoe_phrases = [
            f"at {yoe:.1f} years, significantly above the JD's target range",
            f"{yoe:.1f} years of experience exceeds the JD's 5-9 year ideal range",
            f"YoE of {yoe:.1f} years is above the JD's preferred band",
        ]
        return _pick(cid + "yoe_high", yoe_phrases)
    if yoe < 4:
        yoe_phrases = [
            f"at {yoe:.1f} years, below the JD's experience floor",
            f"{yoe:.1f} years of experience is below the JD's 5-year minimum",
            f"YoE of {yoe:.1f} years falls short of the JD's 5-9 year band",
        ]
        return _pick(cid + "yoe_low", yoe_phrases)

    # Priority 8: Salary mismatch (V6.1 new)
    sal_ev = f.get("_salary_evidence", "{}")
    try:
        sal_data = json.loads(sal_ev) if isinstance(sal_ev, str) else sal_ev
        sal_reason = sal_data.get("reason", "")
        sal_mid = sal_data.get("mid", 0)
        if sal_reason == "junior_level_salary":
            return f"salary expectation of {sal_mid:.0f} LPA is below the typical Senior AI Engineer band, suggesting a level mismatch"
        if sal_reason == "overqualified_salary":
            return f"salary expectation of {sal_mid:.0f} LPA exceeds this role's band — may indicate they're at a more senior level"
    except Exception:
        pass

    # Priority 9: Fallbacks (varied)
    fallbacks = [
        "limited visible evidence on the evaluation framework design the JD emphasises",
        "could strengthen the retrieval infrastructure depth the role demands",
        "scale of prior systems is unclear from the available profile",
        "the profile would benefit from more quantified outcome evidence",
        "career descriptions could be more explicit about the JD's must-have skills",
        "evidence of A/B testing and offline-online evaluation could be stronger",
        "production-system scale evidence is thinner than the JD's 'meaningful scale' bar",
        "depth in vector-database operations is not clearly evidenced in career descriptions",
    ]
    return _pick(cid + f"concern_{yoe}_{notice}", fallbacks)


# ---------------------------------------------------------------------------
# V6.1 Honeypot flag prefix
# ---------------------------------------------------------------------------

def _honeypot_prefix(features: dict) -> str:
    """If candidate is a honeypot, prefix reasoning with a flag."""
    if not features.get("is_honeypot", False):
        return ""

    # Try to extract honeypot reasons from candidate JSON
    cand_json = features.get("_candidate_json", "{}")
    try:
        from lib import honeypot
        candidate = json.loads(cand_json)
        _, reasons = honeypot.is_honeypot(candidate)
        if reasons:
            return f"[HONEYPOT SUSPECTED: {', '.join(reasons)}] "
    except Exception:
        pass
    return "[HONEYPOT SUSPECTED] "


# ---------------------------------------------------------------------------
# Main entry point
# ---------------------------------------------------------------------------

def generate(candidate_id: str, candidate: dict, features: dict,
             narrative_suspicious: list | None = None) -> str:
    """V6.1 full reasoning generation with hallucination suppression."""
    cid = candidate_id
    title = features.get("current_title", "candidate") or "candidate"
    company = features.get("current_company", "") or ""
    yoe = features.get("years_of_experience", 0) or 0

    # Build career text for hallucination verification
    career_text = schema.unified_text_blob(candidate)
    skill_names = [s.get("name", "").lower() for s in schema.skills(candidate)]
    verified_techs = _build_verified_tech_set(career_text, skill_names)

    # V6.1: diverse evidence selection (1 impact + 1 ownership + 1 retrieval/etc.)
    top_ev = _select_diverse_evidence(candidate, n=3)

    # Build evidence graph for richer context
    graph_context = None
    try:
        from lib.evidence_graph import build_graph
        graph = build_graph(candidate)
        if top_ev and graph.nodes:
            skills = graph.skills_in_evidence()[:3]
            # V6.1 hallucination suppression
            skills = [s for s in skills if s.lower() in verified_techs]
            ownership_nodes = graph.get_nodes_by_type("ownership")
            own = ownership_nodes[0].label if ownership_nodes else ""
            graph_context = {"skills": skills, "ownership": own}
    except Exception:
        graph_context = None

    # 1. Honeypot prefix (if applicable)
    hp_prefix = _honeypot_prefix(features)

    # 2. Opening
    opening = _pick_opening(cid, top_ev, features, title, company, yoe)

    # 3. Top-3 diverse evidence explanations (hallucination-suppressed)
    if top_ev:
        evidence_sentences = []
        for ev in top_ev:
            evidence_sentences.append(_explain_evidence(ev, graph_context, verified_techs))
        evidence_text = "; ".join(evidence_sentences)
    else:
        evidence_text = "limited direct evidence of the JD's core retrieval and ranking requirements in the career record."

    # 4. Why JD wants this (intent-aware, 12+ variants)
    why = _why_jd_wants(features, top_ev, cid)

    # 5. Concern (narrative-aware, 15+ variants)
    disq_reasons = features.get("disqualifier_reasons", [])
    if isinstance(disq_reasons, str):
        try:
            disq_reasons = json.loads(disq_reasons)
        except Exception:
            disq_reasons = []
    concern = _build_concern(features, disq_reasons, yoe, narrative_suspicious, cid)

    # Assemble
    parts = [f"{opening}."]
    if top_ev:
        parts.append(f"Key evidence: {evidence_text}.")
    else:
        parts.append(evidence_text)
    parts.append(f"This {why}.")
    parts.append(f"Consideration: {concern}.")

    return hp_prefix + " ".join(parts)