test_adversarial.py

#!/usr/bin/env python3
"""test_adversarial.py — V6 Expanded Adversarial Test Suite

Tests V6-specific capabilities:
  1. Keyword stuffer (many AI skills, zero career context)
  2. Non-engineering title with AI keywords (HR Manager)
  3. Strong candidate (the ideal)
  4. Career chaos (random domain jumps) — V6 narrative test
  5. Title inflation (Junior with 2 YOE claims "Architect") — V6 narrative test
  6. Stable upward career — V6 narrative test
  7. Production owner without impact numbers
  8. Research-only with no production
  9. Honeypot detection
  10. Interaction feature validation — V6
"""
from __future__ import annotations
import json
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
from lib import schema, features, honeypot, evidence
from lib.career_narrative import analyze as analyze_narrative


def make_keyword_stuffer() -> dict:
    return {
        "candidate_id": "CAND_TSTUFF1",
        "profile": {"anonymized_name": "Test Stuffer", "headline": "AI Expert",
                     "summary": "I know everything about AI.", "location": "Pune",
                     "country": "India", "years_of_experience": 7.0,
                     "current_title": "Software Engineer", "current_company": "TCS",
                     "current_company_size": "10001+", "current_industry": "IT Services"},
        "career_history": [{"company": "TCS", "title": "Software Engineer",
            "start_date": "2021-01-01", "end_date": None, "duration_months": 66,
            "is_current": True, "industry": "IT Services", "company_size": "10001+",
            "description": "Worked on various projects using Java and SQL."}],
        "education": [{"institution": "College", "degree": "B.Tech", "field_of_study": "CS",
                        "start_year": 2014, "end_year": 2018}],
        "skills": [
            {"name": "RAG", "proficiency": "expert", "endorsements": 0, "duration_months": 0},
            {"name": "Pinecone", "proficiency": "expert", "endorsements": 0, "duration_months": 0},
            {"name": "FAISS", "proficiency": "expert", "endorsements": 0, "duration_months": 0},
            {"name": "NDCG", "proficiency": "expert", "endorsements": 0, "duration_months": 0},
            {"name": "LoRA", "proficiency": "expert", "endorsements": 0, "duration_months": 0},
            {"name": "BM25", "proficiency": "advanced", "endorsements": 0, "duration_months": 0},
            {"name": "Embeddings", "proficiency": "expert", "endorsements": 0, "duration_months": 0},
        ],
        "redrob_signals": {
            "profile_completeness_score": 95, "signup_date": "2025-01-01",
            "last_active_date": "2026-06-01", "open_to_work_flag": True,
            "recruiter_response_rate": 0.80, "notice_period_days": 30,
            "saved_by_recruiters_30d": 2, "search_appearance_30d": 50,
            "interview_completion_rate": 0.80, "verified_email": True,
            "verified_phone": True, "linkedin_connected": True,
        },
    }


def make_career_chaos() -> dict:
    """V6: Candidate with chaotic career - Marketing -> AI -> CEO -> Intern."""
    return {
        "candidate_id": "CAND_CHAOS1",
        "profile": {"anonymized_name": "Test Chaos", "headline": "AI Architect",
                     "summary": "Diverse career across industries.", "location": "Bangalore",
                     "country": "India", "years_of_experience": 10.0,
                     "current_title": "AI Architect", "current_company": "Unknown Startup",
                     "current_company_size": "1-50", "current_industry": "Technology"},
        "career_history": [
            {"company": "Unknown Startup", "title": "AI Architect",
             "start_date": "2025-01-01", "end_date": None, "duration_months": 6,
             "is_current": True, "industry": "Technology", "company_size": "1-50",
             "description": "Working on AI stuff."},
            {"company": "Random Corp", "title": "CEO",
             "start_date": "2024-01-01", "end_date": "2024-12-01", "duration_months": 11,
             "is_current": False, "industry": "Consulting", "company_size": "1-50",
             "description": "Ran the company."},
            {"company": "SalesForce", "title": "Sales Manager",
             "start_date": "2022-06-01", "end_date": "2023-12-01", "duration_months": 18,
             "is_current": False, "industry": "Sales", "company_size": "10001+",
             "description": "Managed sales team."},
            {"company": "Infosys", "title": "Marketing Intern",
             "start_date": "2022-01-01", "end_date": "2022-05-01", "duration_months": 4,
             "is_current": False, "industry": "IT Services", "company_size": "10001+",
             "description": "Marketing internship."},
            {"company": "TCS", "title": "Junior Developer",
             "start_date": "2018-01-01", "end_date": "2021-12-01", "duration_months": 48,
             "is_current": False, "industry": "IT Services", "company_size": "10001+",
             "description": "Java development."},
        ],
        "education": [{"institution": "College", "degree": "B.Tech", "field_of_study": "CS",
                        "start_year": 2014, "end_year": 2018}],
        "skills": [],
        "redrob_signals": {
            "profile_completeness_score": 80, "last_active_date": "2026-06-10",
            "open_to_work_flag": True, "recruiter_response_rate": 0.50,
            "notice_period_days": 30, "verified_email": True,
        },
    }


def make_title_inflation() -> dict:
    """V6: 2 YOE claiming Principal Architect."""
    return {
        "candidate_id": "CAND_TINFLATE1",
        "profile": {"anonymized_name": "Test Inflate", "headline": "Principal Architect",
                     "summary": "Experienced architect.", "location": "Pune",
                     "country": "India", "years_of_experience": 2.0,
                     "current_title": "Principal Architect", "current_company": "TCS",
                     "current_company_size": "10001+", "current_industry": "IT Services"},
        "career_history": [
            {"company": "TCS", "title": "Principal Architect",
             "start_date": "2024-06-01", "end_date": None, "duration_months": 24,
             "is_current": True, "industry": "IT Services", "company_size": "10001+",
             "description": "Working on architecture."},
            {"company": "Wipro", "title": "Intern",
             "start_date": "2024-01-01", "end_date": "2024-05-01", "duration_months": 4,
             "is_current": False, "industry": "IT Services", "company_size": "10001+",
             "description": "Internship."},
        ],
        "education": [{"institution": "College", "degree": "B.Tech", "field_of_study": "CS",
                        "start_year": 2020, "end_year": 2024}],
        "skills": [],
        "redrob_signals": {"profile_completeness_score": 70, "last_active_date": "2026-06-10",
                           "open_to_work_flag": True, "recruiter_response_rate": 0.60,
                           "notice_period_days": 30},
    }


def make_strong_candidate() -> dict:
    return {
        "candidate_id": "CAND_STRONG1",
        "profile": {"anonymized_name": "Test Strong", "headline": "Senior ML Engineer | Search & Ranking",
                     "summary": "Built production search and ranking systems at scale.",
                     "location": "Pune", "country": "India", "years_of_experience": 7.5,
                     "current_title": "Senior ML Engineer", "current_company": "Flipkart",
                     "current_company_size": "5001-10000", "current_industry": "E-commerce"},
        "career_history": [
            {"company": "Flipkart", "title": "Senior ML Engineer",
             "start_date": "2022-06-01", "end_date": None, "duration_months": 48,
             "is_current": True, "industry": "E-commerce", "company_size": "5001-10000",
             "description": "Owned the product search ranking system serving 50 million users. Improved NDCG by 15% through hybrid retrieval combining BM25 and dense embeddings via FAISS. Reduced p99 latency from 200ms to 45ms. Designed the evaluation framework with NDCG, MRR, and A/B testing pipelines. Led a team of 4 engineers. Architected the embedding drift detection system that reduced retrieval quality regressions by 60%."},
            {"company": "Rippling", "title": "ML Engineer",
             "start_date": "2020-03-01", "end_date": "2022-05-01", "duration_months": 26,
             "is_current": False, "industry": "HR Tech", "company_size": "1001-5000",
             "description": "Built candidate matching system using Elasticsearch and custom scoring. Implemented learning-to-rank with XGBoost improving click-through rate by 22%. Deployed vector search with Pinecone for semantic matching."},
            {"company": "Google", "title": "Software Engineer",
             "start_date": "2018-07-01", "end_date": "2020-02-01", "duration_months": 19,
             "is_current": False, "industry": "Technology", "company_size": "10001+",
             "description": "Worked on search ranking infrastructure. Built query understanding features that improved relevance by 8%. Gained deep experience with information retrieval fundamentals including BM25, query expansion, and relevance feedback loops."},
        ],
        "education": [{"institution": "IIT Bombay", "degree": "B.Tech", "field_of_study": "Computer Science",
                        "start_year": 2014, "end_year": 2018, "grade": "9.2 GPA", "tier": "tier_1"}],
        "skills": [
            {"name": "Python", "proficiency": "expert", "endorsements": 45, "duration_months": 84},
            {"name": "Machine Learning", "proficiency": "expert", "endorsements": 38, "duration_months": 72},
            {"name": "Elasticsearch", "proficiency": "advanced", "endorsements": 22, "duration_months": 48},
            {"name": "FAISS", "proficiency": "advanced", "endorsements": 18, "duration_months": 36},
            {"name": "Pinecone", "proficiency": "advanced", "endorsements": 12, "duration_months": 30},
            {"name": "NLP", "proficiency": "advanced", "endorsements": 30, "duration_months": 60},
        ],
        "redrob_signals": {
            "profile_completeness_score": 92, "last_active_date": "2026-06-15",
            "open_to_work_flag": True, "recruiter_response_rate": 0.85,
            "saved_by_recruiters_30d": 15, "search_appearance_30d": 180,
            "interview_completion_rate": 0.92, "notice_period_days": 25,
            "verified_email": True, "verified_phone": True, "linkedin_connected": True,
            "skill_assessment_scores": {"NLP": 82.0, "Machine Learning": 78.0},
        },
    }


def test_one(name: str, c: dict) -> dict:
    text = schema.unified_text_blob(c)
    is_hp, hp_r = honeypot.is_honeypot(c)
    disq, disq_r = features.disqualifier_penalty(c, text)
    sc, _ = features.skill_coverage(c, text)
    own, _ = features.ownership_hierarchy(c, text)
    imp, _ = features.impact_magnitude(c, text)
    ev = evidence.get_top_evidence(c, 3)
    narrative = analyze_narrative(c)
    return {
        "name": name, "honeypot": is_hp, "hp_reasons": hp_r,
        "disq_penalty": round(disq, 3), "disq_reasons": disq_r,
        "skill_coverage": round(sc, 3), "ownership": round(own, 3),
        "impact_mag": round(imp, 3), "evidence_count": len(ev),
        "narrative_coherence": round(narrative.coherence, 3),
        "narrative_type": narrative.trajectory_type,
        "narrative_suspicious": narrative.suspicious_patterns,
    }


def main():
    print("=== V6 Adversarial Test Suite ===\n")
    t0 = time.time()

    tests = [
        ("Keyword Stuffer", make_keyword_stuffer()),
        ("Career Chaos", make_career_chaos()),
        ("Title Inflation (2 YOE Architect)", make_title_inflation()),
        ("Strong Candidate", make_strong_candidate()),
    ]

    for name, c in tests:
        r = test_one(name, c)
        print(f"--- {name} ({c['candidate_id']}) ---")
        print(f"  Honeypot: {r['honeypot']} {r['hp_reasons']}")
        print(f"  Disq penalty: {r['disq_penalty']} {r['disq_reasons']}")
        print(f"  Skill coverage: {r['skill_coverage']}")
        print(f"  Ownership: {r['ownership']}")
        print(f"  Impact: {r['impact_mag']}")
        print(f"  Evidence: {r['evidence_count']} pieces")
        print(f"  Narrative: coherence={r['narrative_coherence']}, type={r['narrative_type']}")
        print(f"  Narrative suspicious: {r['narrative_suspicious']}")
        print()

    # Assertions
    stuffer = test_one("stuffer", make_keyword_stuffer())
    chaos = test_one("chaos", make_career_chaos())
    inflate = test_one("inflate", make_title_inflation())
    strong = test_one("strong", make_strong_candidate())

    passed = 0
    total = 10

    # V5 tests
    if stuffer["disq_penalty"] <= 0.65:
        print("[PASS] Keyword stuffer penalized"); passed += 1
    else:
        print(f"[FAIL] Keyword stuffer not penalized enough: {stuffer['disq_penalty']}")
    if stuffer["ownership"] < 0.3:
        print("[PASS] Keyword stuffer has low ownership"); passed += 1
    else:
        print(f"[FAIL] Keyword stuffer has high ownership: {stuffer['ownership']}")
    if strong["skill_coverage"] > 0.3:
        print("[PASS] Strong candidate has good skill coverage"); passed += 1
    else:
        print(f"[FAIL] Strong candidate low coverage: {strong['skill_coverage']}")
    if strong["impact_mag"] > 0.5:
        print("[PASS] Strong candidate has high impact"); passed += 1
    else:
        print(f"[FAIL] Strong candidate low impact: {strong['impact_mag']}")
    if strong["ownership"] > 0.5:
        print("[PASS] Strong candidate has high ownership"); passed += 1
    else:
        print(f"[FAIL] Strong candidate low ownership: {strong['ownership']}")

    # V6 narrative tests
    if chaos["narrative_coherence"] < 0.4:
        print("[PASS] Career chaos detected (low coherence)"); passed += 1
    else:
        print(f"[FAIL] Career chaos not detected: coherence={chaos['narrative_coherence']}")
    if any(p in chaos["narrative_suspicious"] for p in ["multiple_regressions", "excessive_domain_hopping", "multiple_career_gaps", "excessive_job_changes"]):
        print("[PASS] Career chaos has suspicious patterns"); passed += 1
    else:
        print(f"[FAIL] Career chaos missing suspicious patterns: {chaos['narrative_suspicious']}")
    if "title_inflation" in inflate["narrative_suspicious"]:
        print("[PASS] Title inflation detected"); passed += 1
    else:
        print(f"[FAIL] Title inflation not detected: {inflate['narrative_suspicious']}")
    if strong["narrative_coherence"] >= 0.5:
        print("[PASS] Strong candidate has reasonable coherence"); passed += 1
    else:
        print(f"[FAIL] Strong candidate low coherence: {strong['narrative_coherence']}")
    if strong["narrative_type"] == "upward":
        print("[PASS] Strong candidate has upward trajectory"); passed += 1
    else:
        print(f"[FAIL] Strong candidate not upward: {strong['narrative_type']}")

    print(f"\n{'='*50}")
    print(f"Passed {passed}/{total} tests in {time.time()-t0:.2f}s")
    if passed == total:
        print("ALL TESTS PASSED")
    else:
        print(f"FAILED {total - passed} tests")
        sys.exit(1)


if __name__ == "__main__":
    main()