app/services/keyword_extractor.py

"""
Keyword extractor: curated tech keyword list + regex matching.

Extracts:
  - Which JD keywords appear in the resume (matched)
  - Which JD keywords are absent from the resume (missing)
  - Which matched keywords appear only weakly / once (weak)
"""
from __future__ import annotations

import re
from collections import Counter
from dataclasses import dataclass


# ── Curated tech keyword catalogue ────────────────────────────────────────
TECH_KEYWORDS: list[str] = [
    # Languages
    "Python", "JavaScript", "TypeScript", "Java", "Kotlin", "Swift", "Go", "Golang",
    "Rust", "C++", "C#", "Ruby", "PHP", "Scala", "R", "MATLAB", "Bash", "Shell",
    "Perl", "Dart", "Elixir", "Haskell", "Clojure",
    # Web / Frontend
    "React", "Next.js", "Vue", "Vue.js", "Angular", "Svelte", "HTML", "CSS",
    "Tailwind", "Bootstrap", "SASS", "SCSS", "Redux", "Zustand", "GraphQL",
    "REST", "REST APIs", "WebSocket", "gRPC",
    # Backend Frameworks
    "FastAPI", "Django", "Flask", "Express", "Node.js", "Spring Boot", "Rails",
    "Laravel", "Gin", "Echo", "Fiber", "Actix",
    # Databases
    "PostgreSQL", "MySQL", "SQLite", "MongoDB", "Cassandra", "DynamoDB", "Redis",
    "Elasticsearch", "Firestore", "Supabase", "PlanetScale", "CockroachDB",
    "SQL", "NoSQL",
    # Cloud & DevOps
    "AWS", "GCP", "Azure", "Docker", "Kubernetes", "K8s", "Terraform", "Ansible",
    "CI/CD", "GitHub Actions", "Jenkins", "CircleCI", "ArgoCD", "Helm",
    "Linux", "Nginx", "Caddy", "Vercel", "Render", "Fly.io", "Railway",
    # AI / ML / Data
    "PyTorch", "TensorFlow", "JAX", "Keras", "scikit-learn", "sklearn",
    "LangChain", "LlamaIndex", "Hugging Face", "Transformers",
    "FAISS", "Chroma", "Pinecone", "Qdrant", "Weaviate",
    "Embeddings", "Vector DB", "Vector Database", "RAG", "LLM",
    "GPT", "OpenAI", "Gemini", "Anthropic", "Claude",
    "BERT", "Sentence Transformers", "spaCy", "NLTK",
    "XGBoost", "LightGBM", "CatBoost", "Random Forest",
    "NLP", "Computer Vision", "CNN", "RNN", "LSTM", "Transformer",
    "Reinforcement Learning", "Fine-tuning", "PEFT", "LoRA",
    "MLflow", "DVC", "Weights & Biases", "wandb",
    # Data Engineering
    "Pandas", "NumPy", "Polars", "Spark", "PySpark", "Airflow", "Kafka",
    "dbt", "Snowflake", "BigQuery", "Redshift", "Databricks",
    "ETL", "Data Pipeline",
    # Testing & Quality
    "Pytest", "Jest", "Cypress", "Playwright", "Selenium", "JUnit",
    "TDD", "BDD", "Unit Testing", "Integration Testing",
    # Architecture
    "Microservices", "System Design", "API Design", "Event-Driven",
    "CQRS", "Event Sourcing", "Message Queue", "RabbitMQ",
    # Tools & Practices
    "Git", "GitHub", "GitLab", "Jira", "Agile", "Scrum", "Kanban",
    "OpenAPI", "Swagger", "Postman", "Linux",
]

# Build a lookup: lowercase → canonical name
_KEYWORD_MAP: dict[str, str] = {kw.lower(): kw for kw in TECH_KEYWORDS}

# Also handle multi-word keywords joined differently (e.g. "ci cd" → "CI/CD")
_ALIASES: dict[str, str] = {
    "ci cd": "CI/CD",
    "ci/cd": "CI/CD",
    "machine learning": "ML",
    "deep learning": "DL",
    "neural network": "DL",
    "node": "Node.js",
    "react.js": "React",
    "nextjs": "Next.js",
    "vuejs": "Vue.js",
    "typescript": "TypeScript",
    "javascript": "JavaScript",
    "postgres": "PostgreSQL",
    "mongo": "MongoDB",
    "k8s": "Kubernetes",
    "golang": "Go",
}


@dataclass
class KeywordResult:
    matched: list[str]
    missing: list[str]
    weak: list[str]          # matched but low frequency (mentioned once)
    jd_keywords: list[str]   # all keywords found in JD
    resume_keywords: list[str]


def extract_keywords(resume_text: str, jd_text: str) -> KeywordResult:
    """
    Find which keywords from the JD are present in the resume.
    """
    jd_kws   = _find_keywords(jd_text)
    res_kws  = _find_keywords(resume_text)
    res_freq = _keyword_frequencies(resume_text)

    jd_set  = set(jd_kws)
    res_set = set(res_kws)

    matched = sorted(jd_set & res_set)
    missing = sorted(jd_set - res_set)

    # "Weak" = matched but mentioned only once in the resume
    weak = [kw for kw in matched if res_freq.get(kw, 0) <= 1]

    return KeywordResult(
        matched=matched,
        missing=missing,
        weak=weak,
        jd_keywords=sorted(jd_set),
        resume_keywords=sorted(res_set),
    )


def _find_keywords(text: str) -> list[str]:
    """Return all canonical tech keywords found in text."""
    text_lower = text.lower()
    found: set[str] = set()

    # Check aliases first
    for alias, canonical in _ALIASES.items():
        if alias in text_lower:
            found.add(canonical)

    # Check keyword map
    for kw_lower, kw_canonical in _KEYWORD_MAP.items():
        # Use word-boundary-like check (avoid matching "react" inside "reactivation")
        pattern = r"(?<![a-z])" + re.escape(kw_lower) + r"(?![a-z])"
        if re.search(pattern, text_lower):
            found.add(kw_canonical)

    return list(found)


def _keyword_frequencies(text: str) -> dict[str, int]:
    """Count occurrences of each keyword in text."""
    text_lower = text.lower()
    freq: Counter[str] = Counter()
    for kw_lower, kw_canonical in _KEYWORD_MAP.items():
        count = len(re.findall(re.escape(kw_lower), text_lower))
        if count:
            freq[kw_canonical] = count
    return dict(freq)