services/bls_coverage.py

"""BLS-driven high-coverage role enrichment.

Ranks occupations by national employment size (BLS OES 2023) and proactively
fills KB gaps for the roles that cover the most workers — before users ever
search for them.

Credit budget: Tavily advanced search, 1 credit per role.
Config: job_query_agent.coverage_enrichment.tavily_daily_budget (default 20)

Offline-safe: hardcoded 2023 BLS OES data is the baseline.
Employment refresh: annual OES flat file download from BLS (no API key needed).
  URL: https://www.bls.gov/oes/special.requests/oesm{YY}nat.zip
  BLS_API_KEY env var: used for CES monthly data in job_market.py (higher rate limits).
  Note: BLS time series API (v2) does NOT support OES — OES is distributed
        as annual Excel flat files only.
"""
from __future__ import annotations

import json
import os
from datetime import date
from pathlib import Path
from typing import Any

# ---------------------------------------------------------------------------
# BLS OES 2023 — top occupations by national employment (thousands)
# Source: BLS Occupational Employment and Wage Statistics, May 2023
# Fields: soc_code, title, employment_k (thousands), search_query, industry
# ---------------------------------------------------------------------------
HIGH_COVERAGE_OCCUPATIONS: list[dict[str, Any]] = [
    # Retail & Sales
    {"soc": "41-2031", "title": "Retail Salesperson",          "emp_k": 4436, "query": "retail salesperson",          "industry": "Retail"},
    {"soc": "41-2011", "title": "Cashier",                     "emp_k": 3327, "query": "cashier",                     "industry": "Retail"},
    {"soc": "41-4011", "title": "Sales Representative (B2B)",  "emp_k": 1521, "query": "sales representative",        "industry": "Retail"},
    {"soc": "41-3021", "title": "Insurance Sales Agent",       "emp_k":  492, "query": "insurance agent",             "industry": "Finance"},
    {"soc": "41-3011", "title": "Advertising Sales Agent",     "emp_k":  150, "query": "advertising sales agent",     "industry": "Media"},
    # Office & Admin
    {"soc": "43-9061", "title": "Office Clerk",                "emp_k": 3074, "query": "office clerk",                "industry": "Government"},
    {"soc": "43-4051", "title": "Customer Service Representative","emp_k": 2884, "query": "customer service representative","industry": "Retail"},
    {"soc": "43-6014", "title": "Administrative Assistant",    "emp_k": 2018, "query": "administrative assistant",    "industry": "Government"},
    {"soc": "43-3031", "title": "Bookkeeping Clerk",           "emp_k": 1481, "query": "bookkeeper",                  "industry": "Finance"},
    {"soc": "43-4171", "title": "Receptionist",                "emp_k":  942, "query": "receptionist",               "industry": "Government"},
    {"soc": "43-1011", "title": "Office Supervisor",           "emp_k":  785, "query": "office supervisor",           "industry": "Government"},
    # Healthcare
    {"soc": "29-1141", "title": "Registered Nurse",            "emp_k": 3171, "query": "registered nurse",           "industry": "Healthcare"},
    {"soc": "31-1121", "title": "Home Health Aide",            "emp_k": 2395, "query": "home health aide",            "industry": "Healthcare"},
    {"soc": "31-1131", "title": "Nursing Assistant",           "emp_k": 1396, "query": "nursing assistant",           "industry": "Healthcare"},
    {"soc": "31-9092", "title": "Medical Assistant",           "emp_k":  782, "query": "medical assistant",           "industry": "Healthcare"},
    {"soc": "29-2061", "title": "Licensed Practical Nurse",    "emp_k":  635, "query": "licensed practical nurse",    "industry": "Healthcare"},
    {"soc": "29-2034", "title": "Radiologic Technologist",     "emp_k":  226, "query": "radiologic technologist",     "industry": "Healthcare"},
    {"soc": "29-1051", "title": "Pharmacist",                  "emp_k":  322, "query": "pharmacist",                  "industry": "Healthcare"},
    {"soc": "11-9111", "title": "Healthcare Manager",          "emp_k":  588, "query": "healthcare manager",          "industry": "Healthcare"},
    # Food & Hospitality
    {"soc": "35-3023", "title": "Fast Food Worker",            "emp_k": 3784, "query": "fast food worker",            "industry": "Hospitality"},
    {"soc": "35-3031", "title": "Waiter / Waitress",           "emp_k": 2218, "query": "waiter",                      "industry": "Hospitality"},
    {"soc": "35-2014", "title": "Restaurant Cook",             "emp_k": 1512, "query": "restaurant cook",             "industry": "Hospitality"},
    {"soc": "35-1011", "title": "Food Service Supervisor",     "emp_k":  994, "query": "food service supervisor",     "industry": "Hospitality"},
    # Logistics & Transport
    {"soc": "53-7062", "title": "Warehouse Worker",            "emp_k": 2596, "query": "warehouse worker",            "industry": "Logistics"},
    {"soc": "53-3032", "title": "Heavy Truck Driver",          "emp_k": 2004, "query": "truck driver",                "industry": "Logistics"},
    {"soc": "53-3033", "title": "Delivery Driver",             "emp_k":  916, "query": "delivery driver",             "industry": "Logistics"},
    # Construction & Maintenance
    {"soc": "47-2061", "title": "Construction Laborer",        "emp_k": 1582, "query": "construction laborer",        "industry": "Construction"},
    {"soc": "37-2011", "title": "Janitor / Cleaner",           "emp_k": 2318, "query": "janitor",                     "industry": "Construction"},
    {"soc": "49-9071", "title": "Maintenance & Repair Worker", "emp_k": 1521, "query": "maintenance technician",      "industry": "Construction"},
    {"soc": "47-1011", "title": "Construction Supervisor",     "emp_k":  720, "query": "construction supervisor",     "industry": "Construction"},
    # Education
    {"soc": "25-2021", "title": "Elementary School Teacher",   "emp_k": 1534, "query": "elementary school teacher",   "industry": "Education"},
    {"soc": "25-2031", "title": "Secondary School Teacher",    "emp_k": 1078, "query": "high school teacher",         "industry": "Education"},
    {"soc": "25-1099", "title": "University Lecturer",         "emp_k":  643, "query": "university lecturer",         "industry": "Education"},
    {"soc": "25-9031", "title": "Instructional Coordinator",   "emp_k":  181, "query": "instructional designer",      "industry": "Education"},
    # Tech
    {"soc": "15-1252", "title": "Software Developer",          "emp_k": 1892, "query": "software developer",         "industry": "Tech"},
    {"soc": "15-1211", "title": "Systems Analyst",             "emp_k":  586, "query": "systems analyst",             "industry": "Tech"},
    {"soc": "15-1232", "title": "IT Support Specialist",       "emp_k":  835, "query": "it support specialist",       "industry": "Tech"},
    {"soc": "15-1244", "title": "Network Administrator",       "emp_k":  344, "query": "network administrator",       "industry": "Tech"},
    {"soc": "15-1254", "title": "Web Developer",               "emp_k":  208, "query": "web developer",               "industry": "Tech"},
    {"soc": "15-2051", "title": "Data Scientist",              "emp_k":  168, "query": "data scientist",              "industry": "Tech"},
    {"soc": "11-3021", "title": "IT Manager",                  "emp_k":  490, "query": "it manager",                  "industry": "Tech"},
    # Finance & Business
    {"soc": "13-2011", "title": "Accountant",                  "emp_k": 1441, "query": "accountant",                  "industry": "Finance"},
    {"soc": "13-1161", "title": "Market Research Analyst",     "emp_k":  792, "query": "market research analyst",     "industry": "Finance"},
    {"soc": "13-2051", "title": "Financial Analyst",           "emp_k":  327, "query": "financial analyst",           "industry": "Finance"},
    {"soc": "13-2082", "title": "Tax Preparer",                "emp_k":   80, "query": "tax preparer",                "industry": "Finance"},
    {"soc": "11-1021", "title": "Operations Manager",          "emp_k": 3057, "query": "operations manager",          "industry": "Government"},
    {"soc": "11-2021", "title": "Marketing Manager",           "emp_k":  386, "query": "marketing manager",           "industry": "Media"},
    {"soc": "13-1071", "title": "HR Specialist",               "emp_k":  861, "query": "hr specialist",               "industry": "Government"},
    # Engineering
    {"soc": "17-2051", "title": "Civil Engineer",              "emp_k":  329, "query": "civil engineer",              "industry": "Construction"},
    {"soc": "17-2141", "title": "Mechanical Engineer",         "emp_k":  303, "query": "mechanical engineer",         "industry": "Manufacturing"},
    {"soc": "17-2071", "title": "Electrical Engineer",         "emp_k":  195, "query": "electrical engineer",         "industry": "Manufacturing"},
    {"soc": "17-2112", "title": "Industrial Engineer",         "emp_k":  299, "query": "industrial engineer",         "industry": "Manufacturing"},
    # Legal & Social
    {"soc": "23-1011", "title": "Lawyer",                      "emp_k":  813, "query": "lawyer",                      "industry": "Legal"},
    {"soc": "23-2011", "title": "Paralegal",                   "emp_k":  372, "query": "paralegal",                   "industry": "Legal"},
    {"soc": "21-1021", "title": "Child / Family Social Worker","emp_k":  342, "query": "social worker",               "industry": "Government"},
    # Security & Protection
    {"soc": "33-9032", "title": "Security Guard",              "emp_k": 1073, "query": "security guard",              "industry": "Government"},
    # Agriculture
    {"soc": "45-2092", "title": "Agricultural Worker",         "emp_k":  870, "query": "farm worker",                 "industry": "Agriculture"},
]

# Sort by employment size (highest first) — this is the enrichment priority order
HIGH_COVERAGE_OCCUPATIONS.sort(key=lambda x: x["emp_k"], reverse=True)

_CACHE_PATH = "data/bls_coverage_cache.json"


def coverage_gaps(
    jobs: list[dict],
    search_cfg: dict,
    *,
    sim_threshold: float | None = None,
    use_live_employment: bool = False,
) -> list[dict]:
    """Return HIGH_COVERAGE_OCCUPATIONS entries not adequately covered in KB.

    'Adequately covered' = find_best_match returns sim >= sim_threshold.
    Default threshold: search_cfg tier_weak (0.55).
    Returns list sorted by employment size (largest gap first).

    use_live_employment=True: pull cached OES data to override hardcoded emp_k.
    """
    import copy
    try:
        import job_radar
    except ImportError:
        return []

    threshold = sim_threshold if sim_threshold is not None else float(
        search_cfg.get("tier_weak", 0.55)
    )

    # Optionally override with live OES employment numbers (from cache)
    live_emp: dict[str, int] = {}
    if use_live_employment:
        cached = _load_emp_cache(_CACHE_PATH)
        live_emp = {k: v for k, v in cached.items() if not k.startswith("_")}

    occupations = HIGH_COVERAGE_OCCUPATIONS
    if live_emp:
        occupations = sorted(
            [{**o, "emp_k": live_emp.get(o["soc"], o["emp_k"] * 1000) // 1000} for o in occupations],
            key=lambda x: x["emp_k"], reverse=True,
        )

    gaps: list[dict] = []
    for occ in occupations:
        _, best = job_radar.find_best_match(
            occ["query"], [copy.deepcopy(j) for j in jobs], search_cfg=search_cfg
        )
        sim = best.get("combined_similarity", 0.0) if best else 0.0
        if sim < threshold:
            gaps.append({**occ, "_current_sim": round(sim, 3), "_best_id": best["id"] if best else None})
    return gaps


def run_coverage_enrichment(
    cfg: dict,
    *,
    daily_budget: int = 20,
    dry_run: bool = False,
) -> dict[str, Any]:
    """Enrich KB with top-employment occupations missing from KB.

    Called from query-agent calibration loop before query rounds.
    Respects daily_budget (Tavily credits). Returns summary dict.
    """
    import copy
    try:
        import job_radar
    except ImportError:
        return {"skipped": True, "reason": "job_radar not available"}

    if not os.environ.get("TAVILY_API_KEY") and not os.environ.get("GROQ_API_KEY"):
        return {"skipped": True, "reason": "no API keys"}

    jr_cfg = cfg.get("job_radar", {})
    kb_path = jr_cfg.get("kb_path", "data/jobs_kb.json")
    search_cfg = job_radar.resolve_search_config(jr_cfg)

    jobs = job_radar.load_knowledge_base(kb_path)
    gaps = coverage_gaps(jobs, search_cfg)

    generated = 0
    failed = 0
    skipped_budget = 0
    results: list[dict] = []

    for occ in gaps:
        if generated >= daily_budget:
            skipped_budget += len(gaps) - generated - failed
            break

        if dry_run:
            results.append({"query": occ["query"], "emp_k": occ["emp_k"], "dry_run": True})
            generated += 1
            continue

        profile = job_radar.generate_job_profile_via_llm(occ["query"], kb_path=kb_path)
        if profile:
            # Reload jobs so next coverage_gaps call sees the new entry
            jobs = job_radar.load_knowledge_base(kb_path)
            results.append({
                "query": occ["query"],
                "emp_k": occ["emp_k"],
                "profile_id": profile["id"],
                "sim_before": occ["_current_sim"],
                "workers_covered_k": occ["emp_k"],
            })
            generated += 1
        else:
            failed += 1

    total_workers_k = sum(r.get("workers_covered_k", 0) for r in results)
    return {
        "gaps_found": len(gaps),
        "generated": generated,
        "failed": failed,
        "skipped_budget": skipped_budget,
        "tavily_credits_used": generated,  # 1 credit per generate_job_profile_via_llm call
        "workers_newly_covered_k": total_workers_k,
        "results": results,
    }


# ---------------------------------------------------------------------------
# Annual OES flat file refresh (no API key needed — BLS publishes publicly)
# URL pattern: https://www.bls.gov/oes/special.requests/oesm{YY}nat.zip
# Released each May; contains national_M{YEAR}_dl.xlsx with TOT_EMP by SOC.
# ---------------------------------------------------------------------------

def refresh_employment_from_bls(
    *,
    year: int | None = None,
    cache_path: str = _CACHE_PATH,
) -> dict[str, int]:
    """Download BLS OES national flat file and extract employment by SOC code.

    Returns {soc_code: employment} (actual count, not thousands).
    Results cached for ~365 days. Falls back to {} on any failure.

    Note: BLS time series API v2 does NOT support OES occupational series.
    OES is annual and distributed as Excel flat files only.
    BLS_API_KEY is used for CES monthly data (job_market.py), not here.
    """
    cache = _load_emp_cache(cache_path)
    cached_date = cache.get("_fetched_at", "")
    if cached_date:
        try:
            age_days = (date.today() - date.fromisoformat(cached_date)).days
            if age_days < 340:  # refresh annually
                return {k: v for k, v in cache.items() if not k.startswith("_")}
        except Exception:
            pass

    if year is None:
        # Use previous year (OES released in May; current year not yet available before May)
        y = date.today().year
        year = y - 1 if date.today().month < 6 else y

    yy = str(year)[2:]  # "2023" → "23"
    url = f"https://www.bls.gov/oes/special.requests/oesm{yy}nat.zip"
    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
        "Referer": "https://www.bls.gov/oes/",
    }

    try:
        import io
        import requests
        import zipfile

        resp = requests.get(url, headers=headers, timeout=60)
        resp.raise_for_status()
        if resp.content[:2] != b"PK":
            return {}  # not a ZIP (got HTML redirect)

        z = zipfile.ZipFile(io.BytesIO(resp.content))
        xlsx_name = next((n for n in z.namelist() if n.endswith(".xlsx")), None)
        if not xlsx_name:
            return {}

        try:
            import openpyxl
        except ImportError:
            return {}  # openpyxl optional

        wb = openpyxl.load_workbook(z.open(xlsx_name), read_only=True, data_only=True)
        ws = wb.active
        rows = ws.iter_rows(values_only=True)
        col_headers = [str(c).strip() if c else "" for c in next(rows)]

        occ_idx = col_headers.index("OCC_CODE")
        emp_idx = col_headers.index("TOT_EMP")
        naics_idx = col_headers.index("NAICS") if "NAICS" in col_headers else None

        # Filter: national cross-industry rows (NAICS = "000000" or "Cross-industry")
        soc_set = {occ["soc"] for occ in HIGH_COVERAGE_OCCUPATIONS}
        result: dict[str, int] = {}
        for row in rows:
            soc = str(row[occ_idx]).strip() if row[occ_idx] else ""
            if soc not in soc_set:
                continue
            # Use cross-industry row (NAICS = 000000)
            naics = str(row[naics_idx]).strip() if naics_idx is not None else "000000"
            if naics not in ("000000", "Cross-industry"):
                continue
            raw = row[emp_idx]
            try:
                emp = int(str(raw).replace(",", ""))
                if soc not in result:  # take first (cross-industry) match
                    result[soc] = emp
            except (TypeError, ValueError):
                pass

        if result:
            result["_fetched_at"] = date.today().isoformat()
            result["_oes_year"] = year
            _save_emp_cache(cache_path, result)
        return {k: v for k, v in result.items() if not k.startswith("_")}

    except Exception:
        return {}


def _load_emp_cache(path: str) -> dict:
    try:
        return json.loads(Path(path).read_text(encoding="utf-8"))
    except Exception:
        return {}


def _save_emp_cache(path: str, data: dict) -> None:
    try:
        Path(path).parent.mkdir(parents=True, exist_ok=True)
        Path(path).write_text(json.dumps(data, indent=2), encoding="utf-8")
    except Exception:
        pass