backend/core/grants/pulse.py

"""
Grant Pulse — classify and list new / planned / closing / current grants.

Pure catalog logic (no network). Used by API, scheduler metrics, firm advisor.
"""
from __future__ import annotations

import os
from datetime import date, datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Sequence

from sqlalchemy.orm import Session

from core.search.deadline_utils import parse_deadline_robust

PULSE_KINDS = frozenset({"new", "planned", "closing", "current", "all"})

DEFAULT_NEW_DAYS = int(os.environ.get("PULSE_NEW_DAYS", "14"))
DEFAULT_CLOSING_DAYS = int(os.environ.get("PULSE_CLOSING_DAYS", "30"))


def _parse_iso_date(value: Any) -> Optional[date]:
    if value is None:
        return None
    if isinstance(value, date) and not isinstance(value, datetime):
        return value
    if isinstance(value, datetime):
        return value.date()
    s = str(value).strip()
    if not s:
        return None
    parsed = parse_deadline_robust(s, "")
    if parsed:
        return parsed
    try:
        return datetime.fromisoformat(s.replace("Z", "+00:00")).date()
    except Exception:
        return None


def _first_seen(grant: Dict[str, Any]) -> Optional[date]:
    raw = grant.get("raw_data") if isinstance(grant.get("raw_data"), dict) else {}
    for key in (
        "first_seen_at",
        "fetched_at",
        "created_at",
        "data_pobrania",
    ):
        d = _parse_iso_date(grant.get(key) or raw.get(key))
        if d:
            return d
    return None


def is_continuous(grant: Dict[str, Any]) -> bool:
    status = str(grant.get("status") or "").lower()
    deadline = str(grant.get("deadline") or "").lower()
    blob = f"{status} {deadline} {grant.get('name') or ''} {grant.get('description') or ''}".lower()
    return any(
        k in blob
        for k in ("ciągły", "ciagly", "continuous", "beztermin", "cały rok", "caly rok")
    )


def classify_pulse_kinds(
    grant: Dict[str, Any],
    *,
    today: Optional[date] = None,
    new_days: int = DEFAULT_NEW_DAYS,
    closing_days: int = DEFAULT_CLOSING_DAYS,
) -> List[str]:
    """
    Return zero or more pulse kinds for a grant dict.
    A grant may be both `new` and `closing`, or `planned` only, etc.
    """
    today = today or date.today()
    kinds: List[str] = []
    status = str(grant.get("status") or "").lower().strip()
    continuous = is_continuous(grant)
    deadline = _parse_iso_date(grant.get("deadline"))
    first_seen = _first_seen(grant)

    closed = status in ("closed", "zakończony", "zakonczony") or bool(
        grant.get("is_outdated_warning")
    )
    if deadline and deadline < today and not continuous:
        closed = True

    if status == "planned" or status in ("planowany", "wkrótce", "zapowiedziany"):
        kinds.append("planned")
    elif not closed and deadline and deadline > today and status not in ("active", "otwarty"):
        # future deadline without active — treat as planned if far start-like wording
        if any(
            k in str(grant.get("name") or "").lower() + str(grant.get("description") or "").lower()
            for k in ("planowany", "wkrótce", "harmonogram", "rusza")
        ):
            kinds.append("planned")

    if not closed and (status in ("active", "otwarty", "trwa", "open") or continuous):
        kinds.append("current")
    elif not closed and deadline and deadline >= today:
        kinds.append("current")

    if not closed and first_seen:
        if first_seen >= today - timedelta(days=max(1, new_days)):
            kinds.append("new")

    if not closed and not continuous and deadline:
        if today <= deadline <= today + timedelta(days=max(1, closing_days)):
            kinds.append("closing")

    # de-dupe preserve order
    seen = set()
    out: List[str] = []
    for k in kinds:
        if k not in seen:
            seen.add(k)
            out.append(k)
    return out


def primary_pulse_kind(grant: Dict[str, Any], **kwargs: Any) -> str:
    kinds = classify_pulse_kinds(grant, **kwargs)
    # priority for single-label views
    for pref in ("closing", "new", "planned", "current"):
        if pref in kinds:
            return pref
    return "other"


def verification_flags(grant: Dict[str, Any], *, today: Optional[date] = None) -> Dict[str, Any]:
    """Claim-level flags for deadline / status / regulation before alerts."""
    today = today or date.today()
    continuous = is_continuous(grant)
    deadline = _parse_iso_date(grant.get("deadline"))
    status = str(grant.get("status") or "").lower()
    reg = (
        grant.get("regulation_url")
        or grant.get("precise_regulation_url")
        or grant.get("eurlex_url")
        or ""
    )
    reg_ok = str(reg).startswith("http")
    grounded = bool(grant.get("regulation_grounded") or grant.get("eurlex_grounded"))
    cred = float(grant.get("source_credibility_score") or 0)
    if cred > 1:
        cred = cred / 100.0

    deadline_ok = continuous or (deadline is not None and deadline >= today)
    status_ok = status in ("active", "planned", "otwarty", "planowany", "open") or continuous
    if deadline and deadline < today and not continuous:
        status_ok = False

    alert_worthy = deadline_ok and status_ok and (reg_ok or grounded) and cred >= 0.35
    return {
        "deadline_verified": bool(deadline_ok),
        "status_verified": bool(status_ok),
        "regulation_verified": bool(reg_ok or grounded),
        "continuous": continuous,
        "credibility_ok": cred >= 0.35,
        "alert_worthy": bool(alert_worthy),
        "reasons": _flag_reasons(deadline_ok, status_ok, reg_ok or grounded, continuous, deadline),
    }


def _flag_reasons(
    deadline_ok: bool,
    status_ok: bool,
    reg_ok: bool,
    continuous: bool,
    deadline: Optional[date],
) -> List[str]:
    reasons: List[str] = []
    if not deadline_ok:
        reasons.append("missing_or_past_deadline")
    if continuous:
        reasons.append("continuous_enrollment")
    if not status_ok:
        reasons.append("status_not_open")
    if not reg_ok:
        reasons.append("regulation_not_grounded")
    if deadline:
        reasons.append(f"deadline={deadline.isoformat()}")
    return reasons


def pulse_item_from_grant(
    grant: Dict[str, Any],
    *,
    today: Optional[date] = None,
    new_days: int = DEFAULT_NEW_DAYS,
    closing_days: int = DEFAULT_CLOSING_DAYS,
) -> Dict[str, Any]:
    today = today or date.today()
    kinds = classify_pulse_kinds(
        grant, today=today, new_days=new_days, closing_days=closing_days
    )
    flags = verification_flags(grant, today=today)
    first_seen = _first_seen(grant)
    return {
        "id": grant.get("id") or grant.get("source_id"),
        "name": grant.get("name"),
        "status": grant.get("status"),
        "deadline": grant.get("deadline") or "",
        "continuous": flags["continuous"],
        "operator": grant.get("operator") or grant.get("program") or grant.get("zrodlo") or "",
        "source": grant.get("source") or "",
        "url": grant.get("url") or grant.get("official_page_url") or "",
        "regulation_url": grant.get("regulation_url")
        or grant.get("precise_regulation_url")
        or "",
        "eurlex_url": grant.get("eurlex_url") or "",
        "celex_id": grant.get("celex_id") or (grant.get("eurlex") or {}).get("celex_id")
        if isinstance(grant.get("eurlex"), dict)
        else grant.get("celex_id"),
        "source_credibility_score": grant.get("source_credibility_score"),
        "data_quality_score": grant.get("data_quality_score"),
        "regulation_grounded": bool(
            grant.get("regulation_grounded") or grant.get("eurlex_grounded")
        ),
        "pulse_kinds": kinds,
        "primary_kind": primary_pulse_kind(
            grant, today=today, new_days=new_days, closing_days=closing_days
        ),
        "first_seen_at": first_seen.isoformat() if first_seen else None,
        "verification": flags,
    }


def _normalize_grant_for_pulse(grant: Dict[str, Any]) -> Dict[str, Any]:
    """Ensure deadline/status honesty before pulse classify (text backfill, ISO)."""
    try:
        from core.search.deadline_utils import apply_deadline_to_grant_dict

        return apply_deadline_to_grant_dict(dict(grant))
    except Exception:
        return dict(grant)


def list_pulse_grants(
    grants: Sequence[Dict[str, Any]],
    *,
    kind: str = "current",
    limit: int = 50,
    today: Optional[date] = None,
    new_days: int = DEFAULT_NEW_DAYS,
    closing_days: int = DEFAULT_CLOSING_DAYS,
) -> Dict[str, Any]:
    """Filter and shape grants for pulse API. kind=all returns mixed current pulse."""
    kind = (kind or "current").lower().strip()
    if kind not in PULSE_KINDS:
        kind = "current"
    today = today or date.today()
    items: List[Dict[str, Any]] = []
    counts = {"new": 0, "planned": 0, "closing": 0, "current": 0}
    quality = {
        "input": 0,
        "with_deadline": 0,
        "deadline_backfilled": 0,
        "with_regulation": 0,
        "past_deadline_closed": 0,
    }

    for g in grants:
        quality["input"] += 1
        had_deadline = bool(str((g or {}).get("deadline") or "").strip())
        g_norm = _normalize_grant_for_pulse(g)
        if str(g_norm.get("deadline") or "").strip():
            quality["with_deadline"] += 1
            if not had_deadline and g_norm.get("deadline_source"):
                quality["deadline_backfilled"] += 1
        reg = (
            g_norm.get("regulation_url")
            or g_norm.get("precise_regulation_url")
            or g_norm.get("eurlex_url")
            or ""
        )
        if str(reg).startswith("http"):
            quality["with_regulation"] += 1
        dl = _parse_iso_date(g_norm.get("deadline"))
        if dl and dl < today and not is_continuous(g_norm):
            quality["past_deadline_closed"] += 1
            # honest status: do not keep "active" past deadline
            if str(g_norm.get("status") or "").lower() in ("active", "otwarty", "open", "trwa"):
                g_norm = dict(g_norm)
                g_norm["status"] = "closed"
                g_norm["is_outdated_warning"] = True
        shaped = pulse_item_from_grant(
            g_norm, today=today, new_days=new_days, closing_days=closing_days
        )
        kinds = shaped["pulse_kinds"]
        for k in kinds:
            if k in counts:
                counts[k] += 1
        if kind == "all":
            if kinds:
                items.append(shaped)
        elif kind in kinds:
            items.append(shaped)

    # Sort: closing first by deadline, then new by first_seen desc, planned by deadline
    def _sort_key(it: Dict[str, Any]):
        dl = it.get("deadline") or "9999-12-31"
        fs = it.get("first_seen_at") or ""
        if kind == "closing":
            return (dl,)
        if kind == "new":
            return (fs,)
        if kind == "planned":
            return (dl,)
        # current / all: prefer closing then new
        prio = 0
        if "closing" in it["pulse_kinds"]:
            prio = 0
        elif "new" in it["pulse_kinds"]:
            prio = 1
        elif "planned" in it["pulse_kinds"]:
            prio = 2
        else:
            prio = 3
        return (prio, dl)

    reverse = kind == "new"
    items.sort(key=_sort_key, reverse=reverse)
    items = items[: max(1, min(int(limit), 200))]

    inp = max(quality["input"], 1)
    return {
        "status": "ok",
        "kind": kind,
        "count": len(items),
        "counts": counts,
        "quality": {
            **quality,
            "deadline_rate": round(quality["with_deadline"] / inp, 3),
            "regulation_rate": round(quality["with_regulation"] / inp, 3),
        },
        "new_days": new_days,
        "closing_days": closing_days,
        "items": items,
        "generated_at": datetime.now(timezone.utc).isoformat(),
    }


def list_pulse_from_db(
    db: Session,
    *,
    kind: str = "current",
    limit: int = 50,
    status: Optional[str] = None,
) -> Dict[str, Any]:
    """Load catalog rows and return pulse listing."""
    from core.grants.catalog_service import nabory_search, search_catalog
    from core.grants.completeness import grant_dict_from_row
    from core.grants.models import Grant

    # Prefer nabory_search (catalog visibility) then raw ORM fallback
    grants: List[Dict[str, Any]] = []
    try:
        kwargs: Dict[str, Any] = {"q": "", "limit": 500}
        if status:
            kwargs["status"] = status
        resp = nabory_search(db, **kwargs)
        nabory = resp.get("nabory") if isinstance(resp, dict) else None
        if nabory:
            grants = list(nabory)
    except Exception:
        grants = []

    if not grants:
        rows = db.query(Grant).limit(500).all()
        grants = [grant_dict_from_row(r) for r in rows]
        if status:
            grants = [g for g in grants if str(g.get("status") or "") == status]

    return list_pulse_grants(grants, kind=kind, limit=limit)


# ── Metrics (in-process; optional Redis later) ───────────────────────────────

_METRICS: Dict[str, float] = {
    "jobs_processed": 0,
    "jobs_failed": 0,
    "eurlex_grounded": 0,
    "eurlex_skipped": 0,
    "pulse_cycles": 0,
    "law_checks": 0,
    "law_changes": 0,
    "deadlines_backfilled": 0,
}


def metrics_increment(key: str, amount: float = 1.0) -> None:
    _METRICS[key] = float(_METRICS.get(key, 0)) + amount


def metrics_snapshot() -> Dict[str, Any]:
    return dict(_METRICS)


def reset_metrics() -> None:
    for k in list(_METRICS.keys()):
        _METRICS[k] = 0