""" Grant Pulse — classify and list new / planned / closing / current grants. Pure catalog logic (no network). Used by API, scheduler metrics, firm advisor. """ from __future__ import annotations import os from datetime import date, datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Sequence from sqlalchemy.orm import Session from core.search.deadline_utils import parse_deadline_robust PULSE_KINDS = frozenset({"new", "planned", "closing", "current", "all"}) DEFAULT_NEW_DAYS = int(os.environ.get("PULSE_NEW_DAYS", "14")) DEFAULT_CLOSING_DAYS = int(os.environ.get("PULSE_CLOSING_DAYS", "30")) def _parse_iso_date(value: Any) -> Optional[date]: if value is None: return None if isinstance(value, date) and not isinstance(value, datetime): return value if isinstance(value, datetime): return value.date() s = str(value).strip() if not s: return None parsed = parse_deadline_robust(s, "") if parsed: return parsed try: return datetime.fromisoformat(s.replace("Z", "+00:00")).date() except Exception: return None def _first_seen(grant: Dict[str, Any]) -> Optional[date]: raw = grant.get("raw_data") if isinstance(grant.get("raw_data"), dict) else {} for key in ( "first_seen_at", "fetched_at", "created_at", "data_pobrania", ): d = _parse_iso_date(grant.get(key) or raw.get(key)) if d: return d return None def is_continuous(grant: Dict[str, Any]) -> bool: status = str(grant.get("status") or "").lower() deadline = str(grant.get("deadline") or "").lower() blob = f"{status} {deadline} {grant.get('name') or ''} {grant.get('description') or ''}".lower() return any( k in blob for k in ("ciągły", "ciagly", "continuous", "beztermin", "cały rok", "caly rok") ) def classify_pulse_kinds( grant: Dict[str, Any], *, today: Optional[date] = None, new_days: int = DEFAULT_NEW_DAYS, closing_days: int = DEFAULT_CLOSING_DAYS, ) -> List[str]: """ Return zero or more pulse kinds for a grant dict. A grant may be both `new` and `closing`, or `planned` only, etc. """ today = today or date.today() kinds: List[str] = [] status = str(grant.get("status") or "").lower().strip() continuous = is_continuous(grant) deadline = _parse_iso_date(grant.get("deadline")) first_seen = _first_seen(grant) closed = status in ("closed", "zakończony", "zakonczony") or bool( grant.get("is_outdated_warning") ) if deadline and deadline < today and not continuous: closed = True if status == "planned" or status in ("planowany", "wkrótce", "zapowiedziany"): kinds.append("planned") elif not closed and deadline and deadline > today and status not in ("active", "otwarty"): # future deadline without active — treat as planned if far start-like wording if any( k in str(grant.get("name") or "").lower() + str(grant.get("description") or "").lower() for k in ("planowany", "wkrótce", "harmonogram", "rusza") ): kinds.append("planned") if not closed and (status in ("active", "otwarty", "trwa", "open") or continuous): kinds.append("current") elif not closed and deadline and deadline >= today: kinds.append("current") if not closed and first_seen: if first_seen >= today - timedelta(days=max(1, new_days)): kinds.append("new") if not closed and not continuous and deadline: if today <= deadline <= today + timedelta(days=max(1, closing_days)): kinds.append("closing") # de-dupe preserve order seen = set() out: List[str] = [] for k in kinds: if k not in seen: seen.add(k) out.append(k) return out def primary_pulse_kind(grant: Dict[str, Any], **kwargs: Any) -> str: kinds = classify_pulse_kinds(grant, **kwargs) # priority for single-label views for pref in ("closing", "new", "planned", "current"): if pref in kinds: return pref return "other" def verification_flags(grant: Dict[str, Any], *, today: Optional[date] = None) -> Dict[str, Any]: """Claim-level flags for deadline / status / regulation before alerts.""" today = today or date.today() continuous = is_continuous(grant) deadline = _parse_iso_date(grant.get("deadline")) status = str(grant.get("status") or "").lower() reg = ( grant.get("regulation_url") or grant.get("precise_regulation_url") or grant.get("eurlex_url") or "" ) reg_ok = str(reg).startswith("http") grounded = bool(grant.get("regulation_grounded") or grant.get("eurlex_grounded")) cred = float(grant.get("source_credibility_score") or 0) if cred > 1: cred = cred / 100.0 deadline_ok = continuous or (deadline is not None and deadline >= today) status_ok = status in ("active", "planned", "otwarty", "planowany", "open") or continuous if deadline and deadline < today and not continuous: status_ok = False alert_worthy = deadline_ok and status_ok and (reg_ok or grounded) and cred >= 0.35 return { "deadline_verified": bool(deadline_ok), "status_verified": bool(status_ok), "regulation_verified": bool(reg_ok or grounded), "continuous": continuous, "credibility_ok": cred >= 0.35, "alert_worthy": bool(alert_worthy), "reasons": _flag_reasons(deadline_ok, status_ok, reg_ok or grounded, continuous, deadline), } def _flag_reasons( deadline_ok: bool, status_ok: bool, reg_ok: bool, continuous: bool, deadline: Optional[date], ) -> List[str]: reasons: List[str] = [] if not deadline_ok: reasons.append("missing_or_past_deadline") if continuous: reasons.append("continuous_enrollment") if not status_ok: reasons.append("status_not_open") if not reg_ok: reasons.append("regulation_not_grounded") if deadline: reasons.append(f"deadline={deadline.isoformat()}") return reasons def pulse_item_from_grant( grant: Dict[str, Any], *, today: Optional[date] = None, new_days: int = DEFAULT_NEW_DAYS, closing_days: int = DEFAULT_CLOSING_DAYS, ) -> Dict[str, Any]: today = today or date.today() kinds = classify_pulse_kinds( grant, today=today, new_days=new_days, closing_days=closing_days ) flags = verification_flags(grant, today=today) first_seen = _first_seen(grant) return { "id": grant.get("id") or grant.get("source_id"), "name": grant.get("name"), "status": grant.get("status"), "deadline": grant.get("deadline") or "", "continuous": flags["continuous"], "operator": grant.get("operator") or grant.get("program") or grant.get("zrodlo") or "", "source": grant.get("source") or "", "url": grant.get("url") or grant.get("official_page_url") or "", "regulation_url": grant.get("regulation_url") or grant.get("precise_regulation_url") or "", "eurlex_url": grant.get("eurlex_url") or "", "celex_id": grant.get("celex_id") or (grant.get("eurlex") or {}).get("celex_id") if isinstance(grant.get("eurlex"), dict) else grant.get("celex_id"), "source_credibility_score": grant.get("source_credibility_score"), "data_quality_score": grant.get("data_quality_score"), "regulation_grounded": bool( grant.get("regulation_grounded") or grant.get("eurlex_grounded") ), "pulse_kinds": kinds, "primary_kind": primary_pulse_kind( grant, today=today, new_days=new_days, closing_days=closing_days ), "first_seen_at": first_seen.isoformat() if first_seen else None, "verification": flags, } def _normalize_grant_for_pulse(grant: Dict[str, Any]) -> Dict[str, Any]: """Ensure deadline/status honesty before pulse classify (text backfill, ISO).""" try: from core.search.deadline_utils import apply_deadline_to_grant_dict return apply_deadline_to_grant_dict(dict(grant)) except Exception: return dict(grant) def list_pulse_grants( grants: Sequence[Dict[str, Any]], *, kind: str = "current", limit: int = 50, today: Optional[date] = None, new_days: int = DEFAULT_NEW_DAYS, closing_days: int = DEFAULT_CLOSING_DAYS, ) -> Dict[str, Any]: """Filter and shape grants for pulse API. kind=all returns mixed current pulse.""" kind = (kind or "current").lower().strip() if kind not in PULSE_KINDS: kind = "current" today = today or date.today() items: List[Dict[str, Any]] = [] counts = {"new": 0, "planned": 0, "closing": 0, "current": 0} quality = { "input": 0, "with_deadline": 0, "deadline_backfilled": 0, "with_regulation": 0, "past_deadline_closed": 0, } for g in grants: quality["input"] += 1 had_deadline = bool(str((g or {}).get("deadline") or "").strip()) g_norm = _normalize_grant_for_pulse(g) if str(g_norm.get("deadline") or "").strip(): quality["with_deadline"] += 1 if not had_deadline and g_norm.get("deadline_source"): quality["deadline_backfilled"] += 1 reg = ( g_norm.get("regulation_url") or g_norm.get("precise_regulation_url") or g_norm.get("eurlex_url") or "" ) if str(reg).startswith("http"): quality["with_regulation"] += 1 dl = _parse_iso_date(g_norm.get("deadline")) if dl and dl < today and not is_continuous(g_norm): quality["past_deadline_closed"] += 1 # honest status: do not keep "active" past deadline if str(g_norm.get("status") or "").lower() in ("active", "otwarty", "open", "trwa"): g_norm = dict(g_norm) g_norm["status"] = "closed" g_norm["is_outdated_warning"] = True shaped = pulse_item_from_grant( g_norm, today=today, new_days=new_days, closing_days=closing_days ) kinds = shaped["pulse_kinds"] for k in kinds: if k in counts: counts[k] += 1 if kind == "all": if kinds: items.append(shaped) elif kind in kinds: items.append(shaped) # Sort: closing first by deadline, then new by first_seen desc, planned by deadline def _sort_key(it: Dict[str, Any]): dl = it.get("deadline") or "9999-12-31" fs = it.get("first_seen_at") or "" if kind == "closing": return (dl,) if kind == "new": return (fs,) if kind == "planned": return (dl,) # current / all: prefer closing then new prio = 0 if "closing" in it["pulse_kinds"]: prio = 0 elif "new" in it["pulse_kinds"]: prio = 1 elif "planned" in it["pulse_kinds"]: prio = 2 else: prio = 3 return (prio, dl) reverse = kind == "new" items.sort(key=_sort_key, reverse=reverse) items = items[: max(1, min(int(limit), 200))] inp = max(quality["input"], 1) return { "status": "ok", "kind": kind, "count": len(items), "counts": counts, "quality": { **quality, "deadline_rate": round(quality["with_deadline"] / inp, 3), "regulation_rate": round(quality["with_regulation"] / inp, 3), }, "new_days": new_days, "closing_days": closing_days, "items": items, "generated_at": datetime.now(timezone.utc).isoformat(), } def list_pulse_from_db( db: Session, *, kind: str = "current", limit: int = 50, status: Optional[str] = None, ) -> Dict[str, Any]: """Load catalog rows and return pulse listing.""" from core.grants.catalog_service import nabory_search, search_catalog from core.grants.completeness import grant_dict_from_row from core.grants.models import Grant # Prefer nabory_search (catalog visibility) then raw ORM fallback grants: List[Dict[str, Any]] = [] try: kwargs: Dict[str, Any] = {"q": "", "limit": 500} if status: kwargs["status"] = status resp = nabory_search(db, **kwargs) nabory = resp.get("nabory") if isinstance(resp, dict) else None if nabory: grants = list(nabory) except Exception: grants = [] if not grants: rows = db.query(Grant).limit(500).all() grants = [grant_dict_from_row(r) for r in rows] if status: grants = [g for g in grants if str(g.get("status") or "") == status] return list_pulse_grants(grants, kind=kind, limit=limit) # ── Metrics (in-process; optional Redis later) ─────────────────────────────── _METRICS: Dict[str, float] = { "jobs_processed": 0, "jobs_failed": 0, "eurlex_grounded": 0, "eurlex_skipped": 0, "pulse_cycles": 0, "law_checks": 0, "law_changes": 0, "deadlines_backfilled": 0, } def metrics_increment(key: str, amount: float = 1.0) -> None: _METRICS[key] = float(_METRICS.get(key, 0)) + amount def metrics_snapshot() -> Dict[str, Any]: return dict(_METRICS) def reset_metrics() -> None: for k in list(_METRICS.keys()): _METRICS[k] = 0