"""F2: batch dossier refresh + readiness distribution stats.""" from __future__ import annotations import logging import os from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from core.grants.completeness import grant_dict_from_row from core.grants.models import Grant from core.grants.program_dossier import dossier_readiness, ensure_program_dossier logger = logging.getLogger(__name__) def compute_readiness_distribution( db: Session, *, limit: int = 5000, statuses: Optional[List[str]] = None, ) -> Dict[str, Any]: """Aggregate ready/partial/blind for catalog grants (no network).""" statuses = statuses or ["active", "planned"] rows = ( db.query(Grant) .filter(Grant.status.in_(statuses)) .limit(limit) .all() ) counts = {"ready": 0, "partial": 0, "blind": 0, "unknown": 0} primary_pdf = 0 pack_sizes: List[int] = [] for row in rows: item = grant_dict_from_row(row) docs = item.get("regulation_documents") if isinstance(item.get("regulation_documents"), list) else [] primary = str(item.get("precise_regulation_url") or item.get("regulation_url") or "") if not docs and item.get("regulation_urls"): docs = [{"url": u, "role": "dokument"} for u in (item.get("regulation_urls") or []) if u] r = dossier_readiness(docs, primary) level = r.get("level") or "unknown" if level in counts: counts[level] += 1 else: counts["unknown"] += 1 if primary.lower().endswith(".pdf"): primary_pdf += 1 pack_sizes.append(len(docs)) n = len(rows) denom = max(n, 1) not_blind = counts["ready"] + counts["partial"] # PLAN §13.5: % active catalog ≠ blind. Only meaningful when n > 0. pct_not_blind = round(100.0 * not_blind / denom, 1) if n > 0 else None return { "total": n, "ready": counts["ready"], "partial": counts["partial"], "blind": counts["blind"], "unknown": counts["unknown"], "levels": dict(counts), "not_blind": not_blind, "pct_ready": round(100.0 * counts["ready"] / denom, 1) if n > 0 else None, "pct_partial": round(100.0 * counts["partial"] / denom, 1) if n > 0 else None, "pct_blind": round(100.0 * counts["blind"] / denom, 1) if n > 0 else None, "pct_not_blind": pct_not_blind, "meets_dod_70_not_blind": bool(n > 0 and pct_not_blind is not None and pct_not_blind >= 70.0), "sample_size": n, "primary_pdf_count": primary_pdf, "pct_primary_pdf": round(100.0 * primary_pdf / denom, 1) if n > 0 else None, "avg_pack_size": round(sum(pack_sizes) / denom, 2) if pack_sizes else 0.0, "flags": { "REQUIRE_DOSSIER_OR_CONSENT": os.environ.get("REQUIRE_DOSSIER_OR_CONSENT", "true"), "ALLOW_STRUCTURE_WITHOUT_REGULATION": os.environ.get( "ALLOW_STRUCTURE_WITHOUT_REGULATION", "true" ), "ENABLE_REGULATION_PACK_FETCH": os.environ.get("ENABLE_REGULATION_PACK_FETCH", "true"), }, } async def batch_refresh_dossiers( db: Session, *, limit: int = 50, only_blind: bool = True, fetch_page: bool = True, ingest: bool = True, ) -> Dict[str, Any]: """ Cron-oriented: ensure dossiers for top active grants missing packs. Env: DOSSIER_BATCH_LIMIT (default 50). """ limit = int(os.environ.get("DOSSIER_BATCH_LIMIT", str(limit))) q = db.query(Grant).filter(Grant.status.in_(["active", "planned"])) rows = q.order_by(Grant.updated_at.desc() if hasattr(Grant, "updated_at") else Grant.id.desc()).limit( limit * 3 ).all() selected: List[Grant] = [] for row in rows: item = grant_dict_from_row(row) docs = item.get("regulation_documents") or [] primary = str(item.get("precise_regulation_url") or item.get("regulation_url") or "") r = dossier_readiness( docs if isinstance(docs, list) else [], primary, ) if only_blind and r.get("level") != "blind": continue selected.append(row) if len(selected) >= limit: break ok = fail = 0 levels_after: Dict[str, int] = {"ready": 0, "partial": 0, "blind": 0} for row in selected: try: d = await ensure_program_dossier( db, grant_row=row, fetch_page=fetch_page, ingest=ingest ) if d.get("ok"): ok += 1 lvl = (d.get("readiness") or {}).get("level") or "blind" levels_after[lvl] = levels_after.get(lvl, 0) + 1 else: fail += 1 except Exception as e: logger.warning("[DossierOps] batch fail %s: %s", row.id, e) fail += 1 return { "processed": len(selected), "ok": ok, "failed": fail, "levels_after": levels_after, "limit": limit, }