backend/core/grants/dossier_ops.py

"""F2: batch dossier refresh + readiness distribution stats."""
from __future__ import annotations

import logging
import os
from typing import Any, Dict, List, Optional

from sqlalchemy.orm import Session

from core.grants.completeness import grant_dict_from_row
from core.grants.models import Grant
from core.grants.program_dossier import dossier_readiness, ensure_program_dossier

logger = logging.getLogger(__name__)


def compute_readiness_distribution(
    db: Session,
    *,
    limit: int = 5000,
    statuses: Optional[List[str]] = None,
) -> Dict[str, Any]:
    """Aggregate ready/partial/blind for catalog grants (no network)."""
    statuses = statuses or ["active", "planned"]
    rows = (
        db.query(Grant)
        .filter(Grant.status.in_(statuses))
        .limit(limit)
        .all()
    )
    counts = {"ready": 0, "partial": 0, "blind": 0, "unknown": 0}
    primary_pdf = 0
    pack_sizes: List[int] = []
    for row in rows:
        item = grant_dict_from_row(row)
        docs = item.get("regulation_documents") if isinstance(item.get("regulation_documents"), list) else []
        primary = str(item.get("precise_regulation_url") or item.get("regulation_url") or "")
        if not docs and item.get("regulation_urls"):
            docs = [{"url": u, "role": "dokument"} for u in (item.get("regulation_urls") or []) if u]
        r = dossier_readiness(docs, primary)
        level = r.get("level") or "unknown"
        if level in counts:
            counts[level] += 1
        else:
            counts["unknown"] += 1
        if primary.lower().endswith(".pdf"):
            primary_pdf += 1
        pack_sizes.append(len(docs))

    n = len(rows)
    denom = max(n, 1)
    not_blind = counts["ready"] + counts["partial"]
    # PLAN §13.5: % active catalog ≠ blind. Only meaningful when n > 0.
    pct_not_blind = round(100.0 * not_blind / denom, 1) if n > 0 else None
    return {
        "total": n,
        "ready": counts["ready"],
        "partial": counts["partial"],
        "blind": counts["blind"],
        "unknown": counts["unknown"],
        "levels": dict(counts),
        "not_blind": not_blind,
        "pct_ready": round(100.0 * counts["ready"] / denom, 1) if n > 0 else None,
        "pct_partial": round(100.0 * counts["partial"] / denom, 1) if n > 0 else None,
        "pct_blind": round(100.0 * counts["blind"] / denom, 1) if n > 0 else None,
        "pct_not_blind": pct_not_blind,
        "meets_dod_70_not_blind": bool(n > 0 and pct_not_blind is not None and pct_not_blind >= 70.0),
        "sample_size": n,
        "primary_pdf_count": primary_pdf,
        "pct_primary_pdf": round(100.0 * primary_pdf / denom, 1) if n > 0 else None,
        "avg_pack_size": round(sum(pack_sizes) / denom, 2) if pack_sizes else 0.0,
        "flags": {
            "REQUIRE_DOSSIER_OR_CONSENT": os.environ.get("REQUIRE_DOSSIER_OR_CONSENT", "true"),
            "ALLOW_STRUCTURE_WITHOUT_REGULATION": os.environ.get(
                "ALLOW_STRUCTURE_WITHOUT_REGULATION", "true"
            ),
            "ENABLE_REGULATION_PACK_FETCH": os.environ.get("ENABLE_REGULATION_PACK_FETCH", "true"),
        },
    }


async def batch_refresh_dossiers(
    db: Session,
    *,
    limit: int = 50,
    only_blind: bool = True,
    fetch_page: bool = True,
    ingest: bool = True,
) -> Dict[str, Any]:
    """
    Cron-oriented: ensure dossiers for top active grants missing packs.
    Env: DOSSIER_BATCH_LIMIT (default 50).
    """
    limit = int(os.environ.get("DOSSIER_BATCH_LIMIT", str(limit)))
    q = db.query(Grant).filter(Grant.status.in_(["active", "planned"]))
    rows = q.order_by(Grant.updated_at.desc() if hasattr(Grant, "updated_at") else Grant.id.desc()).limit(
        limit * 3
    ).all()

    selected: List[Grant] = []
    for row in rows:
        item = grant_dict_from_row(row)
        docs = item.get("regulation_documents") or []
        primary = str(item.get("precise_regulation_url") or item.get("regulation_url") or "")
        r = dossier_readiness(
            docs if isinstance(docs, list) else [],
            primary,
        )
        if only_blind and r.get("level") != "blind":
            continue
        selected.append(row)
        if len(selected) >= limit:
            break

    ok = fail = 0
    levels_after: Dict[str, int] = {"ready": 0, "partial": 0, "blind": 0}
    for row in selected:
        try:
            d = await ensure_program_dossier(
                db, grant_row=row, fetch_page=fetch_page, ingest=ingest
            )
            if d.get("ok"):
                ok += 1
                lvl = (d.get("readiness") or {}).get("level") or "blind"
                levels_after[lvl] = levels_after.get(lvl, 0) + 1
            else:
                fail += 1
        except Exception as e:
            logger.warning("[DossierOps] batch fail %s: %s", row.id, e)
            fail += 1

    return {
        "processed": len(selected),
        "ok": ok,
        "failed": fail,
        "levels_after": levels_after,
        "limit": limit,
    }