src/metadata_normalizer.py

"""
metadata_normalizer.py — Runtime metadata normalization for OpenItaLaw.

Normalizes the heterogeneous field names in doc_metadata.jsonl into canonical
fields at FAISS load time, adds battle_ground detection, pre-republican flags,
and hierarchy_level.
"""

from __future__ import annotations

import logging
import re
from typing import Any

log = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Battle-ground keyword mapping (12 constitutional battlegrounds)
# ---------------------------------------------------------------------------

BATTLE_GROUND_KEYWORDS: dict[str, list[str]] = {
    "LAVORO": [
        "lavoro", "lavorat", "impiego", "occupazione", "disoccupazione",
        "salario", "retribuzione", "ccnl", "sindac", "sciopero",
        "precari", "contratto a termine", "jobs act", "somministrazione",
        "rider", "gig economy", "infortuni", "sicurezza lavoro",
    ],
    "SALUTE": [
        "salute", "sanitari", "ssn", "ospedale", "medic", "farmac",
        "vaccin", "pandemia", "covid", "liste d'attesa", "lea",
        "pronto soccorso", "asl", "azienda sanitaria",
    ],
    "ISTRUZIONE": [
        "istruzione", "scuola", "scolastic", "universit", "docent",
        "insegnament", "studenti", "neet", "dispersione", "formazione",
        "buona scuola", "diritto allo studio", "borse di studio",
    ],
    "UGUAGLIANZA": [
        "uguaglianza", "discriminazione", "pari opportunit", "disuguaglianza",
        "gini", "povertà", "reddito di cittadinanza", "inclusione",
        "genere", "parità", "disabilit", "immigra", "integrazione",
    ],
    "GIUSTIZIA": [
        "giustizia", "processo", "magistrat", "giudice", "tribunale",
        "tar", "corte", "avvocat", "gratuito patrocinio", "detenuti",
        "carcere", "prescrizione", "procedura penale", "procedura civile",
    ],
    "AMBIENTE": [
        "ambiente", "ecologi", "inquinamento", "rifiuti", "clima",
        "energie rinnovabili", "biodiversità", "paesaggio", "acqua",
        "sostenibilit", "emissioni", "condono edilizio", "abusivismo",
    ],
    "WELFARE": [
        "welfare", "pensione", "previdenz", "assistenz", "inps",
        "assegno", "maternità", "disabilità", "non autosufficien",
        "reddito", "invalidità", "indennità",
    ],
    "DEMOCRAZIA": [
        "democrazia", "elettoral", "referendum", "voto", "elezioni",
        "parlamento", "camera", "senato", "astensionismo", "sovranità",
        "partecipazione", "petizione", "iniziativa legislativa",
    ],
    "FISCO": [
        "fisco", "tribut", "imposta", "tasse", "iva", "irpef", "ires",
        "evasione", "flat tax", "progressiv", "agenzia delle entrate",
        "contribuent", "dichiarazione dei redditi",
    ],
    "TRASPARENZA_PA": [
        "trasparenza", "anticorruzione", "anac", "appalti", "corruzione",
        "foia", "accesso civico", "accesso agli atti", "whistleblow",
        "pubblica amministrazione", "conflitto di interessi",
    ],
    "EUROPA": [
        "europa", "ue", "unione europea", "direttiva", "regolamento ue",
        "corte di giustizia", "cedu", "recepimento", "infrazione",
        "trattato", "commissione europea", "parlamento europeo",
    ],
    "DIGITALE": [
        "digitale", "internet", "privacy", "gdpr", "dato personale",
        "intelligenza artificiale", "cybersicurezza", "spid", "cie",
        "pa digitale", "connettività", "banda larga",
    ],
}

# ---------------------------------------------------------------------------
# Canonical field mappings
# ---------------------------------------------------------------------------

_VALIDITY_ALIASES = {
    "validity_status", "vigenza", "stato_vigore", "status_vigenza",
    "abrogazione", "vigente",
}

_TITLE_ALIASES = {
    "title", "titolo", "source_title", "nome", "denominazione",
}

_DATE_ALIASES = {
    "pub_date", "data_pubbl", "enactment_date", "data",
    "data_pubblicazione", "data_entrata_vigore",
}

_URN_ALIASES = {
    "urn", "id", "url", "uri", "identifier", "normattiva_url",
}

# ---------------------------------------------------------------------------
# Normalization logic
# ---------------------------------------------------------------------------

_PRE_REPUBLICAN_YEAR = 1948
_RE_YEAR = re.compile(r"\b(1[0-9]{3}|20[0-9]{2})\b")


def _extract_year(date_str: str) -> int | None:
    """Extract 4-digit year from various date formats."""
    if not date_str:
        return None
    m = _RE_YEAR.search(str(date_str))
    return int(m.group(1)) if m else None


def _normalize_validity(doc: dict) -> str:
    """Resolve validity_status from whichever alias field is present."""
    for alias in _VALIDITY_ALIASES:
        val = doc.get(alias)
        if val and isinstance(val, str) and val.strip():
            raw = val.strip().lower()
            if any(k in raw for k in ("abrogat", "non vigente", "non in vigore")):
                return "abrogato"
            if any(k in raw for k in ("vigente", "in_corso", "in vigore")):
                return "in_corso"
            return raw
    return "sconosciuto"


def _resolve_first(doc: dict, aliases: set[str]) -> str:
    """Return the first non-empty string value for any of the given keys."""
    for key in aliases:
        val = doc.get(key)
        if val and isinstance(val, str) and val.strip():
            return val.strip()
    return ""


def _detect_battle_grounds(doc: dict) -> list[str]:
    """Detect which constitutional battle grounds a document relates to."""
    text = " ".join(
        str(doc.get(k, ""))
        for k in ("title", "titolo", "source_title", "urn", "text", "body")
    ).lower()
    if not text.strip():
        return []
    grounds = []
    for ground, keywords in BATTLE_GROUND_KEYWORDS.items():
        if any(kw in text for kw in keywords):
            grounds.append(ground)
    return grounds


_HIERARCHY = [
    ("costituzione", 1),
    ("legge.costituzionale", 2),
    ("trattato", 2),
    ("regolamento.ue", 3), ("regolamento.ce", 3), ("regolamento.cee", 3),
    ("direttiva.ue", 3), ("direttiva.ce", 3), ("direttiva.cee", 3),
    ("legge", 4),
    ("decreto.legge", 4),
    ("decreto.legislativo", 4),
    ("decreto.del.presidente.della.repubblica", 5),
    ("decreto.del.presidente.del.consiglio", 5),
    ("decreto.ministeriale", 6),
    ("regio.decreto", 4),
    ("regolamento", 7),
    ("circolare", 8),
]


def _infer_hierarchy_level(urn: str, title: str = "") -> int:
    """Return numeric hierarchy level (1=highest) from URN or title."""
    combined = f"{urn} {title}".lower()
    for pattern, level in _HIERARCHY:
        if pattern in combined or pattern.replace(".", " ") in combined:
            return level
    return 9


def normalize_metadata_entry(doc: dict) -> dict:
    """Normalize a single metadata entry in-place and return it.

    Adds canonical fields without removing originals so downstream
    code that references old field names still works.
    """
    # Canonical URN
    doc["urn"] = _resolve_first(doc, _URN_ALIASES) or doc.get("urn", "")

    # Canonical title
    doc["title"] = _resolve_first(doc, _TITLE_ALIASES) or doc.get("title", "")

    # Canonical date
    doc["pub_date"] = _resolve_first(doc, _DATE_ALIASES) or doc.get("pub_date", "")

    # Validity status
    doc["validity_status"] = _normalize_validity(doc)

    # Year and era flags
    year = _extract_year(doc["pub_date"])
    doc["enactment_year"] = year
    doc["is_pre_republican"] = bool(year and year < _PRE_REPUBLICAN_YEAR)

    # Hierarchy level
    doc["hierarchy_level"] = _infer_hierarchy_level(doc["urn"], doc["title"])

    # Battle grounds
    doc["battle_grounds"] = _detect_battle_grounds(doc)

    return doc


def normalize_all_metadata(metadata: list[dict]) -> list[dict]:
    """Normalize entire metadata list in-place. Called once at FAISS load time."""
    count = 0
    for doc in metadata:
        normalize_metadata_entry(doc)
        count += 1
    log.info("Normalized %d metadata entries", count)
    return metadata