""" metadata_normalizer.py — Runtime metadata normalization for OpenItaLaw. Normalizes the heterogeneous field names in doc_metadata.jsonl into canonical fields at FAISS load time, adds battle_ground detection, pre-republican flags, and hierarchy_level. """ from __future__ import annotations import logging import re from typing import Any log = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Battle-ground keyword mapping (12 constitutional battlegrounds) # --------------------------------------------------------------------------- BATTLE_GROUND_KEYWORDS: dict[str, list[str]] = { "LAVORO": [ "lavoro", "lavorat", "impiego", "occupazione", "disoccupazione", "salario", "retribuzione", "ccnl", "sindac", "sciopero", "precari", "contratto a termine", "jobs act", "somministrazione", "rider", "gig economy", "infortuni", "sicurezza lavoro", ], "SALUTE": [ "salute", "sanitari", "ssn", "ospedale", "medic", "farmac", "vaccin", "pandemia", "covid", "liste d'attesa", "lea", "pronto soccorso", "asl", "azienda sanitaria", ], "ISTRUZIONE": [ "istruzione", "scuola", "scolastic", "universit", "docent", "insegnament", "studenti", "neet", "dispersione", "formazione", "buona scuola", "diritto allo studio", "borse di studio", ], "UGUAGLIANZA": [ "uguaglianza", "discriminazione", "pari opportunit", "disuguaglianza", "gini", "povertà", "reddito di cittadinanza", "inclusione", "genere", "parità", "disabilit", "immigra", "integrazione", ], "GIUSTIZIA": [ "giustizia", "processo", "magistrat", "giudice", "tribunale", "tar", "corte", "avvocat", "gratuito patrocinio", "detenuti", "carcere", "prescrizione", "procedura penale", "procedura civile", ], "AMBIENTE": [ "ambiente", "ecologi", "inquinamento", "rifiuti", "clima", "energie rinnovabili", "biodiversità", "paesaggio", "acqua", "sostenibilit", "emissioni", "condono edilizio", "abusivismo", ], "WELFARE": [ "welfare", "pensione", "previdenz", "assistenz", "inps", "assegno", "maternità", "disabilità", "non autosufficien", "reddito", "invalidità", "indennità", ], "DEMOCRAZIA": [ "democrazia", "elettoral", "referendum", "voto", "elezioni", "parlamento", "camera", "senato", "astensionismo", "sovranità", "partecipazione", "petizione", "iniziativa legislativa", ], "FISCO": [ "fisco", "tribut", "imposta", "tasse", "iva", "irpef", "ires", "evasione", "flat tax", "progressiv", "agenzia delle entrate", "contribuent", "dichiarazione dei redditi", ], "TRASPARENZA_PA": [ "trasparenza", "anticorruzione", "anac", "appalti", "corruzione", "foia", "accesso civico", "accesso agli atti", "whistleblow", "pubblica amministrazione", "conflitto di interessi", ], "EUROPA": [ "europa", "ue", "unione europea", "direttiva", "regolamento ue", "corte di giustizia", "cedu", "recepimento", "infrazione", "trattato", "commissione europea", "parlamento europeo", ], "DIGITALE": [ "digitale", "internet", "privacy", "gdpr", "dato personale", "intelligenza artificiale", "cybersicurezza", "spid", "cie", "pa digitale", "connettività", "banda larga", ], } # --------------------------------------------------------------------------- # Canonical field mappings # --------------------------------------------------------------------------- _VALIDITY_ALIASES = { "validity_status", "vigenza", "stato_vigore", "status_vigenza", "abrogazione", "vigente", } _TITLE_ALIASES = { "title", "titolo", "source_title", "nome", "denominazione", } _DATE_ALIASES = { "pub_date", "data_pubbl", "enactment_date", "data", "data_pubblicazione", "data_entrata_vigore", } _URN_ALIASES = { "urn", "id", "url", "uri", "identifier", "normattiva_url", } # --------------------------------------------------------------------------- # Normalization logic # --------------------------------------------------------------------------- _PRE_REPUBLICAN_YEAR = 1948 _RE_YEAR = re.compile(r"\b(1[0-9]{3}|20[0-9]{2})\b") def _extract_year(date_str: str) -> int | None: """Extract 4-digit year from various date formats.""" if not date_str: return None m = _RE_YEAR.search(str(date_str)) return int(m.group(1)) if m else None def _normalize_validity(doc: dict) -> str: """Resolve validity_status from whichever alias field is present.""" for alias in _VALIDITY_ALIASES: val = doc.get(alias) if val and isinstance(val, str) and val.strip(): raw = val.strip().lower() if any(k in raw for k in ("abrogat", "non vigente", "non in vigore")): return "abrogato" if any(k in raw for k in ("vigente", "in_corso", "in vigore")): return "in_corso" return raw return "sconosciuto" def _resolve_first(doc: dict, aliases: set[str]) -> str: """Return the first non-empty string value for any of the given keys.""" for key in aliases: val = doc.get(key) if val and isinstance(val, str) and val.strip(): return val.strip() return "" def _detect_battle_grounds(doc: dict) -> list[str]: """Detect which constitutional battle grounds a document relates to.""" text = " ".join( str(doc.get(k, "")) for k in ("title", "titolo", "source_title", "urn", "text", "body") ).lower() if not text.strip(): return [] grounds = [] for ground, keywords in BATTLE_GROUND_KEYWORDS.items(): if any(kw in text for kw in keywords): grounds.append(ground) return grounds _HIERARCHY = [ ("costituzione", 1), ("legge.costituzionale", 2), ("trattato", 2), ("regolamento.ue", 3), ("regolamento.ce", 3), ("regolamento.cee", 3), ("direttiva.ue", 3), ("direttiva.ce", 3), ("direttiva.cee", 3), ("legge", 4), ("decreto.legge", 4), ("decreto.legislativo", 4), ("decreto.del.presidente.della.repubblica", 5), ("decreto.del.presidente.del.consiglio", 5), ("decreto.ministeriale", 6), ("regio.decreto", 4), ("regolamento", 7), ("circolare", 8), ] def _infer_hierarchy_level(urn: str, title: str = "") -> int: """Return numeric hierarchy level (1=highest) from URN or title.""" combined = f"{urn} {title}".lower() for pattern, level in _HIERARCHY: if pattern in combined or pattern.replace(".", " ") in combined: return level return 9 def normalize_metadata_entry(doc: dict) -> dict: """Normalize a single metadata entry in-place and return it. Adds canonical fields without removing originals so downstream code that references old field names still works. """ # Canonical URN doc["urn"] = _resolve_first(doc, _URN_ALIASES) or doc.get("urn", "") # Canonical title doc["title"] = _resolve_first(doc, _TITLE_ALIASES) or doc.get("title", "") # Canonical date doc["pub_date"] = _resolve_first(doc, _DATE_ALIASES) or doc.get("pub_date", "") # Validity status doc["validity_status"] = _normalize_validity(doc) # Year and era flags year = _extract_year(doc["pub_date"]) doc["enactment_year"] = year doc["is_pre_republican"] = bool(year and year < _PRE_REPUBLICAN_YEAR) # Hierarchy level doc["hierarchy_level"] = _infer_hierarchy_level(doc["urn"], doc["title"]) # Battle grounds doc["battle_grounds"] = _detect_battle_grounds(doc) return doc def normalize_all_metadata(metadata: list[dict]) -> list[dict]: """Normalize entire metadata list in-place. Called once at FAISS load time.""" count = 0 for doc in metadata: normalize_metadata_entry(doc) count += 1 log.info("Normalized %d metadata entries", count) return metadata