normative_metadata.py

from __future__ import annotations

import json
import re
from pathlib import Path
from typing import Any

from utils import normalize_for_search, search_terms_match


PROFILE_SCHEMA = "MCKF-DocumentSemanticProfiles-v1.0"


def load_semantic_profiles(path: Path) -> dict[str, Any]:
    if not path.exists():
        return {"schema": PROFILE_SCHEMA, "documents": {}}
    payload = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(payload, dict) or not isinstance(payload.get("documents", {}), dict):
        raise ValueError("document_semantic_profiles.json must contain a documents object")
    return payload


def document_profile(profiles: dict[str, Any], document_id: str) -> dict[str, Any]:
    return dict((profiles.get("documents", {}) or {}).get(document_id, {}) or {})


def build_article_normative_metadata(
    article: dict[str, Any],
    document: dict[str, Any],
    profile: dict[str, Any] | None = None,
) -> dict[str, Any]:
    """Compile headings plus reviewed document knowledge into a semantic address."""
    profile = profile or {}
    article_id = str(article.get("article_id", "") or "")
    override = dict((profile.get("article_overrides", {}) or {}).get(article_id, {}) or {})
    heading_path = article.get("heading_path", []) or []
    article_heading = _clean_heading(str(article.get("title", "") or ""))
    regulates = str(
        override.get("regulates", "")
        or article_heading
        or _fallback_regulates(article)
        or f"{article_id} yürürlük durumu"
    ).strip()
    display_heading = str(
        override.get("display_heading", "")
        or article_heading
        or regulates
    ).strip()

    domain_path = _unique(
        list(profile.get("domain_path", []) or [])
        + list(document.get("domain_tags", []) or [])
        + [item.get("title", "") for item in heading_path if item.get("title")]
    )
    canonical_concepts = _unique(
        list(override.get("canonical_concepts", []) or [])
        + ([regulates] if regulates else [])
    )
    query_aliases = _unique(
        list(override.get("query_aliases", []) or [])
        + canonical_concepts
        + ([article_heading] if article_heading else [])
    )
    legal_effect_types = _unique(
        list(override.get("legal_effect_types", []) or [])
        + _infer_legal_effect_types(str(article.get("source_text", "") or ""), article_heading)
    )
    subject_classes = _unique(
        list(override.get("subject_classes", []) or [])
        + _infer_subject_classes(str(article.get("source_text", "") or ""), article_heading)
    )
    regulated_situations = _unique(
        list(override.get("regulated_situations", []) or [])
        + ([regulates] if regulates else [])
    )
    exclusions = _unique(list(override.get("exclusions", []) or []))
    legal_operations = _unique(list(override.get("legal_operations", []) or []))
    competent_authorities = _unique(list(override.get("competent_authorities", []) or []))
    operational_actors = _unique(list(override.get("operational_actors", []) or []))
    normative_variables = _extract_profile_variables(
        " ".join(
            [article_heading]
            + [str(item.get("title", "") or "") for item in heading_path]
            + [str(article.get("source_text", "") or "")]
        ),
        profile,
    )
    variable_values = [value for values in normative_variables.values() for value in values]
    variable_search_values = _profile_variable_search_values(normative_variables, profile)

    searchable_parts = (
        domain_path
        + canonical_concepts
        + query_aliases
        + legal_effect_types
        + subject_classes
        + regulated_situations
        + legal_operations
        + competent_authorities
        + operational_actors
        + variable_values
        + variable_search_values
    )
    canonical_address = " > ".join(
        _unique(
            [str(document.get("title", "") or "")]
            + [str(item.get("title", "") or "") for item in heading_path]
            + [article_id, regulates]
        )
    )
    return {
        "schema": "MCKF-NormativeAddress-v1.0",
        "document_id": document.get("document_id", ""),
        "article_id": article_id,
        "document_type": document.get("document_type", ""),
        "article_kind": _article_kind(article_id),
        "heading_path": heading_path,
        "article_heading": article_heading,
        "display_heading": display_heading,
        "domain_path": domain_path,
        "regulates": regulates,
        "canonical_concepts": canonical_concepts,
        "query_aliases": query_aliases,
        "subject_classes": subject_classes,
        "regulated_situations": regulated_situations,
        "legal_effect_types": legal_effect_types,
        "exclusions": exclusions,
        "legal_operations": legal_operations,
        "competent_authorities": competent_authorities,
        "operational_actors": operational_actors,
        "review_notes": list(override.get("review_notes", []) or []),
        # Human-authored, source-locked presentation semantics.  These fields
        # are optional for ordinary retrieval, but they are the only material
        # the deterministic Knowledge Assistant may present as a canonical
        # summary without asking an LLM to interpret the provision.
        "approved_summary": str(override.get("approved_summary", "") or ""),
        "approved_points": list(override.get("approved_points", []) or []),
        "inventory_summary": str(override.get("inventory_summary", "") or ""),
        "topic_memberships": _unique(list(override.get("topic_memberships", []) or [])),
        "effective_from": str(override.get("effective_from", "") or ""),
        "effective_to": str(override.get("effective_to", "") or ""),
        "normative_variables": normative_variables,
        "canonical_address": canonical_address,
        "search_text": " ".join(_unique(searchable_parts)),
        "review_status": override.get("review_status", "derived_from_structure"),
        "provenance": {
            "heading_derived": bool(article_heading or heading_path),
            "document_profile": bool(override),
            "document_variable_schema": bool(profile.get("semantic_dimensions")),
        },
    }


def semantic_address_score(question: str, address: dict[str, Any]) -> float:
    """Score a query against a reviewed canonical address, not raw article text."""
    query = normalize_for_search(question)
    if not query:
        return 0.0
    if _matches_excluded_scope(query, address):
        return 0.0
    aliases = [
        normalize_for_search(str(value))
        for value in (
            list(address.get("query_aliases", []) or [])
            + list(address.get("canonical_concepts", []) or [])
            + list(address.get("regulated_situations", []) or [])
        )
        if value
    ]
    if any(alias and alias in query for alias in aliases):
        return 1.0
    query_terms = _content_terms(query)
    if not query_terms:
        return 0.0
    address_terms = _content_terms(normalize_for_search(str(address.get("search_text", "") or "")))
    if not address_terms:
        return 0.0
    matched = sum(1 for term in query_terms if _term_matches(term, address_terms))
    if len(query_terms) > 1 and matched < 2:
        return 0.0
    return matched / len(query_terms)


def semantic_address_text(address: dict[str, Any]) -> str:
    return str(address.get("search_text", "") or "")


def semantic_address_focus_text(address: dict[str, Any]) -> str:
    """Return only article-specific address terms suitable for lexical indexes."""
    provenance = address.get("provenance", {}) or {}
    if not (provenance.get("document_profile") or provenance.get("heading_derived")):
        return ""
    values = [address.get("article_heading", ""), address.get("regulates", "")]
    values += list(address.get("canonical_concepts", []) or [])
    values += list(address.get("query_aliases", []) or [])
    values += list(address.get("regulated_situations", []) or [])
    if provenance.get("document_profile"):
        values += list(address.get("legal_operations", []) or [])
        values += list(address.get("competent_authorities", []) or [])
        values += list(address.get("operational_actors", []) or [])
    return " ".join(_unique(values))


def _matches_excluded_scope(query: str, address: dict[str, Any]) -> bool:
    """Reject a reviewed address when the question primarily names an excluded concept."""
    query_terms = _content_terms(query)
    if not query_terms:
        return False
    article_id = normalize_for_search(str(address.get("article_id", "") or ""))
    if article_id and article_id in query:
        return False
    positive_terms = _content_terms(
        normalize_for_search(
            " ".join(
                str(value)
                for value in (
                    [address.get("regulates", "")]
                    + list(address.get("canonical_concepts", []) or [])
                    + list(address.get("query_aliases", []) or [])
                    + list(address.get("regulated_situations", []) or [])
                )
                if value
            )
        )
    )
    for exclusion in address.get("exclusions", []) or []:
        exclusion_terms = _content_terms(normalize_for_search(str(exclusion))) - positive_terms
        if len(exclusion_terms) < 2:
            continue
        matched = sum(1 for term in query_terms if _term_matches(term, exclusion_terms))
        if (
            matched >= 2
            and matched / len(query_terms) >= 0.40
            and matched / len(exclusion_terms) >= 0.50
        ):
            return True
    return False


def _clean_heading(value: str) -> str:
    value = re.sub(r"[.:]+\s*\d*\s*$", "", value).strip()
    return value if normalize_for_search(value) != "baslik bulunamadi" else ""


def _fallback_regulates(article: dict[str, Any]) -> str:
    text = re.sub(r"\s+", " ", str(article.get("source_text", "") or "")).strip()
    article_id = str(article.get("article_id", "") or "Madde")
    normalized_source = normalize_for_search(text)
    if "mulga" in normalized_source and not _has_substantive_body(text):
        return f"{article_id} hükmünün mülga olma durumu"
    if "iptal" in normalized_source and not _has_substantive_body(text):
        return f"{article_id} hükmünün iptal durumu"
    text = re.sub(r"^(?:Ek |Geçici )?Madde\s+\w+\s*[-–]?\s*", "", text, flags=re.IGNORECASE)
    text = re.sub(r"^\([^)]*(?:Ek|Değişik|Mülga)[^)]*\)\s*", "", text, flags=re.IGNORECASE)
    text = re.sub(r"^\d{1,3}\s*$", "", text).strip()
    text = text.replace("T.C.", "T.C")
    sentence = re.split(r"(?<=[.!?])\s+", text, maxsplit=1)[0]
    return sentence[:220].rsplit(" ", 1)[0] if len(sentence) > 220 else sentence


def _has_substantive_body(text: str) -> bool:
    body = re.sub(r"^(?:Ek |Geçici )?Madde\s+\w+\s*[-–]?\s*", "", text, flags=re.IGNORECASE)
    body = re.sub(r"^\([^)]*(?:Ek|Değişik|Mülga|İptal)[^)]*\)\s*", "", body, flags=re.IGNORECASE)
    body = re.sub(r"^\d{1,3}\s*$", "", body).strip()
    return len(normalize_for_search(body).split()) >= 4


def _infer_legal_effect_types(text: str, heading: str) -> list[str]:
    normalized = normalize_for_search(f"{heading} {text}")
    patterns = {
        "appointment": ("atanir", "atanır", "atanma", "secilir"),
        "authority_or_duty": ("gorev", "yetki", "sorumlu"),
        "status_restoration": ("yeniden ogren", "yeniden kayit", "ilisigi kesilen", "baslayabilirler"),
        "payment_obligation": ("odenir", "ucret", "ücret", "ödeme", "ödemeler", "katki payi"),
        "eligibility": ("yararlan", "hak kazan", "sartiyla"),
        "sanction": ("ceza", "iptal", "ilisigi kesilir", "ilişiği kesilir"),
        "establishment": ("kurulur", "acilir", "açılır", "teskil edilir", "teşkil edilir"),
        "definition": ("tanim", "tanımlanır", "ifade eder", "denir"),
        "repealed_or_annulled": ("mulga", "iptal"),
    }
    return [effect for effect, markers in patterns.items() if any(marker in normalized for marker in markers)]


def _infer_subject_classes(text: str, heading: str) -> list[str]:
    normalized = normalize_for_search(f"{heading} {text[:600]}")
    subjects = {
        "student": ("ogrenci", "öğrenci"),
        "academic_staff": ("ogretim elemani", "öğretim elemanı", "ogretim uyesi", "öğretim üyesi", "arastirma gorevlisi"),
        "rector": ("rektor", "rektör",),
        "dean": ("dekan",),
        "university": ("universite", "üniversite",),
        "higher_education_institution": ("yuksekogretim kurumu",),
    }
    return [subject for subject, markers in subjects.items() if any(marker in normalized for marker in markers)]


def _extract_profile_variables(text: str, profile: dict[str, Any]) -> dict[str, list[str]]:
    """Apply document-specific semantic dimensions without document-specific code."""
    normalized = normalize_for_search(text)
    extracted: dict[str, list[str]] = {}
    for dimension, values in (profile.get("semantic_dimensions", {}) or {}).items():
        matched: list[str] = []
        for canonical, aliases in (values or {}).items():
            markers = [canonical, *(aliases or [])]
            if any(normalize_for_search(str(marker)) in normalized for marker in markers if marker):
                matched.append(str(canonical))
        if matched:
            extracted[str(dimension)] = _unique(matched)
    return extracted


def _profile_variable_search_values(
    variables: dict[str, list[str]],
    profile: dict[str, Any],
) -> list[str]:
    """Attach institution-maintained aliases to matched canonical variables."""
    dimensions = profile.get("semantic_dimensions", {}) or {}
    searchable: list[str] = []
    for dimension, canonical_values in variables.items():
        configured = dimensions.get(dimension, {}) or {}
        for canonical in canonical_values:
            searchable.append(canonical)
            searchable.extend(str(value) for value in configured.get(canonical, []) or [])
    return _unique(searchable)


def _article_kind(article_id: str) -> str:
    normalized = normalize_for_search(article_id)
    if normalized.startswith("gecici madde"):
        return "geçici_madde"
    if normalized.startswith("ek madde"):
        return "ek_madde"
    return "madde"


def _content_terms(text: str) -> set[str]:
    stopwords = {
        "hangi", "nedir", "nasil", "madde", "maddelerde", "duzenleniyor", "duzenlenir",
        "sayili", "kanun", "kanuna", "yonetmelik", "yonerge", "gore", "ile", "ve", "bir",
    }
    return {term for term in text.split() if len(term) >= 2 and term not in stopwords and not term.isdigit()}


def _term_matches(term: str, candidates: set[str]) -> bool:
    return any(search_terms_match(term, candidate) for candidate in candidates)


def _unique(values: list[Any]) -> list[str]:
    result: list[str] = []
    seen: set[str] = set()
    for value in values:
        text = str(value or "").strip()
        key = normalize_for_search(text)
        if text and key not in seen:
            seen.add(key)
            result.append(text)
    return result