utils.py

import re
import unicodedata
from functools import lru_cache

GENERIC_TERMS = {
    "kanun", "kanunu", "madde", "yuksekogretim", "kurum", "kurumu",
    "kurumlari", "universite", "esaslari", "gorevleri", "yetkileri",
    "sartlari", "nedir", "nelerdir",
}

# Ortak "konu örtüşmesi" hesaplaması için genel/soru-kalıbı kelimeleri.
# Bu liste bilinçli olarak GENERIC_TERMS'ten AYRI tutuldu: GENERIC_TERMS
# başka amaçlarla (ör. terim vurgulama) kullanılıyor olabilir ve kapsamı
# farklı. TOPIC_STOPWORDS yalnızca topic_term_overlap() için, "bu kelime
# hemen hemen her maddede geçebilir, gerçek bir konu sinyali değildir"
# kriteriyle seçildi. Yeni bir yanlış-pozitif/yanlış-negatif örneği
# görüldüğünde önce burayı güncelle — proof_bundle.py ve answering.py
# ayrı ayrı kendi stopword listelerini TUTMAMALI, ikisi de bu fonksiyonu

GENERIC_TERMS = {
    "kanun", "kanunu", "madde", "yuksekogretim", "kurum", "kurumu",
    "kurumlari", "universite", "esaslari", "gorevleri", "yetkileri",
    "sartlari", "nedir", "nelerdir",
}

# Ortak "konu örtüşmesi" hesaplaması için genel/soru-kalıbı kelimeleri.
# Bu liste bilinçli olarak GENERIC_TERMS'ten AYRI tutuldu: GENERIC_TERMS
# başka amaçlarla (ör. terim vurgulama) kullanılıyor olabilir ve kapsamı
# farklı. TOPIC_STOPWORDS yalnızca topic_term_overlap() için, "bu kelime
# hemen hemen her maddede geçebilir, gerçek bir konu sinyali değildir"
# kriteriyle seçildi. Yeni bir yanlış-pozitif/yanlış-negatif örneği
# görüldüğünde önce burayı güncelle — proof_bundle.py ve answering.py
# ayrı ayrı kendi stopword listelerini TUTMAMALI, ikisi de bu fonksiyonu
# çağırmalı; aksi halde aynı hata iki yerde ayrı ayrı yamanır.
TOPIC_STOPWORDS = {
    "hangi", "nedir", "nasil", "kim", "kimler", "kimleri", "ne", "kadar",
    "gore", "sayili", "kanun", "kanuna", "kanunu", "madde", "maddesi",
    "hakkinda", "bilgi", "ver", "verir", "genel", "mevcut", "ilgili",
    "icin", "bir", "ve", "ile", "veya", "bu", "de", "da", "olan", "olarak",
    "kapsar", "duzenler", "neler", "nelerdir", "var", "yok", "midir", "mudur",
}


_SEARCH_TRANSLATION = str.maketrans("çğıöşü", "cgiosu")
_NON_SEARCH_CHARACTER_RE = re.compile(r"[^a-z0-9/\s-]")
_SEARCH_WHITESPACE_RE = re.compile(r"\s+")

_STEMMER = None
try:
    from TurkishStemmer import TurkishStemmer
    _STEMMER = TurkishStemmer()
except ImportError:
    try:
        import snowballstemmer
        _STEMMER = snowballstemmer.stemmer("turkish")
    except ImportError:
        try:
            from trnlp import TrnlpWord
            _STEMMER = "trnlp"
        except ImportError:
            _STEMMER = None


@lru_cache(maxsize=100_000)
def normalize_for_search(text: str) -> str:
    """Normalize Turkish text once and reuse it across retrieval channels.

    Corpus strings are scored repeatedly by BM25, semantic-address, role and
    evidence matchers during one query. The function is pure, so a bounded
    process-local cache removes duplicate Unicode work without changing any
    score or persisted representation.
    """
    text = unicodedata.normalize("NFKD", text or "")
    text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn")
    text = text.lower()
    text = text.translate(_SEARCH_TRANSLATION)
    text = _NON_SEARCH_CHARACTER_RE.sub(" ", text)
    return _SEARCH_WHITESPACE_RE.sub(" ", text).strip()


@lru_cache(maxsize=50_000)
def get_turkish_stem(word: str) -> str:
    """Extract the dictionary/morphological stem of a Turkish word.

    Supports TurkishStemmer, snowballstemmer and trnlp with LRU caching.
    Fallback to normalized word if no stemmer package is installed.
    """
    word_norm = normalize_for_search(word)
    if not word_norm or _STEMMER is None:
        return word_norm
    if len(word_norm) <= 3 or word_norm.isdigit():
        return word_norm
    try:
        if _STEMMER == "trnlp":
            from trnlp import TrnlpWord
            obj = TrnlpWord(word_norm)
            obj.get_stem
            stem = str(getattr(obj, "stem", "") or "").strip()
        elif hasattr(_STEMMER, "stem"):
            stem = _STEMMER.stem(word_norm)
        elif hasattr(_STEMMER, "stemWord"):
            stem = _STEMMER.stemWord(word_norm)
        else:
            stem = word_norm

        if stem and len(stem) >= 2:
            return normalize_for_search(stem)
    except Exception:
        pass
    return word_norm



def search_terms_match(left: str, right: str) -> bool:
    """Match Turkish inflectional variants using TRNLP stems and suffix heuristics.

    Exact and whole-token prefix matches cover ordinary case/possessive suffixes.
    TRNLP stem equivalence provides dictionary-level precision for agglutinative forms.
    The bounded common-stem rule acts as a robust fallback.
    """
    left = normalize_for_search(left)
    right = normalize_for_search(right)
    if not left or not right:
        return False
    if left == right:
        return True

    left_class = _legal_homonym_class(left)
    right_class = _legal_homonym_class(right)
    if left_class and right_class and left_class != right_class:
        return False

    # 1. TRNLP stem equivalence check
    left_stem = get_turkish_stem(left)
    right_stem = get_turkish_stem(right)
    if left_stem and right_stem and left_stem == right_stem:
        return True

    # 2. Suffix heuristic check
    if len(left) >= 4 and right.startswith(left) and _has_turkish_inflection(left, right):
        return True
    if len(right) >= 4 and left.startswith(right) and _has_turkish_inflection(right, left):
        return True

    # 3. Bounded common prefix rule
    common = 0
    for left_char, right_char in zip(left, right):
        if left_char != right_char:
            break
        common += 1
    shorter = min(len(left), len(right))
    if left_class == right_class == "establishment_action":
        return common >= 5
    return common >= 6 and common / max(shorter, 1) >= 0.60


def _legal_homonym_class(term: str) -> str:
    """Disambiguate high-impact Turkish legal homonyms before stemming.

    ``kurul`` as a governing body and ``kurulmak`` as establishment share a
    long orthographic stem but denote different legal objects.  A character
    stemmer must not turn "enstitü nasıl kurulur" into a match for a heading
    about "Kurulunun görevleri".
    """
    if re.fullmatch(
        r"kurul(?:u|un|unun|a|da|dan|ca|unca|lari|larin|larinin|lara|larda|lardan)?",
        term,
    ):
        return "governing_body"
    if re.match(
        r"kurul(?:ur|uyor|du|acak|an|mus|ma|masi|masin|mak|abil)",
        term,
    ):
        return "establishment_action"
    return ""


def _has_turkish_inflection(stem: str, extended: str) -> bool:
    """Accept a token prefix only when the remainder is a plausible suffix.

    Plain ``startswith`` treats proper names such as Gazi/Gaziantep as the
    same entity.  This bounded suffix inventory covers the case, possessive,
    plural and common copular forms needed by legal search while rejecting a
    lexical continuation such as ``antep``.
    """
    if not extended.startswith(stem) or len(extended) <= len(stem):
        return False
    suffix = extended[len(stem) :]
    return bool(
        re.fullmatch(
            r"(?:"
            r"[aeiu]|y[aeiu]|n[aeiu]|s[ai]|"
            r"d[ae]|t[ae]|d[ae]n|t[ae]n|"
            r"l[ae]r(?:[ai]|in|d[ae]|d[ae]n)?|"
            r"[iu]n|n[ıiu]n|inin|unun|sinin|"
            r"d[ai]r|t[ai]r|d[ae]ki|l[ıi]k|l[ıi]gi"
            r")",
            suffix,
        )
    )


def topic_term_overlap(question: str, evidence_texts: list[str], min_term_length: int = 2) -> float:
    """Fuzzy (prefix and TRNLP/stem aware), stopword-filtered overlap calculation.

    Returns 0.5 (neutral) when the question has no usable content terms after filtering.
    """
    raw_query_terms = [
        term for term in normalize_for_search(question).split()
        if len(term) >= min_term_length and term not in TOPIC_STOPWORDS and not term.isdigit()
    ]
    if not raw_query_terms:
        return 0.5

    query_stems = {get_turkish_stem(t) for t in raw_query_terms}
    evidence_tokens = normalize_for_search(" ".join(evidence_texts)).split()
    evidence_stems = {get_turkish_stem(t) for t in evidence_tokens}

    if not evidence_stems:
        return 0.0

    matched_stems = sum(
        1 for q_stem in query_stems
        if q_stem in evidence_stems or any(search_terms_match(q_stem, e_stem) for e_stem in evidence_stems)
    )
    return matched_stems / len(query_stems)


def compact_text(text: str, limit: int = 1100) -> str:
    text = dedupe_repeated_text(text)
    if len(text) <= limit:
        return text
    return text[:limit].rsplit(" ", 1)[0] + "..."


def strip_model_sources(text: str) -> str:
    if not text:
        return ""
    cleaned = text.strip()
    markers = [
        r"^#+\s*Kaynaklar\b", r"^#+\s*İlgili kaynaklar\b", r"^#+\s*Ilgili kaynaklar\b",
        r"^#+\s*İlgili maddeler\b", r"^#+\s*Ilgili maddeler\b", r"^#+\s*Kaynak maddeler\b",
        r"^Kaynaklar\b", r"^İlgili kaynaklar\b", r"^Ilgili kaynaklar\b",
        r"^İlgili maddeler\b", r"^Ilgili maddeler\b", r"^Kaynak maddeler\b", r"^Resmi kaynak\b",
    ]
    for marker in markers:
        match = re.search(marker, cleaned, flags=re.IGNORECASE | re.MULTILINE)
        if match:
            cleaned = cleaned[:match.start()].strip()
            break
    return cleaned


def clean_answer_text(answer: str, question: str = "") -> str:
    text = strip_model_sources(answer or "").strip()
    text = re.sub(r"^\s*(Soru|Question)\s*:\s*.*?\n+", "", text, flags=re.IGNORECASE | re.DOTALL)
    text = re.sub(r"^\s*(Cevap|Answer)\s*:\s*", "", text, flags=re.IGNORECASE)

    q = (question or "").strip()
    if q:
        text = re.sub(rf"^\s*{re.escape(q)}\s*\??\s*", "", text, flags=re.IGNORECASE)

    parts = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
    seen = set()
    cleaned = []
    for part in parts:
        part = dedupe_repeated_sentences(part)
        key = re.sub(r"\s+", " ", part.lower())
        if key in seen:
            continue
        seen.add(key)
        cleaned.append(part)

    return re.sub(r"\n{3,}", "\n\n", "\n\n".join(cleaned)).strip()


def dedupe_repeated_text(text: str) -> str:
    text = re.sub(r"\s+", " ", text or "").strip()
    if not text:
        return ""
    return dedupe_repeated_sentences(text)



def dedupe_repeated_sentences(text: str) -> str:
    text = re.sub(r"\s+", " ", text or "").strip()
    if not text:
        return ""

    sentences = re.split(r"(?<=[.!?])\s+", text)
    cleaned = []
    seen = set()
    previous = ""
    for sentence in sentences:
        sentence = sentence.strip()
        if not sentence:
            continue
        key = normalize_for_search(sentence)
        if key in seen:
            continue
        if previous and _looks_like_near_duplicate(previous, sentence):
            continue
        seen.add(key)
        cleaned.append(sentence)
        previous = sentence
    return " ".join(cleaned)


def _looks_like_near_duplicate(left: str, right: str) -> bool:
    left_tokens = set(normalize_for_search(left).split())
    right_tokens = set(normalize_for_search(right).split())
    if not left_tokens or not right_tokens:
        return False
    overlap = len(left_tokens & right_tokens) / max(min(len(left_tokens), len(right_tokens)), 1)
    return overlap >= 0.86


def query_terms(question: str) -> set[str]:
    stopwords = {
        "2547", "sayili", "kanun", "kanunu", "kanununda", "hangi",
        "nedir", "nelerdir", "olarak", "duzenlenir", "duzenlenmistir",
        "madde", "maddede", "gorev", "gorevleri",
    }
    terms = set()
    for token in normalize_for_search(question).split():
        if len(token) < 4 or token in stopwords:
            continue
        terms.add(token)
        if len(token) >= 6:
            terms.add(token[:6])
        if len(token) >= 5:
            terms.add(token[:5])
    return terms