Blame for turkceocr/postprocess/checksum_decoder.py

"""
TurkceVerify — Algoritmik Sağlama Destekli Çıkarım (Checksum-Guided Constrained Decoding).

Türkiye'ye özgü resmi ve kurumsal kimlik/hesap numaralarını:
1. T.C. Kimlik Numarası (TCKN - 11 hane modüler sağlama + 11. hane çift sayı kısıtı),
2. Vergi Kimlik Numarası (VKN - 10 hane modüler sağlama),
3. TR IBAN (26 karakter ISO 7064 MOD-97-10 + TCMB 10. karakter rezerv '0' kısıtı +
   deterministik kontrol basamağı hesabı),
hem ham metin (OCR görsel karışıklık matrisi üzerinden) hem de doğrudan modelin
logit / olasılık matrisi (Constrained Beam Search) üzerinden doğrular ve onarır.
"""

from __future__ import annotations

import itertools
import re
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Sequence, Tuple

import numpy as np

# OCR motorlarında rakamlarla en sık karışan karakterler ve görsel yakınlık olasılıkları
DIGIT_VISUAL_CONFUSIONS: Dict[str, List[Tuple[str, float]]] = {
    "0": [("0", 0.999), ("8", 0.015), ("6", 0.012), ("9", 0.010)],
    "1": [("1", 0.999), ("7", 0.015), ("4", 0.010)],
    "2": [("2", 0.999), ("7", 0.012), ("3", 0.010), ("8", 0.010)],
    "3": [("3", 0.999), ("8", 0.018), ("5", 0.012), ("2", 0.010)],
    "4": [("4", 0.999), ("1", 0.012), ("9", 0.010)],
    "5": [("5", 0.999), ("6", 0.015), ("3", 0.012), ("8", 0.010)],
    "6": [("6", 0.999), ("8", 0.015), ("5", 0.015), ("0", 0.012)],
    "7": [("7", 0.999), ("1", 0.015), ("2", 0.010)],
    "8": [("8", 0.999), ("0", 0.015), ("3", 0.015), ("6", 0.015), ("9", 0.012), ("2", 0.010)],
    "9": [("9", 0.999), ("8", 0.012), ("0", 0.010), ("4", 0.010)],
    # Harf -> Rakam OCR karışıklıkları (Harf zaten hatalı okuma olduğu için aday dağılımı daha geniştir)
    "O": [("0", 0.95), ("6", 0.35), ("8", 0.35), ("9", 0.20)],
    "o": [("0", 0.95), ("6", 0.35), ("8", 0.25)],
    "D": [("0", 0.92), ("8", 0.25)],
    "Q": [("0", 0.92), ("9", 0.25)],
    "I": [("1", 0.95), ("7", 0.30), ("4", 0.20)],
    "İ": [("1", 0.95), ("7", 0.25)],
    "ı": [("1", 0.95), ("7", 0.25)],
    "l": [("1", 0.95), ("7", 0.25)],
    "|": [("1", 0.95), ("7", 0.25)],
    "Z": [("2", 0.95), ("7", 0.30)],
    "z": [("2", 0.95), ("7", 0.25)],
    "E": [("3", 0.85), ("8", 0.35), ("5", 0.25)],
    "A": [("4", 0.90), ("8", 0.20)],
    "S": [("5", 0.95), ("8", 0.30), ("6", 0.25), ("3", 0.20)],
    "s": [("5", 0.95), ("6", 0.25)],
    "Ş": [("5", 0.92), ("6", 0.25)],
    "ş": [("5", 0.92), ("6", 0.25)],
    "G": [("6", 0.95), ("0", 0.30), ("8", 0.25)],
    "b": [("6", 0.92), ("8", 0.25)],
    "T": [("7", 0.92), ("1", 0.30)],
    "B": [("8", 0.95), ("3", 0.30), ("6", 0.30), ("0", 0.20)],
    "g": [("9", 0.92), ("8", 0.25)],
}

# Tüm görsel karışıklık karakterlerini kapsayan regex karakter sınıfı
CONFUSABLE_DIGIT_CHARS = "0-9ODoQIİıl|ZzEASsŞşGbTBg"


def compute_tckn_check_digits(first_nine: Sequence[int]) -> Tuple[int, int]:
    """İlk 9 haneden TCKN 10. ve 11. kontrol basamaklarını hesaplar."""
    if len(first_nine) != 9 or first_nine[0] == 0:
        raise ValueError("TCKN ilk 9 hanesi geçerli olmalı ve 0 ile başlamamalıdır.")
    odd_sum = first_nine[0] + first_nine[2] + first_nine[4] + first_nine[6] + first_nine[8]
    even_sum = first_nine[1] + first_nine[3] + first_nine[5] + first_nine[7]
    d10 = ((odd_sum * 7) - even_sum) % 10
    d11 = (odd_sum + even_sum + d10) % 10
    return d10, d11


def validate_tckn(tckn: str) -> bool:
    """
    11 haneli T.C. Kimlik Numarasının algoritmik sağlamasını kontrol eder:
    - 11 rakamdan oluşmalı, '0' ile başlamamalı ve 11. hane çift sayı olmalıdır.
    - 10. hane = ((Tek haneler toplamı * 7) - Çift haneler toplamı) mod 10
    - 11. hane = (İlk 10 hane toplamı) mod 10
    """
    cleaned = re.sub(r"\s+", "", tckn)
    if len(cleaned) != 11 or not cleaned.isdigit() or cleaned[0] == "0":
        return False
    digits = [int(c) for c in cleaned]
    if digits[10] % 2 != 0:
        return False
    d10, d11 = compute_tckn_check_digits(digits[:9])
    return digits[9] == d10 and digits[10] == d11


def compute_vkn_check_digit(first_nine: Sequence[int]) -> int:
    """İlk 9 haneden 10 haneli Vergi Kimlik Numarası (VKN) 10. kontrol basamağını hesaplar."""
    if len(first_nine) != 9:
        raise ValueError("VKN ilk 9 hanesi 9 rakam olmalıdır.")
    total = 0
    for i in range(9):
        v = (first_nine[i] + (9 - i)) % 10
        if v == 9:
            c = 9
        else:
            c = (v * (2 ** (9 - i))) % 9
        total += c
    return (10 - (total % 10)) % 10


def validate_vkn(vkn: str) -> bool:
    """10 haneli Vergi Kimlik Numarasının resmi modüler sağlamasını kontrol eder."""
    cleaned = re.sub(r"\s+", "", vkn)
    if len(cleaned) != 10 or not cleaned.isdigit():
        return False
    digits = [int(c) for c in cleaned]
    return digits[9] == compute_vkn_check_digit(digits[:9])


def validate_tr_iban(iban: str, enforce_reserve_zero: bool = True) -> bool:
    """
    26 karakterli TR IBAN ('TR' + 2 kontrol basamağı + 5 banka kodu + 1 rezerv '0' + 16 hesap no)
    ISO 7064 MOD-97-10 algoritmasını ve TCMB rezerv basamağı ('0') kuralını doğrular.
    """
    cleaned = re.sub(r"\s+", "", iban).upper()
    if len(cleaned) != 26 or not cleaned.startswith("TR"):
        return False
    if not cleaned[2:].isdigit():
        return False
    # TCMB Standardı: 10. karakter (indeks 9, banka kodundan sonraki rezerv hanesi) '0' olmalıdır
    if enforce_reserve_zero and cleaned[9] != "0":
        return False
    rearranged = cleaned[4:] + "2927" + cleaned[2:4]
    return int(rearranged) % 97 == 1


def decode_tckn_from_logits(
    digit_probs: np.ndarray,
    top_k_per_pos: int = 3,
) -> Tuple[str, float, bool]:
    """
    Boyutu (11, 10) olan rakam olasılık/logit matrisinden TCKN algoritmik sağlamasını
    geçen en yüksek olasılıklı 11 haneli diziyi (Constrained Beam Search ile) bulur.
    """
    probs = np.asarray(digit_probs, dtype=np.float64)
    if probs.shape != (11, 10):
        raise ValueError(f"TCKN logit matrisi (11, 10) boyutunda olmalıdır, gelen: {probs.shape}")

    shifted = probs - np.max(probs, axis=1, keepdims=True)
    exp_p = np.exp(shifted)
    norm_probs = exp_p / np.sum(exp_p, axis=1, keepdims=True)
    log_probs = np.log(np.clip(norm_probs, 1e-12, 1.0))

    # Kısıt 1: İlk hanede '0' olamaz
    log_probs[0, 0] = -1e9
    # Kısıt 2: 11. hane (indeks 10) her zaman çift sayıdır
    for odd_d in (1, 3, 5, 7, 9):
        log_probs[10, odd_d] = -1e9

    top_indices = np.argsort(-log_probs[:, :10], axis=1)[:, :top_k_per_pos]

    best_candidate: Optional[str] = None
    best_score = -float("inf")

    beam: List[Tuple[float, List[int]]] = [(0.0, [])]
    max_beam_width = 128

    for pos in range(9):
        next_beam: List[Tuple[float, List[int]]] = []
        for score, prefix in beam:
            for d in top_indices[pos]:
                if pos == 0 and d == 0:
                    continue
                next_beam.append((score + float(log_probs[pos, d]), prefix + [int(d)]))
        next_beam.sort(key=lambda x: x[0], reverse=True)
        beam = next_beam[:max_beam_width]

    for score_9, first_nine in beam:
        d10, d11 = compute_tckn_check_digits(first_nine)
        total_score = score_9 + float(log_probs[9, d10]) + float(log_probs[10, d11])
        if total_score > best_score:
            best_score = total_score
            best_candidate = "".join(str(x) for x in first_nine) + f"{d10}{d11}"

    if best_candidate and validate_tckn(best_candidate):
        return best_candidate, float(np.exp(best_score / 11.0)), True

    greedy = "".join(str(int(np.argmax(log_probs[i]))) for i in range(11))
    return greedy, 0.0, validate_tckn(greedy)


def decode_vkn_from_logits(
    digit_probs: np.ndarray,
    top_k_per_pos: int = 3,
) -> Tuple[str, float, bool]:
    """
    Boyutu (10, 10) olan rakam olasılık/logit matrisinden VKN sağlamasını geçen
    en yüksek olasılıklı 10 haneli Vergi Kimlik Numarasını bulur.
    """
    probs = np.asarray(digit_probs, dtype=np.float64)
    if probs.shape != (10, 10):
        raise ValueError(f"VKN logit matrisi (10, 10) boyutunda olmalıdır, gelen: {probs.shape}")

    shifted = probs - np.max(probs, axis=1, keepdims=True)
    exp_p = np.exp(shifted)
    norm_probs = exp_p / np.sum(exp_p, axis=1, keepdims=True)
    log_probs = np.log(np.clip(norm_probs, 1e-12, 1.0))

    top_indices = np.argsort(-log_probs, axis=1)[:, :top_k_per_pos]

    beam: List[Tuple[float, List[int]]] = [(0.0, [])]
    max_beam_width = 128

    for pos in range(9):
        next_beam: List[Tuple[float, List[int]]] = []
        for score, prefix in beam:
            for d in top_indices[pos]:
                next_beam.append((score + float(log_probs[pos, d]), prefix + [int(d)]))
        next_beam.sort(key=lambda x: x[0], reverse=True)
        beam = next_beam[:max_beam_width]

    best_candidate: Optional[str] = None
    best_score = -float("inf")

    for score_9, first_nine in beam:
        d10 = compute_vkn_check_digit(first_nine)
        total_score = score_9 + float(log_probs[9, d10])
        if total_score > best_score:
            best_score = total_score
            best_candidate = "".join(str(x) for x in first_nine) + str(d10)

    if best_candidate and validate_vkn(best_candidate):
        return best_candidate, float(np.exp(best_score / 10.0)), True

    greedy = "".join(str(int(np.argmax(log_probs[i]))) for i in range(10))
    return greedy, 0.0, validate_vkn(greedy)


def decode_iban_from_logits(
    digit_probs_24: np.ndarray,
    top_k_per_pos: int = 3,
) -> Tuple[str, float, bool]:
    """
    TR IBAN'ın 'TR' sonrası 24 rakamı için (24, 10) boyutundaki olasılık matrisinden:
    1. TCMB rezerv basamağını (`pos 7 == 0`) sabitler,
    2. BBAN (`pos 2..23`) içindeki en belirsiz pozisyonlarda top-k arama yaparken
       kontrol basamaklarını (`pos 0, 1`) `98 - (int(bban + "292700") % 97)` ile
       deterministik hesaplayıp gerçek logit olasılığıyla puanlar!
    """
    probs = np.asarray(digit_probs_24, dtype=np.float64)
    if probs.shape != (24, 10):
        raise ValueError(f"TR IBAN rakam matrisi (24, 10) boyutunda olmalıdır, gelen: {probs.shape}")

    shifted = probs - np.max(probs, axis=1, keepdims=True)
    exp_p = np.exp(shifted)
    norm_probs = exp_p / np.sum(exp_p, axis=1, keepdims=True)
    log_probs = np.log(np.clip(norm_probs, 1e-12, 1.0))

    # TCMB rezerv hanesi (24 rakam içinde indeks 7) her zaman '0' olmalıdır
    log_probs[7, 1:] = -1e9

    greedy_digits = [int(np.argmax(log_probs[i])) for i in range(24)]
    greedy_iban = "TR" + "".join(str(d) for d in greedy_digits)
    if validate_tr_iban(greedy_iban):
        mean_prob = float(np.exp(np.mean([log_probs[i, greedy_digits[i]] for i in range(24)])))
        return greedy_iban, mean_prob, True

    # BBAN pozisyonları (indeks 2..23, rezerv indeks 7 hariç) içindeki en belirsiz 5 pozisyonu bul
    bban_positions = [i for i in range(2, 24) if i != 7]
    confidences = [
        (float(log_probs[i, greedy_digits[i]] - np.partition(log_probs[i], -2)[-2]), i)
        for i in bban_positions
    ]
    confidences.sort(key=lambda x: x[0])
    uncertain_set = {pos for _, pos in confidences[:5]}

    options_per_bban: List[List[Tuple[int, float]]] = []
    for i in range(2, 24):
        if i == 7:
            options_per_bban.append([(0, float(log_probs[7, 0]))])
        elif i in uncertain_set:
            top_d = np.argsort(-log_probs[i])[:top_k_per_pos]
            options_per_bban.append([(int(d), float(log_probs[i, d])) for d in top_d])
        else:
            d = greedy_digits[i]
            options_per_bban.append([(d, float(log_probs[i, d]))])

    best_iban = greedy_iban
    best_score = -float("inf")
    found = False

    for bban_combo in itertools.product(*options_per_bban):
        bban_score = sum(item[1] for item in bban_combo)
        bban_str = "".join(str(item[0]) for item in bban_combo)
        # Kontrol basamakları (pos 0 ve pos 1) BBAN'dan %100 deterministik hesaplanır!
        check_val = 98 - (int(bban_str + "292700") % 97)
        c0, c1 = check_val // 10, check_val % 10
        total_score = bban_score + float(log_probs[0, c0]) + float(log_probs[1, c1])
        if total_score > best_score:
            best_score = total_score
            best_iban = f"TR{c0}{c1}{bban_str}"
            found = True

    return best_iban, (float(np.exp(best_score / 24.0)) if found else 0.0), found


def _text_to_digit_prob_matrix(raw_text: str, expected_len: int) -> Optional[np.ndarray]:
    """
    Ham OCR metnini görsel karışıklık önyargılarına (prior) göre
    (expected_len, 10) olasılık matrisine dönüştürür.
    """
    cleaned = re.sub(r"[\s\-]", "", raw_text)
    if len(cleaned) != expected_len:
        return None

    matrix = np.full((expected_len, 10), 0.001, dtype=np.float64)
    for idx, ch in enumerate(cleaned):
        candidates = DIGIT_VISUAL_CONFUSIONS.get(ch)
        if candidates is None:
            if ch.isdigit():
                matrix[idx, int(ch)] = 0.999
            else:
                matrix[idx, :] = 0.1
        else:
            for digit_str, weight in candidates:
                matrix[idx, int(digit_str)] = max(matrix[idx, int(digit_str)], weight)
    return matrix


def repair_tckn_text(raw_tckn: str) -> Tuple[str, bool]:
    """Görsel OCR hataları içeren 11 karakterlik TCKN adayını algoritmik sağlamayla onarır."""
    cleaned = re.sub(r"[\s\-]", "", raw_tckn)
    if validate_tckn(cleaned):
        return cleaned, True

    prob_matrix = _text_to_digit_prob_matrix(cleaned, 11)
    if prob_matrix is None:
        return raw_tckn, False

    repaired, _, is_valid = decode_tckn_from_logits(np.log(prob_matrix), top_k_per_pos=4)
    return (repaired if is_valid else raw_tckn), is_valid


def repair_vkn_text(raw_vkn: str) -> Tuple[str, bool]:
    """Görsel OCR hataları içeren 10 karakterlik VKN adayını algoritmik sağlamayla onarır."""
    cleaned = re.sub(r"[\s\-]", "", raw_vkn)
    if validate_vkn(cleaned):
        return cleaned, True

    prob_matrix = _text_to_digit_prob_matrix(cleaned, 10)
    if prob_matrix is None:
        return raw_vkn, False

    repaired, _, is_valid = decode_vkn_from_logits(np.log(prob_matrix), top_k_per_pos=4)
    return (repaired if is_valid else raw_vkn), is_valid


def repair_tr_iban(raw_iban: str) -> Tuple[str, bool]:
    """Görsel OCR hataları içeren 26 karakterlik TR IBAN adayını MOD-97-10 sağlamasıyla onarır."""
    cleaned = re.sub(r"[\s\-]", "", raw_iban).upper()
    if validate_tr_iban(cleaned):
        return cleaned, True

    if len(cleaned) != 26:
        return raw_iban, False

    body = cleaned[2:]
    prob_matrix = _text_to_digit_prob_matrix(body, 24)
    if prob_matrix is None:
        return raw_iban, False

    repaired, _, is_valid = decode_iban_from_logits(np.log(prob_matrix), top_k_per_pos=3)
    return (repaired if is_valid else raw_iban), is_valid


@dataclass
class VerificationReport:
    """TurkceVerify belge doğrulama ve onarım raporu."""

    original_text: str
    verified_text: str
    repaired_fields: List[Dict[str, str]] = field(default_factory=list)
    valid_tckns: List[str] = field(default_factory=list)
    valid_vkns: List[str] = field(default_factory=list)
    valid_ibans: List[str] = field(default_factory=list)


class TurkceVerify:
    """
    Tam sayfa VLM veya satır OCR çıktılarını tarayarak TCKN, VKN, TR IBAN,
    finansal toplamlar ve Türkçe büyük/küçük harf hatalarını tek geçişte onaran
    birleşik doğrulama motoru.
    """

    def __init__(self, restore_turkish_caps: bool = True) -> None:
        from turkceocr.postprocess.financial_solver import TurkishFinancialSolver
        from turkceocr.postprocess.turkish_casing import TurkishDiacriticRestorer

        self.restore_turkish_caps = restore_turkish_caps
        self.diacritic_restorer = TurkishDiacriticRestorer()
        self.financial_solver = TurkishFinancialSolver()

    def verify_and_repair_document(self, text: str) -> VerificationReport:
        """
        Belge metnindeki `<tckn>`, `<vkn>`, `<iban>` etiketlerini ve etiketlenmemiş
        TCKN/VKN/IBAN kalıplarını bularak algoritmik sağlamayla onarır.
        """
        report = VerificationReport(original_text=text, verified_text=text)
        working = text

        if self.restore_turkish_caps:
            working = self.diacritic_restorer.restore_text(working)

        # 1. Etiketli <tckn>...</tckn> veya "TCKN / T.C. Kimlik No: XXXXXXXXXXX" kalıpları
        def _replace_tagged_tckn(match: re.Match) -> str:
            raw_val = match.group(1).strip()
            repaired, ok = repair_tckn_text(raw_val)
            if ok:
                report.valid_tckns.append(repaired)
                if repaired != raw_val:
                    report.repaired_fields.append(
                        {"type": "TCKN", "original": raw_val, "repaired": repaired}
                    )
            return f"<tckn>{repaired}</tckn>"

        working = re.sub(r"<tckn>(.*?)</tckn>", _replace_tagged_tckn, working)

        def _replace_labeled_tckn(match: re.Match) -> str:
            prefix, raw_val = match.group(1), match.group(2)
            repaired, ok = repair_tckn_text(raw_val)
            if ok:
                if repaired not in report.valid_tckns:
                    report.valid_tckns.append(repaired)
                if repaired != raw_val:
                    report.repaired_fields.append(
                        {"type": "TCKN", "original": raw_val, "repaired": repaired}
                    )
            return f"{prefix}{repaired}"

        working = re.sub(
            rf"((?:T\.?C\.?\s*K[İI]ML[İI]K\s*(?:NO|NUMARASI)?|TCKN)\s*[:\-]?\s*)([{CONFUSABLE_DIGIT_CHARS}]{{11}})\b",
            _replace_labeled_tckn,
            working,
            flags=re.IGNORECASE,
        )

        # 2. Etiketli <vkn>...</vkn> veya "VKN / Vergi No: XXXXXXXXXX" kalıpları
        def _replace_tagged_vkn(match: re.Match) -> str:
            raw_val = match.group(1).strip()
            repaired, ok = repair_vkn_text(raw_val)
            if ok:
                report.valid_vkns.append(repaired)
                if repaired != raw_val:
                    report.repaired_fields.append(
                        {"type": "VKN", "original": raw_val, "repaired": repaired}
                    )
            return f"<vkn>{repaired}</vkn>"

        working = re.sub(r"<vkn>(.*?)</vkn>", _replace_tagged_vkn, working)

        def _replace_labeled_vkn(match: re.Match) -> str:
            prefix, raw_val = match.group(1), match.group(2)
            repaired, ok = repair_vkn_text(raw_val)
            if ok:
                if repaired not in report.valid_vkns:
                    report.valid_vkns.append(repaired)
                if repaired != raw_val:
                    report.repaired_fields.append(
                        {"type": "VKN", "original": raw_val, "repaired": repaired}
                    )
            return f"{prefix}{repaired}"

        working = re.sub(
            rf"((?:VERG[İI]\s*(?:K[İI]ML[İI]K\s*)?(?:NO|NUMARASI)|VKN)\s*[:\-]?\s*)([{CONFUSABLE_DIGIT_CHARS}]{{10}})\b",
            _replace_labeled_vkn,
            working,
            flags=re.IGNORECASE,
        )

        # 3. TR IBAN kalıpları (<iban>...</iban> veya TRxx xxxx ...)
        def _replace_tagged_iban(match: re.Match) -> str:
            raw_val = match.group(1).strip()
            repaired, ok = repair_tr_iban(raw_val)
            if ok:
                report.valid_ibans.append(repaired)
                if repaired != re.sub(r"[\s\-]", "", raw_val).upper():
                    report.repaired_fields.append(
                        {"type": "IBAN", "original": raw_val, "repaired": repaired}
                    )
            return f"<iban>{repaired}</iban>"

        working = re.sub(r"<iban>(.*?)</iban>", _replace_tagged_iban, working)

        def _replace_free_iban(match: re.Match) -> str:
            raw_val = match.group(0)
            compact = re.sub(r"\s+", "", raw_val)
            if len(compact) != 26:
                return raw_val
            repaired, ok = repair_tr_iban(compact)
            if ok:
                if repaired not in report.valid_ibans:
                    report.valid_ibans.append(repaired)
                if repaired != compact.upper():
                    report.repaired_fields.append(
                        {"type": "IBAN", "original": raw_val, "repaired": repaired}
                    )
                return repaired
            return raw_val

        working = re.sub(
            rf"\bTR[{CONFUSABLE_DIGIT_CHARS}]{{2}}(?:\s?[{CONFUSABLE_DIGIT_CHARS}]{{4}}){{5}}\s?[{CONFUSABLE_DIGIT_CHARS}]{{2}}\b",
            _replace_free_iban,
            working,
        )

        report.verified_text = working
        return report


# ASCII uyumluluk takma adı