""" TurkceVerify — Algoritmik Sağlama Destekli Çıkarım (Checksum-Guided Constrained Decoding). Türkiye'ye özgü resmi ve kurumsal kimlik/hesap numaralarını: 1. T.C. Kimlik Numarası (TCKN - 11 hane modüler sağlama + 11. hane çift sayı kısıtı), 2. Vergi Kimlik Numarası (VKN - 10 hane modüler sağlama), 3. TR IBAN (26 karakter ISO 7064 MOD-97-10 + TCMB 10. karakter rezerv '0' kısıtı + deterministik kontrol basamağı hesabı), hem ham metin (OCR görsel karışıklık matrisi üzerinden) hem de doğrudan modelin logit / olasılık matrisi (Constrained Beam Search) üzerinden doğrular ve onarır. """ from __future__ import annotations import itertools import re from dataclasses import dataclass, field from typing import Dict, List, Optional, Sequence, Tuple import numpy as np # OCR motorlarında rakamlarla en sık karışan karakterler ve görsel yakınlık olasılıkları DIGIT_VISUAL_CONFUSIONS: Dict[str, List[Tuple[str, float]]] = { "0": [("0", 0.999), ("8", 0.015), ("6", 0.012), ("9", 0.010)], "1": [("1", 0.999), ("7", 0.015), ("4", 0.010)], "2": [("2", 0.999), ("7", 0.012), ("3", 0.010), ("8", 0.010)], "3": [("3", 0.999), ("8", 0.018), ("5", 0.012), ("2", 0.010)], "4": [("4", 0.999), ("1", 0.012), ("9", 0.010)], "5": [("5", 0.999), ("6", 0.015), ("3", 0.012), ("8", 0.010)], "6": [("6", 0.999), ("8", 0.015), ("5", 0.015), ("0", 0.012)], "7": [("7", 0.999), ("1", 0.015), ("2", 0.010)], "8": [("8", 0.999), ("0", 0.015), ("3", 0.015), ("6", 0.015), ("9", 0.012), ("2", 0.010)], "9": [("9", 0.999), ("8", 0.012), ("0", 0.010), ("4", 0.010)], # Harf -> Rakam OCR karışıklıkları (Harf zaten hatalı okuma olduğu için aday dağılımı daha geniştir) "O": [("0", 0.95), ("6", 0.35), ("8", 0.35), ("9", 0.20)], "o": [("0", 0.95), ("6", 0.35), ("8", 0.25)], "D": [("0", 0.92), ("8", 0.25)], "Q": [("0", 0.92), ("9", 0.25)], "I": [("1", 0.95), ("7", 0.30), ("4", 0.20)], "İ": [("1", 0.95), ("7", 0.25)], "ı": [("1", 0.95), ("7", 0.25)], "l": [("1", 0.95), ("7", 0.25)], "|": [("1", 0.95), ("7", 0.25)], "Z": [("2", 0.95), ("7", 0.30)], "z": [("2", 0.95), ("7", 0.25)], "E": [("3", 0.85), ("8", 0.35), ("5", 0.25)], "A": [("4", 0.90), ("8", 0.20)], "S": [("5", 0.95), ("8", 0.30), ("6", 0.25), ("3", 0.20)], "s": [("5", 0.95), ("6", 0.25)], "Ş": [("5", 0.92), ("6", 0.25)], "ş": [("5", 0.92), ("6", 0.25)], "G": [("6", 0.95), ("0", 0.30), ("8", 0.25)], "b": [("6", 0.92), ("8", 0.25)], "T": [("7", 0.92), ("1", 0.30)], "B": [("8", 0.95), ("3", 0.30), ("6", 0.30), ("0", 0.20)], "g": [("9", 0.92), ("8", 0.25)], } # Tüm görsel karışıklık karakterlerini kapsayan regex karakter sınıfı CONFUSABLE_DIGIT_CHARS = "0-9ODoQIİıl|ZzEASsŞşGbTBg" def compute_tckn_check_digits(first_nine: Sequence[int]) -> Tuple[int, int]: """İlk 9 haneden TCKN 10. ve 11. kontrol basamaklarını hesaplar.""" if len(first_nine) != 9 or first_nine[0] == 0: raise ValueError("TCKN ilk 9 hanesi geçerli olmalı ve 0 ile başlamamalıdır.") odd_sum = first_nine[0] + first_nine[2] + first_nine[4] + first_nine[6] + first_nine[8] even_sum = first_nine[1] + first_nine[3] + first_nine[5] + first_nine[7] d10 = ((odd_sum * 7) - even_sum) % 10 d11 = (odd_sum + even_sum + d10) % 10 return d10, d11 def validate_tckn(tckn: str) -> bool: """ 11 haneli T.C. Kimlik Numarasının algoritmik sağlamasını kontrol eder: - 11 rakamdan oluşmalı, '0' ile başlamamalı ve 11. hane çift sayı olmalıdır. - 10. hane = ((Tek haneler toplamı * 7) - Çift haneler toplamı) mod 10 - 11. hane = (İlk 10 hane toplamı) mod 10 """ cleaned = re.sub(r"\s+", "", tckn) if len(cleaned) != 11 or not cleaned.isdigit() or cleaned[0] == "0": return False digits = [int(c) for c in cleaned] if digits[10] % 2 != 0: return False d10, d11 = compute_tckn_check_digits(digits[:9]) return digits[9] == d10 and digits[10] == d11 def compute_vkn_check_digit(first_nine: Sequence[int]) -> int: """İlk 9 haneden 10 haneli Vergi Kimlik Numarası (VKN) 10. kontrol basamağını hesaplar.""" if len(first_nine) != 9: raise ValueError("VKN ilk 9 hanesi 9 rakam olmalıdır.") total = 0 for i in range(9): v = (first_nine[i] + (9 - i)) % 10 if v == 9: c = 9 else: c = (v * (2 ** (9 - i))) % 9 total += c return (10 - (total % 10)) % 10 def validate_vkn(vkn: str) -> bool: """10 haneli Vergi Kimlik Numarasının resmi modüler sağlamasını kontrol eder.""" cleaned = re.sub(r"\s+", "", vkn) if len(cleaned) != 10 or not cleaned.isdigit(): return False digits = [int(c) for c in cleaned] return digits[9] == compute_vkn_check_digit(digits[:9]) def validate_tr_iban(iban: str, enforce_reserve_zero: bool = True) -> bool: """ 26 karakterli TR IBAN ('TR' + 2 kontrol basamağı + 5 banka kodu + 1 rezerv '0' + 16 hesap no) ISO 7064 MOD-97-10 algoritmasını ve TCMB rezerv basamağı ('0') kuralını doğrular. """ cleaned = re.sub(r"\s+", "", iban).upper() if len(cleaned) != 26 or not cleaned.startswith("TR"): return False if not cleaned[2:].isdigit(): return False # TCMB Standardı: 10. karakter (indeks 9, banka kodundan sonraki rezerv hanesi) '0' olmalıdır if enforce_reserve_zero and cleaned[9] != "0": return False rearranged = cleaned[4:] + "2927" + cleaned[2:4] return int(rearranged) % 97 == 1 def decode_tckn_from_logits( digit_probs: np.ndarray, top_k_per_pos: int = 3, ) -> Tuple[str, float, bool]: """ Boyutu (11, 10) olan rakam olasılık/logit matrisinden TCKN algoritmik sağlamasını geçen en yüksek olasılıklı 11 haneli diziyi (Constrained Beam Search ile) bulur. """ probs = np.asarray(digit_probs, dtype=np.float64) if probs.shape != (11, 10): raise ValueError(f"TCKN logit matrisi (11, 10) boyutunda olmalıdır, gelen: {probs.shape}") shifted = probs - np.max(probs, axis=1, keepdims=True) exp_p = np.exp(shifted) norm_probs = exp_p / np.sum(exp_p, axis=1, keepdims=True) log_probs = np.log(np.clip(norm_probs, 1e-12, 1.0)) # Kısıt 1: İlk hanede '0' olamaz log_probs[0, 0] = -1e9 # Kısıt 2: 11. hane (indeks 10) her zaman çift sayıdır for odd_d in (1, 3, 5, 7, 9): log_probs[10, odd_d] = -1e9 top_indices = np.argsort(-log_probs[:, :10], axis=1)[:, :top_k_per_pos] best_candidate: Optional[str] = None best_score = -float("inf") beam: List[Tuple[float, List[int]]] = [(0.0, [])] max_beam_width = 128 for pos in range(9): next_beam: List[Tuple[float, List[int]]] = [] for score, prefix in beam: for d in top_indices[pos]: if pos == 0 and d == 0: continue next_beam.append((score + float(log_probs[pos, d]), prefix + [int(d)])) next_beam.sort(key=lambda x: x[0], reverse=True) beam = next_beam[:max_beam_width] for score_9, first_nine in beam: d10, d11 = compute_tckn_check_digits(first_nine) total_score = score_9 + float(log_probs[9, d10]) + float(log_probs[10, d11]) if total_score > best_score: best_score = total_score best_candidate = "".join(str(x) for x in first_nine) + f"{d10}{d11}" if best_candidate and validate_tckn(best_candidate): return best_candidate, float(np.exp(best_score / 11.0)), True greedy = "".join(str(int(np.argmax(log_probs[i]))) for i in range(11)) return greedy, 0.0, validate_tckn(greedy) def decode_vkn_from_logits( digit_probs: np.ndarray, top_k_per_pos: int = 3, ) -> Tuple[str, float, bool]: """ Boyutu (10, 10) olan rakam olasılık/logit matrisinden VKN sağlamasını geçen en yüksek olasılıklı 10 haneli Vergi Kimlik Numarasını bulur. """ probs = np.asarray(digit_probs, dtype=np.float64) if probs.shape != (10, 10): raise ValueError(f"VKN logit matrisi (10, 10) boyutunda olmalıdır, gelen: {probs.shape}") shifted = probs - np.max(probs, axis=1, keepdims=True) exp_p = np.exp(shifted) norm_probs = exp_p / np.sum(exp_p, axis=1, keepdims=True) log_probs = np.log(np.clip(norm_probs, 1e-12, 1.0)) top_indices = np.argsort(-log_probs, axis=1)[:, :top_k_per_pos] beam: List[Tuple[float, List[int]]] = [(0.0, [])] max_beam_width = 128 for pos in range(9): next_beam: List[Tuple[float, List[int]]] = [] for score, prefix in beam: for d in top_indices[pos]: next_beam.append((score + float(log_probs[pos, d]), prefix + [int(d)])) next_beam.sort(key=lambda x: x[0], reverse=True) beam = next_beam[:max_beam_width] best_candidate: Optional[str] = None best_score = -float("inf") for score_9, first_nine in beam: d10 = compute_vkn_check_digit(first_nine) total_score = score_9 + float(log_probs[9, d10]) if total_score > best_score: best_score = total_score best_candidate = "".join(str(x) for x in first_nine) + str(d10) if best_candidate and validate_vkn(best_candidate): return best_candidate, float(np.exp(best_score / 10.0)), True greedy = "".join(str(int(np.argmax(log_probs[i]))) for i in range(10)) return greedy, 0.0, validate_vkn(greedy) def decode_iban_from_logits( digit_probs_24: np.ndarray, top_k_per_pos: int = 3, ) -> Tuple[str, float, bool]: """ TR IBAN'ın 'TR' sonrası 24 rakamı için (24, 10) boyutundaki olasılık matrisinden: 1. TCMB rezerv basamağını (`pos 7 == 0`) sabitler, 2. BBAN (`pos 2..23`) içindeki en belirsiz pozisyonlarda top-k arama yaparken kontrol basamaklarını (`pos 0, 1`) `98 - (int(bban + "292700") % 97)` ile deterministik hesaplayıp gerçek logit olasılığıyla puanlar! """ probs = np.asarray(digit_probs_24, dtype=np.float64) if probs.shape != (24, 10): raise ValueError(f"TR IBAN rakam matrisi (24, 10) boyutunda olmalıdır, gelen: {probs.shape}") shifted = probs - np.max(probs, axis=1, keepdims=True) exp_p = np.exp(shifted) norm_probs = exp_p / np.sum(exp_p, axis=1, keepdims=True) log_probs = np.log(np.clip(norm_probs, 1e-12, 1.0)) # TCMB rezerv hanesi (24 rakam içinde indeks 7) her zaman '0' olmalıdır log_probs[7, 1:] = -1e9 greedy_digits = [int(np.argmax(log_probs[i])) for i in range(24)] greedy_iban = "TR" + "".join(str(d) for d in greedy_digits) if validate_tr_iban(greedy_iban): mean_prob = float(np.exp(np.mean([log_probs[i, greedy_digits[i]] for i in range(24)]))) return greedy_iban, mean_prob, True # BBAN pozisyonları (indeks 2..23, rezerv indeks 7 hariç) içindeki en belirsiz 5 pozisyonu bul bban_positions = [i for i in range(2, 24) if i != 7] confidences = [ (float(log_probs[i, greedy_digits[i]] - np.partition(log_probs[i], -2)[-2]), i) for i in bban_positions ] confidences.sort(key=lambda x: x[0]) uncertain_set = {pos for _, pos in confidences[:5]} options_per_bban: List[List[Tuple[int, float]]] = [] for i in range(2, 24): if i == 7: options_per_bban.append([(0, float(log_probs[7, 0]))]) elif i in uncertain_set: top_d = np.argsort(-log_probs[i])[:top_k_per_pos] options_per_bban.append([(int(d), float(log_probs[i, d])) for d in top_d]) else: d = greedy_digits[i] options_per_bban.append([(d, float(log_probs[i, d]))]) best_iban = greedy_iban best_score = -float("inf") found = False for bban_combo in itertools.product(*options_per_bban): bban_score = sum(item[1] for item in bban_combo) bban_str = "".join(str(item[0]) for item in bban_combo) # Kontrol basamakları (pos 0 ve pos 1) BBAN'dan %100 deterministik hesaplanır! check_val = 98 - (int(bban_str + "292700") % 97) c0, c1 = check_val // 10, check_val % 10 total_score = bban_score + float(log_probs[0, c0]) + float(log_probs[1, c1]) if total_score > best_score: best_score = total_score best_iban = f"TR{c0}{c1}{bban_str}" found = True return best_iban, (float(np.exp(best_score / 24.0)) if found else 0.0), found def _text_to_digit_prob_matrix(raw_text: str, expected_len: int) -> Optional[np.ndarray]: """ Ham OCR metnini görsel karışıklık önyargılarına (prior) göre (expected_len, 10) olasılık matrisine dönüştürür. """ cleaned = re.sub(r"[\s\-]", "", raw_text) if len(cleaned) != expected_len: return None matrix = np.full((expected_len, 10), 0.001, dtype=np.float64) for idx, ch in enumerate(cleaned): candidates = DIGIT_VISUAL_CONFUSIONS.get(ch) if candidates is None: if ch.isdigit(): matrix[idx, int(ch)] = 0.999 else: matrix[idx, :] = 0.1 else: for digit_str, weight in candidates: matrix[idx, int(digit_str)] = max(matrix[idx, int(digit_str)], weight) return matrix def repair_tckn_text(raw_tckn: str) -> Tuple[str, bool]: """Görsel OCR hataları içeren 11 karakterlik TCKN adayını algoritmik sağlamayla onarır.""" cleaned = re.sub(r"[\s\-]", "", raw_tckn) if validate_tckn(cleaned): return cleaned, True prob_matrix = _text_to_digit_prob_matrix(cleaned, 11) if prob_matrix is None: return raw_tckn, False repaired, _, is_valid = decode_tckn_from_logits(np.log(prob_matrix), top_k_per_pos=4) return (repaired if is_valid else raw_tckn), is_valid def repair_vkn_text(raw_vkn: str) -> Tuple[str, bool]: """Görsel OCR hataları içeren 10 karakterlik VKN adayını algoritmik sağlamayla onarır.""" cleaned = re.sub(r"[\s\-]", "", raw_vkn) if validate_vkn(cleaned): return cleaned, True prob_matrix = _text_to_digit_prob_matrix(cleaned, 10) if prob_matrix is None: return raw_vkn, False repaired, _, is_valid = decode_vkn_from_logits(np.log(prob_matrix), top_k_per_pos=4) return (repaired if is_valid else raw_vkn), is_valid def repair_tr_iban(raw_iban: str) -> Tuple[str, bool]: """Görsel OCR hataları içeren 26 karakterlik TR IBAN adayını MOD-97-10 sağlamasıyla onarır.""" cleaned = re.sub(r"[\s\-]", "", raw_iban).upper() if validate_tr_iban(cleaned): return cleaned, True if len(cleaned) != 26: return raw_iban, False body = cleaned[2:] prob_matrix = _text_to_digit_prob_matrix(body, 24) if prob_matrix is None: return raw_iban, False repaired, _, is_valid = decode_iban_from_logits(np.log(prob_matrix), top_k_per_pos=3) return (repaired if is_valid else raw_iban), is_valid @dataclass class VerificationReport: """TurkceVerify belge doğrulama ve onarım raporu.""" original_text: str verified_text: str repaired_fields: List[Dict[str, str]] = field(default_factory=list) valid_tckns: List[str] = field(default_factory=list) valid_vkns: List[str] = field(default_factory=list) valid_ibans: List[str] = field(default_factory=list) class TurkceVerify: """ Tam sayfa VLM veya satır OCR çıktılarını tarayarak TCKN, VKN, TR IBAN, finansal toplamlar ve Türkçe büyük/küçük harf hatalarını tek geçişte onaran birleşik doğrulama motoru. """ def __init__(self, restore_turkish_caps: bool = True) -> None: from turkceocr.postprocess.financial_solver import TurkishFinancialSolver from turkceocr.postprocess.turkish_casing import TurkishDiacriticRestorer self.restore_turkish_caps = restore_turkish_caps self.diacritic_restorer = TurkishDiacriticRestorer() self.financial_solver = TurkishFinancialSolver() def verify_and_repair_document(self, text: str) -> VerificationReport: """ Belge metnindeki ``, ``, `` etiketlerini ve etiketlenmemiş TCKN/VKN/IBAN kalıplarını bularak algoritmik sağlamayla onarır. """ report = VerificationReport(original_text=text, verified_text=text) working = text if self.restore_turkish_caps: working = self.diacritic_restorer.restore_text(working) # 1. Etiketli ... veya "TCKN / T.C. Kimlik No: XXXXXXXXXXX" kalıpları def _replace_tagged_tckn(match: re.Match) -> str: raw_val = match.group(1).strip() repaired, ok = repair_tckn_text(raw_val) if ok: report.valid_tckns.append(repaired) if repaired != raw_val: report.repaired_fields.append( {"type": "TCKN", "original": raw_val, "repaired": repaired} ) return f"{repaired}" working = re.sub(r"(.*?)", _replace_tagged_tckn, working) def _replace_labeled_tckn(match: re.Match) -> str: prefix, raw_val = match.group(1), match.group(2) repaired, ok = repair_tckn_text(raw_val) if ok: if repaired not in report.valid_tckns: report.valid_tckns.append(repaired) if repaired != raw_val: report.repaired_fields.append( {"type": "TCKN", "original": raw_val, "repaired": repaired} ) return f"{prefix}{repaired}" working = re.sub( rf"((?:T\.?C\.?\s*K[İI]ML[İI]K\s*(?:NO|NUMARASI)?|TCKN)\s*[:\-]?\s*)([{CONFUSABLE_DIGIT_CHARS}]{{11}})\b", _replace_labeled_tckn, working, flags=re.IGNORECASE, ) # 2. Etiketli ... veya "VKN / Vergi No: XXXXXXXXXX" kalıpları def _replace_tagged_vkn(match: re.Match) -> str: raw_val = match.group(1).strip() repaired, ok = repair_vkn_text(raw_val) if ok: report.valid_vkns.append(repaired) if repaired != raw_val: report.repaired_fields.append( {"type": "VKN", "original": raw_val, "repaired": repaired} ) return f"{repaired}" working = re.sub(r"(.*?)", _replace_tagged_vkn, working) def _replace_labeled_vkn(match: re.Match) -> str: prefix, raw_val = match.group(1), match.group(2) repaired, ok = repair_vkn_text(raw_val) if ok: if repaired not in report.valid_vkns: report.valid_vkns.append(repaired) if repaired != raw_val: report.repaired_fields.append( {"type": "VKN", "original": raw_val, "repaired": repaired} ) return f"{prefix}{repaired}" working = re.sub( rf"((?:VERG[İI]\s*(?:K[İI]ML[İI]K\s*)?(?:NO|NUMARASI)|VKN)\s*[:\-]?\s*)([{CONFUSABLE_DIGIT_CHARS}]{{10}})\b", _replace_labeled_vkn, working, flags=re.IGNORECASE, ) # 3. TR IBAN kalıpları (... veya TRxx xxxx ...) def _replace_tagged_iban(match: re.Match) -> str: raw_val = match.group(1).strip() repaired, ok = repair_tr_iban(raw_val) if ok: report.valid_ibans.append(repaired) if repaired != re.sub(r"[\s\-]", "", raw_val).upper(): report.repaired_fields.append( {"type": "IBAN", "original": raw_val, "repaired": repaired} ) return f"{repaired}" working = re.sub(r"(.*?)", _replace_tagged_iban, working) def _replace_free_iban(match: re.Match) -> str: raw_val = match.group(0) compact = re.sub(r"\s+", "", raw_val) if len(compact) != 26: return raw_val repaired, ok = repair_tr_iban(compact) if ok: if repaired not in report.valid_ibans: report.valid_ibans.append(repaired) if repaired != compact.upper(): report.repaired_fields.append( {"type": "IBAN", "original": raw_val, "repaired": repaired} ) return repaired return raw_val working = re.sub( rf"\bTR[{CONFUSABLE_DIGIT_CHARS}]{{2}}(?:\s?[{CONFUSABLE_DIGIT_CHARS}]{{4}}){{5}}\s?[{CONFUSABLE_DIGIT_CHARS}]{{2}}\b", _replace_free_iban, working, ) report.verified_text = working return report # ASCII uyumluluk takma adı