""" FAZ 2C — `OffsetMap` (π: norm -> raw) Karakter-Piksel İzdüşümü ve 2B Görsel Karartma Motoru (`VisualRedactor`). Özellikler: 1. `OffsetMap`: - `TurkceVerify` veya Unicode normalizasyonu sonrası oluşan `norm_text` üzerindeki `[norm_start, norm_end)` varlık ofsetlerini kaynak `raw_text` üzerindeki `[raw_start, raw_end)` karakter aralığına monotonik dinamik programlama hizalaması ile izdüşürür. 2. 2B Görsel Karartma (`VisualRedactor`): - Metin tabanlı PII (`TR_TCKN`, `TR_TCKN_SUSPECT`, `TR_VKN`, `TR_VKN_SUSPECT`, `TR_IBAN`, `TR_IBAN_SUSPECT`, `PERSON`, `ADDRESS`, `TR_PHONE`, `EMAIL`, `HEALTH_DATA`, `KVKK_SPECIAL` vb.) ve görsel biyometrik/kimlik unsurlarını (``, ``, `VISUAL_PHOTO`, `VISUAL_QRCODE`) geri döndürülemez tam siyah (`#000000`) dikdörtgen + `[KVKK: ]` etiketi ile kapatır. """ from __future__ import annotations import re from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Sequence, Set, Tuple from PIL import Image, ImageDraw from turkceocr.postprocess.entity_engine import ( DEFAULT_REDACTABLE_ENTITY_TYPES, DetectedEntitySpan, UnifiedTurkishEntityEngine, ) @dataclass class OffsetMap: """ π: norm_text [0..N] -> raw_text [0..M] monotonik karakter ofset izdüşüm haritası. """ raw_text: str norm_text: str norm_to_raw: List[int] @classmethod def build(cls, raw_text: str, norm_text: str) -> "OffsetMap": """ `raw_text` ile `norm_text` arasında karakter/token düzeyinde monotonik ofset haritası kurar. Eğer uzunluklar birebir eşitse (1:1 izomorfizm), birim harita (identity map) O(N) sürede üretilir. """ m = len(raw_text) n = len(norm_text) if m == n: return cls(raw_text=raw_text, norm_text=norm_text, norm_to_raw=list(range(n + 1))) if n == 0: return cls(raw_text=raw_text, norm_text=norm_text, norm_to_raw=[0]) if m == 0: return cls(raw_text=raw_text, norm_text=norm_text, norm_to_raw=[0] * (n + 1)) # İki işaretçili açgözlü (greedy) + yerel pencere hizalaması (uzun metinlerde O(N) hız garantisi) mapping = [0] * (n + 1) i = 0 # raw_text indeksi j = 0 # norm_text indeksi raw_fold = raw_text.lower() norm_fold = norm_text.lower() while j < n and i < m: mapping[j] = i if raw_fold[i] == norm_fold[j]: i += 1 j += 1 else: # 12 karakterlik yerel pencerede eşleşme ara found_sync = False for look_raw in range(1, min(12, m - i)): if raw_fold[i + look_raw] == norm_fold[j]: i += look_raw found_sync = True break if not found_sync: for look_norm in range(1, min(12, n - j)): if raw_fold[i] == norm_fold[j + look_norm]: for step in range(look_norm): mapping[j + step] = i j += look_norm found_sync = True break if not found_sync: i += 1 j += 1 while j <= n: mapping[j] = min(i, m) j += 1 # Monotoniklik garantisi for k in range(1, n + 1): if mapping[k] < mapping[k - 1]: mapping[k] = mapping[k - 1] mapping[n] = m return cls(raw_text=raw_text, norm_text=norm_text, norm_to_raw=mapping) def project_span(self, norm_start: int, norm_end: int) -> Tuple[int, int]: """`norm_text` üzerindeki `[norm_start, norm_end)` aralığını `raw_text` ofsetlerine izdüşürür.""" n = len(self.norm_text) s_clamped = max(0, min(norm_start, n)) e_clamped = max(s_clamped, min(norm_end, n)) raw_s = self.norm_to_raw[s_clamped] raw_e = self.norm_to_raw[e_clamped] if raw_e <= raw_s and e_clamped > s_clamped and raw_s < len(self.raw_text): raw_e = min(len(self.raw_text), raw_s + (e_clamped - s_clamped)) return raw_s, raw_e @dataclass class OCRLineBox: """Satır düzeyinde OCR metni, ham metin karakter aralığı ve 2B piksel kutusu `(x1, y1, x2, y2)`.""" text: str bbox: Tuple[int, int, int, int] raw_start: int = 0 raw_end: int = 0 @dataclass class RedactionBox2D: """Karartılan 2B piksel bölgesi kaydı.""" entity_type: str bbox: Tuple[int, int, int, int] label_tag: str text_preview: str = "" tier: str = "VERIFIED_PII" source: str = "text_projection" def to_dict(self) -> Dict[str, Any]: return { "entity_type": self.entity_type, "bbox": list(self.bbox), "label_tag": self.label_tag, "tier": self.tier, "source": self.source, } @dataclass class VisualRedactionResult: """2B Görsel + Metinsel KVKK Karartma Çıktısı.""" redacted_image: Image.Image raw_text: str redacted_text: str entities: List[DetectedEntitySpan] = field(default_factory=list) redaction_boxes: List[RedactionBox2D] = field(default_factory=list) redaction_coverage: float = 1.0 def to_dict(self) -> Dict[str, Any]: return { "redacted_text": self.redacted_text, "entity_count": len(self.entities), "redaction_box_count": len(self.redaction_boxes), "redaction_coverage": round(float(self.redaction_coverage), 4), "entities": [e.to_dict() for e in self.entities], "redaction_boxes": [b.to_dict() for b in self.redaction_boxes], } class VisualRedactor: """ 2B Görsel Belge Karartma Motoru: - Metin içindeki tüm PII ve SUSPECT_PII spanlarını `OffsetMap` üzerinden `OCRLineBox` koordinatlarına izdüşürür. - `` (`VISUAL_SIGNATURE`), `...` (`VISUAL_STAMP`), `VISUAL_PHOTO` ve `VISUAL_QRCODE` görsel bölgelerini tespit eder. - Tüm hassas kutuları geri döndürülemez `#000000` (tam siyah) dolgu ve beyaz `[KVKK: ]` etiketi ile kapatır. """ def __init__( self, entity_engine: Optional[UnifiedTurkishEntityEngine] = None, pad_px: int = 3, ) -> None: self.entity_engine = entity_engine or UnifiedTurkishEntityEngine(use_ner=False) self.pad_px = pad_px @staticmethod def build_synthetic_line_boxes( raw_text: str, image_size: Tuple[int, int], ) -> List[OCRLineBox]: """ Eğer dışarıdan açık satır koordinatları (`OCRLineBox`) verilmemişse, metin satırlarını sayfa yüksekliği ve genişliği boyunca orantısal olarak 2B satır kutularına eşler. """ w, h = image_size lines = raw_text.splitlines(keepends=True) if not lines: return [] num_lines = max(1, len(lines)) margin_x = max(16, int(w * 0.05)) margin_y = max(16, int(h * 0.05)) usable_w = max(60, w - 2 * margin_x) usable_h = max(60, h - 2 * margin_y) line_step = max(16.0, usable_h / float(num_lines)) boxes: List[OCRLineBox] = [] cursor = 0 for idx, line in enumerate(lines): stripped = line.rstrip("\r\n") line_start = cursor line_end = cursor + len(stripped) cursor += len(line) if not stripped.strip(): continue y1 = int(margin_y + idx * line_step) y2 = min(h - 4, int(y1 + max(14.0, line_step * 0.82))) char_ratio = min(1.0, max(0.15, len(stripped) / 75.0)) x1 = margin_x x2 = min(w - margin_x, int(margin_x + usable_w * char_ratio)) boxes.append( OCRLineBox( text=stripped, bbox=(x1, y1, x2, y2), raw_start=line_start, raw_end=line_end, ) ) return boxes @staticmethod def _char_visual_weight(ch: str) -> float: """Orantısal (proportional) fontlarda (Arial/Calibri/Times) karakter genişlik ağırlığı.""" if ch in " .,:;|!iıljtfIİ()[]/'\"-": return 0.48 if ch in "r1": return 0.72 if ch.isdigit(): return 1.08 if ch in "mw": return 1.42 if ch in "MW@%&": return 1.55 if ch.isupper(): return 1.24 return 0.92 def project_char_span_to_boxes( self, raw_start: int, raw_end: int, line_boxes: Sequence[OCRLineBox], image_size: Tuple[int, int], ) -> List[Tuple[int, int, int, int]]: """ `[raw_start, raw_end)` karakter aralığını kesiştiği `OCRLineBox` satırları içinde orantısal tipografik karakter ağırlığıyla 2B piksel kutularına `(bx1, by1, bx2, by2)` dönüştürür. """ w, h = image_size matched_boxes: List[Tuple[int, int, int, int]] = [] for lb in line_boxes: if lb.raw_end <= raw_start or lb.raw_start >= raw_end: continue line_len = max(1, lb.raw_end - lb.raw_start) local_s = max(0, min(line_len, raw_start - lb.raw_start)) local_e = max(local_s, min(line_len, raw_end - lb.raw_start)) if lb.text and len(lb.text) == line_len: weights = [self._char_visual_weight(c) for c in lb.text] total_w = max(1e-6, sum(weights)) rel_s = sum(weights[:local_s]) / total_w rel_e = sum(weights[:local_e]) / total_w else: rel_s = local_s / float(line_len) rel_e = local_e / float(line_len) lx1, ly1, lx2, ly2 = lb.bbox lw = max(8, lx2 - lx1) pad_x = max(self.pad_px, 6) bx1 = max(0, int(lx1 + lw * rel_s) - pad_x) bx2 = min(w, int(lx1 + lw * rel_e) + pad_x) by1 = max(0, ly1 - self.pad_px) by2 = min(h, ly2 + self.pad_px) if bx2 <= bx1 + 4: bx2 = min(w, bx1 + 16) matched_boxes.append((bx1, by1, bx2, by2)) return matched_boxes def redact_document( self, image: Image.Image, raw_text: str, normalized_text: Optional[str] = None, line_boxes: Optional[Sequence[OCRLineBox]] = None, visual_regions: Optional[Sequence[Tuple[str, Tuple[int, int, int, int]]]] = None, redact_types: Optional[Set[str]] = None, ) -> VisualRedactionResult: """ Belge görseli ve OCR metnini birlikte tarayarak: 1. `normalized_text` (veya `raw_text`) üzerinde `UnifiedTurkishEntityEngine` çalıştırır, 2. `OffsetMap` (π: norm -> raw) ile tüm PII varlıklarını `raw_text` ofsetlerine izdüşürür, 3. ``, `...`, `VISUAL_PHOTO`, `VISUAL_QRCODE` görsel bölgelerini ekler, 4. Görsel üzerinde geri döndürülemez `#000000` (tam siyah) karartma uygular. """ target_types = redact_types if redact_types is not None else DEFAULT_REDACTABLE_ENTITY_TYPES analysis_text = normalized_text if normalized_text is not None else raw_text offset_map = OffsetMap.build(raw_text=raw_text, norm_text=analysis_text) report = self.entity_engine.analyze(analysis_text, redact_types=target_types) img_out = image.convert("RGB").copy() w, h = img_out.size active_lines = ( list(line_boxes) if line_boxes is not None else self.build_synthetic_line_boxes(raw_text, (w, h)) ) redaction_boxes: List[RedactionBox2D] = [] # 1. Metin tabanlı PII ve SUSPECT_PII varlıklarının 2B kutulara izdüşümü for ent in report.entities: if ent.entity_type not in target_types: continue raw_s, raw_e = offset_map.project_span(ent.start, ent.end) boxes = self.project_char_span_to_boxes(raw_s, raw_e, active_lines, (w, h)) for bx in boxes: redaction_boxes.append( RedactionBox2D( entity_type=ent.entity_type, bbox=bx, label_tag=f"[KVKK: {ent.entity_type}]", text_preview=ent.text[:16], tier=ent.tier, source=ent.source, ) ) # 2. Yapısal VLM etiketleri (, ...) için 2B görsel karartma for m_sig in re.finditer(r"", raw_text, flags=re.IGNORECASE): for bx in self.project_char_span_to_boxes(m_sig.start(), m_sig.end(), active_lines, (w, h)): redaction_boxes.append( RedactionBox2D( entity_type="VISUAL_SIGNATURE", bbox=bx, label_tag="[KVKK: VISUAL_SIGNATURE]", tier="VERIFIED_VISUAL", source="structural_tag", ) ) for m_stamp in re.finditer(r".*?", raw_text, flags=re.IGNORECASE | re.DOTALL): for bx in self.project_char_span_to_boxes(m_stamp.start(), m_stamp.end(), active_lines, (w, h)): redaction_boxes.append( RedactionBox2D( entity_type="VISUAL_STAMP", bbox=bx, label_tag="[KVKK: VISUAL_STAMP]", tier="VERIFIED_VISUAL", source="structural_tag", ) ) # 3. Açıkça verilen görsel biyometrik/kimlik kutuları (VISUAL_PHOTO, VISUAL_QRCODE, VISUAL_SIGNATURE, VISUAL_STAMP) if visual_regions: for vtype, vbbox in visual_regions: vx1 = max(0, min(w, int(vbbox[0]))) vy1 = max(0, min(h, int(vbbox[1]))) vx2 = max(vx1 + 4, min(w, int(vbbox[2]))) vy2 = max(vy1 + 4, min(h, int(vbbox[3]))) redaction_boxes.append( RedactionBox2D( entity_type=vtype, bbox=(vx1, vy1, vx2, vy2), label_tag=f"[KVKK: {vtype}]", tier="VERIFIED_VISUAL", source="visual_detector", ) ) # 4. Geri döndürülemez #000000 tam siyah dolgu + beyaz [KVKK: ...] etiketi çizimi draw = ImageDraw.Draw(img_out) covered_boxes = 0 for rbox in redaction_boxes: x1, y1, x2, y2 = rbox.bbox draw.rectangle([x1, y1, x2, y2], fill=(0, 0, 0), outline=(0, 0, 0)) if (x2 - x1) >= 75 and (y2 - y1) >= 12: short_tag = rbox.label_tag[: max(8, (x2 - x1) // 6)] draw.text((x1 + 3, y1 + 1), short_tag, fill=(255, 255, 255)) covered_boxes += 1 coverage = 1.0 if len(redaction_boxes) == covered_boxes else (covered_boxes / max(1, len(redaction_boxes))) return VisualRedactionResult( redacted_image=img_out, raw_text=raw_text, redacted_text=report.redacted_text, entities=report.entities, redaction_boxes=redaction_boxes, redaction_coverage=coverage, )