TurkceOCR / turkceocr /data /hf_loader.py

Release TurkceOCR v2.0 (Apache-2.0)

c8b6043 about 8 hours ago

20.4 kB

"""
Hugging Face Açık Kaynak Türkçe OCR Veri Seti Birleştirici, Kalite Filtresi (`Mojibake` &
Çözünürlük Denetimi), Müfredat Öğrenimi (3-Stage Curriculum) ve `TurkceDegrade` Canlı Zenginleştiricisi.
Doğrulanmış Açık Kaynak Türkçe Veri Kaynakları (Kasım 2026):
1. Tam Sayfa Kurumsal / Doküman OCR:
   - `Werea-co/werea-tr-doc-ocr-enterprise-v2`
   - `Werea-co/werea-tr-doc-ocr-synthetic`
   - `HumynLabs/Turkish_Documents_Datasets_PDF`
   - `ytu-ce-cosmos/turkce-kitap`
   - `moganai/turkish-resmi-gazete`
2. Doküman Soru-Cevap (DocVQA):
   - `Ethosoft/TR-DocVQA-Synth` (`question` + `answer` çiftleri `<doc_qa>` şemasıyla korunur)
3. Satır / El Yazısı / Sahne Metni (`TürkçeOCR-Line` için):
   - `emredeveloper/turkish-ocr` (%70 el yazısı, %30 matbu)
   - `selimfirat/bilkent-turkish-writings-dataset`
   - `bartosgaye/thedataset` (T-H-E 156K el yazısı karakter/kelime)
   - `serdaryildiz/turkish-scene-text-recognition-dataset` (TS-TR & STS-TR)
4. Metin/Mevzuat Korpusu (Sentetik Belge Metin Enjeksiyonu için):
   - `CtnkyaABC/turkish-law-corpus`
   - `muhammetakkurt/mevzuat-gov-dataset`
Sızıntı Koruması (Zero Data Contamination):
- `metunlp/ocrturk` (EACL 2026 SIGTURK, 180 gerçek belge) yalnızca kıyaslama/test içindir;
  eğitim havuzuna eklenmesi `BENCHMARK_ONLY_CONTAMINATION_GUARD` tarafından engellenir.
"""

from __future__ import annotations

import math
import random
from collections import Counter
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence, Tuple

from PIL import Image

from turkceocr.data.degradation import TurkceDegrade
from turkceocr.data.pdf_and_table_engine import (
    BornDigitalPDFPairGenerator,
    ComplexTurkishTableRenderer,
)
from turkceocr.data.synthetic_generator import (
    AGGLUTINATIVE_HARD_LEXICON,
    SyntheticDocumentSample,
    TurkishDocumentGenerator,
)

# Eğitim setine kesinlikle karıştırılmaması gereken resmi test benchmarkları
BENCHMARK_ONLY_CONTAMINATION_GUARD = {
    "metunlp/ocrturk",
    "ocrturk",
}

# Yaygın UTF-8 -> Latin-1 Mojibake dönüşüm tablosu (Bozuk dış veri setlerini onarmak için)
TURKISH_MOJIBAKE_MAP: Dict[str, str] = {
    "Ã§": "ç",
    "Ã‡": "Ç",
    "ÄŸ": "ğ",
    "Äž": "Ğ",
    "Ä±": "ı",
    "Ä°": "İ",
    "Ã¶": "ö",
    "Ã–": "Ö",
    "ÅŸ": "ş",
    "Åž": "Ş",
    "Ã¼": "ü",
    "Ãœ": "Ü",
    "Ã¢": "â",
    "Ã‚": "Â",
}


@dataclass
class DatasetSourceSpec:
    """Hugging Face veri seti kaynağı, görev türü, bozulma ön-ayarı ve örnekleme ağırlığı."""

    hf_path: str
    weight: float
    task_type: str = "full_page_ocr"  # "full_page_ocr" | "doc_vqa" | "line_ocr" | "text_corpus"
    image_column: str = "image"
    text_column: str = "text"
    allowed_presets: Tuple[str, ...] = ("corporate_stamped",)
    split: str = "train"


DEFAULT_TURKISH_HF_SOURCES: List[DatasetSourceSpec] = [
    DatasetSourceSpec(
        "Werea-co/werea-tr-doc-ocr-enterprise-v2",
        weight=0.22,
        task_type="full_page_ocr",
        text_column="markdown",
        allowed_presets=("corporate_stamped",),
    ),
    DatasetSourceSpec(
        "Werea-co/werea-tr-doc-ocr-synthetic",
        weight=0.12,
        task_type="full_page_ocr",
        text_column="markdown",
        allowed_presets=("corporate_stamped",),
    ),
    DatasetSourceSpec(
        "Ethosoft/TR-DocVQA-Synth",
        weight=0.16,
        task_type="doc_vqa",
        text_column="answer",
        allowed_presets=("corporate_stamped",),
    ),
    DatasetSourceSpec(
        "HumynLabs/Turkish_Documents_Datasets_PDF",
        weight=0.12,
        task_type="full_page_ocr",
        text_column="text",
        allowed_presets=("corporate_stamped",),
    ),
    DatasetSourceSpec(
        "ytu-ce-cosmos/turkce-kitap",
        weight=0.12,
        task_type="full_page_ocr",
        text_column="text",
        allowed_presets=("corporate_stamped",),
    ),
    DatasetSourceSpec(
        "moganai/turkish-resmi-gazete",
        weight=0.10,
        task_type="full_page_ocr",
        text_column="text",
        allowed_presets=("corporate_stamped",),
    ),
    DatasetSourceSpec(
        "emredeveloper/turkish-ocr",
        weight=0.10,
        task_type="line_ocr",
        text_column="text",
        allowed_presets=("mixed_handwritten_form",),
    ),
    DatasetSourceSpec(
        "selimfirat/bilkent-turkish-writings-dataset",
        weight=0.06,
        task_type="line_ocr",
        text_column="text",
        allowed_presets=("mixed_handwritten_form",),
    ),
]


class TurkishOCRDataQualityFilter:
    """
    Dış kaynaklardan (Hugging Face / PDF kazıma) gelen örneklerde:
    1. UTF-8 Mojibake (`Ã¼ -> ü`, `Ä± -> ı`, `ÅŸ -> ş`) onarımı,
    2. Çözünürlük alt sınırı kontrolü (tam sayfa vs. satır),
    3. Boş veya bozuk (`\\ufffd`) etiket filtrelemesi
    uygulayan kalite kapısı.
    """

    def __init__(
        self,
        min_page_side: int = 256,
        min_line_height: int = 16,
        min_text_chars: int = 2,
    ) -> None:
        self.min_page_side = min_page_side
        self.min_line_height = min_line_height
        self.min_text_chars = min_text_chars

    @staticmethod
    def repair_mojibake(text: str) -> str:
        fixed = text
        for bad, good in TURKISH_MOJIBAKE_MAP.items():
            if bad in fixed:
                fixed = fixed.replace(bad, good)
        return fixed

    def validate_and_clean(
        self,
        image: Image.Image,
        text: str,
        task_type: str = "full_page_ocr",
    ) -> Tuple[bool, str]:
        if not isinstance(text, str):
            return False, ""
        cleaned = self.repair_mojibake(text).strip()
        if len(cleaned) < self.min_text_chars or "\ufffd" in cleaned:
            return False, ""

        w, h = image.size
        if task_type == "line_ocr":
            if h < self.min_line_height or w < self.min_line_height:
                return False, ""
        else:
            if min(w, h) < self.min_page_side:
                return False, ""

        return True, cleaned


def build_stamp_underneath_token_mask(
    full_text: str,
    stamp_underneath_text: str,
    char_Or_token_sequence: Optional[Sequence[str]] = None,
) -> List[float]:
    """
    `TurkceOCRVL.compute_turkish_vl_loss` ve `DiacriticFocalLoss` içinde kaşe altında
    kalan metin bölgesine `2.2x` odak ağırlığı vermek için karakter/token maskesi üretir.
    """
    if not full_text:
        return []
    mask = [0.0] * len(full_text)
    sub = stamp_underneath_text.strip()
    if sub:
        start_idx = full_text.find(sub)
        if start_idx != -1:
            for i in range(start_idx, min(len(full_text), start_idx + len(sub))):
                mask[i] = 1.0
    if char_Or_token_sequence is None:
        return mask

    # Eğer alt-kelime (subword) token dizisi verilmişse karakter maskesini tokenlara izdüşür
    token_mask: List[float] = []
    cursor = 0
    for tok in char_Or_token_sequence:
        t_len = max(1, len(tok))
        span = mask[cursor : min(len(mask), cursor + t_len)]
        token_mask.append(1.0 if any(v > 0.5 for v in span) else 0.0)
        cursor = min(len(mask), cursor + t_len)
    return token_mask


@dataclass
class DatasetSufficiencySummary:
    """Eğitim veri seti yeterlilik, entropi ve Türkçe morfoloji analiz raporu."""

    total_page_samples: int
    total_line_crops: int
    layout_entropy_bits: float
    diacritic_char_ratio: float
    agglutinative_long_word_ratio: float
    stamped_sample_ratio: float
    id_sample_count: int
    ood_sample_count: int
    passes_sufficiency_gate: bool
    notes: List[str] = field(default_factory=list)


class TurkishOCRDatasetMixer:
    """
    Hugging Face veri setlerini ve yerel parametrik `TurkishDocumentGenerator` motorunu
    birleştirip 3 evreli Müfredat Öğrenimi (Curriculum Learning) ve `TurkceDegrade` ile
    zenginleştiren PyTorch uyumlu veri kümesi.
    """

    def __init__(
        self,
        degrade_prob: float = 0.75,
        curriculum_stage: int = 2,
        seed: int = 42,
        synthetic_fallback_size: int = 1000,
    ) -> None:
        self.seed = seed
        self.rng = random.Random(seed)
        self.degrader = TurkceDegrade(seed=seed)
        self.generator = TurkishDocumentGenerator(seed=seed)
        self.table_renderer = ComplexTurkishTableRenderer(seed=seed)
        self.pdf_pair_generator = BornDigitalPDFPairGenerator(seed=seed)
        self.quality_filter = TurkishOCRDataQualityFilter()
        self.synthetic_fallback_size = synthetic_fallback_size
        self._external_samples: List[Dict[str, Any]] = []
        self.curriculum_stage = curriculum_stage
        self.degrade_prob = degrade_prob
        self.set_curriculum_stage(curriculum_stage, custom_degrade_prob=degrade_prob)

    def set_curriculum_stage(
        self, stage: int, custom_degrade_prob: Optional[float] = None
    ) -> None:
        """
        3 Evreli Müfredat Öğrenimi (Curriculum Learning):
        - Evre 1 (Temiz + Hafif Bozulma): `degrade_prob = 0.30`
        - Evre 2 (Yoğun Kaşe, İmza, Termal Fiş): `degrade_prob = 0.75`
        - Evre 3 (Zorlu Morfoloji, Çoklu Şablon + RLVR): `degrade_prob = 0.85`
        """
        if stage not in (1, 2, 3):
            raise ValueError("curriculum_stage 1, 2 veya 3 olmalıdır.")
        self.curriculum_stage = stage
        if custom_degrade_prob is not None:
            self.degrade_prob = custom_degrade_prob
        elif stage == 1:
            self.degrade_prob = 0.30
        elif stage == 2:
            self.degrade_prob = 0.75
        else:
            self.degrade_prob = 0.85

    def load_hf_source(
        self,
        repo_id: str,
        split: str = "train",
        max_samples: Optional[int] = None,
        streaming: bool = True,
    ) -> int:
        """
        İlgili Hugging Face veri setini indirip kalite filtresinden geçirerek ve
        görev tipine göre (`<doc_qa>` veya tam sayfa Markdown) şemalandırarak havuza ekler.
        """
        norm_id = repo_id.strip().lower()
        if any(guard in norm_id for guard in BENCHMARK_ONLY_CONTAMINATION_GUARD):
            raise ValueError(
                f"SIZINTI ENGELİ (Data Contamination Guard): '{repo_id}' resmi bir test "
                "benchmarkıdır ve eğitim veri kümesine dahil edilemez!"
            )

        from datasets import load_dataset

        spec_map = {s.hf_path: s for s in DEFAULT_TURKISH_HF_SOURCES}
        spec = spec_map.get(
            repo_id,
            DatasetSourceSpec(repo_id, weight=0.10, task_type="full_page_ocr"),
        )

        ds = load_dataset(repo_id, split=split, streaming=streaming)
        count = 0
        for row in ds:
            img = row.get("image") or row.get("page_image") or row.get("img")
            if not isinstance(img, Image.Image):
                continue

            # Ethosoft/TR-DocVQA-Synth gibi soru-cevap veri setlerinde soruyu kaybetme!
            if "question" in row and ("answer" in row or "answers" in row):
                q_str = str(row.get("question", "")).strip()
                a_raw = row.get("answer") or row.get("answers")
                a_str = a_raw[0] if isinstance(a_raw, list) and a_raw else str(a_raw or "")
                raw_txt = f"<doc_qa>\nSoru: {q_str}\nCevap: {a_str.strip()}\n</doc_qa>"
                task_type = "doc_vqa"
            else:
                raw_txt = (
                    row.get("markdown")
                    or row.get("text")
                    or row.get("ground_truth")
                    or row.get("label")
                    or ""
                )
                task_type = spec.task_type

            rgb_img = img.convert("RGB")
            is_valid, cleaned_txt = self.quality_filter.validate_and_clean(
                rgb_img, str(raw_txt), task_type=task_type
            )
            if not is_valid:
                continue

            self._external_samples.append(
                {
                    "image": rgb_img,
                    "text": cleaned_txt,
                    "source": repo_id,
                    "task_type": task_type,
                    "allowed_presets": spec.allowed_presets,
                    "weight": spec.weight,
                }
            )
            count += 1
            if max_samples is not None and count >= max_samples:
                break
        return count

    def _select_compatible_preset(
        self, img: Image.Image, allowed_presets: Sequence[str]
    ) -> str:
        """Görüntü en-boy oranına ve veri kaynağına uyumlu bozulma ön-ayarı seçer."""
        if allowed_presets:
            return self.rng.choice(list(allowed_presets))
        w, h = img.size
        aspect = w / max(1.0, float(h))
        if aspect < 0.72:
            return "thermal_pos_receipt"
        if aspect > 1.35:
            return "id_card_camera"
        return "corporate_stamped"

    def __len__(self) -> int:
        if self._external_samples:
            return len(self._external_samples) + self.synthetic_fallback_size
        return self.synthetic_fallback_size

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        if idx < len(self._external_samples):
            item = self._external_samples[idx]
            img: Image.Image = item["image"]
            text: str = item["text"]
            if self.rng.random() < self.degrade_prob:
                preset = self._select_compatible_preset(
                    img, item.get("allowed_presets", ("corporate_stamped",))
                )
                deg = self.degrader.degrade(img, preset=preset)
                img = deg.image
                if deg.stamps_and_signatures:
                    tags = "\n".join(s.xml_tag for s in deg.stamps_and_signatures)
                    text = f"{text}\n{tags}"
            return {
                "image": img,
                "text": text,
                "source": item["source"],
                "task_type": item.get("task_type", "full_page_ocr"),
                "stamp_underneath_mask": [0.0] * len(text),
                "line_crops": [],
            }

        # 6 Şablon Ailesinden Parametrik Türkçe Belge Üretimi
        doc_choice = idx % 6
        apply_deg = self.rng.random() < self.degrade_prob
        # Evre 3'te şablon çeşitliliğini artırmak için %25 oranında çok-sütunlu / alternatif topoloji üret
        use_alt_topology = self.curriculum_stage == 3 and (idx % 5 == 0)

        if doc_choice == 0:
            sample = self.generator.generate_corporate_invoice(
                apply_degradation=apply_deg, is_ood=use_alt_topology
            )
        elif doc_choice == 1:
            sample = self.generator.generate_thermal_pos_receipt(
                apply_degradation=apply_deg, is_ood=use_alt_topology
            )
        elif doc_choice == 2:
            sample = self.generator.generate_mixed_handwritten_form(
                apply_degradation=apply_deg, is_ood=use_alt_topology
            )
        elif doc_choice == 3:
            sample = self.generator.generate_official_gazette_or_contract(
                apply_degradation=apply_deg, is_ood=use_alt_topology
            )
        elif doc_choice == 4:
            sample = self.table_renderer.generate_multi_vat_erp_invoice(
                apply_degradation=apply_deg, is_ood=use_alt_topology
            )
        else:
            sample = self.pdf_pair_generator.generate_two_column_born_digital_page(
                apply_degradation=apply_deg, is_ood=use_alt_topology
            )

        char_mask = build_stamp_underneath_token_mask(
            sample.ground_truth_markdown, sample.stamp_underneath_text
        )
        return {
            "image": sample.image,
            "clean_image": sample.clean_image,
            "text": sample.ground_truth_markdown,
            "plain_text": sample.plain_text,
            "stamp_underneath_text": sample.stamp_underneath_text,
            "stamp_underneath_mask": char_mask,
            "expected_kie": sample.expected_kie,
            "layout_variant": sample.layout_variant,
            "line_crops": sample.line_crops,
            "source": f"synthetic:{sample.doc_type}",
            "task_type": "full_page_ocr",
        }

    def extract_line_training_pairs(
        self, num_documents: int = 24
    ) -> List[Tuple[Image.Image, str]]:
        """
        `TürkçeOCR-Line (SVTRv2-TR)` eğitimi için bozulmuş/kaşeli tam sayfalardan
        gerçekçi satır kırpımları (`(crop_image, line_text)`) üretir.
        """
        pairs: List[Tuple[Image.Image, str]] = []
        for i in range(num_documents):
            item = self[len(self._external_samples) + i]
            for crop_img, line_txt in item.get("line_crops", []):
                pairs.append((crop_img, line_txt))
        return pairs

    def evaluate_data_sufficiency(self, sample_count: int = 32) -> DatasetSufficiencySummary:
        """
        Üretilen/birleştirilen eğitim havuzunun 12 KPI hedefimiz için yeterli
        yerleşim entropisine ($H_{\\text{layout}}$), diakritik yoğunluğuna,
        sondan eklemeli uzun kelime oranına ve kaşe altı kurtarma (`SURR`) çiftlerine
        sahip olup olmadığını nicel olarak denetler.
        """
        layout_counts: Counter[str] = Counter()
        total_chars = 0
        diac_chars = 0
        total_words = 0
        long_words = 0
        stamped_count = 0
        id_count = 0
        ood_count = 0
        total_crops = 0

        diac_set = set("çğıöşüÇĞİÖŞÜâîûÂÎÛ")
        for i in range(sample_count):
            item = self[i]
            txt = item["text"]
            variant = item.get("layout_variant", item.get("source", "external"))
            layout_counts[variant] += 1
            total_crops += len(item.get("line_crops", []))

            if "ood" in str(variant):
                ood_count += 1
            else:
                id_count += 1

            if "<stamp>" in txt or item.get("stamp_underneath_text"):
                stamped_count += 1

            total_chars += len(txt)
            diac_chars += sum(1 for ch in txt if ch in diac_set)

            words = [w.strip(".,;:()[]*#<>\"'") for w in txt.split()]
            words = [w for w in words if w.isalpha()]
            total_words += len(words)
            long_words += sum(1 for w in words if len(w) >= 8)

        # Shannon yerleşim entropisi (bit cinsinden)
        entropy = 0.0
        for cnt in layout_counts.values():
            p = cnt / max(1, sample_count)
            if p > 0:
                entropy -= p * math.log2(p)

        diac_ratio = diac_chars / max(1, total_chars)
        long_word_ratio = long_words / max(1, total_words)
        stamped_ratio = stamped_count / max(1, sample_count)

        notes: List[str] = []
        passes = True
        if entropy < 1.5:
            passes = False
            notes.append(f"Düşük yerleşim entropisi ({entropy:.2f} bit < 1.50 bit): Şablon ezberleme riski!")
        if diac_ratio < 0.05:
            passes = False
            notes.append(f"Düşük Türkçe diakritik yoğunluğu ({diac_ratio:.4f} < 0.0500)!")
        if long_word_ratio < 0.20:
            passes = False
            notes.append(f"Düşük sondan eklemeli uzun kelime oranı ({long_word_ratio:.4f} < 0.2000)!")
        if stamped_ratio < 0.40:
            passes = False
            notes.append(f"Düşük kaşeli belge oranı ({stamped_ratio:.4f} < 0.4000): SURR KPI-7 için yetersiz!")
        if passes:
            notes.append(
                "Eğitim veri havuzu tüm yerleşim entropisi, Türkçe diakritik, morfolojik uzun kelime ve SURR kaşe altı kriterlerini karşılıyor."
            )

        return DatasetSufficiencySummary(
            total_page_samples=sample_count,
            total_line_crops=total_crops,
            layout_entropy_bits=round(entropy, 4),
            diacritic_char_ratio=round(diac_ratio, 4),
            agglutinative_long_word_ratio=round(long_word_ratio, 4),
            stamped_sample_ratio=round(stamped_ratio, 4),
            id_sample_count=id_count,
            ood_sample_count=ood_count,
            passes_sufficiency_gate=passes,
            notes=notes,
        )

	"""
	Hugging Face Açık Kaynak Türkçe OCR Veri Seti Birleştirici, Kalite Filtresi (`Mojibake` &
	Çözünürlük Denetimi), Müfredat Öğrenimi (3-Stage Curriculum) ve `TurkceDegrade` Canlı Zenginleştiricisi.

	Doğrulanmış Açık Kaynak Türkçe Veri Kaynakları (Kasım 2026):
	1. Tam Sayfa Kurumsal / Doküman OCR:
	- `Werea-co/werea-tr-doc-ocr-enterprise-v2`
	- `Werea-co/werea-tr-doc-ocr-synthetic`
	- `HumynLabs/Turkish_Documents_Datasets_PDF`
	- `ytu-ce-cosmos/turkce-kitap`
	- `moganai/turkish-resmi-gazete`
	2. Doküman Soru-Cevap (DocVQA):
	- `Ethosoft/TR-DocVQA-Synth` (`question` + `answer` çiftleri `<doc_qa>` şemasıyla korunur)
	3. Satır / El Yazısı / Sahne Metni (`TürkçeOCR-Line` için):
	- `emredeveloper/turkish-ocr` (%70 el yazısı, %30 matbu)
	- `selimfirat/bilkent-turkish-writings-dataset`
	- `bartosgaye/thedataset` (T-H-E 156K el yazısı karakter/kelime)
	- `serdaryildiz/turkish-scene-text-recognition-dataset` (TS-TR & STS-TR)
	4. Metin/Mevzuat Korpusu (Sentetik Belge Metin Enjeksiyonu için):
	- `CtnkyaABC/turkish-law-corpus`
	- `muhammetakkurt/mevzuat-gov-dataset`

	Sızıntı Koruması (Zero Data Contamination):
	- `metunlp/ocrturk` (EACL 2026 SIGTURK, 180 gerçek belge) yalnızca kıyaslama/test içindir;
	eğitim havuzuna eklenmesi `BENCHMARK_ONLY_CONTAMINATION_GUARD` tarafından engellenir.
	"""

	from __future__ import annotations

	import math
	import random
	from collections import Counter
	from dataclasses import dataclass, field
	from typing import Any, Dict, List, Optional, Sequence, Tuple

	from PIL import Image

	from turkceocr.data.degradation import TurkceDegrade
	from turkceocr.data.pdf_and_table_engine import (
	BornDigitalPDFPairGenerator,
	ComplexTurkishTableRenderer,
	)
	from turkceocr.data.synthetic_generator import (
	AGGLUTINATIVE_HARD_LEXICON,
	SyntheticDocumentSample,
	TurkishDocumentGenerator,
	)

	# Eğitim setine kesinlikle karıştırılmaması gereken resmi test benchmarkları
	BENCHMARK_ONLY_CONTAMINATION_GUARD = {
	"metunlp/ocrturk",
	"ocrturk",
	}

	# Yaygın UTF-8 -> Latin-1 Mojibake dönüşüm tablosu (Bozuk dış veri setlerini onarmak için)
	TURKISH_MOJIBAKE_MAP: Dict[str, str] = {
	"Ã§": "ç",
	"Ã‡": "Ç",
	"ÄŸ": "ğ",
	"Äž": "Ğ",
	"Ä±": "ı",
	"Ä°": "İ",
	"Ã¶": "ö",
	"Ã–": "Ö",
	"ÅŸ": "ş",
	"Åž": "Ş",
	"Ã¼": "ü",
	"Ãœ": "Ü",
	"Ã¢": "â",
	"Ã‚": "Â",
	}


	@dataclass
	class DatasetSourceSpec:
	"""Hugging Face veri seti kaynağı, görev türü, bozulma ön-ayarı ve örnekleme ağırlığı."""

	hf_path: str
	weight: float
	task_type: str = "full_page_ocr" # "full_page_ocr" \| "doc_vqa" \| "line_ocr" \| "text_corpus"
	image_column: str = "image"
	text_column: str = "text"
	allowed_presets: Tuple[str, ...] = ("corporate_stamped",)
	split: str = "train"


	DEFAULT_TURKISH_HF_SOURCES: List[DatasetSourceSpec] = [
	DatasetSourceSpec(
	"Werea-co/werea-tr-doc-ocr-enterprise-v2",
	weight=0.22,
	task_type="full_page_ocr",
	text_column="markdown",
	allowed_presets=("corporate_stamped",),
	),
	DatasetSourceSpec(
	"Werea-co/werea-tr-doc-ocr-synthetic",
	weight=0.12,
	task_type="full_page_ocr",
	text_column="markdown",
	allowed_presets=("corporate_stamped",),
	),
	DatasetSourceSpec(
	"Ethosoft/TR-DocVQA-Synth",
	weight=0.16,
	task_type="doc_vqa",
	text_column="answer",
	allowed_presets=("corporate_stamped",),
	),
	DatasetSourceSpec(
	"HumynLabs/Turkish_Documents_Datasets_PDF",
	weight=0.12,
	task_type="full_page_ocr",
	text_column="text",
	allowed_presets=("corporate_stamped",),
	),
	DatasetSourceSpec(
	"ytu-ce-cosmos/turkce-kitap",
	weight=0.12,
	task_type="full_page_ocr",
	text_column="text",
	allowed_presets=("corporate_stamped",),
	),
	DatasetSourceSpec(
	"moganai/turkish-resmi-gazete",
	weight=0.10,
	task_type="full_page_ocr",
	text_column="text",
	allowed_presets=("corporate_stamped",),
	),
	DatasetSourceSpec(
	"emredeveloper/turkish-ocr",
	weight=0.10,
	task_type="line_ocr",
	text_column="text",
	allowed_presets=("mixed_handwritten_form",),
	),
	DatasetSourceSpec(
	"selimfirat/bilkent-turkish-writings-dataset",
	weight=0.06,
	task_type="line_ocr",
	text_column="text",
	allowed_presets=("mixed_handwritten_form",),
	),
	]


	class TurkishOCRDataQualityFilter:
	"""
	Dış kaynaklardan (Hugging Face / PDF kazıma) gelen örneklerde:
	1. UTF-8 Mojibake (`Ã¼ -> ü`, `Ä± -> ı`, `ÅŸ -> ş`) onarımı,
	2. Çözünürlük alt sınırı kontrolü (tam sayfa vs. satır),
	3. Boş veya bozuk (`\\ufffd`) etiket filtrelemesi
	uygulayan kalite kapısı.
	"""

	def __init__(
	self,
	min_page_side: int = 256,
	min_line_height: int = 16,
	min_text_chars: int = 2,
	) -> None:
	self.min_page_side = min_page_side
	self.min_line_height = min_line_height
	self.min_text_chars = min_text_chars

	@staticmethod
	def repair_mojibake(text: str) -> str:
	fixed = text
	for bad, good in TURKISH_MOJIBAKE_MAP.items():
	if bad in fixed:
	fixed = fixed.replace(bad, good)
	return fixed

	def validate_and_clean(
	self,
	image: Image.Image,
	text: str,
	task_type: str = "full_page_ocr",
	) -> Tuple[bool, str]:
	if not isinstance(text, str):
	return False, ""
	cleaned = self.repair_mojibake(text).strip()
	if len(cleaned) < self.min_text_chars or "\ufffd" in cleaned:
	return False, ""

	w, h = image.size
	if task_type == "line_ocr":
	if h < self.min_line_height or w < self.min_line_height:
	return False, ""
	else:
	if min(w, h) < self.min_page_side:
	return False, ""

	return True, cleaned


	def build_stamp_underneath_token_mask(
	full_text: str,
	stamp_underneath_text: str,
	char_Or_token_sequence: Optional[Sequence[str]] = None,
	) -> List[float]:
	"""
	`TurkceOCRVL.compute_turkish_vl_loss` ve `DiacriticFocalLoss` içinde kaşe altında
	kalan metin bölgesine `2.2x` odak ağırlığı vermek için karakter/token maskesi üretir.
	"""
	if not full_text:
	return []
	mask = [0.0] * len(full_text)
	sub = stamp_underneath_text.strip()
	if sub:
	start_idx = full_text.find(sub)
	if start_idx != -1:
	for i in range(start_idx, min(len(full_text), start_idx + len(sub))):
	mask[i] = 1.0
	if char_Or_token_sequence is None:
	return mask

	# Eğer alt-kelime (subword) token dizisi verilmişse karakter maskesini tokenlara izdüşür
	token_mask: List[float] = []
	cursor = 0
	for tok in char_Or_token_sequence:
	t_len = max(1, len(tok))
	span = mask[cursor : min(len(mask), cursor + t_len)]
	token_mask.append(1.0 if any(v > 0.5 for v in span) else 0.0)
	cursor = min(len(mask), cursor + t_len)
	return token_mask


	@dataclass
	class DatasetSufficiencySummary:
	"""Eğitim veri seti yeterlilik, entropi ve Türkçe morfoloji analiz raporu."""

	total_page_samples: int
	total_line_crops: int
	layout_entropy_bits: float
	diacritic_char_ratio: float
	agglutinative_long_word_ratio: float
	stamped_sample_ratio: float
	id_sample_count: int
	ood_sample_count: int
	passes_sufficiency_gate: bool
	notes: List[str] = field(default_factory=list)


	class TurkishOCRDatasetMixer:
	"""
	Hugging Face veri setlerini ve yerel parametrik `TurkishDocumentGenerator` motorunu
	birleştirip 3 evreli Müfredat Öğrenimi (Curriculum Learning) ve `TurkceDegrade` ile
	zenginleştiren PyTorch uyumlu veri kümesi.
	"""

	def __init__(
	self,
	degrade_prob: float = 0.75,
	curriculum_stage: int = 2,
	seed: int = 42,
	synthetic_fallback_size: int = 1000,
	) -> None:
	self.seed = seed
	self.rng = random.Random(seed)
	self.degrader = TurkceDegrade(seed=seed)
	self.generator = TurkishDocumentGenerator(seed=seed)
	self.table_renderer = ComplexTurkishTableRenderer(seed=seed)
	self.pdf_pair_generator = BornDigitalPDFPairGenerator(seed=seed)
	self.quality_filter = TurkishOCRDataQualityFilter()
	self.synthetic_fallback_size = synthetic_fallback_size
	self._external_samples: List[Dict[str, Any]] = []
	self.curriculum_stage = curriculum_stage
	self.degrade_prob = degrade_prob
	self.set_curriculum_stage(curriculum_stage, custom_degrade_prob=degrade_prob)

	def set_curriculum_stage(
	self, stage: int, custom_degrade_prob: Optional[float] = None
	) -> None:
	"""
	3 Evreli Müfredat Öğrenimi (Curriculum Learning):
	- Evre 1 (Temiz + Hafif Bozulma): `degrade_prob = 0.30`
	- Evre 2 (Yoğun Kaşe, İmza, Termal Fiş): `degrade_prob = 0.75`
	- Evre 3 (Zorlu Morfoloji, Çoklu Şablon + RLVR): `degrade_prob = 0.85`
	"""
	if stage not in (1, 2, 3):
	raise ValueError("curriculum_stage 1, 2 veya 3 olmalıdır.")
	self.curriculum_stage = stage
	if custom_degrade_prob is not None:
	self.degrade_prob = custom_degrade_prob
	elif stage == 1:
	self.degrade_prob = 0.30
	elif stage == 2:
	self.degrade_prob = 0.75
	else:
	self.degrade_prob = 0.85

	def load_hf_source(
	self,
	repo_id: str,
	split: str = "train",
	max_samples: Optional[int] = None,
	streaming: bool = True,
	) -> int:
	"""
	İlgili Hugging Face veri setini indirip kalite filtresinden geçirerek ve
	görev tipine göre (`<doc_qa>` veya tam sayfa Markdown) şemalandırarak havuza ekler.
	"""
	norm_id = repo_id.strip().lower()
	if any(guard in norm_id for guard in BENCHMARK_ONLY_CONTAMINATION_GUARD):
	raise ValueError(
	f"SIZINTI ENGELİ (Data Contamination Guard): '{repo_id}' resmi bir test "
	"benchmarkıdır ve eğitim veri kümesine dahil edilemez!"
	)

	from datasets import load_dataset

	spec_map = {s.hf_path: s for s in DEFAULT_TURKISH_HF_SOURCES}
	spec = spec_map.get(
	repo_id,
	DatasetSourceSpec(repo_id, weight=0.10, task_type="full_page_ocr"),
	)

	ds = load_dataset(repo_id, split=split, streaming=streaming)
	count = 0
	for row in ds:
	img = row.get("image") or row.get("page_image") or row.get("img")
	if not isinstance(img, Image.Image):
	continue

	# Ethosoft/TR-DocVQA-Synth gibi soru-cevap veri setlerinde soruyu kaybetme!
	if "question" in row and ("answer" in row or "answers" in row):
	q_str = str(row.get("question", "")).strip()
	a_raw = row.get("answer") or row.get("answers")
	a_str = a_raw[0] if isinstance(a_raw, list) and a_raw else str(a_raw or "")
	raw_txt = f"<doc_qa>\nSoru: {q_str}\nCevap: {a_str.strip()}\n</doc_qa>"
	task_type = "doc_vqa"
	else:
	raw_txt = (
	row.get("markdown")
	or row.get("text")
	or row.get("ground_truth")
	or row.get("label")
	or ""
	)
	task_type = spec.task_type

	rgb_img = img.convert("RGB")
	is_valid, cleaned_txt = self.quality_filter.validate_and_clean(
	rgb_img, str(raw_txt), task_type=task_type
	)
	if not is_valid:
	continue

	self._external_samples.append(
	{
	"image": rgb_img,
	"text": cleaned_txt,
	"source": repo_id,
	"task_type": task_type,
	"allowed_presets": spec.allowed_presets,
	"weight": spec.weight,
	}
	)
	count += 1
	if max_samples is not None and count >= max_samples:
	break
	return count

	def _select_compatible_preset(
	self, img: Image.Image, allowed_presets: Sequence[str]
	) -> str:
	"""Görüntü en-boy oranına ve veri kaynağına uyumlu bozulma ön-ayarı seçer."""
	if allowed_presets:
	return self.rng.choice(list(allowed_presets))
	w, h = img.size
	aspect = w / max(1.0, float(h))
	if aspect < 0.72:
	return "thermal_pos_receipt"
	if aspect > 1.35:
	return "id_card_camera"
	return "corporate_stamped"

	def __len__(self) -> int:
	if self._external_samples:
	return len(self._external_samples) + self.synthetic_fallback_size
	return self.synthetic_fallback_size

	def __getitem__(self, idx: int) -> Dict[str, Any]:
	if idx < len(self._external_samples):
	item = self._external_samples[idx]
	img: Image.Image = item["image"]
	text: str = item["text"]
	if self.rng.random() < self.degrade_prob:
	preset = self._select_compatible_preset(
	img, item.get("allowed_presets", ("corporate_stamped",))
	)
	deg = self.degrader.degrade(img, preset=preset)
	img = deg.image
	if deg.stamps_and_signatures:
	tags = "\n".join(s.xml_tag for s in deg.stamps_and_signatures)
	text = f"{text}\n{tags}"
	return {
	"image": img,
	"text": text,
	"source": item["source"],
	"task_type": item.get("task_type", "full_page_ocr"),
	"stamp_underneath_mask": [0.0] * len(text),
	"line_crops": [],
	}

	# 6 Şablon Ailesinden Parametrik Türkçe Belge Üretimi
	doc_choice = idx % 6
	apply_deg = self.rng.random() < self.degrade_prob
	# Evre 3'te şablon çeşitliliğini artırmak için %25 oranında çok-sütunlu / alternatif topoloji üret
	use_alt_topology = self.curriculum_stage == 3 and (idx % 5 == 0)

	if doc_choice == 0:
	sample = self.generator.generate_corporate_invoice(
	apply_degradation=apply_deg, is_ood=use_alt_topology
	)
	elif doc_choice == 1:
	sample = self.generator.generate_thermal_pos_receipt(
	apply_degradation=apply_deg, is_ood=use_alt_topology
	)
	elif doc_choice == 2:
	sample = self.generator.generate_mixed_handwritten_form(
	apply_degradation=apply_deg, is_ood=use_alt_topology
	)
	elif doc_choice == 3:
	sample = self.generator.generate_official_gazette_or_contract(
	apply_degradation=apply_deg, is_ood=use_alt_topology
	)
	elif doc_choice == 4:
	sample = self.table_renderer.generate_multi_vat_erp_invoice(
	apply_degradation=apply_deg, is_ood=use_alt_topology
	)
	else:
	sample = self.pdf_pair_generator.generate_two_column_born_digital_page(
	apply_degradation=apply_deg, is_ood=use_alt_topology
	)

	char_mask = build_stamp_underneath_token_mask(
	sample.ground_truth_markdown, sample.stamp_underneath_text
	)
	return {
	"image": sample.image,
	"clean_image": sample.clean_image,
	"text": sample.ground_truth_markdown,
	"plain_text": sample.plain_text,
	"stamp_underneath_text": sample.stamp_underneath_text,
	"stamp_underneath_mask": char_mask,
	"expected_kie": sample.expected_kie,
	"layout_variant": sample.layout_variant,
	"line_crops": sample.line_crops,
	"source": f"synthetic:{sample.doc_type}",
	"task_type": "full_page_ocr",
	}

	def extract_line_training_pairs(
	self, num_documents: int = 24
	) -> List[Tuple[Image.Image, str]]:
	"""
	`TürkçeOCR-Line (SVTRv2-TR)` eğitimi için bozulmuş/kaşeli tam sayfalardan
	gerçekçi satır kırpımları (`(crop_image, line_text)`) üretir.
	"""
	pairs: List[Tuple[Image.Image, str]] = []
	for i in range(num_documents):
	item = self[len(self._external_samples) + i]
	for crop_img, line_txt in item.get("line_crops", []):
	pairs.append((crop_img, line_txt))
	return pairs

	def evaluate_data_sufficiency(self, sample_count: int = 32) -> DatasetSufficiencySummary:
	"""
	Üretilen/birleştirilen eğitim havuzunun 12 KPI hedefimiz için yeterli
	yerleşim entropisine ($H_{\\text{layout}}$), diakritik yoğunluğuna,
	sondan eklemeli uzun kelime oranına ve kaşe altı kurtarma (`SURR`) çiftlerine
	sahip olup olmadığını nicel olarak denetler.
	"""
	layout_counts: Counter[str] = Counter()
	total_chars = 0
	diac_chars = 0
	total_words = 0
	long_words = 0
	stamped_count = 0
	id_count = 0
	ood_count = 0
	total_crops = 0

	diac_set = set("çğıöşüÇĞİÖŞÜâîûÂÎÛ")
	for i in range(sample_count):
	item = self[i]
	txt = item["text"]
	variant = item.get("layout_variant", item.get("source", "external"))
	layout_counts[variant] += 1
	total_crops += len(item.get("line_crops", []))

	if "ood" in str(variant):
	ood_count += 1
	else:
	id_count += 1

	if "<stamp>" in txt or item.get("stamp_underneath_text"):
	stamped_count += 1

	total_chars += len(txt)
	diac_chars += sum(1 for ch in txt if ch in diac_set)

	words = [w.strip(".,;:()[]*#<>\"'") for w in txt.split()]
	words = [w for w in words if w.isalpha()]
	total_words += len(words)
	long_words += sum(1 for w in words if len(w) >= 8)

	# Shannon yerleşim entropisi (bit cinsinden)
	entropy = 0.0
	for cnt in layout_counts.values():
	p = cnt / max(1, sample_count)
	if p > 0:
	entropy -= p * math.log2(p)

	diac_ratio = diac_chars / max(1, total_chars)
	long_word_ratio = long_words / max(1, total_words)
	stamped_ratio = stamped_count / max(1, sample_count)

	notes: List[str] = []
	passes = True
	if entropy < 1.5:
	passes = False
	notes.append(f"Düşük yerleşim entropisi ({entropy:.2f} bit < 1.50 bit): Şablon ezberleme riski!")
	if diac_ratio < 0.05:
	passes = False
	notes.append(f"Düşük Türkçe diakritik yoğunluğu ({diac_ratio:.4f} < 0.0500)!")
	if long_word_ratio < 0.20:
	passes = False
	notes.append(f"Düşük sondan eklemeli uzun kelime oranı ({long_word_ratio:.4f} < 0.2000)!")
	if stamped_ratio < 0.40:
	passes = False
	notes.append(f"Düşük kaşeli belge oranı ({stamped_ratio:.4f} < 0.4000): SURR KPI-7 için yetersiz!")
	if passes:
	notes.append(
	"Eğitim veri havuzu tüm yerleşim entropisi, Türkçe diakritik, morfolojik uzun kelime ve SURR kaşe altı kriterlerini karşılıyor."
	)

	return DatasetSufficiencySummary(
	total_page_samples=sample_count,
	total_line_crops=total_crops,
	layout_entropy_bits=round(entropy, 4),
	diacritic_char_ratio=round(diac_ratio, 4),
	agglutinative_long_word_ratio=round(long_word_ratio, 4),
	stamped_sample_ratio=round(stamped_ratio, 4),
	id_sample_count=id_count,
	ood_sample_count=ood_count,
	passes_sufficiency_gate=passes,
	notes=notes,
	)