code/convert_to_coco.py

#!/usr/bin/env python3
"""
Hitit civi yazisi tablet anotasyonlarini (mark.txt) COCO formatina donusturur.

Kullanim:
    python3 convert_to_coco.py --inspect       # Sadece format kesfet, donusturme
    python3 convert_to_coco.py --convert        # COCO formatina donustur
    python3 convert_to_coco.py --convert --merge-ebl  # eBL verileriyle birlestir

Cikti:
    output/coco_hitit_train.json
    output/coco_hitit_val.json
    output/images/  (symlink'ler)
"""

import json
import os
import sys
import glob
import argparse
import random
from collections import defaultdict
from pathlib import Path

BASE_DIR = Path(__file__).parent
TABLETS_DIR = BASE_DIR / "datasets" / "sources" / "hitit_local"
EBL_DIR = BASE_DIR / "datasets" / "sources" / "ebl_ocr" / "ready-for-training"
OUTPUT_DIR = BASE_DIR / "output"

# ============================================================
# ADIM 1: mark.txt formatini kesfet
# ============================================================

def inspect_mark_file(path):
    """Tek bir mark.txt dosyasinin yapisini analiz et."""
    with open(path, "r", encoding="utf-8") as f:
        raw = f.read()

    # JSON parse
    try:
        data = json.loads(raw)
    except json.JSONDecodeError:
        # Belki UTF-16?
        try:
            with open(path, "r", encoding="utf-16") as f:
                raw = f.read()
            data = json.loads(raw)
        except:
            print(f"  HATA: {path} JSON olarak parse edilemedi")
            print(f"  Ilk 500 karakter: {raw[:500]}")
            return None

    return data


def describe_structure(data, prefix="", depth=0, max_depth=4):
    """JSON yapisini ozetler."""
    indent = "  " * depth
    if depth > max_depth:
        print(f"{indent}...")
        return

    if isinstance(data, dict):
        print(f"{indent}{prefix}dict (anahtarlar: {list(data.keys())})")
        for key in list(data.keys())[:15]:
            val = data[key]
            if isinstance(val, (dict, list)):
                describe_structure(val, prefix=f'"{key}": ', depth=depth + 1, max_depth=max_depth)
            else:
                valstr = str(val)
                if len(valstr) > 100:
                    valstr = valstr[:100] + "..."
                print(f"{indent}  \"{key}\": {type(val).__name__} = {valstr}")
    elif isinstance(data, list):
        print(f"{indent}{prefix}list (uzunluk: {len(data)})")
        if len(data) > 0:
            # Ilk elemani goster
            print(f"{indent}  [0]:")
            describe_structure(data[0], depth=depth + 1, max_depth=max_depth)
            if len(data) > 1:
                print(f"{indent}  ... ({len(data) - 1} eleman daha)")
    else:
        valstr = str(data)
        if len(valstr) > 100:
            valstr = valstr[:100] + "..."
        print(f"{indent}{prefix}{type(data).__name__} = {valstr}")


def inspect_all():
    """Tum mark.txt dosyalarini tara, format ozetini cikar."""
    mark_files = sorted(glob.glob(str(TABLETS_DIR / "*" / "mark.txt")))
    print(f"\n{'='*60}")
    print(f"TOPLAM {len(mark_files)} mark.txt dosyasi bulundu")
    print(f"{'='*60}\n")

    if not mark_files:
        print("HATA: Hicbir mark.txt dosyasi bulunamadi!")
        return

    # Ilk dosyayi detayli incele
    first = mark_files[0]
    print(f"--- {first} ---")
    data = inspect_mark_file(first)
    if data is None:
        return

    print(f"\nDosya boyutu: {os.path.getsize(first) / 1024:.1f} KB")
    print(f"\nYAPI:")
    describe_structure(data)

    # Tum dosyalarin istatistiklerini topla
    print(f"\n{'='*60}")
    print("TUM DOSYALARIN OZETI")
    print(f"{'='*60}")

    total_annotations = 0
    all_names = set()
    all_types = set()
    errors = []

    for mf in mark_files:
        try:
            d = inspect_mark_file(mf)
            if d is None:
                errors.append(mf)
                continue

            annotations = extract_annotations(d)
            total_annotations += len(annotations)

            for ann in annotations:
                if "name" in ann:
                    all_names.add(ann["name"])
                if "text" in ann:
                    all_names.add(ann["text"])
                if "type" in ann:
                    all_types.add(ann["type"])
        except Exception as e:
            errors.append(f"{mf}: {e}")

    print(f"Toplam anotasyon sayisi: {total_annotations}")
    print(f"Benzersiz sinif sayisi: {len(all_names)}")
    print(f"Anotasyon tipleri: {all_types}")
    print(f"Hatali dosyalar: {len(errors)}")
    if errors:
        for e in errors[:10]:
            print(f"  - {e}")

    if all_names:
        sorted_names = sorted(all_names)
        print(f"\nSiniflar (ilk 50):")
        for n in sorted_names[:50]:
            print(f"  - {n}")
        if len(sorted_names) > 50:
            print(f"  ... ve {len(sorted_names) - 50} sinif daha")


# ============================================================
# ADIM 2: Anotasyonlari cikar (format-agnostik)
# ============================================================

def extract_annotations(data):
    """
    JSON yapisindan anotasyonlari cikar.
    Farkli olasi yapilari dener.
    Her anotasyon: {x, y, width, height, name/text, ...}
    """
    annotations = []

    if isinstance(data, list):
        for item in data:
            ann = extract_single_annotation(item)
            if ann:
                annotations.append(ann)

    elif isinstance(data, dict):
        # Ust seviye dict olabilir, icerideki listelerden cikar
        # Olasi anahtarlar: marks, annotations, objects, regions, shapes
        for key in data:
            val = data[key]
            if isinstance(val, list) and len(val) > 0:
                for item in val:
                    ann = extract_single_annotation(item)
                    if ann:
                        annotations.append(ann)

        # Belki dict'in kendisi tek bir anotasyon?
        if not annotations:
            ann = extract_single_annotation(data)
            if ann:
                annotations.append(ann)

    return annotations


def extract_single_annotation(item):
    """Tek bir anotasyon objesinden x, y, w, h ve sinif bilgisi cikar."""
    if not isinstance(item, dict):
        return None

    ann = {}

    # Bounding box bilgisi
    # Durum 1: Dogrudan x, y, width, height
    if "x" in item and "y" in item:
        ann["x"] = float(item["x"])
        ann["y"] = float(item["y"])
        ann["width"] = float(item.get("width", 0))
        ann["height"] = float(item.get("height", 0))

    # Durum 2: "mark" icinde
    elif "mark" in item and isinstance(item["mark"], dict):
        mark = item["mark"]
        if "x" in mark and "y" in mark:
            ann["x"] = float(mark["x"])
            ann["y"] = float(mark["y"])
            ann["width"] = float(mark.get("width", 0))
            ann["height"] = float(mark.get("height", 0))

    # Durum 3: "rect" icinde
    elif "rect" in item and isinstance(item["rect"], dict):
        r = item["rect"]
        if "x" in r and "y" in r:
            ann["x"] = float(r["x"])
            ann["y"] = float(r["y"])
            ann["width"] = float(r.get("width", r.get("w", 0)))
            ann["height"] = float(r.get("height", r.get("h", 0)))

    if "x" not in ann:
        return None

    # Sinif/etiket bilgisi
    for name_key in ["name", "text", "label", "class", "category", "comment"]:
        if name_key in item and item[name_key]:
            ann["name"] = str(item[name_key])
            break
        # "mark" icinde olabilir
        if "mark" in item and isinstance(item["mark"], dict):
            if name_key in item["mark"] and item["mark"][name_key]:
                ann["name"] = str(item["mark"][name_key])
                break

    # ID
    if "id" in item:
        ann["id"] = item["id"]

    # Type
    if "type" in item:
        ann["type"] = item["type"]

    return ann


def extract_image_info(data):
    """JSON verisinden gorsel boyut bilgisini cikar."""
    info = {}

    if isinstance(data, dict):
        for key in ("naturalWidth", "imageWidth", "width"):
            if key in data:
                info["width"] = int(data[key])
                break
        for key in ("naturalHeight", "imageHeight", "height"):
            if key in data:
                info["height"] = int(data[key])
                break
        # Ic icte olabilir
        if "image" in data and isinstance(data["image"], dict):
            img = data["image"]
            for key in ("naturalWidth", "width"):
                if key in img:
                    info["width"] = int(img[key])
                    break
            for key in ("naturalHeight", "height"):
                if key in img:
                    info["height"] = int(img[key])
                    break

    return info


# ============================================================
# ADIM 3: COCO formatina donustur
# ============================================================

def convert_to_coco(val_ratio=0.2, seed=42):
    """Tum mark.txt dosyalarini COCO formatina donustur."""
    mark_files = sorted(glob.glob(str(TABLETS_DIR / "*" / "mark.txt")))
    print(f"\n{len(mark_files)} mark.txt dosyasi isleniyor...\n")

    # Her tablet klasoru icin gorsel dosyasini bul
    all_data = []  # [(tablet_id, image_path, annotations, image_info)]
    category_names = set()

    for mf in mark_files:
        tablet_dir = Path(mf).parent
        tablet_id = tablet_dir.name

        # Gorseli bul (jpg)
        images = list(tablet_dir.glob("*.jpg")) + list(tablet_dir.glob("*.png"))
        if not images:
            print(f"  UYARI: {tablet_id} icin gorsel bulunamadi, atlaniyor")
            continue
        image_path = images[0]

        # Anotasyonlari cikar
        data = inspect_mark_file(mf)
        if data is None:
            continue

        annotations = extract_annotations(data)
        image_info = extract_image_info(data)

        if not annotations:
            print(f"  UYARI: {tablet_id} icin anotasyon bulunamadi")
            continue

        for ann in annotations:
            if "name" in ann:
                category_names.add(ann["name"])

        all_data.append((tablet_id, image_path, annotations, image_info))

    if not all_data:
        print("HATA: Hicbir veri islenmedi!")
        return

    # Kategori mapping olustur
    sorted_categories = sorted(category_names)
    cat_to_id = {name: i for i, name in enumerate(sorted_categories)}

    print(f"\nToplam {len(all_data)} tablet islendi")
    print(f"Toplam {sum(len(d[2]) for d in all_data)} anotasyon")
    print(f"Toplam {len(sorted_categories)} sinif\n")

    # Train/val bolustur
    random.seed(seed)
    indices = list(range(len(all_data)))
    random.shuffle(indices)
    val_count = max(1, int(len(indices) * val_ratio))
    val_indices = set(indices[:val_count])

    train_data = [all_data[i] for i in range(len(all_data)) if i not in val_indices]
    val_data = [all_data[i] for i in range(len(all_data)) if i in val_indices]

    print(f"Train: {len(train_data)} gorsel")
    print(f"Val:   {len(val_data)} gorsel\n")

    # Output klasorlerini olustur
    OUTPUT_DIR.mkdir(exist_ok=True)
    img_dir = OUTPUT_DIR / "images"
    img_dir.mkdir(exist_ok=True)

    # COCO JSON olustur
    for split_name, split_data in [("train", train_data), ("val", val_data)]:
        coco = build_coco_json(split_data, cat_to_id, sorted_categories, img_dir)

        out_path = OUTPUT_DIR / f"coco_hitit_{split_name}.json"
        with open(out_path, "w", encoding="utf-8") as f:
            json.dump(coco, f, ensure_ascii=False, indent=2)
        print(f"Kaydedildi: {out_path}")

    # Kategori listesini ayri kaydet
    cat_path = OUTPUT_DIR / "categories.json"
    with open(cat_path, "w", encoding="utf-8") as f:
        json.dump(
            [{"id": cat_to_id[n], "name": n} for n in sorted_categories],
            f, ensure_ascii=False, indent=2
        )
    print(f"Kaydedildi: {cat_path}")


def build_coco_json(split_data, cat_to_id, sorted_categories, img_dir):
    """COCO formatinda JSON olustur."""
    images = []
    annotations = []
    ann_id = 0

    for img_id, (tablet_id, image_path, anns, image_info) in enumerate(split_data):
        # Gorsel bilgisi
        w = image_info.get("width", 0)
        h = image_info.get("height", 0)

        # Boyut bilinmiyorsa dosyadan okumaya calis
        if w == 0 or h == 0:
            try:
                from PIL import Image
                with Image.open(image_path) as im:
                    w, h = im.size
            except ImportError:
                print(f"  UYARI: PIL yuklu degil, {tablet_id} boyutu bilinmiyor")
            except Exception as e:
                print(f"  UYARI: {tablet_id} gorsel okunamadi: {e}")

        # Gorseli output/images altina sembolik link yap
        ext = image_path.suffix
        new_name = f"tablet_{tablet_id}{ext}"
        link_path = img_dir / new_name
        if not link_path.exists():
            try:
                os.symlink(image_path.resolve(), link_path)
            except OSError:
                # Symlink desteklenmiyorsa kopyala
                import shutil
                shutil.copy2(image_path, link_path)

        images.append({
            "id": img_id,
            "file_name": new_name,
            "height": h,
            "width": w,
        })

        # Anotasyonlar
        for ann in anns:
            cat_name = ann.get("name", "unknown")
            if cat_name not in cat_to_id:
                continue

            x, y = ann["x"], ann["y"]
            bw, bh = ann["width"], ann["height"]
            area = bw * bh

            annotations.append({
                "id": ann_id,
                "image_id": img_id,
                "category_id": cat_to_id[cat_name],
                "bbox": [x, y, bw, bh],
                "area": area,
                "segmentation": [],
                "iscrowd": 0,
            })
            ann_id += 1

    coco = {
        "images": images,
        "annotations": annotations,
        "categories": [{"id": cat_to_id[n], "name": n} for n in sorted_categories],
    }
    return coco


# ============================================================
# ADIM 4: eBL verileriyle birlestir
# ============================================================

def merge_with_ebl():
    """Hitit COCO verisini eBL COCO verisiyle birlestir."""
    hitit_train = OUTPUT_DIR / "coco_hitit_train.json"
    if not hitit_train.exists():
        print("HATA: Once --convert ile Hitit verisini donusturun!")
        return

    # eBL COCO dosyasini bul
    ebl_coco_paths = [
        EBL_DIR / "coco-recognition" / "data" / "coco" / "annotations" / "instances_train2017.json",
        EBL_DIR / "coco-two-stage" / "data" / "coco" / "annotations" / "instances_val2017.json",
    ]

    for ebl_path in ebl_coco_paths:
        if not ebl_path.exists():
            print(f"  eBL dosyasi bulunamadi: {ebl_path}")
            continue

        print(f"\n  eBL dosyasi: {ebl_path}")
        with open(ebl_path, "r") as f:
            ebl_coco = json.load(f)

        with open(hitit_train, "r") as f:
            hitit_coco = json.load(f)

        # Kategori birlesimi
        hitit_cats = {c["name"]: c["id"] for c in hitit_coco["categories"]}
        ebl_cats = {c["name"]: c["id"] for c in ebl_coco["categories"]}

        all_cat_names = sorted(set(hitit_cats.keys()) | set(ebl_cats.keys()))
        merged_cat_to_id = {name: i for i, name in enumerate(all_cat_names)}

        # Hitit anotasyonlarini yeni ID'lerle guncelle
        old_to_new_hitit = {hitit_cats[n]: merged_cat_to_id[n] for n in hitit_cats}
        for ann in hitit_coco["annotations"]:
            ann["category_id"] = old_to_new_hitit[ann["category_id"]]

        # eBL anotasyonlarini yeni ID'lerle guncelle ve ID offset ekle
        img_offset = max(im["id"] for im in hitit_coco["images"]) + 1
        ann_offset = max(a["id"] for a in hitit_coco["annotations"]) + 1 if hitit_coco["annotations"] else 0

        old_to_new_ebl = {ebl_cats[n]: merged_cat_to_id[n] for n in ebl_cats}
        for ann in ebl_coco["annotations"]:
            ann["id"] += ann_offset
            ann["image_id"] += img_offset
            ann["category_id"] = old_to_new_ebl[ann["category_id"]]

        for im in ebl_coco["images"]:
            im["id"] += img_offset

        # Birlestir
        merged = {
            "images": hitit_coco["images"] + ebl_coco["images"],
            "annotations": hitit_coco["annotations"] + ebl_coco["annotations"],
            "categories": [{"id": merged_cat_to_id[n], "name": n} for n in all_cat_names],
        }

        out_name = f"coco_merged_{ebl_path.parent.parent.parent.name}.json"
        out_path = OUTPUT_DIR / out_name
        with open(out_path, "w", encoding="utf-8") as f:
            json.dump(merged, f, ensure_ascii=False, indent=2)

        print(f"  Birlesik: {len(merged['images'])} gorsel, "
              f"{len(merged['annotations'])} anotasyon, "
              f"{len(merged['categories'])} sinif")
        print(f"  Kaydedildi: {out_path}")


# ============================================================
# MAIN
# ============================================================

def main():
    parser = argparse.ArgumentParser(description="Hitit tablet anotasyonlarini COCO formatina donustur")
    parser.add_argument("--inspect", action="store_true", help="Sadece mark.txt formatini kesfet")
    parser.add_argument("--convert", action="store_true", help="COCO formatina donustur")
    parser.add_argument("--merge-ebl", action="store_true", help="eBL verileriyle birlestir")
    parser.add_argument("--val-ratio", type=float, default=0.2, help="Validasyon orani (varsayilan: 0.2)")
    parser.add_argument("--seed", type=int, default=42, help="Random seed")

    args = parser.parse_args()

    if not any([args.inspect, args.convert, args.merge_ebl]):
        # Hicbir flag verilmezse inspect yap
        args.inspect = True

    if args.inspect:
        inspect_all()

    if args.convert:
        convert_to_coco(val_ratio=args.val_ratio, seed=args.seed)

    if args.merge_ebl:
        merge_with_ebl()


if __name__ == "__main__":
    main()