code/src/seq2seq/build_pairs.py

#!/usr/bin/env python3
"""Seq2seq training data builder.

hitit_local mark.txt'den tablet-level image-transliteration pair oluştur.
Her tablet için: image + sign sequence (reading order ile)

Output: datasets/processed/seq2seq_pairs.jsonl
"""
import json, os
from pathlib import Path
from collections import defaultdict

ROOT = Path("/arf/scratch/stakan/hitit-proje")
SOURCES = ROOT / "datasets" / "sources"

def extract_sign_sequence_from_mark(mark_path):
    """mark.txt JSON'dan tablet sign sequence'ı çıkar (reading order'da)."""
    try:
        data = json.load(open(mark_path))
    except Exception:
        return None
    
    spots = data.get('spots', [])
    if not spots:
        return None
    
    # Her spot: title = "<order>_<label>" (örn "1_AN", "2_me")
    # y, x coordinates ile satır + sütun sıralaması
    signs = []
    for spot in spots:
        title = spot.get('title', '')
        # Parse order + label
        if '_' in title:
            order_str, label = title.split('_', 1)
            try:
                order = int(order_str)
            except:
                order = 0
        else:
            order = 0
            label = title
        
        signs.append({
            'order': order,
            'label': label.strip(),
            'x': spot.get('x', 0),
            'y': spot.get('y', 0),
            'w': spot.get('width', 0),
            'h': spot.get('height', 0),
        })
    
    # Reading order: önce satır (y), sonra x
    # Ama 'order' zaten verilmiş
    signs.sort(key=lambda s: s['order'])
    
    return signs

def build_pairs():
    """hitit_local klasöründeki her tablet için pair oluştur."""
    pairs = []
    hitit_root = SOURCES / "hitit_local"
    
    for folder in sorted(hitit_root.iterdir()):
        if not folder.is_dir(): continue
        
        mark_path = folder / "mark.txt"
        if not mark_path.exists(): continue
        
        # Image: klasördeki ilk image
        images = [f for f in folder.iterdir() 
                  if f.suffix.lower() in {'.jpg', '.jpeg', '.png'}]
        if not images: continue
        image_path = images[0]
        
        signs = extract_sign_sequence_from_mark(mark_path)
        if not signs: continue
        
        # Sequence string: "AN me-na-aḫ-ḫa-an-da ki-iš-..."
        # Simple join with space; advanced: cuneiform Unicode output
        sequence_plain = " ".join(s['label'] for s in signs)
        sequence_abz = " ".join(s['label'].upper() for s in signs)
        
        pairs.append({
            'tablet_id': folder.name,
            'image_path': str(image_path),
            'n_signs': len(signs),
            'signs': signs,
            'sequence_plain': sequence_plain,
            'sequence_abz': sequence_abz,
        })
    
    # Output
    out = ROOT / "datasets/processed/seq2seq_pairs.jsonl"
    with open(out, 'w') as f:
        for p in pairs:
            f.write(json.dumps(p, ensure_ascii=False) + '\n')
    
    # Summary
    n = len(pairs)
    n_signs_total = sum(p['n_signs'] for p in pairs)
    print(f"Pairs: {n} tablet")
    print(f"Toplam sign: {n_signs_total:,}")
    print(f"Ortalama: {n_signs_total/max(1,n):.0f} sign/tablet")
    print(f"Output: {out}")
    
    # Uzunluk dağılımı
    lens = sorted([p['n_signs'] for p in pairs])
    if lens:
        print(f"Min: {lens[0]}, median: {lens[len(lens)//2]}, max: {lens[-1]}")
    
    return pairs

if __name__ == '__main__':
    build_pairs()