code/src/tlhdig_integration/tlhdig_exploitation.py

#!/usr/bin/env python3
"""TLHdig 0.2 TRIPLE-PARALLEL Corpus Exploitation.

Zenodo 15459134 (TLHdig 0.2) — foto yok AMA:
- Cuneiform Unicode (𒈨𒈾...) ↔ Transliteration (me-na-...) ↔ Phonetic (menaḫḫanta)
- 315,534 Hittite satır + 16K Akkadian + 11K Hurrian + 6K Hattian + 2K Luwian

Bu corpus ile 5 critical training signal üretilebilir:
1. Character-level Cuneiform → Transliteration seq2seq (text-to-text)
2. Phonetic reconstruction (transliteration → pronunciation)
3. Language ID (Hit/Akk/Hur/Hat/Luw/Sum/Pal)
4. Sign-sequence LM pretraining
5. Cuneiform Unicode ↔ ABZ code mapping (sign identification)
"""
import json, re
from pathlib import Path
from collections import Counter, defaultdict

ROOT = Path("/arf/scratch/stakan/hitit-proje")

def extract_training_corpora():
    """TLHdig'den 5 training corpus üret."""
    
    corpora = {
        'cuneiform_to_transliteration': [],  # Cuneiform Unicode → transliteration
        'transliteration_to_phonetic': [],    # Transliteration → phonetic
        'cuneiform_lm': [],                    # Cuneiform Unicode LM pretraining
        'transliteration_lm': [],              # Transliteration LM (KenLM input)
        'phonetic_lm': [],                     # Phonetic LM
        'language_id': [],                     # (text, lang) classification
        'sign_abz_pairs': [],                  # Cuneiform char → ABZ code (sign ID)
    }
    
    lang_stats = Counter()
    
    with open(ROOT / "datasets/sources/tlhdig/manifest.jsonl") as f:
        for line in f:
            r = json.loads(line)
            extra = r.get('extra') or {}
            if not isinstance(extra, dict): continue
            
            cuneiform = r.get('label_raw') or ''
            translit = r.get('label_norm') or ''
            phonetic = r.get('phonetic_reading') or ''
            lang = extra.get('lang', '?')
            tablet = extra.get('tablet', '')
            
            # Clean lang filter
            lang = lang.split(' ')[0] if lang else '?'
            lang_stats[lang] += 1
            
            # 1. Cuneiform → Transliteration
            if cuneiform and translit:
                # Remove damage markers for clean pairs
                cune_clean = re.sub(r'[▒]', '', cuneiform).strip()
                if cune_clean and translit:
                    corpora['cuneiform_to_transliteration'].append({
                        'cuneiform': cune_clean,
                        'transliteration': translit.strip(),
                        'lang': lang,
                        'tablet': tablet,
                    })
            
            # 2. Transliteration → Phonetic
            if translit and phonetic:
                corpora['transliteration_to_phonetic'].append({
                    'transliteration': translit.strip(),
                    'phonetic': phonetic.strip(),
                    'lang': lang,
                })
            
            # 3-5. LM corpora (language separated)
            if cuneiform:
                corpora['cuneiform_lm'].append({'text': cuneiform, 'lang': lang})
            if translit:
                corpora['transliteration_lm'].append({'text': translit, 'lang': lang})
            if phonetic:
                corpora['phonetic_lm'].append({'text': phonetic, 'lang': lang})
            
            # 6. Language ID
            if translit:
                corpora['language_id'].append({'text': translit, 'lang': lang})
            
            # 7. Cuneiform char → sign mapping
            # Cuneiform sign'ları ayıkla
            for ch in cuneiform:
                # Cuneiform Unicode range: U+12000-U+123FF (Sumero-Akkadian)
                #                          U+12400-U+1247F (Numbers)
                #                          U+12480-U+1254F (Early Dynastic)
                code = ord(ch)
                if 0x12000 <= code <= 0x1254F:
                    corpora['sign_abz_pairs'].append({
                        'char': ch,
                        'codepoint': f"U+{code:05X}",
                        'translit_context': translit,
                        'lang': lang,
                    })
    
    # Write all corpora
    out_dir = ROOT / "datasets/processed/tlhdig_corpora"
    out_dir.mkdir(parents=True, exist_ok=True)
    
    stats = {}
    for name, data in corpora.items():
        out_path = out_dir / f"{name}.jsonl"
        with open(out_path, 'w') as f:
            for item in data:
                f.write(json.dumps(item, ensure_ascii=False) + '\n')
        stats[name] = len(data)
        print(f"  {name}: {len(data):,} kayıt → {out_path.name}")
    
    # Hitit-only parallel corpus (the gold)
    hit_parallel = [item for item in corpora['cuneiform_to_transliteration'] if item['lang'] == 'Hit']
    (out_dir / "hittite_parallel.jsonl").write_text(
        '\n'.join(json.dumps(i, ensure_ascii=False) for i in hit_parallel)
    )
    print(f"\n  ★ hittite_parallel (Hit only): {len(hit_parallel):,}")
    
    # Cuneiform character frequency (ABZ discovery)
    char_freq = Counter()
    for item in corpora['sign_abz_pairs']:
        if item['lang'] == 'Hit':
            char_freq[item['char']] += 1
    
    (out_dir / "cuneiform_char_frequency_hit.json").write_text(
        json.dumps({
            'n_unique_chars': len(char_freq),
            'total_occurrences': sum(char_freq.values()),
            'top_100': [[c, n, f"U+{ord(c):05X}"] for c, n in char_freq.most_common(100)]
        }, indent=2, ensure_ascii=False)
    )
    print(f"  ★ Unique Hittite cuneiform chars: {len(char_freq)}")
    print(f"    Top-10: {[(c, n) for c, n in char_freq.most_common(10)]}")
    
    # Summary
    summary = {
        'source': 'Zenodo 15459134 (TLHdig 0.2 beta)',
        'license': 'CC-BY-4.0',
        'contributions': stats,
        'hittite_parallel_pairs': len(hit_parallel),
        'unique_hittite_cuneiform_chars': len(char_freq),
        'lang_distribution': dict(lang_stats.most_common()),
    }
    with open(out_dir / "summary.json", 'w') as f:
        json.dump(summary, f, indent=2, ensure_ascii=False)

if __name__ == '__main__':
    extract_training_corpora()