code/src/evaluate_e2e.py

#!/usr/bin/env python3
"""End-to-end evaluation — 4-stage pipeline SOTA karşılaştırması.

Metrics:
- Detection: mAP50-95 macro, per-source
- Classification: macro-F1 tier-stratified (head/mid/tail/rare), top-1/top-5
- Open-set: AUROC on oltr_holdout, FPR@95TPR
- Transliteration: CER per language (hit/akk/sum/elx), BLEU, exact match
- E2E tablet: gold set accuracy (3253 hitit_local)
- Per-period breakdown: OH/MH/NH/OB/NA/NB/Ur-III/ACH

Baselines for comparison:
- DeepScribe (RetinaNet+ResNet, replicated)
- CHURRO (Qwen2.5-VL-3B zero-shot)
- CuReD (Kraken transliteration)
- PreP-OCR (ByT5 without Viterbi)
"""
import json, argparse
from pathlib import Path
from collections import defaultdict, Counter
import numpy as np

def compute_map(predictions, ground_truth, iou_threshold=0.5):
    """COCO-style mAP50-95."""
    # Placeholder — gerçek impl için pycocotools kullan
    return {"mAP50": 0.0, "mAP50_95": 0.0}

def compute_macro_f1(preds, targets, tier_map=None):
    """Tier-stratified macro-F1."""
    from sklearn.metrics import f1_score, balanced_accuracy_score
    
    overall = {
        "macro_f1": float(f1_score(targets, preds, average='macro', zero_division=0)),
        "top1_acc": float((preds == targets).mean()),
        "balanced_acc": float(balanced_accuracy_score(targets, preds)),
    }
    if tier_map is None:
        return overall
    
    per_tier = {}
    for tier in ['head','mid','tail','rare']:
        mask = np.array([tier_map.get(t) == tier for t in targets])
        if mask.sum() > 0:
            per_tier[tier] = {
                "n_samples": int(mask.sum()),
                "macro_f1": float(f1_score(targets[mask], preds[mask], average='macro', zero_division=0)),
                "top1_acc": float((preds[mask] == targets[mask]).mean()),
            }
    return {**overall, "per_tier": per_tier}

def compute_openset_auroc(in_energies, out_energies):
    """AUROC for OOD detection (in-dist vs oltr_holdout)."""
    from sklearn.metrics import roc_auc_score
    y = np.concatenate([np.zeros(len(in_energies)), np.ones(len(out_energies))])
    scores = np.concatenate([in_energies, out_energies])
    return float(roc_auc_score(y, scores))

def compute_cer(hypotheses, references):
    """Character Error Rate (Levenshtein-based)."""
    import editdistance
    total_errors = 0
    total_chars = 0
    for h, r in zip(hypotheses, references):
        total_errors += editdistance.eval(h, r)
        total_chars += len(r)
    return total_errors / max(1, total_chars)

def evaluate(args):
    results = {
        "detection": {},
        "classification": {},
        "openset": {},
        "transliteration": {},
        "e2e_tablet": {},
        "baselines": {},
    }
    
    # Gold set yükle
    gold_records = []
    with open(args.gold_set) as f:
        for line in f:
            gold_records.append(json.loads(line))
    print(f"Gold set: {len(gold_records)} kayıt")
    
    # Per-period breakdown için
    period_breakdown = defaultdict(list)
    lang_breakdown = defaultdict(list)
    for r in gold_records:
        period_breakdown[r.get('period', '?')].append(r)
        lang_breakdown[r.get('language', '?')].append(r)
    
    results['gold_set_breakdown'] = {
        'per_period': {p: len(v) for p, v in period_breakdown.items()},
        'per_language': {l: len(v) for l, v in lang_breakdown.items()},
    }
    
    # Hedefler tablosu (PIPELINE.md'den)
    results['targets_vs_actual'] = {
        'detection_mAP50_95': {'target': 0.80, 'deepscribe_baseline': 0.78},
        'classification_macro_f1_head': {'target': 0.92},
        'classification_macro_f1_mid': {'target': 0.75},
        'classification_macro_f1_tail': {'target': 0.50},
        'classification_macro_f1_rare': {'target': 0.25},
        'openset_auroc': {'target': 0.85},
        'cer_hit': {'target': 0.08, 'prep_ocr_baseline': 0.09},
        'cer_akk': {'target': 0.09},
        'cer_sum': {'target': 0.12},
        'cer_elx': {'target': 0.10},
        'churro_baseline_levenshtein': 0.823,  # karşılaştırma
        'deepscribe_top5': 0.80,
    }
    
    out = Path(args.output)
    out.mkdir(parents=True, exist_ok=True)
    with open(out / "evaluation_report.json", 'w') as f:
        json.dump(results, f, indent=2, ensure_ascii=False)
    print(f"Evaluation raporu: {out}/evaluation_report.json")

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--detection-ckpts', nargs='+', help='YOLO per-fold checkpoints')
    ap.add_argument('--classifier-ckpt')
    ap.add_argument('--translit-ckpt')
    ap.add_argument('--gold-set', default='datasets/eval_gold/manifest.jsonl')
    ap.add_argument('--output', default='hitit_ocr/runs/eval_e2e_v1/')
    args = ap.parse_args()
    evaluate(args)

if __name__ == '__main__':
    main()