File size: 8,647 Bytes

5cc40d7

"""
E-Ticaret Musteri Hizmetleri Ozel Benchmark Testi
500 soru x 5 model karsilastirmali test
Google Colab'da calistirilmak uzere tasarlanmistir.

Kullanim:
1. Bu dosyayi ve custom_benchmark.json dosyasini Colab'a yukleyin.
2. Asagidaki pip install komutunu calistirin.
3. Hucreyi calistirin.

!pip install transformers peft accelerate bitsandbytes torch sentencepiece protobuf
"""

import json
import time
import torch
import gc
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel

# =====================================================
# AYARLAR
# =====================================================
BENCHMARK_FILE = "custom_benchmark_final.json"

MODELS = [
    {
        "name": "Qwen2.5-1.5B (Base Model)",
        "model_id": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit",
        "peft_id": None
    },
    {
        "name": "TR-ECommerce-LoRA (Fine-Tuned)",
        "model_id": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit",
        "peft_id": "Mer1Alii/TR-ECommerce-CustomerSupport-LoRA"
    },
    {
        "name": "Gemma-3-1B-IT",
        "model_id": "unsloth/gemma-3-1b-it-unsloth-bnb-4bit",
        "peft_id": None
    },
    {
        "name": "Llama-3.2-1B-Instruct",
        "model_id": "unsloth/Llama-3.2-1B-Instruct-bnb-4bit",
        "peft_id": None
    },
    {
        "name": "Qwen2.5-0.5B-Instruct",
        "model_id": "unsloth/Qwen2.5-0.5B-Instruct-bnb-4bit",
        "peft_id": None
    }
]

# =====================================================
# BENCHMARK VERILERINI YUKLE
# =====================================================
def load_benchmark(filepath):
    with open(filepath, "r", encoding="utf-8") as f:
        data = json.load(f)
    print(f"Toplam {len(data)} soru yuklendi.")
    return data

# =====================================================
# MODEL YUKLE
# =====================================================
def load_model(model_info):
    print(f"\nModel yukleniyor: {model_info['name']}")
    print(f"  Base: {model_info['model_id']}")

    from transformers import BitsAndBytesConfig
    bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16)

    tokenizer = AutoTokenizer.from_pretrained(model_info["model_id"])
    model = AutoModelForCausalLM.from_pretrained(
        model_info["model_id"],
        quantization_config=bnb_config,
        device_map="auto"
    )

    if model_info["peft_id"]:
        print(f"  LoRA: {model_info['peft_id']}")
        model = PeftModel.from_pretrained(model, model_info["peft_id"])

    model.eval()
    return model, tokenizer

# =====================================================
# TEK SORU TEST ET
# =====================================================
def test_single_question(model, tokenizer, question_data):
    q = question_data["question"]
    choices = question_data["choices"]
    correct = question_data["answer"].strip().upper()

    choices_text = "\n".join(choices)

    prompt = f"""Asagidaki e-ticaret musteri hizmetleri sorusunu dikkatlice oku ve dogru cevabi SADECE tek bir harf olarak (A, B, C, D veya E) ver. Aciklama yapma.

Soru: {q}

Secenekler:
{choices_text}

Cevap:"""

    messages = [{"role": "user", "content": prompt}]

    try:
        text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
    except:
        text = prompt

    inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=2048).to(model.device)

    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=10,
            do_sample=False,
            temperature=0.0,
            pad_token_id=tokenizer.eos_token_id
        )

    response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True).strip()

    # Cevaptan harfi cikart
    answer_letter = ""
    for char in response.upper():
        if char in "ABCDE":
            answer_letter = char
            break

    return answer_letter == correct, answer_letter, correct

# =====================================================
# MODEL TEST ET
# =====================================================
def evaluate_model(model, tokenizer, questions, model_name):
    correct_count = 0
    total = len(questions)
    category_scores = {}

    start_time = time.time()

    for i, q in enumerate(questions):
        is_correct, predicted, expected = test_single_question(model, tokenizer, q)
        if is_correct:
            correct_count += 1

        # Kategori bazli skor
        cat = q.get("category", "Bilinmiyor")
        if cat not in category_scores:
            category_scores[cat] = {"correct": 0, "total": 0}
        category_scores[cat]["total"] += 1
        if is_correct:
            category_scores[cat]["correct"] += 1

        if (i + 1) % 50 == 0:
            elapsed = time.time() - start_time
            pct = (correct_count / (i + 1)) * 100
            print(f"  [{i+1}/{total}] Dogru: {correct_count}, Basari: %{pct:.1f} ({elapsed:.0f}sn)")

    elapsed = time.time() - start_time
    accuracy = (correct_count / total) * 100

    print(f"\n  === {model_name} SONUC ===")
    print(f"  Toplam: {total}, Dogru: {correct_count}, Basari: %{accuracy:.2f}")
    print(f"  Sure: {elapsed:.1f} saniye")
    print(f"\n  Kategori Bazli Sonuclar:")
    for cat, scores in category_scores.items():
        cat_pct = (scores['correct'] / scores['total']) * 100
        print(f"    {cat}: %{cat_pct:.1f} ({scores['correct']}/{scores['total']})")

    return {
        "model": model_name,
        "accuracy": accuracy,
        "correct": correct_count,
        "total": total,
        "time": elapsed,
        "categories": category_scores
    }

# =====================================================
# TEMIZLIK
# =====================================================
def cleanup_model(model, tokenizer):
    del model
    del tokenizer
    gc.collect()
    if torch.cuda.is_available():
        torch.cuda.empty_cache()

# =====================================================
# ANA PROGRAM
# =====================================================
def main():
    questions = load_benchmark(BENCHMARK_FILE)
    results = []

    for model_info in MODELS:
        print("\n" + "=" * 60)
        try:
            model, tokenizer = load_model(model_info)
            result = evaluate_model(model, tokenizer, questions, model_info["name"])
            results.append(result)
            cleanup_model(model, tokenizer)
        except Exception as e:
            print(f"  HATA: {model_info['name']} - {e}")
            results.append({
                "model": model_info["name"],
                "accuracy": 0,
                "correct": 0,
                "total": len(questions),
                "time": 0,
                "categories": {}
            })

    # =====================================================
    # OZET TABLO
    # =====================================================
    print("\n" + "=" * 70)
    print("KARSILASTIRMALI SONUC TABLOSU")
    print("=" * 70)
    print(f"{'Model':<35} {'Basari':>8} {'Dogru':>7} {'Sure (sn)':>10}")
    print("-" * 70)
    for r in results:
        print(f"{r['model']:<35} %{r['accuracy']:>6.2f} {r['correct']:>5}/{r['total']} {r['time']:>9.1f}")
    print("=" * 70)

    # Kategori bazli tablo
    print("\nKATEGORI BAZLI KARSILASTIRMA")
    print("=" * 70)
    all_cats = set()
    for r in results:
        all_cats.update(r.get("categories", {}).keys())

    for cat in sorted(all_cats):
        print(f"\n  {cat}:")
        for r in results:
            cs = r.get("categories", {}).get(cat, {"correct": 0, "total": 0})
            if cs["total"] > 0:
                pct = (cs["correct"] / cs["total"]) * 100
                print(f"    {r['model']:<33} %{pct:.1f} ({cs['correct']}/{cs['total']})")

    # Sonuclari dosyaya kaydet
    with open("benchmark_sonuclari.json", "w", encoding="utf-8") as f:
        json.dump(results, f, ensure_ascii=False, indent=2)
    print("\nSonuclar benchmark_sonuclari.json dosyasina kaydedildi.")

    # Model karti icin MD tablosu olustur
    md_lines = []
    md_lines.append("| Model | Basari (%) | Dogru | Toplam Soru | Sure (sn) |")
    md_lines.append("| :--- | :--- | :--- | :--- | :--- |")
    for r in results:
        md_lines.append(f"| {r['model']} | **%{r['accuracy']:.2f}** | {r['correct']} | {r['total']} | {r['time']:.1f} |")

    with open("model_karti_tablosu.md", "w", encoding="utf-8") as f:
        f.write("\n".join(md_lines))
    print("Model karti tablosu model_karti_tablosu.md dosyasina kaydedildi.")

if __name__ == "__main__":
    main()