""" E-Ticaret Musteri Hizmetleri Ozel Benchmark Testi 500 soru x 5 model karsilastirmali test Google Colab'da calistirilmak uzere tasarlanmistir. Kullanim: 1. Bu dosyayi ve custom_benchmark.json dosyasini Colab'a yukleyin. 2. Asagidaki pip install komutunu calistirin. 3. Hucreyi calistirin. !pip install transformers peft accelerate bitsandbytes torch sentencepiece protobuf """ import json import time import torch import gc from transformers import AutoModelForCausalLM, AutoTokenizer from peft import PeftModel # ===================================================== # AYARLAR # ===================================================== BENCHMARK_FILE = "custom_benchmark_final.json" MODELS = [ { "name": "Qwen2.5-1.5B (Base Model)", "model_id": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", "peft_id": None }, { "name": "TR-ECommerce-LoRA (Fine-Tuned)", "model_id": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", "peft_id": "Mer1Alii/TR-ECommerce-CustomerSupport-LoRA" }, { "name": "Gemma-3-1B-IT", "model_id": "unsloth/gemma-3-1b-it-unsloth-bnb-4bit", "peft_id": None }, { "name": "Llama-3.2-1B-Instruct", "model_id": "unsloth/Llama-3.2-1B-Instruct-bnb-4bit", "peft_id": None }, { "name": "Qwen2.5-0.5B-Instruct", "model_id": "unsloth/Qwen2.5-0.5B-Instruct-bnb-4bit", "peft_id": None } ] # ===================================================== # BENCHMARK VERILERINI YUKLE # ===================================================== def load_benchmark(filepath): with open(filepath, "r", encoding="utf-8") as f: data = json.load(f) print(f"Toplam {len(data)} soru yuklendi.") return data # ===================================================== # MODEL YUKLE # ===================================================== def load_model(model_info): print(f"\nModel yukleniyor: {model_info['name']}") print(f" Base: {model_info['model_id']}") from transformers import BitsAndBytesConfig bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16) tokenizer = AutoTokenizer.from_pretrained(model_info["model_id"]) model = AutoModelForCausalLM.from_pretrained( model_info["model_id"], quantization_config=bnb_config, device_map="auto" ) if model_info["peft_id"]: print(f" LoRA: {model_info['peft_id']}") model = PeftModel.from_pretrained(model, model_info["peft_id"]) model.eval() return model, tokenizer # ===================================================== # TEK SORU TEST ET # ===================================================== def test_single_question(model, tokenizer, question_data): q = question_data["question"] choices = question_data["choices"] correct = question_data["answer"].strip().upper() choices_text = "\n".join(choices) prompt = f"""Asagidaki e-ticaret musteri hizmetleri sorusunu dikkatlice oku ve dogru cevabi SADECE tek bir harf olarak (A, B, C, D veya E) ver. Aciklama yapma. Soru: {q} Secenekler: {choices_text} Cevap:""" messages = [{"role": "user", "content": prompt}] try: text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) except: text = prompt inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=2048).to(model.device) with torch.no_grad(): outputs = model.generate( **inputs, max_new_tokens=10, do_sample=False, temperature=0.0, pad_token_id=tokenizer.eos_token_id ) response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True).strip() # Cevaptan harfi cikart answer_letter = "" for char in response.upper(): if char in "ABCDE": answer_letter = char break return answer_letter == correct, answer_letter, correct # ===================================================== # MODEL TEST ET # ===================================================== def evaluate_model(model, tokenizer, questions, model_name): correct_count = 0 total = len(questions) category_scores = {} start_time = time.time() for i, q in enumerate(questions): is_correct, predicted, expected = test_single_question(model, tokenizer, q) if is_correct: correct_count += 1 # Kategori bazli skor cat = q.get("category", "Bilinmiyor") if cat not in category_scores: category_scores[cat] = {"correct": 0, "total": 0} category_scores[cat]["total"] += 1 if is_correct: category_scores[cat]["correct"] += 1 if (i + 1) % 50 == 0: elapsed = time.time() - start_time pct = (correct_count / (i + 1)) * 100 print(f" [{i+1}/{total}] Dogru: {correct_count}, Basari: %{pct:.1f} ({elapsed:.0f}sn)") elapsed = time.time() - start_time accuracy = (correct_count / total) * 100 print(f"\n === {model_name} SONUC ===") print(f" Toplam: {total}, Dogru: {correct_count}, Basari: %{accuracy:.2f}") print(f" Sure: {elapsed:.1f} saniye") print(f"\n Kategori Bazli Sonuclar:") for cat, scores in category_scores.items(): cat_pct = (scores['correct'] / scores['total']) * 100 print(f" {cat}: %{cat_pct:.1f} ({scores['correct']}/{scores['total']})") return { "model": model_name, "accuracy": accuracy, "correct": correct_count, "total": total, "time": elapsed, "categories": category_scores } # ===================================================== # TEMIZLIK # ===================================================== def cleanup_model(model, tokenizer): del model del tokenizer gc.collect() if torch.cuda.is_available(): torch.cuda.empty_cache() # ===================================================== # ANA PROGRAM # ===================================================== def main(): questions = load_benchmark(BENCHMARK_FILE) results = [] for model_info in MODELS: print("\n" + "=" * 60) try: model, tokenizer = load_model(model_info) result = evaluate_model(model, tokenizer, questions, model_info["name"]) results.append(result) cleanup_model(model, tokenizer) except Exception as e: print(f" HATA: {model_info['name']} - {e}") results.append({ "model": model_info["name"], "accuracy": 0, "correct": 0, "total": len(questions), "time": 0, "categories": {} }) # ===================================================== # OZET TABLO # ===================================================== print("\n" + "=" * 70) print("KARSILASTIRMALI SONUC TABLOSU") print("=" * 70) print(f"{'Model':<35} {'Basari':>8} {'Dogru':>7} {'Sure (sn)':>10}") print("-" * 70) for r in results: print(f"{r['model']:<35} %{r['accuracy']:>6.2f} {r['correct']:>5}/{r['total']} {r['time']:>9.1f}") print("=" * 70) # Kategori bazli tablo print("\nKATEGORI BAZLI KARSILASTIRMA") print("=" * 70) all_cats = set() for r in results: all_cats.update(r.get("categories", {}).keys()) for cat in sorted(all_cats): print(f"\n {cat}:") for r in results: cs = r.get("categories", {}).get(cat, {"correct": 0, "total": 0}) if cs["total"] > 0: pct = (cs["correct"] / cs["total"]) * 100 print(f" {r['model']:<33} %{pct:.1f} ({cs['correct']}/{cs['total']})") # Sonuclari dosyaya kaydet with open("benchmark_sonuclari.json", "w", encoding="utf-8") as f: json.dump(results, f, ensure_ascii=False, indent=2) print("\nSonuclar benchmark_sonuclari.json dosyasina kaydedildi.") # Model karti icin MD tablosu olustur md_lines = [] md_lines.append("| Model | Basari (%) | Dogru | Toplam Soru | Sure (sn) |") md_lines.append("| :--- | :--- | :--- | :--- | :--- |") for r in results: md_lines.append(f"| {r['model']} | **%{r['accuracy']:.2f}** | {r['correct']} | {r['total']} | {r['time']:.1f} |") with open("model_karti_tablosu.md", "w", encoding="utf-8") as f: f.write("\n".join(md_lines)) print("Model karti tablosu model_karti_tablosu.md dosyasina kaydedildi.") if __name__ == "__main__": main()