evaluate.py

import json
import torch
from sklearn.metrics import classification_report, accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.utils.multiclass import unique_labels
from transformers import AutoTokenizer, AutoModelForSequenceClassification



with open("intents_augmented.json", encoding="utf-8") as f:
    data = json.load(f)

sentences = []
labels = []
label2id = {}
id2label = {}

for i, intent in enumerate(data["intents"]):
    tag = intent["tag"]
    label2id[tag] = i
    id2label[i] = tag
    for pattern in intent["patterns"]:
        sentences.append(pattern)
        labels.append(i)


_, val_texts, _, val_labels = train_test_split(
    sentences, labels, test_size=0.2, random_state=42
)



MODEL_PATH = "./intent_model"   
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
model = AutoModelForSequenceClassification.from_pretrained(MODEL_PATH)
model.eval()  


val_encodings = tokenizer(val_texts, truncation=True, padding=True, max_length=128, return_tensors="pt")



with torch.no_grad():
    outputs = model(**val_encodings)
    preds = torch.argmax(outputs.logits, dim=1)



print(f" الدقة: {accuracy_score(val_labels, preds) * 100:.2f}%")
print("\n تقرير التصنيف:")

labels_used = unique_labels(val_labels, preds)
print(classification_report(
    val_labels,
    preds,
    labels=labels_used,
    target_names=[id2label[i] for i in labels_used],
    zero_division=0
))



print("\n أمثلة على الأخطاء:")
for text, true, pred in zip(val_texts, val_labels, preds):
    if true != pred:
        print(f"- السؤال: {text} | متوقع: {id2label[true]} | تنبؤ: {id2label[int(pred)]}")