evaluate_qwen_and_plot.py

import os
import pandas as pd
import torch
import time
import json
import re
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix

def load_qwen_model():
    device = "mps" if torch.backends.mps.is_available() else "cpu"
    print(f"Loading on {device}...")
    model_id = "Qwen/Qwen2.5-0.5B-Instruct"
    adapter_path = "qwen2.5_local_mac_lora"
    
    tokenizer = AutoTokenizer.from_pretrained(model_id)
    tokenizer.pad_token = tokenizer.eos_token
    
    base_model = AutoModelForCausalLM.from_pretrained(
        model_id,
        torch_dtype=torch.float16,
        low_cpu_mem_usage=True
    ).to(device)
    model = PeftModel.from_pretrained(base_model, adapter_path)
    return model, tokenizer, device

def evaluate_on_dataset(model, tokenizer, device, df, text_col='text'):
    preds = []
    times = []
    
    for idx, row in df.iterrows():
        text = row[text_col]
        messages = [{"role": "user", "content": f"Analyse le sentiment de ce tweet : '{text}'"}]
        prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        inputs = tokenizer(prompt, return_tensors="pt").to(device)
        
        start_time = time.time()
        with torch.no_grad():
            outputs = model.generate(**inputs, max_new_tokens=15, temperature=0.1)
        inf_time = time.time() - start_time
        
        response = tokenizer.decode(outputs[0][len(inputs["input_ids"][0]):], skip_special_tokens=True).strip()
        
        if "positive" in response.lower():
            pred = 1
        else:
            pred = 0
            
        preds.append(pred)
        times.append(inf_time)
        
        if idx % 50 == 0:
            print(f"Processed {idx}/{len(df)}...")
            
    return preds, sum(times)/len(times)

def main():
    model, tokenizer, device = load_qwen_model()
    
    # 1. Benchmark V1
    print("Evaluating Benchmark V1...")
    df_v1 = pd.read_csv("data/benchmark_results.csv")
    preds, avg_time = evaluate_on_dataset(model, tokenizer, device, df_v1)
    df_v1['qwen_pred'] = preds
    df_v1.to_csv("data/benchmark_results.csv", index=False)
    
    qwen_acc_v1 = accuracy_score(df_v1['true_sentiment'], df_v1['qwen_pred'])
    qwen_f1_v1 = f1_score(df_v1['true_sentiment'], df_v1['qwen_pred'])
    
    keras_acc_v1 = accuracy_score(df_v1['true_sentiment'], df_v1['keras_pred'])
    llm_acc_v1 = accuracy_score(df_v1['true_sentiment'], df_v1['llm_pred'])
    keras_f1_v1 = f1_score(df_v1['true_sentiment'], df_v1['keras_pred'])
    llm_f1_v1 = f1_score(df_v1['true_sentiment'], df_v1['llm_pred'])
    
    # 2. Benchmark V2 (Modern)
    print("Evaluating Modern Benchmark V2...")
    df_v2 = pd.read_csv("data/modern_benchmark_results.csv")
    preds_v2, avg_time_v2 = evaluate_on_dataset(model, tokenizer, device, df_v2)
    df_v2['qwen_pred'] = preds_v2
    df_v2.to_csv("data/modern_benchmark_results.csv", index=False)
    
    qwen_acc_v2 = accuracy_score(df_v2['true_sentiment'], df_v2['qwen_pred'])
    keras_acc_v2 = accuracy_score(df_v2['true_sentiment'], df_v2['keras_pred'])
    llm_acc_v2 = accuracy_score(df_v2['true_sentiment'], df_v2['llm_pred'])
    
    # 3. Human Ground Truth
    print("Evaluating Human Ground Truth...")
    df_human = pd.read_csv("data/human_ground_truth.csv")
    preds_human, _ = evaluate_on_dataset(model, tokenizer, device, df_human)
    df_human['qwen_pred'] = preds_human
    df_human.to_csv("data/human_ground_truth.csv", index=False)
    
    qwen_human = accuracy_score(df_human['human_label'], df_human['qwen_pred'])
    
    # Generate JSON for app.py
    metrics = {
        "qwen_acc_v1": qwen_acc_v1,
        "qwen_f1_v1": qwen_f1_v1,
        "qwen_acc_v2": qwen_acc_v2,
        "qwen_human": qwen_human,
        "qwen_time": avg_time,
        "keras_acc_v1": keras_acc_v1,
        "llm_acc_v1": llm_acc_v1,
        "keras_f1_v1": keras_f1_v1,
        "llm_f1_v1": llm_f1_v1,
        "keras_acc_v2": keras_acc_v2,
        "llm_acc_v2": llm_acc_v2
    }
    with open("qwen_metrics.json", "w") as f:
        json.dump(metrics, f)
        
    # Plot 1: Accuracy V1
    plt.figure(figsize=(10, 6))
    models = ['Keras (Baseline)', 'Gemma 3 1B', 'Qwen 2.5 0.5B']
    accuracies = [keras_acc_v1*100, llm_acc_v1*100, qwen_acc_v1*100]
    sns.barplot(x=models, y=accuracies, palette=['#ff5e62', '#00df89', '#00c3ff'])
    plt.title('Accuracy Comparaison (Sentiment140 V1)')
    plt.ylabel('Accuracy (%)')
    plt.ylim(0, 100)
    for i, acc in enumerate(accuracies):
        plt.text(i, acc + 1, f'{acc:.1f}%', ha='center', fontweight='bold')
    plt.savefig('assets/accuracy_comparison.png', dpi=300, bbox_inches='tight')
    
    # Plot 2: Speed
    keras_time = 0.00013
    llm_time = 0.56
    plt.figure(figsize=(10, 6))
    times_plt = [keras_time, llm_time, avg_time]
    sns.barplot(x=models, y=times_plt, palette=['#ff5e62', '#00df89', '#00c3ff'])
    plt.title("Temps d'inférence moyen par tweet (secondes)")
    plt.ylabel('Temps (s)')
    for i, t in enumerate(times_plt):
        plt.text(i, t + 0.02, f'{t:.3f}s', ha='center', fontweight='bold')
    plt.savefig('assets/speed_comparison.png', dpi=300, bbox_inches='tight')
    
    # Plot 3: Robustness V2
    plt.figure(figsize=(10, 6))
    accuracies_v2 = [keras_acc_v2*100, llm_acc_v2*100, qwen_acc_v2*100]
    sns.barplot(x=models, y=accuracies_v2, palette=['#ff5e62', '#00df89', '#00c3ff'])
    plt.title('Sensibilité au Data Drift (Accuracy sur Tweets Modernes V2)')
    plt.ylabel('Accuracy (%)')
    plt.ylim(0, 100)
    for i, acc in enumerate(accuracies_v2):
        plt.text(i, acc + 1, f'{acc:.1f}%', ha='center', fontweight='bold')
    plt.savefig('assets/modern_accuracy_comparison.png', dpi=300, bbox_inches='tight')

    # Plot 4: Confusion Matrices
    fig, axes = plt.subplots(1, 3, figsize=(18, 5))
    cm_keras = confusion_matrix(df_v1['true_sentiment'], df_v1['keras_pred'])
    cm_llm = confusion_matrix(df_v1['true_sentiment'], df_v1['llm_pred'])
    cm_qwen = confusion_matrix(df_v1['true_sentiment'], df_v1['qwen_pred'])
    
    sns.heatmap(cm_keras, annot=True, fmt='d', cmap='Reds', ax=axes[0])
    axes[0].set_title('Keras Confusion Matrix')
    axes[0].set_xlabel('Predicted')
    axes[0].set_ylabel('True')
    
    sns.heatmap(cm_llm, annot=True, fmt='d', cmap='Greens', ax=axes[1])
    axes[1].set_title('Gemma 3 Confusion Matrix')
    axes[1].set_xlabel('Predicted')
    axes[1].set_ylabel('True')
    
    sns.heatmap(cm_qwen, annot=True, fmt='d', cmap='Blues', ax=axes[2])
    axes[2].set_title('Qwen Confusion Matrix')
    axes[2].set_xlabel('Predicted')
    axes[2].set_ylabel('True')
    
    plt.tight_layout()
    plt.savefig('assets/confusion_matrices.png', dpi=300, bbox_inches='tight')
    
    print("Done! Check assets/ and data/ directories.")

if __name__ == "__main__":
    main()