import os import pandas as pd import torch import time import json import re from transformers import AutoModelForCausalLM, AutoTokenizer from peft import PeftModel import matplotlib.pyplot as plt import seaborn as sns from sklearn.metrics import accuracy_score, f1_score, confusion_matrix def load_qwen_model(): device = "mps" if torch.backends.mps.is_available() else "cpu" print(f"Loading on {device}...") model_id = "Qwen/Qwen2.5-0.5B-Instruct" adapter_path = "qwen2.5_local_mac_lora" tokenizer = AutoTokenizer.from_pretrained(model_id) tokenizer.pad_token = tokenizer.eos_token base_model = AutoModelForCausalLM.from_pretrained( model_id, torch_dtype=torch.float16, low_cpu_mem_usage=True ).to(device) model = PeftModel.from_pretrained(base_model, adapter_path) return model, tokenizer, device def evaluate_on_dataset(model, tokenizer, device, df, text_col='text'): preds = [] times = [] for idx, row in df.iterrows(): text = row[text_col] messages = [{"role": "user", "content": f"Analyse le sentiment de ce tweet : '{text}'"}] prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) inputs = tokenizer(prompt, return_tensors="pt").to(device) start_time = time.time() with torch.no_grad(): outputs = model.generate(**inputs, max_new_tokens=15, temperature=0.1) inf_time = time.time() - start_time response = tokenizer.decode(outputs[0][len(inputs["input_ids"][0]):], skip_special_tokens=True).strip() if "positive" in response.lower(): pred = 1 else: pred = 0 preds.append(pred) times.append(inf_time) if idx % 50 == 0: print(f"Processed {idx}/{len(df)}...") return preds, sum(times)/len(times) def main(): model, tokenizer, device = load_qwen_model() # 1. Benchmark V1 print("Evaluating Benchmark V1...") df_v1 = pd.read_csv("data/benchmark_results.csv") preds, avg_time = evaluate_on_dataset(model, tokenizer, device, df_v1) df_v1['qwen_pred'] = preds df_v1.to_csv("data/benchmark_results.csv", index=False) qwen_acc_v1 = accuracy_score(df_v1['true_sentiment'], df_v1['qwen_pred']) qwen_f1_v1 = f1_score(df_v1['true_sentiment'], df_v1['qwen_pred']) keras_acc_v1 = accuracy_score(df_v1['true_sentiment'], df_v1['keras_pred']) llm_acc_v1 = accuracy_score(df_v1['true_sentiment'], df_v1['llm_pred']) keras_f1_v1 = f1_score(df_v1['true_sentiment'], df_v1['keras_pred']) llm_f1_v1 = f1_score(df_v1['true_sentiment'], df_v1['llm_pred']) # 2. Benchmark V2 (Modern) print("Evaluating Modern Benchmark V2...") df_v2 = pd.read_csv("data/modern_benchmark_results.csv") preds_v2, avg_time_v2 = evaluate_on_dataset(model, tokenizer, device, df_v2) df_v2['qwen_pred'] = preds_v2 df_v2.to_csv("data/modern_benchmark_results.csv", index=False) qwen_acc_v2 = accuracy_score(df_v2['true_sentiment'], df_v2['qwen_pred']) keras_acc_v2 = accuracy_score(df_v2['true_sentiment'], df_v2['keras_pred']) llm_acc_v2 = accuracy_score(df_v2['true_sentiment'], df_v2['llm_pred']) # 3. Human Ground Truth print("Evaluating Human Ground Truth...") df_human = pd.read_csv("data/human_ground_truth.csv") preds_human, _ = evaluate_on_dataset(model, tokenizer, device, df_human) df_human['qwen_pred'] = preds_human df_human.to_csv("data/human_ground_truth.csv", index=False) qwen_human = accuracy_score(df_human['human_label'], df_human['qwen_pred']) # Generate JSON for app.py metrics = { "qwen_acc_v1": qwen_acc_v1, "qwen_f1_v1": qwen_f1_v1, "qwen_acc_v2": qwen_acc_v2, "qwen_human": qwen_human, "qwen_time": avg_time, "keras_acc_v1": keras_acc_v1, "llm_acc_v1": llm_acc_v1, "keras_f1_v1": keras_f1_v1, "llm_f1_v1": llm_f1_v1, "keras_acc_v2": keras_acc_v2, "llm_acc_v2": llm_acc_v2 } with open("qwen_metrics.json", "w") as f: json.dump(metrics, f) # Plot 1: Accuracy V1 plt.figure(figsize=(10, 6)) models = ['Keras (Baseline)', 'Gemma 3 1B', 'Qwen 2.5 0.5B'] accuracies = [keras_acc_v1*100, llm_acc_v1*100, qwen_acc_v1*100] sns.barplot(x=models, y=accuracies, palette=['#ff5e62', '#00df89', '#00c3ff']) plt.title('Accuracy Comparaison (Sentiment140 V1)') plt.ylabel('Accuracy (%)') plt.ylim(0, 100) for i, acc in enumerate(accuracies): plt.text(i, acc + 1, f'{acc:.1f}%', ha='center', fontweight='bold') plt.savefig('assets/accuracy_comparison.png', dpi=300, bbox_inches='tight') # Plot 2: Speed keras_time = 0.00013 llm_time = 0.56 plt.figure(figsize=(10, 6)) times_plt = [keras_time, llm_time, avg_time] sns.barplot(x=models, y=times_plt, palette=['#ff5e62', '#00df89', '#00c3ff']) plt.title("Temps d'inférence moyen par tweet (secondes)") plt.ylabel('Temps (s)') for i, t in enumerate(times_plt): plt.text(i, t + 0.02, f'{t:.3f}s', ha='center', fontweight='bold') plt.savefig('assets/speed_comparison.png', dpi=300, bbox_inches='tight') # Plot 3: Robustness V2 plt.figure(figsize=(10, 6)) accuracies_v2 = [keras_acc_v2*100, llm_acc_v2*100, qwen_acc_v2*100] sns.barplot(x=models, y=accuracies_v2, palette=['#ff5e62', '#00df89', '#00c3ff']) plt.title('Sensibilité au Data Drift (Accuracy sur Tweets Modernes V2)') plt.ylabel('Accuracy (%)') plt.ylim(0, 100) for i, acc in enumerate(accuracies_v2): plt.text(i, acc + 1, f'{acc:.1f}%', ha='center', fontweight='bold') plt.savefig('assets/modern_accuracy_comparison.png', dpi=300, bbox_inches='tight') # Plot 4: Confusion Matrices fig, axes = plt.subplots(1, 3, figsize=(18, 5)) cm_keras = confusion_matrix(df_v1['true_sentiment'], df_v1['keras_pred']) cm_llm = confusion_matrix(df_v1['true_sentiment'], df_v1['llm_pred']) cm_qwen = confusion_matrix(df_v1['true_sentiment'], df_v1['qwen_pred']) sns.heatmap(cm_keras, annot=True, fmt='d', cmap='Reds', ax=axes[0]) axes[0].set_title('Keras Confusion Matrix') axes[0].set_xlabel('Predicted') axes[0].set_ylabel('True') sns.heatmap(cm_llm, annot=True, fmt='d', cmap='Greens', ax=axes[1]) axes[1].set_title('Gemma 3 Confusion Matrix') axes[1].set_xlabel('Predicted') axes[1].set_ylabel('True') sns.heatmap(cm_qwen, annot=True, fmt='d', cmap='Blues', ax=axes[2]) axes[2].set_title('Qwen Confusion Matrix') axes[2].set_xlabel('Predicted') axes[2].set_ylabel('True') plt.tight_layout() plt.savefig('assets/confusion_matrices.png', dpi=300, bbox_inches='tight') print("Done! Check assets/ and data/ directories.") if __name__ == "__main__": main()