evaluate/verify_generalization.py

"""
SCRIPT DE VÉRIFICATION DE LA GÉNÉRALISATION (ANTI-DATA LEAKAGE)
----------------------------------------------------------------
Ce script :
1. Génère 50 tweets modernes inédits (OOD, slang 2026, emojis) jamais vus à l'entraînement.
2. Sauvegarde ces tweets dans 'unseen_tweets_test.csv'.
3. Lance l'inférence avec Qwen de base (Zero-Shot) et Qwen + LoRA (Fine-Tuned) sur votre GPU Mac.
4. Calcule l'Accuracy pour vérifier s'il y a surapprentissage ou vraie généralisation.
"""

import os
import csv
import time
import pandas as pd
import numpy as np
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
from sklearn.metrics import accuracy_score
from tensorflow import keras
from tensorflow.keras.preprocessing.text import tokenizer_from_json
from tensorflow.keras.preprocessing.sequence import pad_sequences

# 1. Génération des 50 tweets inédits (25 positifs, 25 négatifs)
unseen_tweets = [
    # --- 25 POSITIFS (Label: 4 dans Sentiment140) ---
    (4, "This new anime series is an absolute banger! Best animation I've seen in years! 🤩🎬"),
    (4, "My coffee this morning tastes like liquid gold. Happy Tuesday everyone! ☕✨"),
    (4, "Just got tickets to see my favorite artist live! I am literally screaming! 🎫🔥"),
    (4, "Shoutout to the kind stranger who helped me carry my groceries today. Pure kindness! 🥺❤️"),
    (4, "Our puppy finally slept through the entire night. Absolute win for us! 🐕💤"),
    (4, "This new skincare routine is a total glow up. My skin has never looked so clear! 🧴✨"),
    (4, "Absolutely in love with this cozy rainy weather. Time to read with a warm blanket. 🌧️📖"),
    (4, "Our team just got funded! Hard work and long nights finally paid off. 🚀💼"),
    (4, "Baking fresh chocolate chip cookies and the whole apartment smells like heaven! 🍪😋"),
    (4, "Nothing beats the feeling of leaving the office early on a Friday afternoon! ☀️🏃‍♂️"),
    (4, "This new AI noise-canceling feature makes my calls so quiet, it's black magic! 🤫🎧"),
    (4, "Finding a pristine vintage vinyl record at the flea market today made my week! 📻❤️"),
    (4, "Spent the day hiking in the mountains. Fresh air is the best medicine. ⛰️🌲"),
    (4, "This local restaurant serves the best vegan tacos. Literally a culinary masterpiece! 🌮🌱"),
    (4, "Got a surprise bonus at work today! Hard work is rewarding. 🎉"),
    (4, "The sunset over the beach tonight is absolutely breathtaking. Feeling so peaceful. 🌅🌊"),
    (4, "Passed my driving test on the first try! Absolutely thrilled! 🚗💨"),
    (4, "Our neighborhood garden is finally blooming, so many beautiful flowers! 🌸🐝"),
    (4, "The new update to my favorite video game is so polished and fun to play! 🎮✨"),
    (4, "Getting a warm hug from my little brother today was exactly what I needed. 🥰"),
    (4, "This homemade soup is so comforting on a cold winter night. 🍲✨"),
    (4, "Finished my final exam of the semester! Freedom tastes so good! 🎓🎉"),
    (4, "The support team was so helpful, they resolved my issue in under 2 minutes. Outstanding! 👏"),
    (4, "Learning to play the piano has been so therapeutic and rewarding. 🎹🎵"),
    (4, "Just made a perfect espresso shot with a beautiful crema. Little joys! ☕😊"),

    # --- 25 NÉGATIFS (Label: 0 dans Sentiment140) ---
    (0, "Spent 4 hours updating my operating system and now my printer won't connect. Cringe! ☠️🖨️"),
    (0, "Lying in bed with a terrible fever while it is beautiful outside. Worst timing ever. 🤒"),
    (0, "Ordered a salad and found a literal bug crawling on the lettuce. Absolutely disgusted! 🥗🤮"),
    (0, "My internet connection keeps dropping every 5 minutes during my Zoom meeting. So frustrating! 🌐🤬"),
    (0, "The zipper on my brand new winter jacket just snapped in half. Cheap quality! 🧥😤"),
    (0, "wasting two hours of my life watching a movie with the worst plot twist in history. Waste! 🎬🗑️"),
    (0, "The customer service representative hung up on me after putting me on hold for an hour. 📞💀"),
    (0, "Woke up at 3 AM to the sound of drilling next door. I am so exhausted. 🔨😭"),
    (0, "Accidentally spilled hot tea all over my clean white keyboard. RIP hardware. ☕☠️"),
    (0, "The bus is delayed by 40 minutes and it's freezing cold outside. Fantastic morning. 🥶🚌"),
    (0, "This new app update is completely bloated with ads. It's completely ruined. 📱🤮"),
    (0, "Having a massive migraine during a busy workday is the absolute worst feeling. 🤕"),
    (0, "The parcel was left in the rain and the contents are completely soaked. Thanks a lot! 📦🌧️"),
    (0, "My favorite cozy cafe just got replaced by a generic corporate bank. Heartbreaking. 🏦💔"),
    (0, "Tried to make pancakes but they turned out like burnt rubber. Kitchen fail. 🥞😭"),
    (0, "Left my umbrella at the office and got completely drenched on the walk home. 🌧️🚶‍♂️"),
    (0, "This hotel room smells like stale cigarettes despite being marked as non-smoking. 🏨🤢"),
    (0, "wasted my whole morning trying to cancel a subscription through a broken chatbot. 🤖"),
    (0, "The heel of my left shoe broke while crossing a busy intersection. Extremely embarrassing! 🥿"),
    (0, "My new headphones have zero bass and sound like a tin can. Instant return. 🎧🗑️"),
    (0, "Spent hours cooking a complex meal only to drop the entire plate on the floor. Crying. 😭🍝"),
    (0, "The constant notifications from work group chats are driving me absolutely insane. 💬🤯"),
    (0, "Ordered an iced latte and they gave me hot milk with zero coffee. Absolute scam! 🥛🤮"),
    (0, "Trying to study but my neighbor's dog won't stop barking since 8 AM. 🐕🔊"),
    (0, "Lost my wallet with all my cards today. The administrative nightmare begins. 💳☠️")
]

# Sauvegarde des 50 tweets dans unseen_tweets_test.csv
csv_path = "data/unseen_tweets_test.csv"
with open(csv_path, "w", newline="", encoding="utf-8") as f:
    writer = csv.writer(f)
    for row in unseen_tweets:
        # Format standard: [sentiment, id, date, query, user, text]
        writer.writerow([row[0], 999999, "Mon May 19 10:30:00 PDT 2026", "NO_QUERY", "unseen_tester", row[1]])

print(f"=== [Générateur] 50 tweets inédits sauvegardés dans '{csv_path}' ===", flush=True)

# 2. Inférence avec PyTorch GPU MPS
device = "mps" if torch.backends.mps.is_available() else "cpu"
print(f"Périphérique d'inférence active : {device.upper()}")

MODEL_ID = "Qwen/Qwen2.5-0.5B-Instruct"
LORA_PATH = "./qwen2.5_local_mac_lora"

print("\n--- Chargement du tokenizer et du modèle de base ---", flush=True)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
tokenizer.pad_token = tokenizer.eos_token

model_base = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    torch_dtype=torch.float16,
    low_cpu_mem_usage=True
).to(device)

def evaluate_on_unseen(model, label_desc):
    print(f"\nÉvaluation en cours sur les 50 tweets inédits pour : {label_desc}...", flush=True)
    preds = []
    
    for idx, row in enumerate(unseen_tweets):
        true_sentiment = 1 if row[0] == 4 else 0
        text = row[1]
        
        messages = [
            {"role": "user", "content": f"Analyse le sentiment de ce tweet : \"{text}\""}
        ]
        formatted = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        inputs = tokenizer(formatted, return_tensors="pt").to(device)
        
        with torch.no_grad():
            outputs = model.generate(
                **inputs,
                max_new_tokens=20,
                pad_token_id=tokenizer.eos_token_id,
                eos_token_id=tokenizer.eos_token_id
            )
            
        response_text = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True).lower()
        
        # Binarisation
        pred = 1 if "positive" in response_text or '"positive"' in response_text or "1" in response_text else 0
        preds.append(pred)
        
    true_labels = [1 if r[0] == 4 else 0 for r in unseen_tweets]
    acc = accuracy_score(true_labels, preds)
    return acc

# A. Évaluer le modèle de base (Zero-Shot)
base_acc = evaluate_on_unseen(model_base, "Qwen 2.5 0.5B (Zero-Shot)")

# B. Évaluer le modèle Fine-Tuned (LoRA)
print(f"\n--- Application locale des adaptateurs LoRA depuis {LORA_PATH} ---", flush=True)
model_lora = PeftModel.from_pretrained(model_base, os.path.abspath(LORA_PATH)).to(device)

lora_acc = evaluate_on_unseen(model_lora, "Qwen 2.5 0.5B + LoRA (Fine-Tuned)")

# C. Évaluer le modèle Keras historique
print("\n--- Chargement du modèle Keras historique et de son tokenizer ---", flush=True)
keras_model_path = "keras_baseline/model1_simple_neural_network.keras"
tokenizer_path = "keras_baseline/tokenizer_simple_neural_network.json"

keras_acc = 0.0
if os.path.exists(keras_model_path) and os.path.exists(tokenizer_path):
    try:
        keras_model = keras.models.load_model(keras_model_path)
        with open(tokenizer_path, "r", encoding="utf-8") as f:
            tokenizer_json_str = f.read()
        keras_tokenizer = tokenizer_from_json(tokenizer_json_str)

        print("Évaluation en cours sur les 50 tweets inédits pour : Keras (Projet 7)...", flush=True)
        texts = [row[1] for row in unseen_tweets]
        seqs = keras_tokenizer.texts_to_sequences(texts)
        padded = pad_sequences(seqs, maxlen=50, padding='post', truncating='post')
        
        # Inférence directe eager pour éviter le deadlock Keras sur Mac
        keras_probs_tensor = keras_model(padded)
        keras_probs = keras_probs_tensor.numpy().ravel()
        keras_preds = (keras_probs > 0.5).astype(int)
        
        true_labels = [1 if r[0] == 4 else 0 for r in unseen_tweets]
        keras_acc = accuracy_score(true_labels, keras_preds)
    except Exception as e:
        print(f"Erreur lors de l'évaluation Keras : {str(e)}", flush=True)
else:
    print("Modèle Keras introuvable, évaluation ignorée.", flush=True)

# 3. Résultats finaux de généralisation
print("\n" + "="*65)
print("          VERDICT DE LA CAPACITÉ DE GÉNÉRALISATION (ANTI-LEAKAGE)")
print("="*65)
print(f"Modèle Keras (Baseline V1)    : {keras_acc*100:.2f}% d'Accuracy")
print(f"Modèle Zero-Shot (Base Qwen)  : {base_acc*100:.2f}% d'Accuracy")
print(f"Modèle Fine-Tuned (LoRA Qwen) : {lora_acc*100:.2f}% d'Accuracy sur l'INÉDIT")
print("-"*65)

if lora_acc >= 0.90:
    print("🎉 DIAGNOSTIC : SUCCÈS TOTAL DE GÉNÉRALISATION !")
    print("Le modèle n'a PAS fait de surapprentissage (overfitting). Il a réellement")
    print("appris les règles sémantiques et s'adapte parfaitement à l'inédit.")
else:
    print("⚠️ DIAGNOSTIC : ATTENTION AU SURAPPRENTISSAGE (OVERFITTING) !")
    print("Le modèle s'effondre face à l'inédit. Les adaptateurs ont appris par cœur.")
print("="*65 + "\n")