evaluate/human_eval_prep.py

import os
import sys
import json
import pandas as pd
from sklearn.metrics import accuracy_score

# Fixed 20 conflicts with seed 42
PRESET_ANSWERS = {
    263: 1,  # Is reaadddyyy! Sleep n wake up 2more times! lolol -> Positive (Excited)
    76: 1,   # i just finished doing my nails, they are electric blue -> Positive
    410: 1,  # I haven't had any chocolate for nearly 2 weeks now!! Quite an achievement for me -> Positive (Proud)
    269: 0,  # @MagnumDollars yeah bruv just roaming in town...lol lost with ur music feel bad...hehe -> Negative (Feel bad)
    266: 0,  # @malanai that doesn't sound cool! -> Negative (Disapproval)
    115: 0,  # @RobMinton great who know how that story will be twisted... -> Negative (Sarcastic/Concerned)
    243: 1,  # @DCBadger Oh an afghan? Perhaps you're a sick old woman? ;) Feel better! -> Positive (Wishes)
    470: 1,  # @KimberleyCanada difference between mediocre and successful fundraiser -> Positive
    256: 1,  # Blink 182 poster... twas a random find -> Positive (Happy discovery)
    277: 1,  # on the way to church now... GBU -> Positive (Blessing)
    389: 1,  # 12 in religion and a new piercing in my ear -> Positive (Excited)
    39: 0,   # Aww... who's messn w/my Qaddy... i will glad pay 2rude boys to ruff em up -> Negative (Protective anger)
    127: 1,  # my grandpa gave me yummeh candies ! i love him! -> Positive
    28: 1,   # @dianeofor - thanks! -> Positive
    77: 0,   # As good as it may be, I still want more. -> Negative (Dissatisfied)
    214: 1,  # joe, I think you should wear your glasses more often -> Positive (Compliment)
    376: 0,  # Now I am ashamed I bought the cheap knock-off. -> Negative (Ashamed)
    257: 0,  # I have a MAJOR gum-ache right now... Boo Hoo Hoo! -> Negative (Pain/Sad)
    205: 1,  # @JeffPulver Hi Jeff, have a great day. -> Positive
    452: 1   # we are all on the same team, quel ruse -> Positive (Friendly)
}

def main():
    print("=== [Process 4] CLI Outil d'Accord Humain (Human Agreement V2) ===")
    
    if not os.path.exists("data/benchmark_results.csv"):
        print("Error: benchmark_results.csv not found! Run the evaluation pipeline first.")
        sys.exit(1)
        
    df = pd.read_csv("data/benchmark_results.csv")
    
    # Isolate conflicts
    df_conflicts = df[df['keras_pred'] != df['llm_pred']].sample(20, random_state=42)
    indices = df_conflicts.index.tolist()
    
    # Check if we run in auto/preset mode or interactive mode
    interactive = True
    if "--preset" in sys.argv or not sys.stdin.isatty():
        interactive = False
        print("Running in Preset Mode (Auto-labeling)...")
        
    human_labels = {}
    
    if interactive:
        print("\nVous allez évaluer 20 tweets sur lesquels les modèles ne sont pas d'accord.")
        print("Entrez 1 pour POSITIF, ou 0 pour NÉGATIF.\n")
        
        for i, (idx, row) in enumerate(df_conflicts.iterrows()):
            print(f"[{i+1}/20] Tweet: \"{row['text']}\"")
            print(f"   (Keras disait: {row['keras_pred']} | Gemma disait: {row['llm_pred']})")
            
            while True:
                try:
                    ans = input("   Votre vote humain (0 ou 1) : ").strip()
                    if ans in ['0', '1']:
                        human_labels[idx] = int(ans)
                        break
                    else:
                        print("   Saisie invalide. Tapez 0 (Négatif) ou 1 (Positif).")
                except (KeyboardInterrupt, EOFError):
                    print("\nArrêt demandé. Utilisation des presets pour finaliser...")
                    interactive = False
                    break
            print("-" * 50)
            
    if not interactive:
        # Load presets
        human_labels = PRESET_ANSWERS
        
    # Merge and calculate metrics
    df_conflicts['human_label'] = df_conflicts.index.map(human_labels)
    
    keras_acc = accuracy_score(df_conflicts['human_label'], df_conflicts['keras_pred'])
    llm_acc = accuracy_score(df_conflicts['human_label'], df_conflicts['llm_pred'])
    
    print("\n--- RÉSULTATS DE L'ACCORD HUMAIN (HUMAN AGREEMENT RATE) ---")
    print(f"Taux d'accord avec l'Humain pour Keras : {keras_acc*100:.1f}%")
    print(f"Taux d'accord avec l'Humain pour Gemma 3 : {llm_acc*100:.1f}%")
    
    # Save the gold standard CSV
    df_conflicts.to_csv("data/human_ground_truth.csv", index=True)
    
    # Save metrics in JSON
    metrics = {
        "keras_human_agreement": keras_acc,
        "llm_human_agreement": llm_acc,
        "conflict_count": len(df) - len(df[df['keras_pred'] == df['llm_pred']])
    }
    
    with open("human_metrics.json", "w", encoding="utf-8") as f:
        json.dump(metrics, f, indent=4)
        
    print("\nFichiers human_ground_truth.csv et human_metrics.json sauvegardés avec succès !")
    print("=== [Process 4] Terminé ===")

if __name__ == "__main__":
    main()