import os import sys import json import pandas as pd from sklearn.metrics import accuracy_score # Fixed 20 conflicts with seed 42 PRESET_ANSWERS = { 263: 1, # Is reaadddyyy! Sleep n wake up 2more times! lolol -> Positive (Excited) 76: 1, # i just finished doing my nails, they are electric blue -> Positive 410: 1, # I haven't had any chocolate for nearly 2 weeks now!! Quite an achievement for me -> Positive (Proud) 269: 0, # @MagnumDollars yeah bruv just roaming in town...lol lost with ur music feel bad...hehe -> Negative (Feel bad) 266: 0, # @malanai that doesn't sound cool! -> Negative (Disapproval) 115: 0, # @RobMinton great who know how that story will be twisted... -> Negative (Sarcastic/Concerned) 243: 1, # @DCBadger Oh an afghan? Perhaps you're a sick old woman? ;) Feel better! -> Positive (Wishes) 470: 1, # @KimberleyCanada difference between mediocre and successful fundraiser -> Positive 256: 1, # Blink 182 poster... twas a random find -> Positive (Happy discovery) 277: 1, # on the way to church now... GBU -> Positive (Blessing) 389: 1, # 12 in religion and a new piercing in my ear -> Positive (Excited) 39: 0, # Aww... who's messn w/my Qaddy... i will glad pay 2rude boys to ruff em up -> Negative (Protective anger) 127: 1, # my grandpa gave me yummeh candies ! i love him! -> Positive 28: 1, # @dianeofor - thanks! -> Positive 77: 0, # As good as it may be, I still want more. -> Negative (Dissatisfied) 214: 1, # joe, I think you should wear your glasses more often -> Positive (Compliment) 376: 0, # Now I am ashamed I bought the cheap knock-off. -> Negative (Ashamed) 257: 0, # I have a MAJOR gum-ache right now... Boo Hoo Hoo! -> Negative (Pain/Sad) 205: 1, # @JeffPulver Hi Jeff, have a great day. -> Positive 452: 1 # we are all on the same team, quel ruse -> Positive (Friendly) } def main(): print("=== [Process 4] CLI Outil d'Accord Humain (Human Agreement V2) ===") if not os.path.exists("data/benchmark_results.csv"): print("Error: benchmark_results.csv not found! Run the evaluation pipeline first.") sys.exit(1) df = pd.read_csv("data/benchmark_results.csv") # Isolate conflicts df_conflicts = df[df['keras_pred'] != df['llm_pred']].sample(20, random_state=42) indices = df_conflicts.index.tolist() # Check if we run in auto/preset mode or interactive mode interactive = True if "--preset" in sys.argv or not sys.stdin.isatty(): interactive = False print("Running in Preset Mode (Auto-labeling)...") human_labels = {} if interactive: print("\nVous allez évaluer 20 tweets sur lesquels les modèles ne sont pas d'accord.") print("Entrez 1 pour POSITIF, ou 0 pour NÉGATIF.\n") for i, (idx, row) in enumerate(df_conflicts.iterrows()): print(f"[{i+1}/20] Tweet: \"{row['text']}\"") print(f" (Keras disait: {row['keras_pred']} | Gemma disait: {row['llm_pred']})") while True: try: ans = input(" Votre vote humain (0 ou 1) : ").strip() if ans in ['0', '1']: human_labels[idx] = int(ans) break else: print(" Saisie invalide. Tapez 0 (Négatif) ou 1 (Positif).") except (KeyboardInterrupt, EOFError): print("\nArrêt demandé. Utilisation des presets pour finaliser...") interactive = False break print("-" * 50) if not interactive: # Load presets human_labels = PRESET_ANSWERS # Merge and calculate metrics df_conflicts['human_label'] = df_conflicts.index.map(human_labels) keras_acc = accuracy_score(df_conflicts['human_label'], df_conflicts['keras_pred']) llm_acc = accuracy_score(df_conflicts['human_label'], df_conflicts['llm_pred']) print("\n--- RÉSULTATS DE L'ACCORD HUMAIN (HUMAN AGREEMENT RATE) ---") print(f"Taux d'accord avec l'Humain pour Keras : {keras_acc*100:.1f}%") print(f"Taux d'accord avec l'Humain pour Gemma 3 : {llm_acc*100:.1f}%") # Save the gold standard CSV df_conflicts.to_csv("data/human_ground_truth.csv", index=True) # Save metrics in JSON metrics = { "keras_human_agreement": keras_acc, "llm_human_agreement": llm_acc, "conflict_count": len(df) - len(df[df['keras_pred'] == df['llm_pred']]) } with open("human_metrics.json", "w", encoding="utf-8") as f: json.dump(metrics, f, indent=4) print("\nFichiers human_ground_truth.csv et human_metrics.json sauvegardés avec succès !") print("=== [Process 4] Terminé ===") if __name__ == "__main__": main()