Upload 6 files

ferret_faithfullness.py

@@ -0,0 +1,157 @@
+import os
+import gc
+import pandas as pd
+import numpy as np
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from ferret import Benchmark
+from scipy.stats import rankdata
+import torch
+
+# ================================================================
+# CONFIGURATION
+# ================================================================
+
+# Set your Hugging Face model repo name here
+hf_model_name = "PLACE_YOUR_MODEL_NAME"
+
+# CSV test file (expected in current directory)
+test_file = "test.csv"
+
+# Batch sizes
+prediction_batch_size = 64
+ferret_batch_size = 1
+
+# Label mapping
+label_map = {
+    "Negative": 0,
+    "Neutral": 1,
+    "Positive": 2
+}
+
+# Device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+print(f"[INFO] Using device: {device}")
+
+# ================================================================
+# LOAD TEST DATA
+# ================================================================
+
+if not os.path.exists(test_file):
+    raise FileNotFoundError(f"Test file not found: {test_file}")
+
+df_full = pd.read_csv(test_file)
+df_full["final_label"] = df_full["final_label"].astype("category")
+df_full["final_label_numeric"] = df_full["final_label"].map(label_map)
+
+texts_all = df_full["Content"].tolist()
+labels_all = df_full["final_label"].tolist()
+print(f"[INFO] ✅ Loaded test data: {len(df_full)} rows.")
+
+# ================================================================
+# PIPELINE FOR SINGLE MODEL (MATCHED ONLY)
+# ================================================================
+
+print(f"\n==============================")
+print(f"[INFO] 🚀 Starting pipeline for model: {hf_model_name}")
+print(f"==============================")
+
+# -----------------------------
+# LOAD MODEL & TOKENIZER
+# -----------------------------
+tokenizer = AutoTokenizer.from_pretrained(hf_model_name)
+model = AutoModelForSequenceClassification.from_pretrained(
+    hf_model_name,
+    trust_remote_code=True,
+    use_safetensors=True
+)
+model = model.to(device)
+model.eval()
+print("[INFO] ✅ Model loaded.")
+
+# -----------------------------
+# PREDICTIONS
+# -----------------------------
+predictions = []
+print("[INFO] 🔎 Predicting on test set...")
+
+for i in tqdm(range(0, len(texts_all), prediction_batch_size), desc="Predicting"):
+    batch_texts = texts_all[i : i + prediction_batch_size]
+    inputs = tokenizer(
+        batch_texts,
+        padding=True,
+        truncation=True,
+        return_tensors="pt",
+        max_length=256
+    )
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+    with torch.no_grad():
+        outputs = model(**inputs)
+        preds = torch.argmax(outputs.logits, dim=1).cpu().tolist()
+    predictions.extend(preds)
+
+# Store predictions
+df = df_full.copy()
+df["prediction"] = predictions
+df["prediction"] = df["prediction"].astype("int8")
+
+predictions_filename = f"{hf_model_name.replace('/', '_')}_predictions.csv"
+df.to_csv(predictions_filename, index=False)
+print(f"[INFO] ✅ Predictions saved to {predictions_filename}.")
+
+# -----------------------------
+# SPLIT MATCHED ONLY
+# -----------------------------
+matched_df = df[df["prediction"] == df["final_label_numeric"]].reset_index(drop=True)
+print(f"[INFO] ✅ {len(matched_df)} matched rows retained.")
+
+# Save matched for records
+matched_df.to_csv(f"{hf_model_name.replace('/', '_')}_matched.csv", index=False)
+
+# -----------------------------
+# FERRET ON MATCHED
+# -----------------------------
+if len(matched_df) > 0:
+    ferret_rows = []
+    bench = Benchmark(model, tokenizer)
+    print(f"[INFO] 🚀 Running FERRET on matched rows...")
+    for i in tqdm(range(0, len(matched_df), ferret_batch_size), desc="FERRET (Matched)"):
+        batch = matched_df.iloc[i : i + ferret_batch_size]
+        for _, row in batch.iterrows():
+            text = row["Content"]
+            label = int(row["final_label_numeric"])
+            try:
+                explanations = bench.explain(text, target=label)
+                evaluations = bench.evaluate_explanations(explanations, target=label)
+            except Exception as ex:
+                print(f"[WARN] FERRET failed on matched text: {text}\nReason: {ex}")
+                continue
+            ferret_row = {
+                "Text": text,
+                "final_label": row["final_label"],
+                "final_label_numeric": label,
+                "Annotations": row.get("Annotations", ""),
+                "Rationale": row.get("Rationale", ""),
+            }
+            if explanations:
+                ferret_row["Tokens"] = " ".join(explanations[0].tokens)
+            for expl, evaluation in zip(explanations, evaluations):
+                explainer_name = expl.explainer
+                scores = expl.scores
+                ranks = rankdata(-np.array(scores), method="min")
+                ferret_row[f"{explainer_name}_ImportanceScores"] = " ".join(map(str, scores))
+                ferret_row[f"{explainer_name}_RankVector"] = " ".join(map(str, ranks))
+                if evaluation and hasattr(evaluation, "evaluation_scores"):
+                    for eval_score in evaluation.evaluation_scores:
+                        ferret_row[f"{explainer_name}_{eval_score.name}"] = float(eval_score.score)
+            ferret_rows.append(ferret_row)
+            del explanations
+            del evaluations
+            gc.collect()
+    ferret_filename = f"{hf_model_name.replace('/', '_')}_ferret_matched.csv"
+    pd.DataFrame(ferret_rows).to_csv(ferret_filename, index=False)
+    print(f"[INFO] ✅ FERRET results saved to {ferret_filename}.")
+else:
+    print("[INFO] ⚠ No matched rows to run FERRET on.")
+
+print("[INFO] ✅ Pipeline finished for matched rows only!")

ferret_plausibility.py

@@ -0,0 +1,313 @@
+import os
+import ferret
+from ferret import Benchmark
+import csv
+import gc
+import pandas as pd
+import numpy as np
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch
+
+# ==============================
+# File path and Data Loading
+# ==============================
+input_file = "modelname_ferret_matched.csv"  # Columns: 'Text', 'final_label', 'final_label_numeric', 'Annotations', 'Rationale', plus FERRET columns
+
+if not os.path.exists(input_file):
+    raise FileNotFoundError(f"Input file not found: {input_file}")
+
+df = pd.read_csv(input_file)
+
+# ==============================
+# Model and Plausibility Configs
+# ==============================
+hf_model_names = [
+    #keep your model names as "model_name"
+]
+label_map = {"Negative": 0, "Neutral": 1, "Positive": 2}
+inv_label_map = {v: k for k, v in label_map.items()}
+max_length = 128
+
+# ==============================
+# Rationales to Attention Vectors
+# ==============================
+def generate_attention_vectors_from_rationales(df, tokenizer, max_length=128):
+    """
+    For each row, generate attention vectors for each annotator.
+    Vector is zero if annotator's label does not match final label.
+    """
+    all_attention_vectors = []
+    token_lengths = []
+    max_annotators = 0
+
+    for _, row in df.iterrows():
+        text = str(row["Text"])
+        final_label_id = row["final_label_numeric"]
+        final_label = inv_label_map[final_label_id]
+
+        encoding = tokenizer(
+            text,
+            add_special_tokens=True,
+            return_offsets_mapping=True,
+            return_attention_mask=False,
+            return_token_type_ids=False,
+            max_length=max_length,
+            truncation=True
+        )
+        offsets = encoding["offset_mapping"]
+        real_token_indices = [i for i, (start, end) in enumerate(offsets) if start != end and start >= 0]
+        num_real_tokens = len(real_token_indices)
+        token_lengths.append(num_real_tokens)
+
+        annotations = str(row["Annotations"]).split("|")
+        rationales = str(row["Rationale"]).split("|")
+        max_annotators = max(max_annotators, len(annotations))
+
+        row_attention_vectors = []
+
+        for annot_label, annot_rationale in zip(annotations, rationales):
+            vec = np.zeros(num_real_tokens, dtype=np.float32)
+            # Set all zeros and skip if annotator label does not match final label
+            if not annot_label.strip() or annot_label.split("-")[0].strip() != final_label:
+                row_attention_vectors.append(vec)
+                continue
+
+            spans = [s.strip() for s in annot_rationale.split(",") if s.strip()]
+            if not spans:
+                row_attention_vectors.append(vec)
+                continue
+
+            for span_text in spans:
+                start = 0
+                while True:
+                    idx = text.find(span_text, start)
+                    if idx < 0:
+                        break
+                    span_start, span_end = idx, idx + len(span_text)
+                    for vec_idx, tok_idx in enumerate(real_token_indices):
+                        tok_start, tok_end = offsets[tok_idx]
+                        if tok_end > span_start and tok_start < span_end:
+                            vec[vec_idx] = 1.0
+                    start = idx + 1
+
+            row_attention_vectors.append(vec)
+
+        all_attention_vectors.append(row_attention_vectors)
+
+    # Write attention vectors to new columns
+    for i in range(max_annotators):
+        col_name = f"embert_attention_{i+1}"
+        col_vectors = []
+        for row_vecs, num_tokens in zip(all_attention_vectors, token_lengths):
+            if i < len(row_vecs):
+                vec_str = " ".join(f"{int(v)}" for v in row_vecs[i])
+            else:
+                vec_str = " ".join(["0"] * num_tokens)
+            col_vectors.append(vec_str)
+        df[col_name] = col_vectors
+
+    return df
+
+# ==============================
+# Explanation class for FERRET
+# ==============================
+class Explanation:
+    def __init__(self, text, tokens, scores, explainer, target):
+        self.text = text
+        self.tokens = tokens
+        self.scores = np.array(scores, dtype=np.float32)
+        self.explainer = explainer
+        self.target = target
+
+    def __repr__(self):
+        return f"Explanation(text={self.text!r}, tokens={self.tokens}, scores=array({self.scores}, dtype=float32), explainer={self.explainer!r}, target={self.target})"
+
+# ==============================
+# DEVICE SETUP
+# ==============================
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+print(f"[INFO] Using device: {device}")
+
+# ==============================
+# FERRET PIPELINE
+# ==============================
+for hf_model_name in hf_model_names:
+    print(f"\n==============================")
+    print(f"[INFO] Starting pipeline for model: {hf_model_name}")
+    print(f"==============================")
+
+    # Load model and tokenizer
+    model = AutoModelForSequenceClassification.from_pretrained(
+        hf_model_name,
+        trust_remote_code=True,
+        use_safetensors=True
+    )
+    model.to(device)
+    model.eval()
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        hf_model_name,
+        trust_remote_code=True,
+        use_safetensors=True
+    )
+
+    bench = Benchmark(model, tokenizer)
+
+    df = generate_attention_vectors_from_rationales(df, tokenizer)
+
+    # List of explainers you want to use
+    explainer_names = [
+        "Partition SHAP", "LIME", "Gradient", "Gradient (x Input)",
+        "Integrated Gradient", "Integrated Gradient (x Input)"
+    ]
+
+    ferret_filename = f"{hf_model_name.replace('/', '_')}_ferret_plausibility.csv"
+    header_written = os.path.exists(ferret_filename)
+
+    # To ensure no empty cells: collect all possible output columns
+    all_fieldnames = set(["Index", "Text", "final_label", "final_label_numeric", "Annotations", "Rationale"])
+
+    # --- MAIN LOOP ---
+    for idx in tqdm(range(len(df)), desc="FERRET (Plausibility Only)"):
+        row = df.iloc[idx]
+
+        ferret_row = {
+            "Index": idx,
+            "Text": row["Text"],
+            "final_label": row["final_label"],
+            "final_label_numeric": int(row["final_label_numeric"]),
+            "Annotations": row.get("Annotations", ""),
+            "Rationale": row.get("Rationale", ""),
+        }
+
+        # Prepare explanations for all explainers
+        row_explanations = {}
+        for explainer_name in explainer_names:
+            score_col = f"{explainer_name}_ImportanceScores"
+            tokens_col = "Tokens"
+            if pd.notna(row.get(score_col)) and pd.notna(row.get(tokens_col)):
+                try:
+                    scores = [float(score) for score in str(row[score_col]).split()]
+                    tokens = str(row[tokens_col]).split()
+                    target_label = int(row["final_label_numeric"])
+                    row_explanations[explainer_name] = Explanation(
+                        text=row["Text"], tokens=tokens, scores=scores,
+                        explainer=explainer_name, target=target_label
+                    )
+                except Exception as e:
+                    print(f"Could not create explanation for explainer {explainer_name} at index {idx}: {e}")
+                    continue
+
+        # Discover available metrics for plausibility
+        available_metrics = set()
+        if row_explanations:
+            first_explainer = next(iter(row_explanations.keys()))
+            first_explanation = row_explanations[first_explainer]
+            for test_annot_idx in range(3):
+                test_attn_col = f"embert_attention_{test_annot_idx+1}"
+                test_human_rationale_str = str(row.get(test_attn_col, ""))
+                test_human_rationale = [int(v) for v in test_human_rationale_str.split() if v.isdigit()]
+                if any(test_human_rationale):
+                    try:
+                        test_plaus_eval = bench.evaluate_explanations(
+                            [first_explanation],
+                            human_rationale=test_human_rationale,
+                            target=first_explanation.target,
+                            skip_faithfulness=True
+                        )
+                        if test_plaus_eval and len(test_plaus_eval) > 0:
+                            test_eval_obj = test_plaus_eval[0]
+                            if hasattr(test_eval_obj, "evaluation_scores") and test_eval_obj.evaluation_scores:
+                                for score in test_eval_obj.evaluation_scores:
+                                    if score.name in ['auprc_plau', 'token_f1_plau', 'token_iou_plau']:
+                                        available_metrics.add(score.name)
+                                break
+                    except Exception as e:
+                        print(f"Error discovering metrics with {first_explainer} and annotator {test_annot_idx+1}: {e}")
+                        continue
+
+        print(f"Row {idx}: Using FERRET plausibility metrics: {list(available_metrics)}")
+
+        # --- Evaluate plausibility for each explainer/annotator combination ---
+        for explainer_name in explainer_names:
+            if explainer_name not in row_explanations:
+                for annot_idx in range(3):
+                    for metric in available_metrics:
+                        colname = f"{explainer_name}Annotator{annot_idx+1}{metric}"
+                        ferret_row[colname] = "N/A"
+                        all_fieldnames.add(colname)
+                continue
+
+            explanation = row_explanations[explainer_name]
+            label = explanation.target
+
+            for annot_idx in range(3):
+                attn_col = f"embert_attention_{annot_idx+1}"
+                human_rationale_str = str(row.get(attn_col, ""))
+                human_rationale = [int(v) for v in human_rationale_str.split() if v.isdigit()]
+
+                annot_labels_list = str(row["Annotations"]).split("|")
+                if annot_idx < len(annot_labels_list):
+                    annot_label_str = annot_labels_list[annot_idx].split("-")[0].strip()
+                else:
+                    annot_label_str = ""
+
+                final_label_str = inv_label_map[label]
+                for metric in available_metrics:
+                    colname = f"{explainer_name}Annotator{annot_idx+1}{metric}"
+                    all_fieldnames.add(colname)
+
+                if annot_label_str != final_label_str:
+                    for metric in available_metrics:
+                        ferret_row[f"{explainer_name}Annotator{annot_idx+1}{metric}"] = "N/A"
+                    continue
+
+                if any(human_rationale):
+                    try:
+                        plaus_eval = bench.evaluate_explanations(
+                            [explanation],
+                            human_rationale=human_rationale,
+                            target=label,
+                            skip_faithfulness=True
+                        )
+                        if plaus_eval and len(plaus_eval) > 0:
+                            eval_obj = plaus_eval[0]
+                            if hasattr(eval_obj, "evaluation_scores") and eval_obj.evaluation_scores:
+                                for score in eval_obj.evaluation_scores:
+                                    if score.name in ['auprc_plau', 'token_f1_plau', 'token_iou_plau']:
+                                        ferret_row[f"{explainer_name}Annotator{annot_idx+1}{score.name}"] = float(score.score)
+                    except Exception as e:
+                        print(f"Error evaluating {explainer_name} for annotator {annot_idx+1} at index {idx}: {e}")
+                        for metric in available_metrics:
+                            ferret_row[f"{explainer_name}Annotator{annot_idx+1}{metric}"] = "N/A"
+                else:
+                    for metric in available_metrics:
+                        ferret_row[f"{explainer_name}Annotator{annot_idx+1}{metric}"] = "N/A"
+
+        # --- Ensure no empty cells: fill missing columns with "N/A" ---
+        for col in all_fieldnames:
+            if col not in ferret_row:
+                ferret_row[col] = "N/A"
+
+        # === SAVE THIS ROW TO CSV IMMEDIATELY ===
+        write_header = not header_written
+        with open(ferret_filename, mode='a', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=list(all_fieldnames))
+            if write_header:
+                writer.writeheader()
+                header_written = True
+            writer.writerow(ferret_row)
+
+    print(f"[INFO] FERRET plausibility results saved row-wise to {ferret_filename}.")
+
+    # --- Memory cleanup ---
+    print(f"[INFO] Cleaning up memory for model {hf_model_name}...")
+    del bench, model, tokenizer, df
+    gc.collect()
+    if device.type == "cuda":
+        torch.cuda.empty_cache()
+
+# ==============================
+# End of pipeline
+# ==============================

hyperparameter_tuning_for_rationale.py

@@ -0,0 +1,326 @@
+import csv
+import os
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import Dataset, DataLoader
+from transformers import AutoTokenizer, AutoModel
+from sklearn.metrics import f1_score, roc_auc_score, accuracy_score, precision_recall_fscore_support
+import itertools
+import warnings
+import random
+
+def set_seed(seed=13):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+
+set_seed(13)
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+# --- CONFIG ---
+param_grid = {
+    "learning_rate": [1e-5, 2e-5, 3e-5, 4e-5, 5e-5],
+    "batch_size": [16, 32, 64],
+    "optimizer": ["Adam"],
+    "lambda": [0.2, 0.3, 0.4, 0.5, 0.6, 0.7]
+}
+num_epochs = 7
+max_length = 128
+model_name = "bert-base-multilingual-cased"
+num_labels = 3
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# --- LOAD DATA ---
+train_df = pd.read_csv("train.csv")
+val_df = pd.read_csv("val.csv")
+
+valid_labels = {"Negative": 0, "Neutral": 1, "Positive": 2}
+train_df = train_df[train_df["final_label"].isin(valid_labels.keys())]
+val_df = val_df[val_df["final_label"].isin(valid_labels.keys())]
+if train_df.empty:
+    raise ValueError("Train dataset empty after filtering.")
+if val_df.empty:
+    raise ValueError("Validation dataset empty after filtering.")
+
+# --- INITIALIZE TOKENIZER & ADD EMOJIS ---
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+emoji_path = "emoji.csv"  # adjust path if needed
+if os.path.exists(emoji_path):
+    emoji_df = pd.read_csv(emoji_path)
+    emoji_list = emoji_df["emoji"].dropna().astype(str).str.strip().tolist()
+    existing_vocab = set(tokenizer.get_vocab().keys())
+    emoji_set = set(emoji_list) - existing_vocab
+    if emoji_set:
+        tokenizer.add_tokens(list(emoji_set))
+        print(f"Added {len(emoji_set)} new emoji tokens to the tokenizer.")
+    else:
+        print("No new emojis to add.")
+else:
+    print(f"Emoji file not found at: {emoji_path}")
+
+# --- FUNCTIONS ---
+
+def generate_attention_vectors_from_rationales(df, tokenizer, epsilon=1e-8):
+    attention_vectors = []
+    for _, row in df.iterrows():
+        text = str(row["Content"])
+        final_label = str(row["final_label"]).strip()
+        encoding = tokenizer(text, add_special_tokens=False, return_offsets_mapping=True)
+        offsets = encoding["offset_mapping"]
+        num_tokens = len(offsets)
+        avg_vector = np.zeros(num_tokens, dtype=np.float32)
+        annotations = str(row.get("Annotations", "")).split("|")
+        rationales = str(row.get("Rationale", "")).split("|")
+        annot_vectors = []
+        for annot_label, annot_rationale in zip(annotations, rationales):
+            if not annot_label:
+                continue
+            if annot_label.split("-")[0].strip() != final_label:
+                continue
+            spans = [s.strip() for s in annot_rationale.split(",") if s.strip()]
+            if not spans:
+                continue
+            vec = np.zeros(num_tokens, dtype=np.float32)
+            for span_text in spans:
+                start = 0
+                while True:
+                    idx = text.find(span_text, start)
+                    if idx < 0:
+                        break
+                    span_start, span_end = idx, idx + len(span_text)
+                    for i, (tok_start, tok_end) in enumerate(offsets):
+                        if tok_end > span_start and tok_start < span_end:
+                            vec[i] = 1.0
+                    start = idx + 1
+            if vec.sum() > 0:
+                annot_vectors.append(vec)
+        if annot_vectors:
+            avg_vector = np.mean(annot_vectors, axis=0)
+            avg_vector = np.where(avg_vector == 0, epsilon, avg_vector)
+        attn_str = " ".join(f"{v:.8f}" for v in avg_vector)
+        attention_vectors.append(attn_str)
+    df["embert_attention"] = attention_vectors
+    return df
+
+class RationaleDataset(Dataset):
+    def __init__(self, df, tokenizer, max_length=128, label_mapping=None):
+        self.df = df
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.label_mapping = label_mapping
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, idx):
+        row = self.df.iloc[idx]
+        text = row["Content"]
+        label = self.label_mapping[row["final_label"]]
+        encoding = self.tokenizer(
+            text, padding="max_length", truncation=True,
+            max_length=self.max_length, return_tensors="pt"
+        )
+        rationale_raw = [float(x) for x in row["embert_attention"].split()] \
+            if pd.notna(row["embert_attention"]) and row["embert_attention"].strip() else []
+        rationale_vector = np.concatenate([
+            np.array([0.0], dtype=np.float32),
+            np.array(rationale_raw, dtype=np.float32),
+            np.array([0.0], dtype=np.float32)
+        ])
+        rationale_vector = rationale_vector[:self.max_length]
+        if len(rationale_vector) < self.max_length:
+            rationale_vector = np.pad(rationale_vector, (0, self.max_length - len(rationale_vector)), constant_values=0.0)
+        rationale_tensor = torch.tensor(rationale_vector, dtype=torch.float32)
+        if torch.sum(rationale_tensor) == 0.0:
+            has_rationale = False
+            rationale_probs = torch.ones(self.max_length, dtype=torch.float32) / self.max_length
+        else:
+            has_rationale = True
+            rationale_probs = torch.softmax(rationale_tensor, dim=0)
+        return (
+            encoding["input_ids"].squeeze(0),
+            encoding["attention_mask"].squeeze(0),
+            rationale_probs,
+            torch.tensor(label, dtype=torch.long),
+            torch.tensor(has_rationale, dtype=torch.bool)
+        )
+
+class RationaleModel(nn.Module):
+    def __init__(self, model_name, num_labels):
+        super().__init__()
+        self.bert = AutoModel.from_pretrained(model_name, output_attentions=True)
+        self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels)
+
+    def forward(self, input_ids, attention_mask):
+        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
+        cls_output = outputs.last_hidden_state[:, 0, :]
+        logits = self.classifier(cls_output)
+        last_layer_attn = outputs.attentions[-1]  # (batch, heads, seq, seq)
+        cls_attn = last_layer_attn[:, :, 0, :]    # (batch, heads, seq)
+        cls_attn_avg = cls_attn.mean(dim=1)       # (batch, seq)
+        return logits, cls_attn_avg
+
+def evaluate_model(model, val_loader, criterion_cls, device):
+    model.eval()
+    total_val_loss = 0.0
+    all_preds = []
+    all_labels = []
+    all_probs = []
+    with torch.no_grad():
+        for batch in val_loader:
+            input_ids, attention_mask, _, labels, _ = [b.to(device) for b in batch]
+            logits, _ = model(input_ids, attention_mask)
+            loss = criterion_cls(logits, labels)
+            total_val_loss += loss.item()
+            probs = torch.softmax(logits, dim=1)
+            preds = torch.argmax(probs, dim=1)
+            all_preds.extend(preds.cpu().numpy())
+            all_labels.extend(labels.cpu().numpy())
+            all_probs.extend(probs.cpu().numpy())
+    avg_val_loss = total_val_loss / len(val_loader)
+    # Overall metrics
+    accuracy = accuracy_score(all_labels, all_preds)
+    f1_macro = f1_score(all_labels, all_preds, average="macro")
+    try:
+        y_true_oh = np.eye(num_labels)[all_labels]
+        auroc_ovr = roc_auc_score(y_true_oh, all_probs, multi_class="ovr")
+    except Exception:
+        auroc_ovr = -1.0
+    # Class-wise metrics
+    class_wise_metrics = {}
+    target_names = sorted(valid_labels, key=valid_labels.get)
+    precision, recall, f1_per_class, _ = precision_recall_fscore_support(all_labels, all_preds, average=None, labels=[valid_labels[label_name] for label_name in target_names])
+    for i, label_name in enumerate(target_names):
+        class_wise_metrics[f"{label_name}_precision"] = precision[i]
+        class_wise_metrics[f"{label_name}_recall"] = recall[i]
+        class_wise_metrics[f"{label_name}_f1"] = f1_per_class[i]
+        # Per-class accuracy: of true class samples, how many were predicted correctly
+        idx = np.array(all_labels) == valid_labels[label_name]
+        if idx.sum() > 0:
+            acc = (np.array(all_preds)[idx] == valid_labels[label_name]).sum() / idx.sum()
+        else:
+            acc = -1.0
+        class_wise_metrics[f"{label_name}_accuracy"] = acc
+        # Class-wise AUROC
+        try:
+            binary_labels = (np.array(all_labels) == valid_labels[label_name]).astype(int)
+            class_probs = np.array(all_probs)[:, valid_labels[label_name]]
+            if len(np.unique(binary_labels)) > 1:
+                class_wise_metrics[f"{label_name}_auroc"] = roc_auc_score(binary_labels, class_probs)
+            else:
+                class_wise_metrics[f"{label_name}_auroc"] = -1.0
+        except Exception:
+            class_wise_metrics[f"{label_name}_auroc"] = -1.0
+    return avg_val_loss, accuracy, f1_macro, auroc_ovr, class_wise_metrics
+
+def train_model(model, train_loader, val_loader, num_epochs, device, lambda_attn=1.0, optimizer=None, learning_rate=2e-5, results_writer=None, results_file_handle=None, params=None):
+    criterion_cls = nn.CrossEntropyLoss()
+    criterion_kl = nn.KLDivLoss(reduction="batchmean")
+    if optimizer is None:
+        optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+    for epoch in range(num_epochs):
+        model.train()
+        total_train_loss = 0.0
+        for batch in train_loader:
+            input_ids, attention_mask, rationale_probs, labels, has_rationale = [b.to(device) for b in batch]
+            optimizer.zero_grad()
+            logits, model_attention = model(input_ids, attention_mask)
+            loss_cls = criterion_cls(logits, labels)
+            loss = loss_cls
+            if has_rationale.any():
+                model_attn_batch = model_attention[has_rationale]
+                rationale_batch = rationale_probs[has_rationale]
+                log_model_attn = torch.log(model_attn_batch + 1e-8)
+                loss_kl = criterion_kl(log_model_attn, rationale_batch)
+                loss += lambda_attn * loss_kl
+            loss.backward()
+            optimizer.step()
+            total_train_loss += loss.item()
+        avg_train_loss = total_train_loss / len(train_loader)
+        val_loss, val_acc, val_f1_macro, val_auroc_ovr, class_wise_metrics = evaluate_model(model, val_loader, criterion_cls, device)
+        print(f"Epoch {epoch+1} | Train Loss: {avg_train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.4f} | Val F1 (Macro): {val_f1_macro:.4f} | Val AUROC (OvR): {val_auroc_ovr:.4f}")
+        sorted_labels = sorted(valid_labels, key=valid_labels.get)
+        for label_name in sorted_labels:
+            print(f"  {label_name}: P={class_wise_metrics[f'{label_name}_precision']:.4f}, R={class_wise_metrics[f'{label_name}_recall']:.4f}, F1={class_wise_metrics[f'{label_name}_f1']:.4f}, Acc={class_wise_metrics[f'{label_name}_accuracy']:.4f}, AUROC={class_wise_metrics[f'{label_name}_auroc']:.4f}")
+        if results_writer and results_file_handle:
+            row_data = [
+                params["learning_rate"], 
+                params["batch_size"], 
+                params["optimizer"], 
+                params["lambda"], 
+                epoch + 1,
+                avg_train_loss, 
+                val_loss, 
+                val_acc, 
+                val_f1_macro, 
+                val_auroc_ovr
+            ]
+            for label_name in sorted_labels:
+                row_data.extend([
+                    class_wise_metrics[f"{label_name}_precision"],
+                    class_wise_metrics[f"{label_name}_recall"],
+                    class_wise_metrics[f"{label_name}_f1"],
+                    class_wise_metrics[f"{label_name}_accuracy"],
+                    class_wise_metrics[f"{label_name}_auroc"]
+                ])
+            results_writer.writerow(row_data)
+            results_file_handle.flush()
+            os.fsync(results_file_handle.fileno())
+
+# --- PREPARE DATASETS ---
+print("Generating attention vectors for training data...")
+train_df = generate_attention_vectors_from_rationales(train_df, tokenizer)
+print("Generating attention vectors for validation data...")
+val_df = generate_attention_vectors_from_rationales(val_df, tokenizer)
+
+train_dataset = RationaleDataset(train_df, tokenizer, max_length, label_mapping=valid_labels)
+val_dataset = RationaleDataset(val_df, tokenizer, max_length, label_mapping=valid_labels)
+
+# --- GRID SEARCH LOOP ---
+keys, values = zip(*param_grid.items())
+param_combinations = [dict(zip(keys, v)) for v in itertools.product(*values)]
+results_file = "grid_results_detailed.csv"
+headers = ["learning_rate", "batch_size", "optimizer", "lambda", "epoch", "train_loss", "val_loss", "val_accuracy", "val_f1_macro", "val_auroc_ovr"]
+sorted_labels = sorted(valid_labels, key=valid_labels.get)
+for label_name in sorted_labels:
+    headers.extend([f"{label_name}_precision", f"{label_name}_recall", f"{label_name}_f1", f"{label_name}_accuracy", f"{label_name}_auroc"])
+with open(results_file, mode="w", newline="") as f:
+    writer = csv.writer(f)
+    writer.writerow(headers)
+    for params in param_combinations:
+        print("\nRunning:", params)
+        learning_rate = params["learning_rate"]
+        batch_size = params["batch_size"]
+        optimizer_type = params["optimizer"]
+        lambda_attn = params["lambda"]
+        model = RationaleModel(model_name=model_name, num_labels=num_labels).to(device)
+        if 'emoji_set' in locals() and len(emoji_set) > 0:
+            model.bert.resize_token_embeddings(len(tokenizer))
+        if optimizer_type == "Adam":
+            optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+        else:
+            raise ValueError("Unsupported optimizer")
+        train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, generator=torch.Generator().manual_seed(13))
+        val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
+        train_model(
+            model=model,
+            train_loader=train_loader,
+            val_loader=val_loader,
+            num_epochs=num_epochs,
+            device=device,
+            lambda_attn=lambda_attn,
+            optimizer=optimizer,
+            learning_rate=learning_rate,
+            results_writer=writer,
+            results_file_handle=f,
+            params=params
+        )
+print("Grid search complete. Results saved to:", results_file)

hyperparameter_tuning_without_rationale.py

@@ -0,0 +1,165 @@
+import os
+import torch
+import random
+import numpy as np
+import pandas as pd
+import itertools
+from datetime import datetime
+from torch.utils.data import Dataset, DataLoader
+from transformers import BertTokenizer, BertForSequenceClassification
+from torch.optim import Adam
+from sklearn.metrics import f1_score, roc_auc_score, precision_recall_fscore_support, accuracy_score
+
+# Reproducibility
+def set_seed(seed=13):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+set_seed(13)
+
+# Configurations
+param_grid = {
+    "learning_rate": [1e-5, 2e-5, 3e-5, 4e-5, 5e-5],
+    "batch_size": [16, 32, 64]
+}
+num_epochs = 10
+max_length = 128
+model_name = "bert-base-multilingual-cased"
+num_labels = 3
+
+# Tokenizer + Emoji Extension
+emoji_df = pd.read_csv("emoji.csv")
+emoji_list = emoji_df.iloc[:, 0].dropna().astype(str).unique().tolist()
+
+tokenizer = BertTokenizer.from_pretrained(model_name)
+new_tokens = list(set(emoji_list) - set(tokenizer.vocab.keys()))
+if new_tokens:
+    tokenizer.add_tokens(new_tokens)
+    print(f"Added {len(new_tokens)} emojis to tokenizer.")
+
+# Data loading
+train_df = pd.read_csv("train.csv")
+val_df = pd.read_csv("val.csv")
+
+valid_labels = {"Negative": 0, "Neutral": 1, "Positive": 2}
+train_df = train_df[train_df["final_label"].isin(valid_labels)]
+val_df = val_df[val_df["final_label"].isin(valid_labels)]
+
+class CustomDataset(Dataset):
+    def __init__(self, dataframe, tokenizer, max_length):
+        self.dataframe = dataframe.reset_index(drop=True)
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+
+    def __len__(self):
+        return len(self.dataframe)
+
+    def __getitem__(self, idx):
+        row = self.dataframe.iloc[idx]
+        text = row["Content"]
+        label = valid_labels[row["final_label"]]
+        encoding = self.tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            max_length=self.max_length,
+            return_tensors="pt"
+        )
+        return (
+            encoding["input_ids"].squeeze(0),
+            encoding["attention_mask"].squeeze(0),
+            torch.tensor(label, dtype=torch.long)
+        )
+
+train_dataset = CustomDataset(train_df, tokenizer, max_length)
+val_dataset = CustomDataset(val_df, tokenizer, max_length)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Results directory and file
+os.makedirs("results", exist_ok=True)
+results_path = "results/grid_search_metrics.csv"
+
+if not os.path.exists(results_path):
+    with open(results_path, "w") as f:
+        f.write("timestamp,learning_rate,batch_size,epoch,val_macro_f1,val_auroc,"
+                "acc_negative,prec_negative,rec_negative,f1_negative,"
+                "acc_neutral,prec_neutral,rec_neutral,f1_neutral,"
+                "acc_positive,prec_positive,rec_positive,f1_positive\n")
+
+# Hyperparameter grid search
+for lr, bs in itertools.product(param_grid["learning_rate"], param_grid["batch_size"]):
+    print(f"\nStarting config: LR={lr}, Batch Size={bs}")
+    set_seed(13)
+    train_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True)
+    val_loader = DataLoader(val_dataset, batch_size=bs)
+
+    model = BertForSequenceClassification.from_pretrained(model_name, num_labels=num_labels).to(device)
+    if new_tokens:
+        model.resize_token_embeddings(len(tokenizer))
+
+    optimizer = Adam(model.parameters(), lr=lr)
+
+    for epoch in range(1, num_epochs + 1):
+        model.train()
+        for batch in train_loader:
+            input_ids, attention_mask, labels = [b.to(device) for b in batch]
+            optimizer.zero_grad()
+            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
+            outputs.loss.backward()
+            optimizer.step()
+
+        # Evaluation
+        model.eval()
+        val_preds, val_probs, val_labels = [], [], []
+        with torch.no_grad():
+            for batch in val_loader:
+                input_ids, attention_mask, labels = [b.to(device) for b in batch]
+                logits = model(input_ids, attention_mask=attention_mask).logits
+                probs = torch.softmax(logits, dim=1).cpu().numpy()
+                preds = torch.argmax(logits, axis=1).cpu().tolist()
+
+                val_probs.extend(probs)
+                val_preds.extend(preds)
+                val_labels.extend(labels.cpu().tolist())
+
+        val_macro_f1 = f1_score(val_labels, val_preds, average="macro")
+        val_auroc = roc_auc_score(
+            np.eye(num_labels)[val_labels],
+            np.array(val_probs),
+            average="macro",
+            multi_class="ovr"
+        )
+
+        # Label-wise metrics
+        report = precision_recall_fscore_support(val_labels, val_preds, labels=[0, 1, 2], zero_division=0)
+        acc_per_label = []
+        for i in range(num_labels):
+            idx = np.array(val_labels) == i
+            correct = (np.array(val_preds)[idx] == i).sum()
+            total = idx.sum()
+            acc = correct / total if total > 0 else 0
+            acc_per_label.append(acc)
+
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        row = [
+            timestamp, lr, bs, epoch, f"{val_macro_f1:.4f}", f"{val_auroc:.4f}"
+        ]
+        for i in range(num_labels):
+            row.extend([
+                f"{acc_per_label[i]:.4f}",
+                f"{report[0][i]:.4f}",  # precision
+                f"{report[1][i]:.4f}",  # recall
+                f"{report[2][i]:.4f}"   # f1
+            ])
+
+        with open(results_path, "a") as f:
+            f.write(",".join(map(str, row)) + "\n")
+
+        print(f"[Epoch {epoch}] LR={lr}, BS={bs} | F1={val_macro_f1:.4f} | AUROC={val_auroc:.4f}")
+
+print(f"\nGrid Search Complete. Results saved to: {results_path}")

model_training_with_rationale.py

@@ -0,0 +1,318 @@
+import csv
+import os
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from sklearn.metrics import f1_score, roc_auc_score, accuracy_score, precision_recall_fscore_support
+import warnings
+import random
+
+def set_seed(seed=13):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+
+set_seed(13)
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+# --- CONFIG ---
+model_name = "bert-base-multilingual-cased"   # Set your model name here
+num_epochs = 4
+max_length = 128
+num_labels = 3
+learning_rate = 2e-5
+batch_size = 64
+optimizer_type = "Adam"
+lambda_attn = 0.6
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# --- LOAD DATA ---
+train_df = pd.read_csv("train.csv")
+val_df = pd.read_csv("val.csv")
+test_df = pd.read_csv("test.csv")
+valid_labels = {"Negative": 0, "Neutral": 1, "Positive": 2}
+train_df = train_df[train_df["final_label"].isin(valid_labels.keys())]
+val_df = val_df[val_df["final_label"].isin(valid_labels.keys())]
+test_df = test_df[test_df["final_label"].isin(valid_labels.keys())]
+if train_df.empty:
+    raise ValueError("Train dataset empty after filtering.")
+if val_df.empty:
+    raise ValueError("Validation dataset empty after filtering.")
+
+# --- FUNCTIONS ---
+def generate_attention_vectors_from_rationales(df, tokenizer, epsilon=1e-8):
+    attention_vectors = []
+    for _, row in df.iterrows():
+        text = str(row["Content"])
+        final_label = str(row["final_label"]).strip()
+        encoding = tokenizer(text, add_special_tokens=False, return_offsets_mapping=True)
+        offsets = encoding["offset_mapping"]
+        num_tokens = len(offsets)
+        avg_vector = np.zeros(num_tokens, dtype=np.float32)
+        annotations = str(row.get("Annotations", "")).split("|")
+        rationales = str(row.get("Rationale", "")).split("|")
+        annot_vectors = []
+        for annot_label, annot_rationale in zip(annotations, rationales):
+            if not annot_label:
+                continue
+            if annot_label.split("-")[0].strip() != final_label:
+                continue
+            spans = [s.strip() for s in annot_rationale.split(",") if s.strip()]
+            if not spans:
+                continue
+            vec = np.zeros(num_tokens, dtype=np.float32)
+            for span_text in spans:
+                start = 0
+                while True:
+                    idx = text.find(span_text, start)
+                    if idx < 0:
+                        break
+                    span_start, span_end = idx, idx + len(span_text)
+                    for i, (tok_start, tok_end) in enumerate(offsets):
+                        if tok_end > span_start and tok_start < span_end:
+                            vec[i] = 1.0
+                    start = idx + 1
+            if vec.sum() > 0:
+                annot_vectors.append(vec)
+        if annot_vectors:
+            avg_vector = np.mean(annot_vectors, axis=0)
+            avg_vector = np.where(avg_vector == 0, epsilon, avg_vector)
+        attn_str = " ".join(f"{v:.8f}" for v in avg_vector)
+        attention_vectors.append(attn_str)
+    df["embert_attention"] = attention_vectors
+    return df
+
+class RationaleDataset(Dataset):
+    def __init__(self, df, tokenizer, max_length=128, label_mapping=None):
+        self.df = df
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.label_mapping = label_mapping
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, idx):
+        row = self.df.iloc[idx]
+        text = row["Content"]
+        label = self.label_mapping[row["final_label"]]
+        encoding = self.tokenizer(
+            text, padding="max_length", truncation=True,
+            max_length=self.max_length, return_tensors="pt"
+        )
+        rationale_raw = [float(x) for x in row["embert_attention"].split()] \
+            if pd.notna(row["embert_attention"]) and row["embert_attention"].strip() else []
+        rationale_vector = np.concatenate([
+            np.array([0.0], dtype=np.float32),
+            np.array(rationale_raw, dtype=np.float32),
+            np.array([0.0], dtype=np.float32)
+        ])
+        rationale_vector = rationale_vector[:self.max_length]
+        if len(rationale_vector) < self.max_length:
+            rationale_vector = np.pad(rationale_vector, (0, self.max_length - len(rationale_vector)), constant_values=0.0)
+        rationale_tensor = torch.tensor(rationale_vector, dtype=torch.float32)
+        if torch.sum(rationale_tensor) == 0.0:
+            has_rationale = False
+            rationale_probs = torch.ones(self.max_length, dtype=torch.float32) / self.max_length
+        else:
+            has_rationale = True
+            rationale_probs = torch.softmax(rationale_tensor, dim=0)
+        return (
+            encoding["input_ids"].squeeze(0),
+            encoding["attention_mask"].squeeze(0),
+            rationale_probs,
+            torch.tensor(label, dtype=torch.long),
+            torch.tensor(has_rationale, dtype=torch.bool)
+        )
+
+class RationaleModel(nn.Module):
+    def __init__(self, model_name, num_labels):
+        super().__init__()
+        self.bert = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels, output_attentions=True)
+    def forward(self, input_ids, attention_mask):
+        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
+        logits = outputs.logits
+        last_layer_attn = outputs.attentions[-1]  # (batch, heads, seq, seq)
+        cls_attn = last_layer_attn[:, :, 0, :]    # (batch, heads, seq)
+        cls_attn_avg = cls_attn.mean(dim=1)       # (batch, seq)
+        return logits, cls_attn_avg
+
+def evaluate_model(model, val_loader, criterion_cls, device, valid_labels, num_labels):
+    model.eval()
+    total_val_loss = 0.0
+    all_preds = []
+    all_labels = []
+    all_probs = []
+    with torch.no_grad():
+        for batch in val_loader:
+            input_ids, attention_mask, _, labels, _ = [b.to(device) for b in batch]
+            logits, _ = model(input_ids, attention_mask)
+            loss = criterion_cls(logits, labels)
+            total_val_loss += loss.item()
+            probs = torch.softmax(logits, dim=1)
+            preds = torch.argmax(probs, dim=1)
+            all_preds.extend(preds.cpu().numpy())
+            all_labels.extend(labels.cpu().numpy())
+            all_probs.extend(probs.cpu().numpy())
+    avg_val_loss = total_val_loss / len(val_loader)
+    all_labels_np = np.array(all_labels)
+    all_preds_np = np.array(all_preds)
+    all_probs_np = np.array(all_probs)
+    accuracy = accuracy_score(all_labels_np, all_preds_np)
+    f1_macro = f1_score(all_labels_np, all_preds_np, average="macro")
+    try:
+        y_true_oh = np.eye(num_labels)[all_labels_np]
+        auroc_ovr = roc_auc_score(y_true_oh, all_probs_np, multi_class="ovr")
+    except:
+        auroc_ovr = -1.0
+    class_wise_metrics = {}
+    target_names = sorted(valid_labels, key=valid_labels.get)
+    label_indices = [valid_labels[label_name] for label_name in target_names]
+    precision, recall, f1_per_class, support = precision_recall_fscore_support(
+        all_labels_np, all_preds_np, labels=label_indices, average=None)
+    for i, label_name in enumerate(target_names):
+        label_id = valid_labels[label_name]
+        class_wise_metrics[f"{label_name}_precision"] = precision[i]
+        class_wise_metrics[f"{label_name}_recall"] = recall[i]
+        class_wise_metrics[f"{label_name}_f1"] = f1_per_class[i]
+        label_mask = all_labels_np == label_id
+        correct_preds = np.sum((all_preds_np == label_id) & label_mask)
+        total_label = np.sum(label_mask)
+        if total_label > 0:
+            class_wise_metrics[f"{label_name}_accuracy"] = correct_preds / total_label
+        else:
+            class_wise_metrics[f"{label_name}_accuracy"] = -1.0
+        try:
+            binary_labels = (all_labels_np == label_id).astype(int)
+            class_probs = all_probs_np[:, label_id]
+            if len(np.unique(binary_labels)) > 1:
+                class_wise_metrics[f"{label_name}_auroc"] = roc_auc_score(binary_labels, class_probs)
+            else:
+                class_wise_metrics[f"{label_name}_auroc"] = -1.0
+        except:
+            class_wise_metrics[f"{label_name}_auroc"] = -1.0
+    return avg_val_loss, accuracy, f1_macro, auroc_ovr, class_wise_metrics
+
+def train_model(model, train_loader, val_loader, num_epochs, device, lambda_attn=1.0, optimizer=None, learning_rate=2e-5, results_writer=None, results_file_handle=None):
+    criterion_cls = nn.CrossEntropyLoss()
+    criterion_kl = nn.KLDivLoss(reduction="batchmean")
+    if optimizer is None:
+        optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+    for epoch in range(num_epochs):
+        model.train()
+        total_train_loss = 0.0
+        for batch in train_loader:
+            input_ids, attention_mask, rationale_probs, labels, has_rationale = [b.to(device) for b in batch]
+            optimizer.zero_grad()
+            logits, model_attention = model(input_ids, attention_mask)
+            loss_cls = criterion_cls(logits, labels)
+            loss = loss_cls
+            if has_rationale.any():
+                model_attn_batch = model_attention[has_rationale]
+                rationale_batch = rationale_probs[has_rationale]
+                log_model_attn = torch.log(model_attn_batch + 1e-8)
+                loss_kl = criterion_kl(log_model_attn, rationale_batch)
+                loss += lambda_attn * loss_kl
+            loss.backward()
+            optimizer.step()
+            total_train_loss += loss.item()
+        avg_train_loss = total_train_loss / len(train_loader)
+        val_loss, val_acc, val_f1_macro, val_auroc_ovr, class_wise_metrics = evaluate_model(model, val_loader, criterion_cls, device, valid_labels, num_labels)
+        print(f"Epoch {epoch+1} | Train Loss: {avg_train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.4f} | Val F1 (Macro): {val_f1_macro:.4f} | Val AUROC (OvR): {val_auroc_ovr:.4f}")
+        sorted_labels = sorted(valid_labels, key=valid_labels.get)
+        for label_name in sorted_labels:
+            print(f"  {label_name}: P={class_wise_metrics[f'{label_name}_precision']:.4f}, R={class_wise_metrics[f'{label_name}_recall']:.4f}, F1={class_wise_metrics[f'{label_name}_f1']:.4f}, Acc={class_wise_metrics[f'{label_name}_accuracy']:.4f}, AUROC={class_wise_metrics[f'{label_name}_auroc']:.4f}")
+
+        if results_writer and results_file_handle:
+            row_data = [
+                learning_rate, 
+                batch_size, 
+                optimizer_type, 
+                lambda_attn, 
+                epoch + 1,
+                avg_train_loss, 
+                val_loss, 
+                val_acc, 
+                val_f1_macro, 
+                val_auroc_ovr
+            ]
+            for label_name in sorted_labels:
+                row_data.extend([
+                    class_wise_metrics[f"{label_name}_precision"],
+                    class_wise_metrics[f"{label_name}_recall"],
+                    class_wise_metrics[f"{label_name}_f1"],
+                    class_wise_metrics[f"{label_name}_accuracy"],
+                    class_wise_metrics[f"{label_name}_auroc"]
+                ])
+            results_writer.writerow(row_data)
+            results_file_handle.flush()
+            os.fsync(results_file_handle.fileno())
+
+# --- OUTPUT FOLDERS ---
+csv_output_dir = "csv_outputs"
+os.makedirs(csv_output_dir, exist_ok=True)
+results_file = os.path.join(csv_output_dir, "results_detailed.csv")
+headers = ["learning_rate", "batch_size", "optimizer", "lambda", "epoch", "train_loss", "val_loss", "val_accuracy", "val_f1_macro", "val_auroc_ovr"]
+sorted_labels = sorted(valid_labels, key=valid_labels.get)
+for label in sorted_labels:
+    headers.extend([f"{label}_precision", f"{label}_recall", f"{label}_f1", f"{label}_accuracy", f"{label}_auroc"])
+
+# --- INITIALIZE TOKENIZER & ADD EMOJIS ---
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+emoji_path = "emoji.csv"
+if os.path.exists(emoji_path):
+    emoji_df = pd.read_csv(emoji_path)
+    emoji_list = emoji_df["emoji"].dropna().astype(str).str.strip().tolist()
+    existing_vocab = set(tokenizer.get_vocab().keys())
+    emoji_set = set(emoji_list) - existing_vocab
+    if emoji_set:
+        tokenizer.add_tokens(list(emoji_set))
+        print(f"Added {len(emoji_set)} new emoji tokens to the tokenizer.")
+    else:
+        print("No new emojis to add.")
+else:
+    print(f"Emoji file not found at: {emoji_path}")
+
+# --- PREPARE DATASETS ---
+print("Generating attention vectors for training data...")
+train_df_model = generate_attention_vectors_from_rationales(train_df.copy(), tokenizer)
+print("Generating attention vectors for validation data...")
+val_df_model = generate_attention_vectors_from_rationales(val_df.copy(), tokenizer)
+
+train_dataset = RationaleDataset(train_df_model, tokenizer, max_length, label_mapping=valid_labels)
+val_dataset = RationaleDataset(val_df_model, tokenizer, max_length, label_mapping=valid_labels)
+train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, generator=torch.Generator().manual_seed(13))
+val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
+
+# --- CSV Setup ---
+with open(results_file, mode="w", newline="") as f:
+    writer = csv.writer(f)
+    writer.writerow(headers)
+    model = RationaleModel(model_name=model_name, num_labels=num_labels).to(device)
+    if 'emoji_set' in locals() and len(emoji_set) > 0:
+        model.bert.resize_token_embeddings(len(tokenizer))
+    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+    train_model(
+        model=model,
+        train_loader=train_loader,
+        val_loader=val_loader,
+        num_epochs=num_epochs,
+        device=device,
+        lambda_attn=lambda_attn,
+        optimizer=optimizer,
+        learning_rate=learning_rate,
+        results_writer=writer,
+        results_file_handle=f
+    )
+    # Save final model and tokenizer
+    model.bert.save_pretrained("model_outputs")
+    tokenizer.save_pretrained("model_outputs")
+    print(f"Final model and tokenizer saved to model_outputs")

model_training_without_rationale.py

@@ -0,0 +1,251 @@
+import torch
+from torch.utils.data import Dataset, DataLoader
+from torch.optim import Adam
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import pandas as pd
+import numpy as np
+import os
+import warnings
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.metrics import (
+    f1_score, roc_auc_score, accuracy_score,
+    precision_recall_fscore_support, confusion_matrix
+)
+
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+# --- CONFIG ---
+args_dict = {
+    "batch_size": 64,
+    "num_epochs": 4,
+    "learning_rate": 2e-5,
+    "max_length": 128,
+    "model_name": "bert-base-multilingual-cased",#replace with your model_name
+    "num_labels": 3,
+    "save_dir": "./saved_model"
+}
+os.makedirs(args_dict["save_dir"], exist_ok=True)
+
+# --- LABEL MAPPING ---
+label_mapping = {"Negative": 0, "Neutral": 1, "Positive": 2}
+label2name = {v: k for k, v in label_mapping.items()}
+label_ids = list(label2name.keys())
+
+# --- LOAD DATA ---
+train_df = pd.read_csv("train.csv")
+val_df = pd.read_csv("val.csv")
+test_df = pd.read_csv("test.csv")
+emoji_df = pd.read_csv("emoji.csv")
+
+# --- FILTER INVALID LABELS ---
+train_df = train_df[train_df["final_label"].isin(label_mapping)]
+val_df = val_df[val_df["final_label"].isin(label_mapping)]
+test_df = test_df[test_df["final_label"].isin(label_mapping)]
+
+# --- TOKENIZER ---
+tokenizer = AutoTokenizer.from_pretrained(args_dict["model_name"])
+emoji_list = emoji_df["emoji"].dropna().astype(str).str.strip().tolist()
+emoji_set = set(emoji_list) - set(tokenizer.vocab.keys())
+if emoji_set:
+    tokenizer.add_tokens(list(emoji_set))
+    print(f"Added {len(emoji_set)} emojis to tokenizer.")
+
+# --- MODEL ---
+model = AutoModelForSequenceClassification.from_pretrained(
+    args_dict["model_name"], num_labels=args_dict["num_labels"]
+)
+model.resize_token_embeddings(len(tokenizer))
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model.to(device)
+
+# --- DATASET ---
+class SimpleTextDataset(Dataset):
+    def __init__(self, dataframe, tokenizer, max_length=128):
+        self.dataframe = dataframe
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+
+    def __len__(self):
+        return len(self.dataframe)
+
+    def __getitem__(self, idx):
+        row = self.dataframe.iloc[idx]
+        text = row["Content"]
+        label = label_mapping[row["final_label"]]
+        encoding = self.tokenizer(
+            text, padding="max_length", truncation=True,
+            max_length=self.max_length, return_tensors="pt"
+        )
+        return (
+            encoding["input_ids"].squeeze(0),
+            encoding["attention_mask"].squeeze(0),
+            torch.tensor(label, dtype=torch.long),
+            text
+        )
+
+# --- DATALOADERS ---
+train_loader = DataLoader(SimpleTextDataset(train_df, tokenizer), batch_size=args_dict["batch_size"], shuffle=True)
+val_loader = DataLoader(SimpleTextDataset(val_df, tokenizer), batch_size=args_dict["batch_size"])
+test_loader = DataLoader(SimpleTextDataset(test_df, tokenizer), batch_size=args_dict["batch_size"])
+
+# --- TRAINING ---
+optimizer = Adam(model.parameters(), lr=args_dict["learning_rate"])
+val_metrics_history = []
+
+for epoch in range(1, args_dict["num_epochs"] + 1):
+    model.train()
+    total_loss = 0
+    for batch in train_loader:
+        input_ids, attn_mask, labels, _ = [x.to(device) for x in batch[:3]]
+        optimizer.zero_grad()
+        outputs = model(input_ids, attention_mask=attn_mask, labels=labels)
+        outputs.loss.backward()
+        optimizer.step()
+        total_loss += outputs.loss.item()
+    avg_train_loss = total_loss / len(train_loader)
+
+    # --- VALIDATION ---
+    model.eval()
+    val_preds, val_labels, val_loss = [], [], 0
+    with torch.no_grad():
+        for batch in val_loader:
+            input_ids, attn_mask, labels, _ = [x.to(device) for x in batch[:3]]
+            outputs = model(input_ids, attention_mask=attn_mask, labels=labels)
+            val_preds.extend(outputs.logits.argmax(dim=1).cpu().numpy())
+            val_labels.extend(labels.cpu().numpy())
+            val_loss += outputs.loss.item()
+    val_loss /= len(val_loader)
+    val_acc = accuracy_score(val_labels, val_preds)
+    val_f1 = f1_score(val_labels, val_preds, average="weighted")
+    try:
+        val_auroc = roc_auc_score(
+            pd.get_dummies(val_labels), pd.get_dummies(val_preds),
+            average="weighted", multi_class="ovo"
+        )
+    except:
+        val_auroc = float("nan")
+
+    # --- Label-wise Metrics ---
+    prec, rec, f1, supp = precision_recall_fscore_support(val_labels, val_preds, labels=[0,1,2])
+    labelwise = {}
+    for i in [0, 1, 2]:
+        idx = np.array(val_labels) == i
+        if idx.sum() > 0:
+            acc = (np.array(val_preds)[idx] == i).sum() / idx.sum()
+        else:
+            acc = 0.0
+        labelwise[label2name[i]] = {
+            "val_acc": acc,
+            "val_f1": f1[i],
+            "val_precision": prec[i],
+            "val_recall": rec[i],
+            "val_support": supp[i]
+        }
+
+    val_metrics_history.append({
+        "epoch": epoch,
+        "train_loss": avg_train_loss,
+        "val_loss": val_loss,
+        "val_accuracy": val_acc,
+        "val_f1": val_f1,
+        "val_auroc": val_auroc,
+        **{f"{label}_{m}": labelwise[label][m]
+           for label in labelwise for m in labelwise[label]}
+    })
+
+    print(f"Epoch {epoch}: Train Loss={avg_train_loss:.4f} | Val Acc={val_acc:.4f} | Val F1={val_f1:.4f} | AUROC={val_auroc:.4f}")
+
+model.save_pretrained(args_dict["save_dir"])
+tokenizer.save_pretrained(args_dict["save_dir"])
+print(f"Last model saved after epoch {args_dict['num_epochs']}")
+# --- SAVE VAL METRICS ---
+pd.DataFrame(val_metrics_history).to_csv("val_metrics_detailed.csv", index=False)
+
+# --- LOAD BEST MODEL ---
+model = AutoModelForSequenceClassification.from_pretrained(args_dict["save_dir"]).to(device)
+tokenizer = AutoTokenizer.from_pretrained(args_dict["save_dir"])
+
+# --- TEST EVAL ---
+model.eval()
+all_preds, all_labels, all_sentences, all_tokens = [], [], [], []
+test_loss = 0
+
+with torch.no_grad():
+    for batch in test_loader:
+        input_ids, attn_mask, labels, sentences = batch
+        input_ids, attn_mask, labels = input_ids.to(device), attn_mask.to(device), labels.to(device)
+        outputs = model(input_ids, attention_mask=attn_mask, labels=labels)
+        test_loss += outputs.loss.item()
+        preds = outputs.logits.argmax(dim=1)
+        all_preds.extend(preds.cpu().numpy())
+        all_labels.extend(labels.cpu().numpy())
+        all_sentences.extend(sentences)
+        all_tokens.extend(tokenizer.batch_decode(input_ids.cpu(), skip_special_tokens=True))
+
+test_loss /= len(test_loader)
+test_acc = accuracy_score(all_labels, all_preds)
+test_f1 = f1_score(all_labels, all_preds, average="weighted")
+try:
+    test_auroc = roc_auc_score(pd.get_dummies(all_labels), pd.get_dummies(all_preds), average="weighted", multi_class="ovo")
+except:
+    test_auroc = float("nan")
+
+# --- LABEL-WISE TEST METRICS ---
+prec, rec, f1, supp = precision_recall_fscore_support(all_labels, all_preds, labels=[0,1,2])
+label_metrics = {
+    "Label": [], "Accuracy": [], "F1": [], "Precision": [], "Recall": [], "Support": []
+}
+for i in [0, 1, 2]:
+    idx = np.array(all_labels) == i
+    if idx.sum() > 0:
+        acc = (np.array(all_preds)[idx] == i).sum() / idx.sum()
+    else:
+        acc = 0.0
+    label_name = label2name[i]
+    label_metrics["Label"].append(label_name)
+    label_metrics["Accuracy"].append(acc)
+    label_metrics["F1"].append(f1[i])
+    label_metrics["Precision"].append(prec[i])
+    label_metrics["Recall"].append(rec[i])
+    label_metrics["Support"].append(supp[i])
+pd.DataFrame(label_metrics).to_csv("labelwise_test_metrics.csv", index=False)
+
+# --- OVERALL TEST METRICS CSV ---
+pd.DataFrame([{
+    "Test Loss": test_loss,
+    "Test Accuracy": test_acc,
+    "Test F1 Score": test_f1,
+    "Test AUROC": test_auroc
+}]).to_csv("overall_test_metrics.csv", index=False)
+
+# --- TEST PREDICTIONS ---
+pd.DataFrame({
+    "Content": all_sentences,
+    "Tokens": all_tokens,
+    "final_label": [label2name[l] for l in all_labels],
+    "predicted_label": [label2name[p] for p in all_preds]
+}).to_csv("test_predictions.csv", index=False)
+
+# --- CONFUSION MATRIX ---
+conf_matrix = confusion_matrix(all_labels, all_preds, labels=[0, 1, 2])
+conf_matrix_df = pd.DataFrame(conf_matrix, index=[label2name[i] for i in [0,1,2]],
+                              columns=[label2name[i] for i in [0,1,2]])
+conf_matrix_df.to_csv("confusion_matrix.csv")
+
+# --- CONFUSION MATRIX PLOT ---
+plt.figure(figsize=(6, 5))
+sns.heatmap(conf_matrix_df, annot=True, fmt='d', cmap='Blues')
+plt.title("Confusion Matrix")
+plt.ylabel("True Label")
+plt.xlabel("Predicted Label")
+plt.tight_layout()
+plt.savefig("confusion_matrix.png")
+plt.close()
+
+# --- DONE ---
+print("\n=== FINAL TEST METRICS ===")
+print(f"Test Accuracy : {test_acc:.4f}")
+print(f"Test F1       : {test_f1:.4f}")
+print(f"Test AUROC    : {test_auroc:.4f}")
+print("All test metrics, predictions, and confusion matrix saved.")