import argparse import json import math import os import sys import time from collections import Counter sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) import torch import wandb from datasets import load_dataset from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer from src.part3.decoding import beam_search_decode, greedy_decode, top_k_decoding, top_p_decoding def compute_unigram_metrics(pred_text, ref_text): pred_tokens = pred_text.strip().lower().split() ref_tokens = ref_text.strip().lower().split() if len(pred_tokens) == 0 or len(ref_tokens) == 0: return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.0} pred_counts = Counter(pred_tokens) ref_counts = Counter(ref_tokens) overlap = sum(min(pred_counts[token], ref_counts[token]) for token in pred_counts) precision = overlap / len(pred_tokens) recall = overlap / len(ref_tokens) f1 = 0.0 accuracy = 0.0 if (precision + recall) > 0: f1 = (2 * precision * recall) / (precision + recall) else: f1 = 0.0 if pred_text.strip().lower() == ref_text.strip().lower(): accuracy = 1.0 else: accuracy = 0.0 return { "precision": precision, "recall": recall, "f1": f1, "accuracy": accuracy, } @torch.no_grad() def compute_conditional_perplexity(model, tokenizer, prompt, continuation, device): full_text = prompt + " " + continuation enc_full = tokenizer(full_text, return_tensors = "pt").to(device) enc_prompt = tokenizer(prompt, return_tensors = "pt") prompt_len = enc_prompt["input_ids"].shape[1] labels = enc_full["input_ids"].clone() labels[:, :prompt_len] = -100 outputs = model(**enc_full, labels = labels) loss = outputs.loss.item() return math.exp(loss) def run_evaluation(model_name = "EleutherAI/pythia-160m", num_samples = 1000, max_new_tokens = 40, device = "cuda" if torch.cuda.is_available() else "cpu", output_file = "part3_results.json", use_wandb = True, wandb_project = "ANLP-Assignment-2-Part3", wandb_run_name = "decoding_eval"): if use_wandb: wandb.init( project = wandb_project, name = wandb_run_name, config = { "model_name": model_name, "num_samples": num_samples, "max_new_tokens": max_new_tokens, "device": device, }, ) print(f"Loading model and tokenizer '{model_name}' on {device}") tokenizer = AutoTokenizer.from_pretrained(model_name) if tokenizer.pad_token_id is None: tokenizer.pad_token = tokenizer.eos_token model = AutoModelForCausalLM.from_pretrained(model_name).to(device) model.eval() print(f"Loading ROCStories test split (selecting {num_samples} samples)") dataset = load_dataset("hamishivi/ROCStories", split = "test") samples = dataset.select(range(min(num_samples, len(dataset)))) configurations = { "Greedy": lambda m, inp, eos: greedy_decode(m, inp, max_new_tokens = max_new_tokens, eos_token_id = eos), "Top-k (k = 50, T = 0.7)": lambda m, inp, eos: top_k_decoding(m, inp, max_new_tokens = max_new_tokens, k = 50, temperature = 0.7, eos_token_id = eos), "Top-p (p = 0.9, T = 0.7)": lambda m, inp, eos: top_p_decoding(m, inp, max_new_tokens = max_new_tokens, p = 0.9, temperature = 0.7, eos_token_id = eos), "Beam Search (B = 1)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 1, max_new_tokens = max_new_tokens, eos_token_id = eos), "Beam Search (B = 2)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 2, max_new_tokens = max_new_tokens, eos_token_id = eos), "Beam Search (B = 4)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 4, max_new_tokens = max_new_tokens, eos_token_id = eos), } results = {} eos_id = tokenizer.eos_token_id print("\nComputing baseline Reference Perplexity over ground truth continuations...") ref_ppls = [] for sample in tqdm(samples, desc = "Reference PPL"): ppl = compute_conditional_perplexity(model, tokenizer, sample["prompt"], sample["continuation"], device) ref_ppls.append(ppl) avg_ref_ppl = sum(ref_ppls) / len(ref_ppls) print(f"Average Reference Perplexity: {avg_ref_ppl:.2f}") if use_wandb: wandb.log({"reference_ppl": avg_ref_ppl}) for name, decode_fn in configurations.items(): print(f"\n--- Evaluating: {name} ---") metrics_list = [] gen_ppls = [] start_time = time.perf_counter() for sample in tqdm(samples, desc = name): prompt = sample["prompt"] ref_continuation = sample["continuation"] inputs = tokenizer(prompt, return_tensors = "pt").to(device) prompt_len = inputs["input_ids"].shape[1] output_ids = decode_fn(model, inputs["input_ids"], eos_id) gen_ids = output_ids[:, prompt_len:] gen_text = tokenizer.decode(gen_ids[0], skip_special_tokens = True) metrics = compute_unigram_metrics(gen_text, ref_continuation) metrics_list.append(metrics) if len(gen_text.strip()) > 0: gen_ppl = compute_conditional_perplexity(model, tokenizer, prompt, gen_text, device) gen_ppls.append(gen_ppl) total_time = time.perf_counter() - start_time avg_precision = sum(m["precision"] for m in metrics_list) / len(metrics_list) avg_recall = sum(m["recall"] for m in metrics_list) / len(metrics_list) avg_f1 = sum(m["f1"] for m in metrics_list) / len(metrics_list) avg_acc = sum(m["accuracy"] for m in metrics_list) / len(metrics_list) avg_gen_ppl = sum(gen_ppls) / len(gen_ppls) if gen_ppls else float("nan") results[name] = { "Time (s)": round(total_time, 2), "PPL (Generated)": round(avg_gen_ppl, 2), "Accuracy": round(avg_acc, 4), "Precision": round(avg_precision, 4), "Recall": round(avg_recall, 4), "F1": round(avg_f1, 4), } print( f"Result for {name}: Time={total_time:.2f}s | Gen PPL={avg_gen_ppl:.2f} | " f"Acc={avg_acc:.4f} | Prec={avg_precision:.4f} | Rec={avg_recall:.4f} | F1={avg_f1:.4f}" ) if use_wandb: wandb.log({ f"{name}/time_seconds": total_time, f"{name}/gen_ppl": avg_gen_ppl, f"{name}/accuracy": avg_acc, f"{name}/precision": avg_precision, f"{name}/recall": avg_recall, f"{name}/f1": avg_f1, }) print("\n" + "=" * 80) print(f"{'Decoding Strategy':<25} | {'Time (s)':<10} | {'Gen PPL':<10} | {'Acc':<8} | {'Prec':<8} | {'Rec':<8} | {'F1':<8}") print("=" * 80) for name, data in results.items(): print(f"{name:<25} | {data['Time (s)']:<10} | {data['PPL (Generated)']:<10} | {data['Accuracy']:<8} | {data['Precision']:<8} | {data['Recall']:<8} | {data['F1']:<8}") print("=" * 80) with open(output_file, "w") as f: json.dump({"reference_ppl": avg_ref_ppl, "results": results}, f, indent=2) print(f"\nResults successfully saved to {output_file}") if use_wandb: summary_table = wandb.Table(columns=["Strategy", "Time (s)", "Gen PPL", "Accuracy", "Precision", "Recall", "F1"]) for name, data in results.items(): summary_table.add_data( name, data["Time (s)"], data["PPL (Generated)"], data["Accuracy"], data["Precision"], data["Recall"], data["F1"], ) wandb.log({"decoding_summary_table": summary_table}) beam_timing_table = wandb.Table(columns=["Beam Width", "Time (s)"]) for b, b_name in [(1, "Beam Search (B=1)"), (2, "Beam Search (B=2)"), (4, "Beam Search (B=4)")]: if b_name in results: beam_timing_table.add_data(b, results[b_name]["Time (s)"]) wandb.log({ "beam_timing_curve": wandb.plot.line( beam_timing_table, "Beam Width", "Time (s)", title="Beam Search Time vs Beam Width" ) }) wandb.finish() if __name__ == "__main__": parser = argparse.ArgumentParser(description="Evaluate Task 3 Decoding Strategies") parser.add_argument("--num_samples", type=int, default=1000, help="Number of test samples (min 1000)") parser.add_argument("--max_new_tokens", type=int, default=40, help="Max tokens to generate") parser.add_argument("--output_file", type=str, default="task3_results.json", help="Path to save JSON results") parser.add_argument("--wandb", action="store_true", default=True, help="Log metrics to Weights & Biases") parser.add_argument("--no_wandb", action="store_false", dest="wandb", help="Disable WandB logging") parser.add_argument("--wandb_project", type=str, default="ANLP-Assignment-2-Part3", help="WandB project name") parser.add_argument("--wandb_run_name", type=str, default="decoding_evaluation", help="WandB run name") args = parser.parse_args() run_evaluation( num_samples=args.num_samples, max_new_tokens=args.max_new_tokens, output_file=args.output_file, use_wandb=args.wandb, wandb_project=args.wandb_project, wandb_run_name=args.wandb_run_name, )