anlp-a2-models / src /part3 /evaluate.py
Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
9.58 kB
import argparse
import json
import math
import os
import sys
import time
from collections import Counter
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
import torch
import wandb
from datasets import load_dataset
from tqdm import tqdm
from transformers import AutoModelForCausalLM, AutoTokenizer
from src.part3.decoding import beam_search_decode, greedy_decode, top_k_decoding, top_p_decoding
def compute_unigram_metrics(pred_text, ref_text):
pred_tokens = pred_text.strip().lower().split()
ref_tokens = ref_text.strip().lower().split()
if len(pred_tokens) == 0 or len(ref_tokens) == 0:
return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.0}
pred_counts = Counter(pred_tokens)
ref_counts = Counter(ref_tokens)
overlap = sum(min(pred_counts[token], ref_counts[token]) for token in pred_counts)
precision = overlap / len(pred_tokens)
recall = overlap / len(ref_tokens)
f1 = 0.0
accuracy = 0.0
if (precision + recall) > 0:
f1 = (2 * precision * recall) / (precision + recall)
else:
f1 = 0.0
if pred_text.strip().lower() == ref_text.strip().lower():
accuracy = 1.0
else:
accuracy = 0.0
return {
"precision": precision,
"recall": recall,
"f1": f1,
"accuracy": accuracy,
}
@torch.no_grad()
def compute_conditional_perplexity(model, tokenizer, prompt, continuation, device):
full_text = prompt + " " + continuation
enc_full = tokenizer(full_text, return_tensors = "pt").to(device)
enc_prompt = tokenizer(prompt, return_tensors = "pt")
prompt_len = enc_prompt["input_ids"].shape[1]
labels = enc_full["input_ids"].clone()
labels[:, :prompt_len] = -100
outputs = model(**enc_full, labels = labels)
loss = outputs.loss.item()
return math.exp(loss)
def run_evaluation(model_name = "EleutherAI/pythia-160m", num_samples = 1000, max_new_tokens = 40,
device = "cuda" if torch.cuda.is_available() else "cpu", output_file = "part3_results.json", use_wandb = True,
wandb_project = "ANLP-Assignment-2-Part3", wandb_run_name = "decoding_eval"):
if use_wandb:
wandb.init(
project = wandb_project,
name = wandb_run_name,
config = {
"model_name": model_name,
"num_samples": num_samples,
"max_new_tokens": max_new_tokens,
"device": device,
},
)
print(f"Loading model and tokenizer '{model_name}' on {device}")
tokenizer = AutoTokenizer.from_pretrained(model_name)
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
model.eval()
print(f"Loading ROCStories test split (selecting {num_samples} samples)")
dataset = load_dataset("hamishivi/ROCStories", split = "test")
samples = dataset.select(range(min(num_samples, len(dataset))))
configurations = {
"Greedy": lambda m, inp, eos: greedy_decode(m, inp, max_new_tokens = max_new_tokens, eos_token_id = eos),
"Top-k (k = 50, T = 0.7)": lambda m, inp, eos: top_k_decoding(m, inp, max_new_tokens = max_new_tokens, k = 50, temperature = 0.7, eos_token_id = eos),
"Top-p (p = 0.9, T = 0.7)": lambda m, inp, eos: top_p_decoding(m, inp, max_new_tokens = max_new_tokens, p = 0.9, temperature = 0.7, eos_token_id = eos),
"Beam Search (B = 1)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 1, max_new_tokens = max_new_tokens, eos_token_id = eos),
"Beam Search (B = 2)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 2, max_new_tokens = max_new_tokens, eos_token_id = eos),
"Beam Search (B = 4)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 4, max_new_tokens = max_new_tokens, eos_token_id = eos),
}
results = {}
eos_id = tokenizer.eos_token_id
print("\nComputing baseline Reference Perplexity over ground truth continuations...")
ref_ppls = []
for sample in tqdm(samples, desc = "Reference PPL"):
ppl = compute_conditional_perplexity(model, tokenizer, sample["prompt"], sample["continuation"], device)
ref_ppls.append(ppl)
avg_ref_ppl = sum(ref_ppls) / len(ref_ppls)
print(f"Average Reference Perplexity: {avg_ref_ppl:.2f}")
if use_wandb:
wandb.log({"reference_ppl": avg_ref_ppl})
for name, decode_fn in configurations.items():
print(f"\n--- Evaluating: {name} ---")
metrics_list = []
gen_ppls = []
start_time = time.perf_counter()
for sample in tqdm(samples, desc = name):
prompt = sample["prompt"]
ref_continuation = sample["continuation"]
inputs = tokenizer(prompt, return_tensors = "pt").to(device)
prompt_len = inputs["input_ids"].shape[1]
output_ids = decode_fn(model, inputs["input_ids"], eos_id)
gen_ids = output_ids[:, prompt_len:]
gen_text = tokenizer.decode(gen_ids[0], skip_special_tokens = True)
metrics = compute_unigram_metrics(gen_text, ref_continuation)
metrics_list.append(metrics)
if len(gen_text.strip()) > 0:
gen_ppl = compute_conditional_perplexity(model, tokenizer, prompt, gen_text, device)
gen_ppls.append(gen_ppl)
total_time = time.perf_counter() - start_time
avg_precision = sum(m["precision"] for m in metrics_list) / len(metrics_list)
avg_recall = sum(m["recall"] for m in metrics_list) / len(metrics_list)
avg_f1 = sum(m["f1"] for m in metrics_list) / len(metrics_list)
avg_acc = sum(m["accuracy"] for m in metrics_list) / len(metrics_list)
avg_gen_ppl = sum(gen_ppls) / len(gen_ppls) if gen_ppls else float("nan")
results[name] = {
"Time (s)": round(total_time, 2),
"PPL (Generated)": round(avg_gen_ppl, 2),
"Accuracy": round(avg_acc, 4),
"Precision": round(avg_precision, 4),
"Recall": round(avg_recall, 4),
"F1": round(avg_f1, 4),
}
print(
f"Result for {name}: Time={total_time:.2f}s | Gen PPL={avg_gen_ppl:.2f} | "
f"Acc={avg_acc:.4f} | Prec={avg_precision:.4f} | Rec={avg_recall:.4f} | F1={avg_f1:.4f}"
)
if use_wandb:
wandb.log({
f"{name}/time_seconds": total_time,
f"{name}/gen_ppl": avg_gen_ppl,
f"{name}/accuracy": avg_acc,
f"{name}/precision": avg_precision,
f"{name}/recall": avg_recall,
f"{name}/f1": avg_f1,
})
print("\n" + "=" * 80)
print(f"{'Decoding Strategy':<25} | {'Time (s)':<10} | {'Gen PPL':<10} | {'Acc':<8} | {'Prec':<8} | {'Rec':<8} | {'F1':<8}")
print("=" * 80)
for name, data in results.items():
print(f"{name:<25} | {data['Time (s)']:<10} | {data['PPL (Generated)']:<10} | {data['Accuracy']:<8} | {data['Precision']:<8} | {data['Recall']:<8} | {data['F1']:<8}")
print("=" * 80)
with open(output_file, "w") as f:
json.dump({"reference_ppl": avg_ref_ppl, "results": results}, f, indent=2)
print(f"\nResults successfully saved to {output_file}")
if use_wandb:
summary_table = wandb.Table(columns=["Strategy", "Time (s)", "Gen PPL", "Accuracy", "Precision", "Recall", "F1"])
for name, data in results.items():
summary_table.add_data(
name,
data["Time (s)"],
data["PPL (Generated)"],
data["Accuracy"],
data["Precision"],
data["Recall"],
data["F1"],
)
wandb.log({"decoding_summary_table": summary_table})
beam_timing_table = wandb.Table(columns=["Beam Width", "Time (s)"])
for b, b_name in [(1, "Beam Search (B=1)"), (2, "Beam Search (B=2)"), (4, "Beam Search (B=4)")]:
if b_name in results:
beam_timing_table.add_data(b, results[b_name]["Time (s)"])
wandb.log({
"beam_timing_curve": wandb.plot.line(
beam_timing_table, "Beam Width", "Time (s)", title="Beam Search Time vs Beam Width"
)
})
wandb.finish()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Evaluate Task 3 Decoding Strategies")
parser.add_argument("--num_samples", type=int, default=1000, help="Number of test samples (min 1000)")
parser.add_argument("--max_new_tokens", type=int, default=40, help="Max tokens to generate")
parser.add_argument("--output_file", type=str, default="task3_results.json", help="Path to save JSON results")
parser.add_argument("--wandb", action="store_true", default=True, help="Log metrics to Weights & Biases")
parser.add_argument("--no_wandb", action="store_false", dest="wandb", help="Disable WandB logging")
parser.add_argument("--wandb_project", type=str, default="ANLP-Assignment-2-Part3", help="WandB project name")
parser.add_argument("--wandb_run_name", type=str, default="decoding_evaluation", help="WandB run name")
args = parser.parse_args()
run_evaluation(
num_samples=args.num_samples,
max_new_tokens=args.max_new_tokens,
output_file=args.output_file,
use_wandb=args.wandb,
wandb_project=args.wandb_project,
wandb_run_name=args.wandb_run_name,
)