Download src/part3/evaluate.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 9.58 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part3/evaluate.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part3/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part3/evaluate.py
9.58 kB
| import argparse | |
| import json | |
| import math | |
| import os | |
| import sys | |
| import time | |
| from collections import Counter | |
| sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) | |
| import torch | |
| import wandb | |
| from datasets import load_dataset | |
| from tqdm import tqdm | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| from src.part3.decoding import beam_search_decode, greedy_decode, top_k_decoding, top_p_decoding | |
| def compute_unigram_metrics(pred_text, ref_text): | |
| pred_tokens = pred_text.strip().lower().split() | |
| ref_tokens = ref_text.strip().lower().split() | |
| if len(pred_tokens) == 0 or len(ref_tokens) == 0: | |
| return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.0} | |
| pred_counts = Counter(pred_tokens) | |
| ref_counts = Counter(ref_tokens) | |
| overlap = sum(min(pred_counts[token], ref_counts[token]) for token in pred_counts) | |
| precision = overlap / len(pred_tokens) | |
| recall = overlap / len(ref_tokens) | |
| f1 = 0.0 | |
| accuracy = 0.0 | |
| if (precision + recall) > 0: | |
| f1 = (2 * precision * recall) / (precision + recall) | |
| else: | |
| f1 = 0.0 | |
| if pred_text.strip().lower() == ref_text.strip().lower(): | |
| accuracy = 1.0 | |
| else: | |
| accuracy = 0.0 | |
| return { | |
| "precision": precision, | |
| "recall": recall, | |
| "f1": f1, | |
| "accuracy": accuracy, | |
| } | |
| def compute_conditional_perplexity(model, tokenizer, prompt, continuation, device): | |
| full_text = prompt + " " + continuation | |
| enc_full = tokenizer(full_text, return_tensors = "pt").to(device) | |
| enc_prompt = tokenizer(prompt, return_tensors = "pt") | |
| prompt_len = enc_prompt["input_ids"].shape[1] | |
| labels = enc_full["input_ids"].clone() | |
| labels[:, :prompt_len] = -100 | |
| outputs = model(**enc_full, labels = labels) | |
| loss = outputs.loss.item() | |
| return math.exp(loss) | |
| def run_evaluation(model_name = "EleutherAI/pythia-160m", num_samples = 1000, max_new_tokens = 40, | |
| device = "cuda" if torch.cuda.is_available() else "cpu", output_file = "part3_results.json", use_wandb = True, | |
| wandb_project = "ANLP-Assignment-2-Part3", wandb_run_name = "decoding_eval"): | |
| if use_wandb: | |
| wandb.init( | |
| project = wandb_project, | |
| name = wandb_run_name, | |
| config = { | |
| "model_name": model_name, | |
| "num_samples": num_samples, | |
| "max_new_tokens": max_new_tokens, | |
| "device": device, | |
| }, | |
| ) | |
| print(f"Loading model and tokenizer '{model_name}' on {device}") | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| if tokenizer.pad_token_id is None: | |
| tokenizer.pad_token = tokenizer.eos_token | |
| model = AutoModelForCausalLM.from_pretrained(model_name).to(device) | |
| model.eval() | |
| print(f"Loading ROCStories test split (selecting {num_samples} samples)") | |
| dataset = load_dataset("hamishivi/ROCStories", split = "test") | |
| samples = dataset.select(range(min(num_samples, len(dataset)))) | |
| configurations = { | |
| "Greedy": lambda m, inp, eos: greedy_decode(m, inp, max_new_tokens = max_new_tokens, eos_token_id = eos), | |
| "Top-k (k = 50, T = 0.7)": lambda m, inp, eos: top_k_decoding(m, inp, max_new_tokens = max_new_tokens, k = 50, temperature = 0.7, eos_token_id = eos), | |
| "Top-p (p = 0.9, T = 0.7)": lambda m, inp, eos: top_p_decoding(m, inp, max_new_tokens = max_new_tokens, p = 0.9, temperature = 0.7, eos_token_id = eos), | |
| "Beam Search (B = 1)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 1, max_new_tokens = max_new_tokens, eos_token_id = eos), | |
| "Beam Search (B = 2)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 2, max_new_tokens = max_new_tokens, eos_token_id = eos), | |
| "Beam Search (B = 4)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 4, max_new_tokens = max_new_tokens, eos_token_id = eos), | |
| } | |
| results = {} | |
| eos_id = tokenizer.eos_token_id | |
| print("\nComputing baseline Reference Perplexity over ground truth continuations...") | |
| ref_ppls = [] | |
| for sample in tqdm(samples, desc = "Reference PPL"): | |
| ppl = compute_conditional_perplexity(model, tokenizer, sample["prompt"], sample["continuation"], device) | |
| ref_ppls.append(ppl) | |
| avg_ref_ppl = sum(ref_ppls) / len(ref_ppls) | |
| print(f"Average Reference Perplexity: {avg_ref_ppl:.2f}") | |
| if use_wandb: | |
| wandb.log({"reference_ppl": avg_ref_ppl}) | |
| for name, decode_fn in configurations.items(): | |
| print(f"\n--- Evaluating: {name} ---") | |
| metrics_list = [] | |
| gen_ppls = [] | |
| start_time = time.perf_counter() | |
| for sample in tqdm(samples, desc = name): | |
| prompt = sample["prompt"] | |
| ref_continuation = sample["continuation"] | |
| inputs = tokenizer(prompt, return_tensors = "pt").to(device) | |
| prompt_len = inputs["input_ids"].shape[1] | |
| output_ids = decode_fn(model, inputs["input_ids"], eos_id) | |
| gen_ids = output_ids[:, prompt_len:] | |
| gen_text = tokenizer.decode(gen_ids[0], skip_special_tokens = True) | |
| metrics = compute_unigram_metrics(gen_text, ref_continuation) | |
| metrics_list.append(metrics) | |
| if len(gen_text.strip()) > 0: | |
| gen_ppl = compute_conditional_perplexity(model, tokenizer, prompt, gen_text, device) | |
| gen_ppls.append(gen_ppl) | |
| total_time = time.perf_counter() - start_time | |
| avg_precision = sum(m["precision"] for m in metrics_list) / len(metrics_list) | |
| avg_recall = sum(m["recall"] for m in metrics_list) / len(metrics_list) | |
| avg_f1 = sum(m["f1"] for m in metrics_list) / len(metrics_list) | |
| avg_acc = sum(m["accuracy"] for m in metrics_list) / len(metrics_list) | |
| avg_gen_ppl = sum(gen_ppls) / len(gen_ppls) if gen_ppls else float("nan") | |
| results[name] = { | |
| "Time (s)": round(total_time, 2), | |
| "PPL (Generated)": round(avg_gen_ppl, 2), | |
| "Accuracy": round(avg_acc, 4), | |
| "Precision": round(avg_precision, 4), | |
| "Recall": round(avg_recall, 4), | |
| "F1": round(avg_f1, 4), | |
| } | |
| print( | |
| f"Result for {name}: Time={total_time:.2f}s | Gen PPL={avg_gen_ppl:.2f} | " | |
| f"Acc={avg_acc:.4f} | Prec={avg_precision:.4f} | Rec={avg_recall:.4f} | F1={avg_f1:.4f}" | |
| ) | |
| if use_wandb: | |
| wandb.log({ | |
| f"{name}/time_seconds": total_time, | |
| f"{name}/gen_ppl": avg_gen_ppl, | |
| f"{name}/accuracy": avg_acc, | |
| f"{name}/precision": avg_precision, | |
| f"{name}/recall": avg_recall, | |
| f"{name}/f1": avg_f1, | |
| }) | |
| print("\n" + "=" * 80) | |
| print(f"{'Decoding Strategy':<25} | {'Time (s)':<10} | {'Gen PPL':<10} | {'Acc':<8} | {'Prec':<8} | {'Rec':<8} | {'F1':<8}") | |
| print("=" * 80) | |
| for name, data in results.items(): | |
| print(f"{name:<25} | {data['Time (s)']:<10} | {data['PPL (Generated)']:<10} | {data['Accuracy']:<8} | {data['Precision']:<8} | {data['Recall']:<8} | {data['F1']:<8}") | |
| print("=" * 80) | |
| with open(output_file, "w") as f: | |
| json.dump({"reference_ppl": avg_ref_ppl, "results": results}, f, indent=2) | |
| print(f"\nResults successfully saved to {output_file}") | |
| if use_wandb: | |
| summary_table = wandb.Table(columns=["Strategy", "Time (s)", "Gen PPL", "Accuracy", "Precision", "Recall", "F1"]) | |
| for name, data in results.items(): | |
| summary_table.add_data( | |
| name, | |
| data["Time (s)"], | |
| data["PPL (Generated)"], | |
| data["Accuracy"], | |
| data["Precision"], | |
| data["Recall"], | |
| data["F1"], | |
| ) | |
| wandb.log({"decoding_summary_table": summary_table}) | |
| beam_timing_table = wandb.Table(columns=["Beam Width", "Time (s)"]) | |
| for b, b_name in [(1, "Beam Search (B=1)"), (2, "Beam Search (B=2)"), (4, "Beam Search (B=4)")]: | |
| if b_name in results: | |
| beam_timing_table.add_data(b, results[b_name]["Time (s)"]) | |
| wandb.log({ | |
| "beam_timing_curve": wandb.plot.line( | |
| beam_timing_table, "Beam Width", "Time (s)", title="Beam Search Time vs Beam Width" | |
| ) | |
| }) | |
| wandb.finish() | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description="Evaluate Task 3 Decoding Strategies") | |
| parser.add_argument("--num_samples", type=int, default=1000, help="Number of test samples (min 1000)") | |
| parser.add_argument("--max_new_tokens", type=int, default=40, help="Max tokens to generate") | |
| parser.add_argument("--output_file", type=str, default="task3_results.json", help="Path to save JSON results") | |
| parser.add_argument("--wandb", action="store_true", default=True, help="Log metrics to Weights & Biases") | |
| parser.add_argument("--no_wandb", action="store_false", dest="wandb", help="Disable WandB logging") | |
| parser.add_argument("--wandb_project", type=str, default="ANLP-Assignment-2-Part3", help="WandB project name") | |
| parser.add_argument("--wandb_run_name", type=str, default="decoding_evaluation", help="WandB run name") | |
| args = parser.parse_args() | |
| run_evaluation( | |
| num_samples=args.num_samples, | |
| max_new_tokens=args.max_new_tokens, | |
| output_file=args.output_file, | |
| use_wandb=args.wandb, | |
| wandb_project=args.wandb_project, | |
| wandb_run_name=args.wandb_run_name, | |
| ) | |