Download src/part1/evaluate.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 7.03 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part1/evaluate.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part1/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part1/evaluate.py
7.03 kB
| import torch | |
| import torch.nn as nn | |
| from torch.utils.data import DataLoader | |
| import argparse | |
| import math | |
| import os | |
| import json | |
| import matplotlib.pyplot as plt | |
| import seaborn as sns | |
| from tqdm import tqdm | |
| from .model import DecoderOnlyTransformer | |
| from .dataset import prepare_data, train_or_load_tokenizer, Translation_Dataset | |
| from .train import evaluate, compute_translation_bleu | |
| def evaluate_language_usage(model, pairs, tokenizer, seq_len = 128, device = "cuda"): | |
| model.reset_moe_usage() | |
| model.set_moe_tracking(True) | |
| dataset = Translation_Dataset(pairs, tokenizer, max_seq_len = seq_len) | |
| loader = DataLoader(dataset, batch_size = 32, shuffle = False) | |
| model.eval() | |
| with torch.no_grad(): | |
| for batch in loader: | |
| input_ids = batch["input_ids"].to(device) | |
| _ = model(input_ids) | |
| raw_counts = model.get_moe_usage() | |
| total_per_layer = raw_counts.sum(dim = -1, keepdim = True).float() | |
| normalized = (raw_counts.float() / (total_per_layer + 1e-9)).cpu().numpy() | |
| model.set_moe_tracking(False) | |
| model.reset_moe_usage() | |
| return normalized | |
| def plot_language_heatmap(vi_usage, ja_usage, variant, save_path): | |
| num_layers, num_experts = vi_usage.shape | |
| y_labels = [f"Layer {i + 1}" for i in range(num_layers)] | |
| if variant == 4: | |
| x_labels = [f"Routed {i}" for i in range(num_experts - 1)] + ["Shared"] | |
| else: | |
| x_labels = [f"Expert {i}" for i in range(num_experts)] | |
| vmax = max(vi_usage.max(), ja_usage.max()) | |
| fig, axes = plt.subplots(1, 2, figsize = (12,5), sharey = True) | |
| sns.heatmap(vi_usage, ax = axes[0], annot = True, fmt = ".2f", cmap = "Blues", vmin = 0.0, vmax = vmax, xticklabels = x_labels, yticklabels = y_labels, cbar = False) | |
| axes[0].set_title(f"Variant {variant}: Vietnamese Activations", fontsize = 12, fontweight = "bold") | |
| axes[0].set_xlabel("Experts") | |
| axes[0].set_ylabel("Transformer Layers") | |
| sns.heatmap(ja_usage, ax = axes[1], annot = True, fmt = ".2f", cmap = "Blues", vmin = 0.0, vmax = vmax, xticklabels = x_labels, yticklabels = y_labels, cbar = True, cbar_kws = {"label": "Activation Proportion"}) | |
| axes[1].set_title(f"Variant {variant}: Japanese Activations", fontsize = 12, fontweight = "bold") | |
| axes[1].set_xlabel("Experts") | |
| plt.suptitle(f"Language Specialization Heatmap (Variant {variant})", fontsize = 14, y = 1.02) | |
| plt.tight_layout() | |
| os.makedirs(os.path.dirname(save_path), exist_ok = True) | |
| plt.savefig(save_path, dpi = 300, bbox_inches = "tight") | |
| plt.close() | |
| print(f"Saved heatmap to: {save_path}") | |
| def run_evaluation(args): | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| os.makedirs(args.output_dir, exist_ok = True) | |
| print("Loading test dataset and tokenizer...") | |
| _, test_pairs, raw_ds = prepare_data(num_samples = args.num_samples, test_size = args.test_size) | |
| tokenizer = train_or_load_tokenizer(raw_ds, vocab_size = args.vocab_size) | |
| vi_test = [p for p in test_pairs if p[0] == "vi"] | |
| ja_test = [p for p in test_pairs if p[0] == "ja"] | |
| print(f"Test split: Total = {len(test_pairs)} | vi = {len(vi_test)} | ja = {len(ja_test)}") | |
| test_ds = Translation_Dataset(test_pairs, tokenizer, max_seq_len = args.seq_len) | |
| test_loader = DataLoader(test_ds, batch_size = args.batch_size, shuffle = False) | |
| results = {} | |
| for v in range(1, 6): | |
| checkpoint_path = os.path.join(args.save_dir, f"best_variant_{v}.pt") | |
| if not os.path.exists(checkpoint_path): | |
| print(f"Skipping variant {v}: {checkpoint_path} not found") | |
| continue | |
| print(f"\n{'=' * 55}\n Evaluating Variant {v}({checkpoint_path})\n{'=' * 55}") | |
| model = DecoderOnlyTransformer(vocab_size = len(tokenizer), d_model = args.d_model, n_heads = args.n_heads, d_ff = args.d_ff, n_layers = args.n_layers, variant = v, max_seq_len = args.seq_len).to(device) | |
| state_dict = torch.load(checkpoint_path, map_location = device) | |
| model.load_state_dict(state_dict) | |
| model.eval() | |
| print("Computing test loss and perplexity...") | |
| test_loss, test_ppl = evaluate(model, test_loader, device = device) | |
| print("Computing BLEU scores...") | |
| bleu_overall = compute_translation_bleu(model, test_pairs, tokenizer, max_samples = args.bleu_samples, device = device) | |
| bleu_vi = compute_translation_bleu(model, vi_test, tokenizer, max_samples = args.bleu_samples // 2, device = device) | |
| bleu_ja = compute_translation_bleu(model, ja_test, tokenizer, max_samples = args.bleu_samples // 2, device = device) | |
| print(f"Results -> Loss: {test_loss:.4f} | PPL: {test_ppl:.2f} | BLEU Total: {bleu_overall:.2f} (Vi: {bleu_vi:.2f}, Ja: {bleu_ja:.2f})") | |
| if v > 1: | |
| print("Analyzing expert routing patterns per language...") | |
| vi_usage = evaluate_language_usage(model, vi_test, tokenizer, seq_len = args.seq_len, device = device) | |
| ja_usage = evaluate_language_usage(model, ja_test, tokenizer, seq_len = args.seq_len, device = device) | |
| heatmap_path = os.path.join(args.output_dir, f"heatmap_variant_{v}.png") | |
| plot_language_heatmap(vi_usage, ja_usage, variant = v, save_path = heatmap_path) | |
| results[f"variant_{v}"] = { | |
| "test_loss": round(float(test_loss), 4), | |
| "test_perplexity": round(float(test_ppl), 2), | |
| "bleu_overall": round(float(bleu_overall), 2), | |
| "bleu_vi_en": round(float(bleu_vi), 2), | |
| "bleu_ja_en": round(float(bleu_ja), 2), | |
| } | |
| results_file = os.path.join(args.output_dir, "task1_results.json") | |
| with open(results_file, "w") as f: | |
| json.dump(results, f, indent = 4) | |
| print(f"\nAll results saved to: {results_file}") | |
| print("\n" + "=" * 65) | |
| print(f"{'Variant':<12} | {'PPL':<8} | {'BLEU (All)':<12} | {'BLEU (Vi)':<10} | {'BLEU (Ja)':<10}") | |
| print("-" * 65) | |
| for v_name, metrics in results.items(): | |
| print(f"{v_name:<12} | {metrics['test_perplexity']:<8.2f} | {metrics['bleu_overall']:<12.2f} | {metrics['bleu_vi_en']:<10.2f} | {metrics['bleu_ja_en']:<10.2f}") | |
| print("=" * 65) | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--save_dir", type=str, default="checkpoints_part1") | |
| parser.add_argument("--output_dir", type=str, default="assets/part1_results") | |
| parser.add_argument("--d_model", type=int, default=384) | |
| parser.add_argument("--n_heads", type=int, default=6) | |
| parser.add_argument("--d_ff", type=int, default=1536) | |
| parser.add_argument("--n_layers", type=int, default=6) | |
| parser.add_argument("--vocab_size", type=int, default=8192) | |
| parser.add_argument("--seq_len", type=int, default=128) | |
| parser.add_argument("--batch_size", type=int, default=32) | |
| parser.add_argument("--num_samples", type=int, default=50000) | |
| parser.add_argument("--test_size", type=int, default=2000) | |
| parser.add_argument("--bleu_samples", type=int, default=100) | |
| args = parser.parse_args() | |
| run_evaluation(args) |