from datasets import load_dataset
import evaluate

perplexity = evaluate.load("perplexity", module_type="metric")
dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")
input_texts = [text for text in dataset["text"] if text.strip() != ""]
results = perplexity.compute(
    model_id='tinycompany/ShawtyIsBad-bgem3',
    predictions=input_texts,
    batch_size=1,
    add_start_token=False,
    device="cuda"
)
print("Mean Perplexity:", results["mean_perplexity"])