backend/nlp/evaluators.py

import torch
from transformers import Trainer
from ..nlp.trainers import TextDataset

def evaluate_nlp_model(model, tokenizer, texts, labels):
    encodings = tokenizer(texts, truncation=True, padding=True, max_length=512, return_tensors="pt")
    dataset = TextDataset(encodings, labels)

    trainer = Trainer(model=model)
    predictions = trainer.predict(dataset)
    preds = torch.argmax(torch.tensor(predictions.predictions), axis=1)

    accuracy = (preds == torch.tensor(labels)).float().mean().item()
    return {"accuracy": accuracy}