File size: 1,387 Bytes
96a8ab7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
from preprocess_data import remove_html,remove_links
import torch
from model import BERT_Arch
from transformers import AutoModel,BertTokenizerFast
import numpy as np
def evaluate(data,device):
bert = AutoModel.from_pretrained('bert-base-uncased')
data = [data]
for param in bert.parameters():
param.requires_grad = False
model = BERT_Arch(bert)
map_location = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.load_state_dict(torch.load("/content/model.pth", weights_only=True,map_location=map_location))
model = model.to(device)
data = [remove_html(i) for i in data]
data = [remove_links(i) for i in data]
tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
tokenized = tokenizer.batch_encode_plus(data,
max_length = 25,
pad_to_max_length=True,
truncation=True
)
tokenized_seq = torch.tensor(tokenized['input_ids'])
tokenized_mask = torch.tensor(tokenized['attention_mask'])
with torch.no_grad():
preds = model(tokenized_seq.to(device), tokenized_mask.to(device))
preds = preds.detach().cpu().numpy()
pred = np.argmax(preds, axis = 1)
return pred |