Spaces:
Runtime error
Runtime error
File size: 1,911 Bytes
639ef39 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | import pandas as pd
import torch
from torch.utils.data import Dataset
class Dataset(Dataset):
""" Load dataset from file csv"""
def __init__(self, vocab, csv_fpath=None, tokenized_fpath=None):
"""
@param vocab (Vocabulary)
@param csv_fpath (str)
@param tokenized_fpath (str)
"""
self.vocab = vocab
self.pad_idx = vocab["<pad>"]
df = pd.read_csv(csv_fpath)
self.sentiments_list = list(df.sentiment)
self.reviews_list = list(df.vi_review)
sentiments_type = list(set(self.sentiments_list))
sentiments_type.sort()
self.sentiment2id = {sentiment: i for i, sentiment in enumerate(sentiments_type)}
if tokenized_fpath:
self.tokenized_reviews = torch.load(tokenized_fpath)
else:
self.tokenized_reviews = self.vocab.tokenize_corpus(self.reviews_list)
self.tensor_data = self.vocab.corpus_to_tensor(self.tokenized_reviews, is_tokenized=True)
self.tensor_label = torch.tensor([self.sentiment2id[sentiment] for sentiment in self.sentiments_list],
dtype=torch.float64)
def __len__(self):
return len(self.tensor_data)
def __getitem__(self, idx):
return self.tensor_data[idx], self.tensor_label[idx]
def collate_fn(self, examples):
examples = sorted(examples, key=lambda e: len(e[0]), reverse=True)
reviews = [e[0] for e in examples]
reviews = torch.nn.utils.rnn.pad_sequence(reviews,
batch_first=False,
padding_value=self.pad_idx)
reviews_lengths = torch.tensor([len(e[0]) for e in examples])
sentiments = torch.tensor([e[1] for e in examples])
return {"reviews": (reviews, reviews_lengths), "sentiments": sentiments}
|