anlp-a2-models / src /part1 /dataset.py
Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
3.1 kB
import torch
from torch.utils.data import Dataset
from datasets import load_dataset
from tokenizers import ByteLevelBPETokenizer
from transformers import PreTrainedTokenizerFast
import os
import random
def train_or_load_tokenizer(dataset, vocab_size = 8192, save_dir = "tokenizer_part1"):
if os.path.exists(save_dir):
print(f"Loading existing tokenizer from {save_dir}...")
return PreTrainedTokenizerFast.from_pretrained(save_dir)
def batch_iterator(max_samples = 50000):
for i, row in enumerate(dataset):
if i >= max_samples:
break
yield row["en"]
yield row["vi"]
yield row["ja"]
special_tokens = ["<pad>", "<unk>", "<eos>", "<|vi|>", "<|ja|>", "<|en|>"]
tokenizer = ByteLevelBPETokenizer()
tokenizer.train_from_iterator(batch_iterator(), vocab_size = vocab_size, min_frequency = 2, special_tokens = special_tokens)
os.makedirs(save_dir, exist_ok = True)
hf_tokenizer = PreTrainedTokenizerFast(tokenizer_object = tokenizer, pad_token = "<pad>", unk_token = "<unk>", eos_token = "<eos>", bos_token = "<eos>")
hf_tokenizer.save_pretrained(save_dir)
return hf_tokenizer
def prepare_data(num_samples = 50000, test_size = 2000):
raw_ds = load_dataset("belumind/en-vi-ja-curated-500k-triplets", split = f"train[:{num_samples}]")
pairs = []
for row in raw_ds:
en = row["en"].strip()
vi = row["vi"].strip()
ja = row["ja"].strip()
if not en or not vi or not ja:
continue
pairs.append(("vi", vi, en))
pairs.append(("ja", ja, en))
random.seed(42)
random.shuffle(pairs)
test_pairs = pairs[:test_size]
train_pairs = pairs[test_size:]
return train_pairs, test_pairs, raw_ds
class Translation_Dataset(Dataset):
def __init__(self, data_pairs, tokenizer, max_seq_len = 128):
self.data_pairs = data_pairs
self.tokenizer = tokenizer
self.max_seq_len = max_seq_len
def __len__(self):
return len(self.data_pairs)
def __getitem__(self, idx):
lang, src_text, tgt_text = self.data_pairs[idx]
prompt_str = f"<|{lang}|> {src_text} <|en|>"
target_str = f"{tgt_text}<|eos|>"
prompt_ids = self.tokenizer.encode(prompt_str, add_special_tokens = False)
target_ids = self.tokenizer.encode(target_str, add_special_tokens = False)
full_ids = prompt_ids + target_ids
full_ids = full_ids[: self.max_seq_len + 1]
input_ids = full_ids[:-1]
targets = full_ids[1:]
prompt_len = len(prompt_ids)
mask_len = min(prompt_len - 1, len(targets))
targets = [-100] * mask_len + targets[mask_len:]
pad_len = self.max_seq_len - len(input_ids)
if pad_len > 0:
input_ids = input_ids + [self.tokenizer.pad_token_id] * pad_len
targets = targets + [-100] * pad_len
return {
"input_ids": torch.tensor(input_ids, dtype = torch.long),
"targets": torch.tensor(targets, dtype = torch.long)
}