import torch from torch.utils.data import Dataset from datasets import load_dataset from tokenizers import ByteLevelBPETokenizer from transformers import PreTrainedTokenizerFast import os import random def train_or_load_tokenizer(dataset, vocab_size = 8192, save_dir = "tokenizer_part1"): if os.path.exists(save_dir): print(f"Loading existing tokenizer from {save_dir}...") return PreTrainedTokenizerFast.from_pretrained(save_dir) def batch_iterator(max_samples = 50000): for i, row in enumerate(dataset): if i >= max_samples: break yield row["en"] yield row["vi"] yield row["ja"] special_tokens = ["", "", "", "<|vi|>", "<|ja|>", "<|en|>"] tokenizer = ByteLevelBPETokenizer() tokenizer.train_from_iterator(batch_iterator(), vocab_size = vocab_size, min_frequency = 2, special_tokens = special_tokens) os.makedirs(save_dir, exist_ok = True) hf_tokenizer = PreTrainedTokenizerFast(tokenizer_object = tokenizer, pad_token = "", unk_token = "", eos_token = "", bos_token = "") hf_tokenizer.save_pretrained(save_dir) return hf_tokenizer def prepare_data(num_samples = 50000, test_size = 2000): raw_ds = load_dataset("belumind/en-vi-ja-curated-500k-triplets", split = f"train[:{num_samples}]") pairs = [] for row in raw_ds: en = row["en"].strip() vi = row["vi"].strip() ja = row["ja"].strip() if not en or not vi or not ja: continue pairs.append(("vi", vi, en)) pairs.append(("ja", ja, en)) random.seed(42) random.shuffle(pairs) test_pairs = pairs[:test_size] train_pairs = pairs[test_size:] return train_pairs, test_pairs, raw_ds class Translation_Dataset(Dataset): def __init__(self, data_pairs, tokenizer, max_seq_len = 128): self.data_pairs = data_pairs self.tokenizer = tokenizer self.max_seq_len = max_seq_len def __len__(self): return len(self.data_pairs) def __getitem__(self, idx): lang, src_text, tgt_text = self.data_pairs[idx] prompt_str = f"<|{lang}|> {src_text} <|en|>" target_str = f"{tgt_text}<|eos|>" prompt_ids = self.tokenizer.encode(prompt_str, add_special_tokens = False) target_ids = self.tokenizer.encode(target_str, add_special_tokens = False) full_ids = prompt_ids + target_ids full_ids = full_ids[: self.max_seq_len + 1] input_ids = full_ids[:-1] targets = full_ids[1:] prompt_len = len(prompt_ids) mask_len = min(prompt_len - 1, len(targets)) targets = [-100] * mask_len + targets[mask_len:] pad_len = self.max_seq_len - len(input_ids) if pad_len > 0: input_ids = input_ids + [self.tokenizer.pad_token_id] * pad_len targets = targets + [-100] * pad_len return { "input_ids": torch.tensor(input_ids, dtype = torch.long), "targets": torch.tensor(targets, dtype = torch.long) }