Download src/part1/dataset.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 3.1 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part1/dataset.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part1/dataset.py
-
curl -L -o dataset.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part1/dataset.py
3.1 kB
| import torch | |
| from torch.utils.data import Dataset | |
| from datasets import load_dataset | |
| from tokenizers import ByteLevelBPETokenizer | |
| from transformers import PreTrainedTokenizerFast | |
| import os | |
| import random | |
| def train_or_load_tokenizer(dataset, vocab_size = 8192, save_dir = "tokenizer_part1"): | |
| if os.path.exists(save_dir): | |
| print(f"Loading existing tokenizer from {save_dir}...") | |
| return PreTrainedTokenizerFast.from_pretrained(save_dir) | |
| def batch_iterator(max_samples = 50000): | |
| for i, row in enumerate(dataset): | |
| if i >= max_samples: | |
| break | |
| yield row["en"] | |
| yield row["vi"] | |
| yield row["ja"] | |
| special_tokens = ["<pad>", "<unk>", "<eos>", "<|vi|>", "<|ja|>", "<|en|>"] | |
| tokenizer = ByteLevelBPETokenizer() | |
| tokenizer.train_from_iterator(batch_iterator(), vocab_size = vocab_size, min_frequency = 2, special_tokens = special_tokens) | |
| os.makedirs(save_dir, exist_ok = True) | |
| hf_tokenizer = PreTrainedTokenizerFast(tokenizer_object = tokenizer, pad_token = "<pad>", unk_token = "<unk>", eos_token = "<eos>", bos_token = "<eos>") | |
| hf_tokenizer.save_pretrained(save_dir) | |
| return hf_tokenizer | |
| def prepare_data(num_samples = 50000, test_size = 2000): | |
| raw_ds = load_dataset("belumind/en-vi-ja-curated-500k-triplets", split = f"train[:{num_samples}]") | |
| pairs = [] | |
| for row in raw_ds: | |
| en = row["en"].strip() | |
| vi = row["vi"].strip() | |
| ja = row["ja"].strip() | |
| if not en or not vi or not ja: | |
| continue | |
| pairs.append(("vi", vi, en)) | |
| pairs.append(("ja", ja, en)) | |
| random.seed(42) | |
| random.shuffle(pairs) | |
| test_pairs = pairs[:test_size] | |
| train_pairs = pairs[test_size:] | |
| return train_pairs, test_pairs, raw_ds | |
| class Translation_Dataset(Dataset): | |
| def __init__(self, data_pairs, tokenizer, max_seq_len = 128): | |
| self.data_pairs = data_pairs | |
| self.tokenizer = tokenizer | |
| self.max_seq_len = max_seq_len | |
| def __len__(self): | |
| return len(self.data_pairs) | |
| def __getitem__(self, idx): | |
| lang, src_text, tgt_text = self.data_pairs[idx] | |
| prompt_str = f"<|{lang}|> {src_text} <|en|>" | |
| target_str = f"{tgt_text}<|eos|>" | |
| prompt_ids = self.tokenizer.encode(prompt_str, add_special_tokens = False) | |
| target_ids = self.tokenizer.encode(target_str, add_special_tokens = False) | |
| full_ids = prompt_ids + target_ids | |
| full_ids = full_ids[: self.max_seq_len + 1] | |
| input_ids = full_ids[:-1] | |
| targets = full_ids[1:] | |
| prompt_len = len(prompt_ids) | |
| mask_len = min(prompt_len - 1, len(targets)) | |
| targets = [-100] * mask_len + targets[mask_len:] | |
| pad_len = self.max_seq_len - len(input_ids) | |
| if pad_len > 0: | |
| input_ids = input_ids + [self.tokenizer.pad_token_id] * pad_len | |
| targets = targets + [-100] * pad_len | |
| return { | |
| "input_ids": torch.tensor(input_ids, dtype = torch.long), | |
| "targets": torch.tensor(targets, dtype = torch.long) | |
| } | |