Spaces:
Sleeping
Sleeping
| import tiktoken | |
| import torch | |
| from torch.utils.data import DataLoader, Dataset | |
| STOIC_FILES = [ | |
| "Stoic Knowledge/The Project Gutenberg.txt", | |
| "Stoic Knowledge/Letters.txt", | |
| "Stoic Knowledge/Seneca_s Morals.txt", | |
| "Stoic Knowledge/Discourses.txt", | |
| "Stoic Knowledge/Enchridion.txt", | |
| ] | |
| def load_text(files=STOIC_FILES): | |
| combined = "" | |
| for fname in files: | |
| with open(fname, "r", encoding="utf-8") as f: | |
| combined += f.read() + "\n<|endoftext|>\n" | |
| return combined | |
| class GPTDatasetV1(Dataset): | |
| def __init__(self, text, tokenizer, max_length, stride): | |
| self.input_ids = [] | |
| self.target_ids = [] | |
| token_ids = tokenizer.encode(text, allowed_special={"<|endoftext|>"}) | |
| if len(token_ids) <= max_length: | |
| raise ValueError("Tokenized text must contain at least max_length + 1 tokens.") | |
| for start_idx in range(0, len(token_ids) - max_length, stride): | |
| input_chunk = token_ids[start_idx : start_idx + max_length] | |
| target_chunk = token_ids[start_idx + 1 : start_idx + max_length + 1] | |
| self.input_ids.append(torch.tensor(input_chunk)) | |
| self.target_ids.append(torch.tensor(target_chunk)) | |
| def __len__(self): | |
| return len(self.input_ids) | |
| def __getitem__(self, idx): | |
| return self.input_ids[idx], self.target_ids[idx] | |
| def create_dataloader_v1(text, batch_size=4, max_length=256, stride=128, | |
| shuffle=True, drop_last=True, num_workers=0): | |
| tokenizer = tiktoken.get_encoding("gpt2") | |
| dataset = GPTDatasetV1(text, tokenizer, max_length, stride) | |
| return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, | |
| drop_last=drop_last, num_workers=num_workers) | |