Spaces:
Sleeping
Sleeping
File size: 1,762 Bytes
8257f60 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | import tiktoken
import torch
from torch.utils.data import DataLoader, Dataset
STOIC_FILES = [
"Stoic Knowledge/The Project Gutenberg.txt",
"Stoic Knowledge/Letters.txt",
"Stoic Knowledge/Seneca_s Morals.txt",
"Stoic Knowledge/Discourses.txt",
"Stoic Knowledge/Enchridion.txt",
]
def load_text(files=STOIC_FILES):
combined = ""
for fname in files:
with open(fname, "r", encoding="utf-8") as f:
combined += f.read() + "\n<|endoftext|>\n"
return combined
class GPTDatasetV1(Dataset):
def __init__(self, text, tokenizer, max_length, stride):
self.input_ids = []
self.target_ids = []
token_ids = tokenizer.encode(text, allowed_special={"<|endoftext|>"})
if len(token_ids) <= max_length:
raise ValueError("Tokenized text must contain at least max_length + 1 tokens.")
for start_idx in range(0, len(token_ids) - max_length, stride):
input_chunk = token_ids[start_idx : start_idx + max_length]
target_chunk = token_ids[start_idx + 1 : start_idx + max_length + 1]
self.input_ids.append(torch.tensor(input_chunk))
self.target_ids.append(torch.tensor(target_chunk))
def __len__(self):
return len(self.input_ids)
def __getitem__(self, idx):
return self.input_ids[idx], self.target_ids[idx]
def create_dataloader_v1(text, batch_size=4, max_length=256, stride=128,
shuffle=True, drop_last=True, num_workers=0):
tokenizer = tiktoken.get_encoding("gpt2")
dataset = GPTDatasetV1(text, tokenizer, max_length, stride)
return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle,
drop_last=drop_last, num_workers=num_workers)
|