tokenizer.py

import torch

text = open("all_data.txt", "r", encoding="utf-8").read()

tokens = [ord(c) % 256 for c in text]

torch.save(torch.tensor(tokens, dtype=torch.long), "tokens.pt")

print("Saved", len(tokens), "tokens")