dbd41fe
1
2
3
4
5
6
7
8
9
10
import torch text = open("all_data.txt", "r", encoding="utf-8").read() tokens = [ord(c) % 256 for c in text] torch.save(torch.tensor(tokens, dtype=torch.long), "tokens.pt") print("Saved", len(tokens), "tokens")
