Upload 2 files

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 banner.png filter=lfs diff=lfs merge=lfs -text
+data/chats/data.txt filter=lfs diff=lfs merge=lfs -text

data/chats/data.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e63d662bc5d7c7c523107d36c744074ac86530b3353065b53b814625ee849b9
+size 2154471817

data/chats/prepare.py

@@ -0,0 +1,41 @@
+import os
+import pickle
+import numpy as np
+
+input_file_path = os.path.join('data', 'chats', 'data.txt')
+
+with open(input_file_path, 'r') as f:
+    data = f.read()
+
+chars = sorted(list(set(data)))
+vocab_size = len(chars)
+
+stoi = {ch: i for i, ch in enumerate(chars)}
+itos = {i: ch for i, ch in enumerate(chars)}
+
+def encode(s):
+    return [stoi[c] for c in s]
+
+def decode(l):
+    return ''.join([itos[i] for i in l])
+
+n = len(data)
+train_data = data[:int(n*0.9)]
+val_data = data[int(n*0.9):]
+
+train_ids = encode(train_data)
+val_ids = encode(val_data)
+
+train_ids = np.array(train_ids, dtype=np.uint16)
+val_ids = np.array(val_ids, dtype=np.uint16)
+
+train_ids.tofile(os.path.join('data', 'chats', 'train.bin'))
+val_ids.tofile(os.path.join('data', 'chats', 'val.bin'))
+
+meta = {
+    'vocab_size': vocab_size,
+    'itos': itos,
+    'stoi': stoi,
+}
+with open(os.path.join('data', 'chats', 'meta.pkl'), 'wb') as f:
+    pickle.dump(meta, f)