Update 4 files

- /tokenizer.py
- /dataset.py
- /tokenizer.cli.py
- /trainer.cli.py

dataset.py

@@ -11,5 +11,11 @@ class Dataset:
         self.text = ''.join(s for s in self.dataset['train']['text']).encode('ascii', 'ignore').decode('ascii')
 
 
-    def Batch(self, ids):
+    def __iadd__(self, value):
+        attr_name = value.__name__ if hasattr(value, '__name__') else type(value).__name__.lower()
+        setattr(self, attr_name, value)
+        return self
+
+
+    def batch(self, value): # TODO: Implement
         pass

tokenizer.cli.py

@@ -0,0 +1 @@
+# TODO: Implement

tokenizer.py

@@ -0,0 +1,148 @@
+from collections import Counter
+import struct
+import re
+
+
+
+class Token:
+    def __init__(self, byte, prev):
+        self.byte = byte
+        self.prev = prev
+
+
+    def pack(self):
+        if not 0 <= ord(self.byte) <= 255:
+            raise ValueError(f"Byte value is out of range, got {self.byte} ({ord(self.byte)})")
+
+        return struct.pack("=B H", ord(self.byte), self.prev)
+
+
+    def __str__(self):
+        return f"{self.byte}, {self.prev}"
+
+    def to_binary(self):
+        return self.pack()
+
+
+
+
+class Tokenizer:
+    def __init__(self):
+        self.vocab = [Token(chr(i), 0) for i in range(256)]  # define base vocab from ASCII values
+
+
+    def find(self, byte, prev):
+        for i in range(prev, self.vocab_size):
+            token = self.vocab[i]
+            if token.byte == byte and token.prev == prev:
+                return i
+
+        return 0
+
+
+    def append(self, byte, prev):
+        token = self.find(byte, prev)
+        if token:
+            return token
+
+        self.vocab.append(Token(byte, prev))
+        return self.vocab_size - 1
+
+
+    def encode_one(self, text):
+        prev = 0
+
+        for i in range(len(text)):
+            byte = text[i]
+            token = self.find(byte, prev)
+
+            if token == 0:
+                return prev, text[i:]
+
+            prev = token
+
+        return prev, ''
+
+
+    def encode(self, text):
+        ids = []
+
+        while text:
+            token, text = self.encode_one(text)
+            ids.append(token)
+
+        return ids
+
+
+    def decode_one(self, token):
+        text = ""
+
+        while token:
+            text += self.vocab[token].byte
+            token = self.vocab[token].prev
+
+        return text[::-1]
+
+
+    def decode(self, ids):
+        text = ""
+
+        for token in ids:
+            text += self.decode_one(token)
+
+        return text
+
+
+    def add_special(self, text):
+        #print(f"Encoding string: {text}")
+        token = ord(text[0])
+        for byte in text[1:]:
+            token = self.append(byte, token)
+            #print(f"Working on byte {byte}")
+
+
+    @property
+    def vocab_size(self):
+        return len(self.vocab)
+
+
+    def __str__(self):
+        return '[' + ', '.join(str(token) for token in self.vocab) + ']'
+
+
+    def to_file(self, file):
+        with open(file, 'ab') as f:
+            for token in self.vocab:
+                  f.write(token.to_binary())
+
+
+    def from_file(self, file):
+        self.clear()
+        with open(file, 'rb') as f:
+            while True:
+                try:
+                    data = f.read(3)
+                    token = Token.from_binary(data)
+                    self.vocab += token
+                except ValueError:
+                    break
+
+
+    def train(self, text, max_length=32000):
+        words = text.split()
+        words = [' ' + ''.join(re.findall(r'\w', word)) for word in words]
+        words = [word for word in words if len(word) >= 2]
+
+        word_freq = Counter(words)
+        sorted_words = sorted(word_freq, key=lambda x: (-word_freq[x], x))
+
+        for word in sorted_words:
+            if self.vocab_size > max_length:
+                break
+
+            self.add_special(word)
+            print(f"adding word: {word} | current vocab size: {self.vocab_size} | max length: {max_length}")
+
+
+    def c_encode(self, text): #TODO: Implement
+        return []

trainer.cli.py

@@ -6,7 +6,7 @@ from logger import Wandb
 
 from trainer import Trainer
 from dataset import Dataset
-#from tokenizer import Tokenizer
+from tokenizer import Tokenizer
 
 
 
@@ -27,6 +27,12 @@ if __name__ == '__main__':
     
     dataset = Dataset(config.dataset)
 
-    #tokenizer = Tokenizer()
+    tokenizer = Tokenizer()
+    tokenizer.train(dataset.text, max_length=config.tokenizer.max_length)
+    ids = tokenizer.c_encode(dataset.text)
+    
+    dataset += ids
+    dataset.batch(ids)
 
-    trainer = Trainer(config)
+    trainer = Trainer(config)
+    trainer.train(dataset)