Upload 4 files

tokenizer.py

@@ -0,0 +1,26 @@
+from collections import defaultdict
+import pickle
+
+class CaptionTokenizer:
+    def __init__(self, tok_to_ind, ind_to_tok, vocab_freq, min_freq=5):
+        self.tok_to_ind = tok_to_ind
+        self.ind_to_tok = ind_to_tok
+        self.vocab_freq = vocab_freq
+        self.min_freq = min_freq
+        self.vocab_size = len(tok_to_ind)
+        
+    def encode(self, text):
+        return [self.tok_to_ind[tok] if tok in self.tok_to_ind and self.vocab_freq[tok] >= self.min_freq else 0 
+                for tok in text.split()]
+                
+    def decode(self, indices):
+        return " ".join([self.ind_to_tok[idx] for idx in indices if idx in self.ind_to_tok and idx != 0])
+        
+    def save(self, path):
+        with open(path, 'wb') as f:
+            pickle.dump(self, f)
+            
+    @classmethod
+    def load(cls, path):
+        with open(path, 'rb') as f:
+            return pickle.load(f)