Upload tokenizer.py with huggingface_hub

tokenizer.py

@@ -0,0 +1,36 @@
+import os
+
+class CharacterTokenizer:
+    def __init__(self):
+        self.chars = []
+        self.vocab_size = 0
+        self.stoi = {} 
+        self.itos = {} 
+        self.special_tokens = ["<|pad|>", "<|user|>", "<|assistant|>", "<|end|>"]
+
+    def fit(self, text: str):
+        chars = sorted(list(set(text)))
+        self.chars = self.special_tokens + chars
+        self.vocab_size = len(self.chars)
+        self.stoi = { ch:i for i,ch in enumerate(self.chars) }
+        self.itos = { i:ch for i,ch in enumerate(self.chars) }
+        print(f"Vocabulario SSM creado. Tamaño: {self.vocab_size}")
+
+    def encode(self, s: str) -> list[int]:
+        return [self.stoi[c] for c in s if c in self.stoi]
+
+    def decode(self, l: list[int]) -> str:
+        return ''.join([self.itos[i] for i in l])
+
+def load_books(data_path: str) -> str:
+    all_text = ""
+    if not os.path.exists(data_path):
+        return "Error: Carpeta de datos no encontrada."
+    for filename in os.listdir(data_path):
+        if filename.endswith(".txt"):
+            try:
+                with open(os.path.join(data_path, filename), 'r', encoding='utf-8') as f:
+                    all_text += f.read() + "\n"
+            except:
+                continue
+    return all_text