Upload centurio.py

centurio.py

@@ -0,0 +1,140 @@
+import json
+import os
+from typing import List, Optional
+from transformers import PreTrainedTokenizer
+
+
+class CenturioTokenizer(PreTrainedTokenizer):
+    vocab_files_names = {"vocab_file": "centurio_vocab.json"}
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file=None,
+        unk_token="<unk>",
+        bos_token="<s>",
+        eos_token="</s>",
+        pad_token="<pad>",
+        sep_token="<sep>",
+        cls_token="<cls>",
+        mask_token="<mask>",
+        space_token="▁",
+        **kwargs
+    ):
+        self.space_token = space_token
+        self._vocab = {}
+        self._inv_vocab = {}
+
+        super().__init__(
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            sep_token=sep_token,
+            cls_token=cls_token,
+            mask_token=mask_token,
+            **kwargs
+        )
+
+        if vocab_file is not None:
+            self._load_vocab(vocab_file)
+        else:
+            self._build_default_vocab()
+
+    def _build_default_vocab(self):
+        special_tokens = [
+            self.unk_token, self.bos_token, self.eos_token,
+            self.pad_token, self.sep_token, self.cls_token,
+            self.mask_token, self.space_token
+        ]
+        self._vocab = {token: i for i, token in enumerate(special_tokens)}
+        self._inv_vocab = {i: token for token, i in self._vocab.items()}
+
+    def _load_vocab(self, vocab_file):
+        with open(vocab_file, "r", encoding="utf-8") as f:
+            self._vocab = json.load(f)
+        self._inv_vocab = {v: k for k, v in self._vocab.items()}
+
+    def get_vocab(self):
+        return self._vocab.copy()
+
+    @property
+    def vocab_size(self):
+        return len(self._vocab)
+
+    def _tokenize(self, text: str) -> List[str]:
+        text = text.replace(" ", self.space_token)
+        tokens = []
+        current = ""
+        for ch in text:
+            if ch.isalnum() or ch in "абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ":
+                current += ch
+            else:
+                if current:
+                    tokens.append(current)
+                    current = ""
+                tokens.append(ch if ch != self.space_token else self.space_token)
+        if current:
+            tokens.append(current)
+        return tokens
+
+    def _convert_token_to_id(self, token: str) -> int:
+        return self._vocab.get(token, self._vocab.get(self.unk_token, 0))
+
+    def _convert_id_to_token(self, index: int) -> str:
+        return self._inv_vocab.get(index, self.unk_token)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple:
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory)
+        vocab_file = os.path.join(
+            save_directory,
+            (filename_prefix + "-" if filename_prefix else "") + "centurio_vocab.json"
+        )
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            json.dump(self._vocab, f, ensure_ascii=False, indent=2)
+        return (vocab_file,)
+
+    def build_vocab_from_corpus(self, corpus: List[str], min_freq: int = 2):
+        from collections import Counter
+        token_counter = Counter()
+        for text in corpus:
+            tokens = self._tokenize(text)
+            token_counter.update(tokens)
+
+        special_tokens = [
+            self.unk_token, self.bos_token, self.eos_token,
+            self.pad_token, self.sep_token, self.cls_token,
+            self.mask_token, self.space_token
+        ]
+        new_vocab = {token: i for i, token in enumerate(special_tokens)}
+        idx = len(new_vocab)
+
+        for token, freq in token_counter.items():
+            if freq >= min_freq and token not in new_vocab:
+                new_vocab[token] = idx
+                idx += 1
+
+        self._vocab = new_vocab
+        self._inv_vocab = {v: k for k, v in self._vocab.items()}
+
+if __name__ == "__main__":
+    corpus = [
+        "Привет, как дела!",
+        "Я учу немецкий язык.",
+        "Морфемы помогают понять структуру слов."
+    ]
+
+    tokenizer = CenturioTokenizer()
+    tokenizer.build_vocab_from_corpus(corpus, min_freq=1)
+    tokenizer.save_pretrained("./centurio_model")
+
+    for text in corpus:
+        tokens = tokenizer.tokenize(text)
+        ids = tokenizer.encode(text)
+        back = tokenizer.decode(ids)
+        print(f"\nTEXT   : {text}")
+        print(f"TOKENS : {tokens}")
+        print(f"IDS    : {ids}")
+        print(f"BACK   : {back}")
+        print(f"VOCAB  : {tokenizer.vocab_size}")