Update tokenization_milli.py

tokenization_milli.py

@@ -4,6 +4,7 @@ import pickle
 from typing import List, Optional, Tuple
 from transformers import PreTrainedTokenizer
 from tokenizers import Tokenizer
+from huggingface_hub import hf_hub_download
 
 class MiLLiTokenizer(PreTrainedTokenizer):
     """miLLi 1.0: Azerbaijani Hybrid Tokenizer with Phonological Restoration"""
@@ -12,30 +13,42 @@ class MiLLiTokenizer(PreTrainedTokenizer):
 
     def __init__(
         self,
-        trie_file=None,
-        bpe_file=None,
+        repo_id="elshadrahimov/miLLi-1.0", # Repozitoriyanızın adı bura yazılır
+        trie_file="milli_trie_v1.pkl",
+        bpe_file="milli_bpe_v1.json",
         unk_token="<UNK>",
         pad_token="<PAD>",
         eos_token="<EOS>",
         **kwargs
     ):
-        # Faylların olduğu qovluğu tapırıq (Hugging Face cache qovluğu)
-        current_dir = os.path.dirname(__file__)
 
-        # Fayl yollarını mütləq yola çeviririk
-        resolved_trie_path = os.path.join(current_dir, "milli_trie_v1.pkl")
-        resolved_bpe_path = os.path.join(current_dir, "milli_bpe_v1.json")
+        local_dir = os.path.dirname(__file__)
+        local_trie_path = os.path.join(local_dir, trie_file)
+        local_bpe_path = os.path.join(local_dir, bpe_file)
 
-        # 1. Trie lüğətini yükləyirik
-        if not os.path.exists(resolved_trie_path):
-            raise FileNotFoundError(f"Trie faylı tapılmadı: {resolved_trie_path}")
-        with open(resolved_trie_path, "rb") as f:
+        if os.path.exists(local_trie_path):
+            self.trie_path = local_trie_path
+        else:
+            try:
+                self.trie_path = hf_hub_download(repo_id=repo_id, filename=trie_file)
+            except Exception as e:
+                raise FileNotFoundError(f"Trie faylını Hub-dan yükləmək olmadı: {e}")
+
+        if os.path.exists(local_bpe_path):
+            self.bpe_path = local_bpe_path
+        else:
+
+            try:
+                self.bpe_path = hf_hub_download(repo_id=repo_id, filename=bpe_file)
+            except Exception as e:
+                raise FileNotFoundError(f"BPE faylını Hub-dan yükləmək olmadı: {e}")
+
+        
+        with open(self.trie_path, "rb") as f:
             self.trie = pickle.load(f)
 
-        # 2. BPE modelini yükləyirik
-        if not os.path.exists(resolved_bpe_path):
-            raise FileNotFoundError(f"BPE faylı tapılmadı: {resolved_bpe_path}")
-        self.bpe_backend = Tokenizer.from_file(resolved_bpe_path)
+        
+        self.bpe_backend = Tokenizer.from_file(self.bpe_path)
 
         super().__init__(
             unk_token=unk_token,