Update gptx_tokenizer.py

gptx_tokenizer.py

@@ -66,9 +66,9 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
     
     def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optional[Path]:
         if not os.path.isfile(config_path):
-            config_path = try_to_load_from_cache(repo_id=REPO_ID, filename=Path(config_path).name)
+            config_path = try_to_load_from_cache(repo_id=repo_id, filename=Path(config_path).name)
             if not config_path:
-                config_path = self._download_config_from_hub()
+                config_path = self._download_config_from_hub(repo_id=repo_id)
 
         return config_path
 
@@ -89,19 +89,16 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
             OSError: If the model file cannot be loaded or downloaded.
         """
         if not os.path.isfile(model_file_or_name):
-            model_file_or_name = try_to_load_from_cache(repo_id=REPO_ID, filename=Path(model_file_or_name).name)
+            model_file_or_name = try_to_load_from_cache(repo_id=repo_id, filename=Path(model_file_or_name).name)
             if not model_file_or_name:
-                model_file_or_name = self._download_model_from_hub()
+                model_file_or_name = self._download_model_from_hub(repo_id=repo_id)
 
         try:
             return spm.SentencePieceProcessor(model_file=model_file_or_name)
         except Exception as e:
             raise OSError(f"Failed to load tokenizer model: {str(e)}")
 
-    def _download_model_from_hub(self) -> Optional[str]:
-        if repo_id is None:
-            raise ValueError("repo_id must be provided if model_file_or_name is not a local file")
-
+    def _download_model_from_hub(self, repo_id: str) -> Optional[str]:
         try:
             # List all files in the repo
             repo_files = list_repo_files(repo_id)
@@ -123,7 +120,7 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
 
         return model_file_or_name
 
-    def _download_config_from_hub(self):
+    def _download_config_from_hub(self, repo_id: str):
         if repo_id is None:
             raise ValueError("repo_id must be provided if config_path is not a local file")