Update gptx_tokenizer.py

gptx_tokenizer.py

@@ -103,28 +103,28 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
             ValueError: If repo_id is not provided when model_file_or_name is not a file.
             OSError: If the model file cannot be loaded or downloaded.
         """
-        if not os.path.isfile(model_file_or_name):
-            if repo_id is None:
-                raise ValueError("repo_id must be provided if model_file_or_name is not a local file")
+        # if not os.path.isfile(model_file_or_name):
+        #     if repo_id is None:
+        #         raise ValueError("repo_id must be provided if model_file_or_name is not a local file")
             
-            try:
-                # List all files in the repo
-                repo_files = list_repo_files(repo_id)
+        #     try:
+        #         # List all files in the repo
+        #         repo_files = list_repo_files(repo_id)
                 
-                # Find the tokenizer model file
-                tokenizer_files = [f for f in repo_files if f.endswith('.model')]
-                if not tokenizer_files:
-                    raise FileNotFoundError(f"No .model file found in repository {repo_id}")
+        #         # Find the tokenizer model file
+        #         tokenizer_files = [f for f in repo_files if f.endswith('.model')]
+        #         if not tokenizer_files:
+        #             raise FileNotFoundError(f"No .model file found in repository {repo_id}")
                 
-                # Use the first .model file found
-                model_file = tokenizer_files[0]
-                print(f"Found tokenizer model file: {model_file}")
+        #         # Use the first .model file found
+        #         model_file = tokenizer_files[0]
+        #         print(f"Found tokenizer model file: {model_file}")
                 
-                # Download the file
-                model_file_or_name = hf_hub_download(repo_id=repo_id, filename=model_file)
-                print(f"Downloaded tokenizer model to: {model_file_or_name}")
-            except Exception as e:
-                raise OSError(f"Failed to download tokenizer model: {str(e)}")
+        #         # Download the file
+        #         model_file_or_name = hf_hub_download(repo_id=repo_id, filename=model_file)
+        #         print(f"Downloaded tokenizer model to: {model_file_or_name}")
+        #     except Exception as e:
+        #         raise OSError(f"Failed to download tokenizer model: {str(e)}")
 
         try:
             return spm.SentencePieceProcessor(model_file=model_file_or_name)
@@ -182,10 +182,10 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
         if config_path is None:
             config_path = str(Path(cp_path) / TOKENIZER_CONFIG_FILE)
 
-        if os.path.isfile(config_path):
-            self.tokenizer_config = self.load_json(Path(config_path))
-        else: # Load from repo
-            self.tokenizer_config = self.load_json(Path(self.find_tokenizer_config(Path(config_path), repo_id=REPO_ID)))
+        # if os.path.isfile(config_path):
+        self.tokenizer_config = self.load_json(Path(config_path))
+        # else: # Load from repo
+            # self.tokenizer_config = self.load_json(Path(self.find_tokenizer_config(Path(config_path), repo_id=REPO_ID)))
 
     @property
     def vocab_size(self) -> int: