Update isoformer_tokenizer.py

isoformer_tokenizer.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 """Tokenization classes for ESM."""
 import os
+from huggingface_hub import hf_hub_download
 from typing import List, Optional
 
 #from transformers.models.esm.tokenization_esm import PreTrainedTokenizer
@@ -38,17 +39,30 @@ class IsoformerTokenizer(PreTrainedTokenizer):
         self,
         **kwargs
     ):
-        print(f"\n>>> DEBUG: IsoformerTokenizer __init__ received kwargs: {kwargs}")
-        pretrained_model_path = kwargs.get("pretrained_model_name_or_path", "")
-        print(f">>> DEBUG: Determined pretrained_model_path: '{pretrained_model_path}'")
-        
-        dna_vocab_path = os.path.join(pretrained_model_path, "dna_vocab_list.txt")
-        rna_vocab_path = os.path.join(pretrained_model_path, "rna_vocab_list.txt")
-        protein_vocab_path = os.path.join(pretrained_model_path, "protein_vocab_list.txt")
-        print(f">>> DEBUG: dna_vocab_path will be: '{dna_vocab_path}'") # Add this
-        print(f">>> DEBUG: Checking if dna_vocab_path exists: {os.path.exists(dna_vocab_path)}")
-        print(f">>> DEBUG: Checking if dna_vocab_list.txt exists in CWD: {os.path.exists('dna_vocab_list.txt')}")
-        
+        # Get the model ID from kwargs
+        model_id = kwargs.get("name_or_path", None) # This will be "InstaDeepAI/isoformer"
+
+        # Use hf_hub_download to get the local path to each vocabulary file.
+        # This function intelligently uses the local cache if the file is already downloaded.
+        if model_id:
+            try:
+                dna_vocab_path = hf_hub_download(repo_id=model_id, filename="dna_vocab_list.txt")
+                rna_vocab_path = hf_hub_download(repo_id=model_id, filename="rna_vocab_list.txt")
+                protein_vocab_path = hf_hub_download(repo_id=model_id, filename="protein_vocab_list.txt")
+            except Exception as e:
+                # Fallback in case hf_hub_download fails (e.g., if model_id was a local path not a Hub ID)
+                # This fallback might not be perfect for all edge cases, but covers the common local loading.
+                print(f"Warning: Failed to resolve model files via hf_hub_download. Attempting local fallback. Error: {e}")
+                dna_vocab_path = os.path.join(model_id, "dna_vocab_list.txt")
+                rna_vocab_path = os.path.join(model_id, "rna_vocab_list.txt")
+                protein_vocab_path = os.path.join(model_id, "protein_vocab_list.txt")
+        else:
+            # Fallback if model_id is not found (unlikely for AutoTokenizer.from_pretrained)
+            print("Warning: Could not determine model_id from kwargs. Falling back to relative paths.")
+            dna_vocab_path = "dna_vocab_list.txt"
+            rna_vocab_path = "rna_vocab_list.txt"
+            protein_vocab_path = "protein_vocab_list.txt"
+
         dna_hf_tokenizer = EsmTokenizer(dna_vocab_path, model_max_length=196608)
         dna_hf_tokenizer.eos_token = None  # Stops the tokenizer adding an EOS/SEP token at the end
         dna_hf_tokenizer.init_kwargs["eos_token"] = None  # Ensures it doesn't come back when reloading