Upload app.py

app.py

@@ -97,7 +97,39 @@ def load_tokenizer():
     global tokenizer
     try:
         if tokenizer is None:
-            tokenizer = RobertaTokenizer.from_pretrained(HF_MODEL_ID)
+            # Download tokenizer files from the tokenizer_files subdirectory
+            print("Downloading tokenizer files...")
+            tokenizer_files = [
+                "tokenizer_files/vocab.json",
+                "tokenizer_files/merges.txt",
+                "tokenizer_files/tokenizer_config.json",
+                "tokenizer_files/special_tokens_map.json"
+            ]
+            
+            # Download all tokenizer files
+            for file_path in tokenizer_files:
+                hf_hub_download(
+                    repo_id=HF_MODEL_ID,
+                    filename=file_path,
+                    cache_dir=None
+                )
+            
+            # Get the snapshot directory path by downloading the model file (already done)
+            # or by downloading any file and getting its parent directory
+            # The tokenizer files are in tokenizer_files/ subdirectory of the snapshot
+            model_path = hf_hub_download(
+                repo_id=HF_MODEL_ID,
+                filename=MODEL_FILENAME,
+                cache_dir=None
+            )
+            snapshot_dir = os.path.dirname(model_path)
+            tokenizer_dir = os.path.join(snapshot_dir, "tokenizer_files")
+            
+            print(f"Loading tokenizer from: {tokenizer_dir}")
+            
+            # Load tokenizer from the local tokenizer_files directory
+            tokenizer = RobertaTokenizer.from_pretrained(tokenizer_dir)
+            print("Tokenizer loaded successfully!")
         return tokenizer
     except Exception as e:
         raise RuntimeError(f"Error loading tokenizer: {str(e)}")