Update tokenizer.py

tokenizer.py

@@ -1,10 +1,10 @@
 import json
 from pathlib import Path
 from typing import Optional, Union
+import os
 
 import torch
 
-
 class Tokenizer:
     def __init__(self, checkpoint_dir: Union[Path, str]) -> None:
         checkpoint_dir = Path(checkpoint_dir)
@@ -17,6 +17,10 @@ class Tokenizer:
 
         # some checkpoints have both files, `.model` takes precedence
         print(f'tokenizer.py checkpoint_dir is : {checkpoint_dir}')
+        print(f'checking the file : {(checkpoint_dir / "tokenizer.json").is_file()}')
+        print(f'Current working directory is : {os.getcwd()}')
+        curr_dir = os.getcwd()
+        print(f'contents in pwd are : {os.listdir(curr_dir)}')
         if (vocabulary_path := checkpoint_dir / "tokenizer.model").is_file():
             from sentencepiece import SentencePieceProcessor