Upload tokenizer

tokenizer.py

@@ -1,9 +1,10 @@
 import re
 import os
-import json
 from typing import Any, Dict, List, Optional, Tuple, Union
 from transformers import PreTrainedTokenizer
 from transformers.utils import logging
+from pathlib import Path
+import json
 
 logger = logging.get_logger(__name__)
 
@@ -17,10 +18,7 @@ def load_json(path: str) -> Union[Dict, List]:
     Returns:
         Union[Dict, List]: The parsed content of the JSON file, which could be a dictionary or a list.
     """
-    full_path = path
-    if not os.path.exists(full_path):
-        # carica relativo alla posizione del file tokenizer.py
-        full_path = os.path.join(os.path.dirname(__file__), path)
+    full_path = Path(__file__).parent / path
     with open(full_path, "r", encoding="utf-8") as f:
         return json.load(f)