Upload folder using huggingface_hub

processing_action_tokenizer.py

@@ -155,13 +155,4 @@ class UniversalActionProcessor(ProcessorMixin):
             min_token=min_token,
             time_horizon=time_horizon,
             action_dim=action_dim,
-        )
-        
-from huggingface_hub import HfApi
-api = HfApi()
-
-api.upload_folder(
-    folder_path="/data2/czhenghao/infinity_125M/libero_action_tokenizer",
-    repo_id="Felix-Zhenghao/Libero-FAST",
-    repo_type="model",
-)
+        )

processor_config.json

@@ -1,8 +1,11 @@
 {
-  "action_dim": 7,
-  "min_token": -32,
+  "action_dim": null,
+  "auto_map": {
+    "AutoProcessor": "processing_action_tokenizer.UniversalActionProcessor"
+  },
+  "min_token": -354,
   "processor_class": "UniversalActionProcessor",
   "scale": 10,
-  "time_horizon": 10,
+  "time_horizon": null,
   "vocab_size": 2048
 }

tokenizer_config.json

@@ -1,8 +1,11 @@
 {
   "added_tokens_decoder": {},
-  "clean_up_tokenization_spaces": false,
-  "extra_special_tokens": {},
+  "auto_map": {
+    "AutoProcessor": "processing_action_tokenizer.UniversalActionProcessor"
+  },
+  "clean_up_tokenization_spaces": true,
   "model_max_length": 1000000000000000019884624838656,
   "processor_class": "UniversalActionProcessor",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }
+