Upload folder using huggingface_hub

config.json

@@ -6,6 +6,7 @@
   "base_model_name": "roberta-base",
   "num_ai_classes": 62,
   "classifier_dropout": 0.1,
+  "tokenizer_class": "RobertaTokenizerFast",
   "id2label": {
     "0": "human",
     "1": "ai"

prepare_hf_artifacts_light.py

@@ -62,6 +62,7 @@ def main():
         "base_model_name": "roberta-base",
         "num_ai_classes": num_ai_classes,
         "classifier_dropout": 0.1,
+        "tokenizer_class": "RobertaTokenizerFast",
         "id2label": {"0": "human", "1": "ai"},
         "label2id": {"human": 0, "ai": 1},
         "auto_map": {
@@ -79,6 +80,27 @@ def main():
 
     _download_roberta_tokenizer_files(root)
 
+    tokenizer_config = {
+        "tokenizer_class": "RobertaTokenizerFast",
+        "model_max_length": 512,
+        "padding_side": "right",
+        "truncation_side": "right",
+    }
+    with open(root / "tokenizer_config.json", "w", encoding="utf-8") as file:
+        json.dump(tokenizer_config, file, indent=2)
+
+    special_tokens = {
+        "bos_token": "<s>",
+        "eos_token": "</s>",
+        "unk_token": "<unk>",
+        "sep_token": "</s>",
+        "pad_token": "<pad>",
+        "cls_token": "<s>",
+        "mask_token": "<mask>",
+    }
+    with open(root / "special_tokens_map.json", "w", encoding="utf-8") as file:
+        json.dump(special_tokens, file, indent=2)
+
     print("HF artifacts generated: config.json, pytorch_model.bin, tokenizer files")
 
 

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "sep_token": "</s>",
+  "pad_token": "<pad>",
+  "cls_token": "<s>",
+  "mask_token": "<mask>"
+}

tokenizer_config.json

@@ -1 +1,6 @@
-{"model_max_length": 512}
+{
+  "tokenizer_class": "RobertaTokenizerFast",
+  "model_max_length": 512,
+  "padding_side": "right",
+  "truncation_side": "right"
+}