Upload tokenizer_config.json with huggingface_hub

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "tokenizer_class": "SentencePieceProcessor",
+  "model_type": "bpe",
+  "vocab_size": 12000,
+  "pad_id": 0,
+  "bos_id": 1,
+  "eos_id": 2,
+  "unk_id": 3,
+  "character_coverage": 0.9999,
+  "byte_fallback": false,
+  "split_digits": false,
+  "normalization_rule_name": "identity",
+  "reserved_symbols_count": 6806,
+  "domains": [
+    "ROS",
+    "HTTP",
+    "MQTT",
+    "DB",
+    "WORKFLOW",
+    "ECOMMERCE",
+    "VEHICLE",
+    "HOME",
+    "CAL",
+    "FILE"
+  ],
+  "structural_tags": [
+    "<SCHEMA>",
+    "</SCHEMA>",
+    "<TASK>",
+    "</TASK>",
+    "<DOMAIN>",
+    "</DOMAIN>",
+    "<JSON>",
+    "</JSON>",
+    "<ACTION>",
+    "</ACTION>",
+    "<META>",
+    "</META>"
+  ],
+  "license": "apache-2.0",
+  "organization": "AMFORGE"
+}