Upload tokenizer_config.json with huggingface_hub

tokenizer_config.json

@@ -1,16 +1,13 @@
 {
-  "tokenizer_class": "SentencePieceProcessor",
-  "model_type": "bpe",
+  "name": "SAM Tokenizer",
+  "architecture": "NexusBPE",
+  "organization": "AMFORGE",
+  "version": "1.0",
   "vocab_size": 12000,
   "pad_id": 0,
   "bos_id": 1,
   "eos_id": 2,
   "unk_id": 3,
-  "character_coverage": 0.9999,
-  "byte_fallback": false,
-  "split_digits": false,
-  "normalization_rule_name": "identity",
-  "reserved_symbols_count": 6806,
   "domains": [
     "ROS",
     "HTTP",
@@ -23,13 +20,11 @@
     "CAL",
     "FILE"
   ],
-  "structural_tags": [
+  "structural_markers": [
     "<SCHEMA>",
     "</SCHEMA>",
     "<TASK>",
     "</TASK>",
-    "<DOMAIN>",
-    "</DOMAIN>",
     "<JSON>",
     "</JSON>",
     "<ACTION>",
@@ -37,6 +32,12 @@
     "<META>",
     "</META>"
   ],
+  "guarantees": [
+    "atomic numerics in supported ranges",
+    "atomic domain markers and structural tags",
+    "deterministic encoding",
+    "bit-perfect roundtrip on structured payloads"
+  ],
   "license": "apache-2.0",
-  "organization": "AMFORGE"
+  "homepage": "https://huggingface.co/AMFORGE/sam_tokenizer"
 }