⚠️ DO NOT MERGE: feat(transformers): Add native Transformers config

config.json

@@ -24,13 +24,17 @@
   "pack_qkv": true,
   "pad_token_id": 1,
   "position_embedding_type": "rope",
-  "rope_scaling": null,
-  "rope_theta": 160000,
   "torch_dtype": "float32",
   "transformers_version": "4.39.3",
   "type_vocab_size": 1,
   "unpad_inputs": "true",
   "use_memory_efficient_attention": "true",
-  "matryoshka_dimensions": [256],
-  "vocab_size": 250048
-}
+  "matryoshka_dimensions": [
+    256
+  ],
+  "vocab_size": 250048,
+  "rope_parameters": {
+    "rope_theta": 160000,
+    "rope_type": "default"
+  }
+}