Upload config.json with huggingface_hub

config.json

@@ -1,20 +1,31 @@
 {
-  "architectures": [
-    "LlamaForCausalLM"
-  ],
-  "model_type": "llama",
-  "vocab_size": 32000,
-  "hidden_size": 512,
-  "intermediate_size": 2048,
-  "num_hidden_layers": 6,
-  "num_attention_heads": 8,
-  "max_position_embeddings": 1024,
-  "rms_norm_eps": 1e-6,
-  "rope_theta": 10000.0,
-  "use_cache": true,
-  "torch_dtype": "float32",
-  "transformers_version": "4.0.0",
-  "openllm_version": "0.1.0",
-  "training_steps": 10000,
-  "model_size": "small"
+  "model_config": {
+    "model_name": "OpenLLM-Small-10k",
+    "model_size": "small",
+    "vocab_size": 32000,
+    "n_layer": 6,
+    "n_head": 8,
+    "n_embd": 512,
+    "block_size": 1024,
+    "dropout": 0.1,
+    "bias": true,
+    "training_info": {
+      "step": 10000,
+      "best_loss": 5.22,
+      "model_type": "gpt-small"
+    }
+  },
+  "tokenizer_config": {
+    "type": "sentencepiece",
+    "vocab_size": 32000,
+    "model_file": "data/tokenizer/tokenizer.model"
+  },
+  "training_config": {
+    "learning_rate": 0.0001,
+    "batch_size": 4,
+    "gradient_accumulation_steps": 4,
+    "max_steps": 10000,
+    "warmup_steps": 100,
+    "weight_decay": 0.01
+  }
 }