vuminhtue commited on
Commit
0aed5d0
·
verified ·
1 Parent(s): 8364424

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +17 -0
config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "Qwen3",
3
+ "architecture": "Qwen3-0.6B",
4
+ "training_dataset": "TinyStories",
5
+ "vocab_size": 151936,
6
+ "context_length": 40960,
7
+ "emb_dim": 1024,
8
+ "n_heads": 16,
9
+ "n_layers": 28,
10
+ "hidden_dim": 3072,
11
+ "head_dim": 128,
12
+ "qk_norm": true,
13
+ "n_kv_groups": 8,
14
+ "rope_base": 1000000.0,
15
+ "max_iters": 200000,
16
+ "tokenizer": "gpt2"
17
+ }