WICKED4950 commited on
Commit
718d18b
·
verified ·
1 Parent(s): de0c7cd

Upload understanding_good_stuff.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. understanding_good_stuff.json +20 -0
understanding_good_stuff.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "arc": {
3
+ "num_heads": 20,
4
+ "attention_dim": 1280,
5
+ "vocab_size": 43012,
6
+ "num_blocks": 20,
7
+ "ff_dim": 2560
8
+ },
9
+ "attn": {
10
+ "attn_chunks": 1,
11
+ "gqa_repeats": 5,
12
+ "theeta": 2500
13
+ },
14
+ "moe": {
15
+ "num_experts": 1,
16
+ "top_k": 1,
17
+ "capacity": 1.0
18
+ },
19
+ "dropout_rate": 0.05
20
+ }