Hugging Face's logo

AverageMetaheuristicsEnjoyer
/
shared-experts

Model card Files
xet
 Community
shared-experts
File size: 1,209 Bytes
04e3bbe
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
{
 "train_loss": [],
 "val_loss": [],
 "val_pp": [],
 "val_acc": [],
 "args": {
  "seed": 0,
  "data_seed": 1337,
  "device": "cuda:0",
  "dtype": "bfloat16",
  "compile": true,
  "vocab_size": 50257,
  "block_size": 1024,
  "n_layer": 12,
  "n_embd": 768,
  "n_head": 12,
  "kv_lora_rank": 192,
  "q_lora_rank": 256,
  "rope_dim": 64,
  "n_experts": 32,
  "n_experts_per_token": 2,
  "expert_intermediate_size": 296,
  "shared_expert_intermediate_size": 296,
  "mtp_num_heads": 0,
  "dropout": 0.0,
  "bias": true,
  "gate_bias": true,
  "gate_score_func": "softmax",
  "aux_loss_weight": 0.0,
  "mtp_loss_weight": 0.3,
  "opt": "adamw",
  "lr": 0.0003,
  "weight_decay": 0.1,
  "beta1": 0.9,
  "beta2": 0.95,
  "grad_clip": 1.0,
  "batch_size": 32,
  "acc_steps": 8,
  "scheduler": "cos",
  "iterations": 12800,
  "warmup_steps": 1280,
  "min_lr_ratio": 0.1,
  "wsd_fract_decay": 0.1,
  "wsd_final_lr_scale": 0.0,
  "decay_type": "cosine",
  "eval_interval": 200,
  "eval_batches": 32,
  "full_eval_at": [],
  "log_interval": 50,
  "diag_interval": 200,
  "diag_batches": 4,
  "permanent_ckpt_interval": 3200,
  "latest_ckpt_interval": 500,
  "resume_from": null,
  "auto_resume": true,
  "wandb": true
 }
}