Hugging Face's logo

ASzecsenyi
/
VQLM

Model card Files
xet
Community
ASzecsenyi commited on
Commit
f40772e
·
verified ·
1 Parent(s): 70fd72b

Upload gpt_6x256_d75hudtl/meta_002870.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. gpt_6x256_d75hudtl/meta_002870.json +155 -0
gpt_6x256_d75hudtl/meta_002870.json ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 2870,
3
+ "val_bpb": 1.171798266579121,
4
+ "model_config": {
5
+ "run": "gpt_6x256",
6
+ "wandb_group": null,
7
+ "seed": 42,
8
+ "device_type": "cuda",
9
+ "config": "gpt5m",
10
+ "depth": 6,
11
+ "aspect_ratio": 64,
12
+ "model_dim": 256,
13
+ "head_dim": 128,
14
+ "sequence_len": 1024,
15
+ "vocab_size": 16384,
16
+ "clustering_init_num_samples": 0,
17
+ "rva_blocks": [],
18
+ "recurrent_vocab_sizes": [],
19
+ "rva_balance_loss_weight": 0.0,
20
+ "rva_balance_loss_temperature": 1.0,
21
+ "kla_blocks": [],
22
+ "mamba_blocks": [],
23
+ "gdn_blocks": [],
24
+ "gla_blocks": [],
25
+ "moe_blocks": [],
26
+ "d_state": 16,
27
+ "mamba_params": true,
28
+ "kla_kernel": true,
29
+ "mimo_rank": 1,
30
+ "skip_around_kla": true,
31
+ "decoder_mlp": false,
32
+ "use_reparametrisation_trick": true,
33
+ "num_experts": 4,
34
+ "moe_top_k": 2,
35
+ "moe_capacity_factor": 1.0,
36
+ "moe_aux_loss_weight": 0.01,
37
+ "moe_embedding_experts": false,
38
+ "num_iterations": 2870,
39
+ "target_flops": -1.0,
40
+ "target_param_data_ratio": 20,
41
+ "data_dir": "base_data",
42
+ "device_batch_size": 64,
43
+ "total_batch_size": 524288,
44
+ "use_muon": true,
45
+ "embedding_lr": 0.3,
46
+ "unembedding_lr": 0.004,
47
+ "grad_clip": 1.0,
48
+ "weight_decay": 0.0,
49
+ "matrix_lr": 0.02,
50
+ "adam_beta1": 0.8,
51
+ "adam_beta2": 0.95,
52
+ "warmup_ratio": 0.05,
53
+ "warmdown_ratio": 0.4,
54
+ "final_lr_frac": 0.0,
55
+ "resume_from_step": -1,
56
+ "eval_every": 250,
57
+ "eval_tokens": 10485760,
58
+ "core_metric_every": -1,
59
+ "core_metric_max_per_task": 500,
60
+ "sample_every": 250,
61
+ "save_every": 1000,
62
+ "push_checkpoints_to_hub": true,
63
+ "use_profiler": false,
64
+ "profile_step": 2,
65
+ "profile_micro_step": 0,
66
+ "memory_history_max_entries": 10000,
67
+ "profile_rva_codebook_usage": false,
68
+ "model_tag": "gpt_6x256",
69
+ "stage": null,
70
+ "n_layer": 6,
71
+ "n_head": 2,
72
+ "n_kv_head": 2,
73
+ "n_embd": 256
74
+ },
75
+ "user_config": {
76
+ "run": "gpt_6x256",
77
+ "wandb_group": null,
78
+ "seed": 42,
79
+ "device_type": "cuda",
80
+ "config": "gpt5m",
81
+ "depth": 6,
82
+ "aspect_ratio": 64,
83
+ "model_dim": 256,
84
+ "head_dim": 128,
85
+ "sequence_len": 1024,
86
+ "vocab_size": 16384,
87
+ "clustering_init_num_samples": 0,
88
+ "rva_blocks": [],
89
+ "recurrent_vocab_sizes": [],
90
+ "rva_balance_loss_weight": 0.0,
91
+ "rva_balance_loss_temperature": 1.0,
92
+ "kla_blocks": [],
93
+ "mamba_blocks": [],
94
+ "gdn_blocks": [],
95
+ "gla_blocks": [],
96
+ "moe_blocks": [],
97
+ "d_state": 16,
98
+ "mamba_params": true,
99
+ "kla_kernel": true,
100
+ "mimo_rank": 1,
101
+ "skip_around_kla": true,
102
+ "decoder_mlp": false,
103
+ "use_reparametrisation_trick": true,
104
+ "num_experts": 4,
105
+ "moe_top_k": 2,
106
+ "moe_capacity_factor": 1.0,
107
+ "moe_aux_loss_weight": 0.01,
108
+ "moe_embedding_experts": false,
109
+ "num_iterations": 2870,
110
+ "target_flops": -1.0,
111
+ "target_param_data_ratio": 20,
112
+ "data_dir": "base_data",
113
+ "device_batch_size": 64,
114
+ "total_batch_size": 524288,
115
+ "use_muon": true,
116
+ "embedding_lr": 0.3,
117
+ "unembedding_lr": 0.004,
118
+ "grad_clip": 1.0,
119
+ "weight_decay": 0.0,
120
+ "matrix_lr": 0.02,
121
+ "adam_beta1": 0.8,
122
+ "adam_beta2": 0.95,
123
+ "warmup_ratio": 0.05,
124
+ "warmdown_ratio": 0.4,
125
+ "final_lr_frac": 0.0,
126
+ "resume_from_step": -1,
127
+ "eval_every": 250,
128
+ "eval_tokens": 10485760,
129
+ "core_metric_every": -1,
130
+ "core_metric_max_per_task": 500,
131
+ "sample_every": 250,
132
+ "save_every": 1000,
133
+ "push_checkpoints_to_hub": true,
134
+ "use_profiler": false,
135
+ "profile_step": 2,
136
+ "profile_micro_step": 0,
137
+ "memory_history_max_entries": 10000,
138
+ "profile_rva_codebook_usage": false,
139
+ "model_tag": "gpt_6x256",
140
+ "stage": null
141
+ },
142
+ "stage": null,
143
+ "device_batch_size": 64,
144
+ "sequence_len": 1024,
145
+ "dataloader_state_dict": {
146
+ "pq_idx": 25,
147
+ "rg_idx": 30
148
+ },
149
+ "loop_state": {
150
+ "min_val_bpb": 1.171798266579121,
151
+ "smooth_train_loss": 3.546073663632608,
152
+ "smooth_aux_loss": 0.0,
153
+ "total_training_time": 783.1575973033905
154
+ }
155
+ }