Hugging Face's logo

ASzecsenyi
/
VQLM

Model card Files
xet
Community
ASzecsenyi commited on
Commit
bd2e9b8
·
verified ·
1 Parent(s): 7ab3cfa

Upload stage_5_7mg7hekf/meta_001000.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. stage_5_7mg7hekf/meta_001000.json +177 -0
stage_5_7mg7hekf/meta_001000.json ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 1000,
3
+ "val_bpb": 2.675782267041374,
4
+ "model_config": {
5
+ "run": "rva_6x256/stage_5",
6
+ "wandb_group": "rva_6x256",
7
+ "seed": 42,
8
+ "device_type": "cuda",
9
+ "config": "rva5m",
10
+ "depth": 6,
11
+ "aspect_ratio": 64,
12
+ "model_dim": 256,
13
+ "head_dim": 16,
14
+ "sequence_len": 1024,
15
+ "vocab_size": 265,
16
+ "clustering_init_num_samples": 0,
17
+ "rva_blocks": [
18
+ 0,
19
+ 1,
20
+ 2,
21
+ 3,
22
+ 4,
23
+ 5
24
+ ],
25
+ "recurrent_vocab_sizes": [
26
+ 8192,
27
+ 8192,
28
+ 8192,
29
+ 8192,
30
+ 8192,
31
+ 8192
32
+ ],
33
+ "kla_blocks": [],
34
+ "mamba_blocks": [],
35
+ "gdn_blocks": [],
36
+ "gla_blocks": [],
37
+ "moe_blocks": [],
38
+ "d_state": 16,
39
+ "mamba_params": true,
40
+ "kla_kernel": true,
41
+ "mimo_rank": 1,
42
+ "skip_around_kla": true,
43
+ "decoder_mlp": false,
44
+ "use_reparametrisation_trick": true,
45
+ "num_experts": 4,
46
+ "moe_top_k": 2,
47
+ "moe_capacity_factor": 1.0,
48
+ "moe_aux_loss_weight": 0.01,
49
+ "moe_embedding_experts": false,
50
+ "num_iterations": -1,
51
+ "target_flops": -1.0,
52
+ "target_param_data_ratio": 20,
53
+ "data_dir": "base_data",
54
+ "device_batch_size": 8,
55
+ "total_batch_size": 524288,
56
+ "use_muon": true,
57
+ "embedding_lr": 0.3,
58
+ "unembedding_lr": 0.004,
59
+ "grad_clip": 1.0,
60
+ "weight_decay": 0.0,
61
+ "matrix_lr": 0.02,
62
+ "adam_beta1": 0.8,
63
+ "adam_beta2": 0.95,
64
+ "warmup_ratio": 0.05,
65
+ "warmdown_ratio": 0.4,
66
+ "final_lr_frac": 0.0,
67
+ "resume_from_step": -1,
68
+ "eval_every": 250,
69
+ "eval_tokens": 10485760,
70
+ "core_metric_every": -1,
71
+ "core_metric_max_per_task": 500,
72
+ "sample_every": 250,
73
+ "save_every": 1000,
74
+ "push_checkpoints_to_hub": true,
75
+ "use_profiler": false,
76
+ "profile_step": 2,
77
+ "profile_micro_step": 0,
78
+ "memory_history_max_entries": 10000,
79
+ "model_tag": "rva_6x256/stage_5",
80
+ "stage": 5,
81
+ "n_layer": 6,
82
+ "n_head": 16,
83
+ "n_kv_head": 16,
84
+ "n_embd": 256
85
+ },
86
+ "user_config": {
87
+ "run": "rva_6x256/stage_5",
88
+ "wandb_group": "rva_6x256",
89
+ "seed": 42,
90
+ "device_type": "cuda",
91
+ "config": "rva5m",
92
+ "depth": 6,
93
+ "aspect_ratio": 64,
94
+ "model_dim": 256,
95
+ "head_dim": 16,
96
+ "sequence_len": 1024,
97
+ "vocab_size": 265,
98
+ "clustering_init_num_samples": 0,
99
+ "rva_blocks": [
100
+ 0,
101
+ 1,
102
+ 2,
103
+ 3,
104
+ 4,
105
+ 5
106
+ ],
107
+ "recurrent_vocab_sizes": [
108
+ 8192,
109
+ 8192,
110
+ 8192,
111
+ 8192,
112
+ 8192,
113
+ 8192
114
+ ],
115
+ "kla_blocks": [],
116
+ "mamba_blocks": [],
117
+ "gdn_blocks": [],
118
+ "gla_blocks": [],
119
+ "moe_blocks": [],
120
+ "d_state": 16,
121
+ "mamba_params": true,
122
+ "kla_kernel": true,
123
+ "mimo_rank": 1,
124
+ "skip_around_kla": true,
125
+ "decoder_mlp": false,
126
+ "use_reparametrisation_trick": true,
127
+ "num_experts": 4,
128
+ "moe_top_k": 2,
129
+ "moe_capacity_factor": 1.0,
130
+ "moe_aux_loss_weight": 0.01,
131
+ "moe_embedding_experts": false,
132
+ "num_iterations": -1,
133
+ "target_flops": -1.0,
134
+ "target_param_data_ratio": 20,
135
+ "data_dir": "base_data",
136
+ "device_batch_size": 8,
137
+ "total_batch_size": 524288,
138
+ "use_muon": true,
139
+ "embedding_lr": 0.3,
140
+ "unembedding_lr": 0.004,
141
+ "grad_clip": 1.0,
142
+ "weight_decay": 0.0,
143
+ "matrix_lr": 0.02,
144
+ "adam_beta1": 0.8,
145
+ "adam_beta2": 0.95,
146
+ "warmup_ratio": 0.05,
147
+ "warmdown_ratio": 0.4,
148
+ "final_lr_frac": 0.0,
149
+ "resume_from_step": -1,
150
+ "eval_every": 250,
151
+ "eval_tokens": 10485760,
152
+ "core_metric_every": -1,
153
+ "core_metric_max_per_task": 500,
154
+ "sample_every": 250,
155
+ "save_every": 1000,
156
+ "push_checkpoints_to_hub": true,
157
+ "use_profiler": false,
158
+ "profile_step": 2,
159
+ "profile_micro_step": 0,
160
+ "memory_history_max_entries": 10000,
161
+ "model_tag": "rva_6x256/stage_5",
162
+ "stage": 5
163
+ },
164
+ "stage": 5,
165
+ "device_batch_size": 8,
166
+ "sequence_len": 1024,
167
+ "dataloader_state_dict": {
168
+ "pq_idx": 2,
169
+ "rg_idx": 4
170
+ },
171
+ "loop_state": {
172
+ "min_val_bpb": 2.675782267041374,
173
+ "smooth_train_loss": 1944.0993725320152,
174
+ "smooth_aux_loss": 1942.2091441590833,
175
+ "total_training_time": 3078.493688583374
176
+ }
177
+ }