Hugging Face's logo

ASzecsenyi
/
VQLM

Model card Files
xet
 Community
ASzecsenyi commited on
Commit
2b1b6ed
·
verified ·
1 Parent(s): f1141eb

Upload rva_6x256_3km37620/meta_001748.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. rva_6x256_3km37620/meta_001748.json +164 -0
rva_6x256_3km37620/meta_001748.json ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 1748,
3
+ "val_bpb": 4.524454016929745,
4
+ "model_config": {
5
+ "run": "rva_6x256",
6
+ "wandb_group": null,
7
+ "seed": 42,
8
+ "device_type": "cuda",
9
+ "config": "rva5m",
10
+ "depth": 6,
11
+ "aspect_ratio": 64,
12
+ "model_dim": 256,
13
+ "head_dim": 16,
14
+ "sequence_len": 1024,
15
+ "vocab_size": 265,
16
+ "clustering_init_num_samples": 0,
17
+ "rva_blocks": [
18
+ -1
19
+ ],
20
+ "recurrent_vocab_sizes": [
21
+ 8192,
22
+ 8192,
23
+ 8192,
24
+ 8192,
25
+ 8192,
26
+ 8192
27
+ ],
28
+ "kla_blocks": [],
29
+ "mamba_blocks": [],
30
+ "gdn_blocks": [],
31
+ "gla_blocks": [],
32
+ "moe_blocks": [],
33
+ "d_state": 16,
34
+ "mamba_params": true,
35
+ "kla_kernel": true,
36
+ "mimo_rank": 1,
37
+ "skip_around_kla": true,
38
+ "decoder_mlp": false,
39
+ "use_reparametrisation_trick": true,
40
+ "num_experts": 4,
41
+ "moe_top_k": 2,
42
+ "moe_capacity_factor": 1.0,
43
+ "moe_aux_loss_weight": 0.01,
44
+ "moe_embedding_experts": false,
45
+ "num_iterations": -1,
46
+ "target_flops": -1.0,
47
+ "target_param_data_ratio": 20,
48
+ "data_dir": "base_data",
49
+ "device_batch_size": 2,
50
+ "total_batch_size": 524288,
51
+ "use_muon": true,
52
+ "embedding_lr": 0.3,
53
+ "unembedding_lr": 0.004,
54
+ "grad_clip": 1.0,
55
+ "weight_decay": 0.0,
56
+ "matrix_lr": 0.02,
57
+ "adam_beta1": 0.8,
58
+ "adam_beta2": 0.95,
59
+ "warmup_ratio": 0.05,
60
+ "warmdown_ratio": 0.4,
61
+ "final_lr_frac": 0.0,
62
+ "resume_from_step": -1,
63
+ "eval_every": 250,
64
+ "eval_tokens": 10485760,
65
+ "core_metric_every": -1,
66
+ "core_metric_max_per_task": 500,
67
+ "sample_every": 250,
68
+ "save_every": 1000,
69
+ "push_checkpoints_to_hub": true,
70
+ "use_profiler": false,
71
+ "profile_step": 2,
72
+ "profile_micro_step": 0,
73
+ "memory_history_max_entries": 10000,
74
+ "model_tag": "rva_6x256",
75
+ "n_layer": 6,
76
+ "n_head": 16,
77
+ "n_kv_head": 16,
78
+ "n_embd": 256
79
+ },
80
+ "user_config": {
81
+ "run": "rva_6x256",
82
+ "wandb_group": null,
83
+ "seed": 42,
84
+ "device_type": "cuda",
85
+ "config": "rva5m",
86
+ "depth": 6,
87
+ "aspect_ratio": 64,
88
+ "model_dim": 256,
89
+ "head_dim": 16,
90
+ "sequence_len": 1024,
91
+ "vocab_size": 265,
92
+ "clustering_init_num_samples": 0,
93
+ "rva_blocks": [
94
+ -1
95
+ ],
96
+ "recurrent_vocab_sizes": [
97
+ 8192,
98
+ 8192,
99
+ 8192,
100
+ 8192,
101
+ 8192,
102
+ 8192
103
+ ],
104
+ "kla_blocks": [],
105
+ "mamba_blocks": [],
106
+ "gdn_blocks": [],
107
+ "gla_blocks": [],
108
+ "moe_blocks": [],
109
+ "d_state": 16,
110
+ "mamba_params": true,
111
+ "kla_kernel": true,
112
+ "mimo_rank": 1,
113
+ "skip_around_kla": true,
114
+ "decoder_mlp": false,
115
+ "use_reparametrisation_trick": true,
116
+ "num_experts": 4,
117
+ "moe_top_k": 2,
118
+ "moe_capacity_factor": 1.0,
119
+ "moe_aux_loss_weight": 0.01,
120
+ "moe_embedding_experts": false,
121
+ "num_iterations": -1,
122
+ "target_flops": -1.0,
123
+ "target_param_data_ratio": 20,
124
+ "data_dir": "base_data",
125
+ "device_batch_size": 2,
126
+ "total_batch_size": 524288,
127
+ "use_muon": true,
128
+ "embedding_lr": 0.3,
129
+ "unembedding_lr": 0.004,
130
+ "grad_clip": 1.0,
131
+ "weight_decay": 0.0,
132
+ "matrix_lr": 0.02,
133
+ "adam_beta1": 0.8,
134
+ "adam_beta2": 0.95,
135
+ "warmup_ratio": 0.05,
136
+ "warmdown_ratio": 0.4,
137
+ "final_lr_frac": 0.0,
138
+ "resume_from_step": -1,
139
+ "eval_every": 250,
140
+ "eval_tokens": 10485760,
141
+ "core_metric_every": -1,
142
+ "core_metric_max_per_task": 500,
143
+ "sample_every": 250,
144
+ "save_every": 1000,
145
+ "push_checkpoints_to_hub": true,
146
+ "use_profiler": false,
147
+ "profile_step": 2,
148
+ "profile_micro_step": 0,
149
+ "memory_history_max_entries": 10000,
150
+ "model_tag": "rva_6x256"
151
+ },
152
+ "device_batch_size": 2,
153
+ "sequence_len": 1024,
154
+ "dataloader_state_dict": {
155
+ "pq_idx": 3,
156
+ "rg_idx": 32
157
+ },
158
+ "loop_state": {
159
+ "min_val_bpb": 4.115631892357812,
160
+ "smooth_train_loss": 7.251877900050826,
161
+ "smooth_aux_loss": 4.109446192006799,
162
+ "total_training_time": 27046.127122163773
163
+ }
164
+ }