Hugging Face's logo

aixk
/
twiny-stack-L01

Text Generation
ONNX
Safetensors
twiny
Model card Files
xet
 Community
aixk commited on
Commit
ad9a566
·
1 Parent(s): 0e80d75

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. slots/2/checkpoint-376/config.json +24 -0
  2. slots/2/checkpoint-376/trainer_state.json +167 -0
slots/2/checkpoint-376/config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "TwinyForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "dtype": "float32",
7
+ "hidden_dropout": 0.0,
8
+ "hidden_size": 768,
9
+ "initializer_range": 0.02,
10
+ "intermediate_size": 3072,
11
+ "max_position_embeddings": 128,
12
+ "model_type": "twiny",
13
+ "neftune_alpha": 0.0,
14
+ "num_attention_heads": 12,
15
+ "num_hidden_layers": 1,
16
+ "num_key_value_heads": 3,
17
+ "qk_norm": true,
18
+ "rezero_init": 1.0,
19
+ "rms_norm_eps": 1e-06,
20
+ "rope_theta": 10000.0,
21
+ "transformers_version": "5.0.0",
22
+ "use_cache": false,
23
+ "vocab_size": 32000
24
+ }
slots/2/checkpoint-376/trainer_state.json ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.01967996503665786,
6
+ "eval_steps": 500,
7
+ "global_step": 376,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 2.617016627215141e-05,
14
+ "grad_norm": 122.33340454101562,
15
+ "learning_rate": 5e-05,
16
+ "loss": 419.9996032714844,
17
+ "step": 1
18
+ },
19
+ {
20
+ "epoch": 0.0005234033254430282,
21
+ "grad_norm": 90.82410430908203,
22
+ "learning_rate": 4.999999511980259e-05,
23
+ "loss": 428.18197471217104,
24
+ "step": 20
25
+ },
26
+ {
27
+ "epoch": 0.0010468066508860564,
28
+ "grad_norm": 59.52641296386719,
29
+ "learning_rate": 4.999997943828395e-05,
30
+ "loss": 366.3947265625,
31
+ "step": 40
32
+ },
33
+ {
34
+ "epoch": 0.0015702099763290846,
35
+ "grad_norm": 41.61277389526367,
36
+ "learning_rate": 4.999995294193233e-05,
37
+ "loss": 325.6545166015625,
38
+ "step": 60
39
+ },
40
+ {
41
+ "epoch": 0.0020936133017721128,
42
+ "grad_norm": 46.67556381225586,
43
+ "learning_rate": 4.999991563075919e-05,
44
+ "loss": 310.3648193359375,
45
+ "step": 80
46
+ },
47
+ {
48
+ "epoch": 0.002617016627215141,
49
+ "grad_norm": 47.65824890136719,
50
+ "learning_rate": 4.999986750478065e-05,
51
+ "loss": 303.49443359375,
52
+ "step": 100
53
+ },
54
+ {
55
+ "epoch": 0.003140419952658169,
56
+ "grad_norm": 62.748138427734375,
57
+ "learning_rate": 4.999980856401755e-05,
58
+ "loss": 322.389990234375,
59
+ "step": 120
60
+ },
61
+ {
62
+ "epoch": 0.007327646556202395,
63
+ "grad_norm": 50.97892761230469,
64
+ "learning_rate": 4.999895523943923e-05,
65
+ "loss": 290.35420365767044,
66
+ "step": 140
67
+ },
68
+ {
69
+ "epoch": 0.008374453207088451,
70
+ "grad_norm": 71.21157836914062,
71
+ "learning_rate": 4.99986329623214e-05,
72
+ "loss": 286.067724609375,
73
+ "step": 160
74
+ },
75
+ {
76
+ "epoch": 0.009421259857974507,
77
+ "grad_norm": 76.37614440917969,
78
+ "learning_rate": 4.999826742820622e-05,
79
+ "loss": 279.012451171875,
80
+ "step": 180
81
+ },
82
+ {
83
+ "epoch": 0.010468066508860565,
84
+ "grad_norm": 76.57452392578125,
85
+ "learning_rate": 4.999785863772618e-05,
86
+ "loss": 287.710888671875,
87
+ "step": 200
88
+ },
89
+ {
90
+ "epoch": 0.01151487315974662,
91
+ "grad_norm": 55.99504852294922,
92
+ "learning_rate": 4.999740659158865e-05,
93
+ "loss": 283.03916015625,
94
+ "step": 220
95
+ },
96
+ {
97
+ "epoch": 0.012561679810632677,
98
+ "grad_norm": 61.760135650634766,
99
+ "learning_rate": 4.9996911290575844e-05,
100
+ "loss": 263.22587890625,
101
+ "step": 240
102
+ },
103
+ {
104
+ "epoch": 0.013608486461518732,
105
+ "grad_norm": 51.249114990234375,
106
+ "learning_rate": 4.999637273554481e-05,
107
+ "loss": 261.2483642578125,
108
+ "step": 260
109
+ },
110
+ {
111
+ "epoch": 0.01465529311240479,
112
+ "grad_norm": 53.85161590576172,
113
+ "learning_rate": 4.9995790927427446e-05,
114
+ "loss": 257.092822265625,
115
+ "step": 280
116
+ },
117
+ {
118
+ "epoch": 0.015702099763290844,
119
+ "grad_norm": 55.01472473144531,
120
+ "learning_rate": 4.99951658672305e-05,
121
+ "loss": 256.8466796875,
122
+ "step": 300
123
+ },
124
+ {
125
+ "epoch": 0.016748906414176902,
126
+ "grad_norm": 57.187171936035156,
127
+ "learning_rate": 4.9994497556035567e-05,
128
+ "loss": 251.65341796875,
129
+ "step": 320
130
+ },
131
+ {
132
+ "epoch": 0.01779571306506296,
133
+ "grad_norm": 56.99523162841797,
134
+ "learning_rate": 4.9993785994999074e-05,
135
+ "loss": 250.6625244140625,
136
+ "step": 340
137
+ },
138
+ {
139
+ "epoch": 0.018842519715949014,
140
+ "grad_norm": 51.036258697509766,
141
+ "learning_rate": 4.999303118535229e-05,
142
+ "loss": 246.926318359375,
143
+ "step": 360
144
+ }
145
+ ],
146
+ "logging_steps": 20,
147
+ "max_steps": 47765,
148
+ "num_input_tokens_seen": 0,
149
+ "num_train_epochs": 3,
150
+ "save_steps": 1000000000,
151
+ "stateful_callbacks": {
152
+ "TrainerControl": {
153
+ "args": {
154
+ "should_epoch_stop": false,
155
+ "should_evaluate": false,
156
+ "should_log": false,
157
+ "should_save": true,
158
+ "should_training_stop": false
159
+ },
160
+ "attributes": {}
161
+ }
162
+ },
163
+ "total_flos": 389541321031680.0,
164
+ "train_batch_size": 1,
165
+ "trial_name": null,
166
+ "trial_params": null
167
+ }