Hugging Face's logo

legesher
/
language-decoded-lora

Text Generation
Transformers
Safetensors
lora
aya
tiny-aya
multilingual
code
legesher
tiny-aya-expedition
language-decoded
unsloth
arxiv:2603.11510
arxiv:2211.15533
arxiv:2510.09591
arxiv:1809.05053
arxiv:2308.16884
arxiv:2106.06937
arxiv:2210.03057
Model card Files
xet
 Community
23

condition-5-es-5k-c4ai-aya-expanse-32b-seed123

#20
by rafaym - opened
base: refs/heads/main
←
from: refs/pr/20
Discussion Files changed
+257
-0
Files changed (3) hide show
  1. tiny-aya-base/condition-5-es-5k-c4ai-aya-expanse-32b-seed123/adapter_config.json +54 -0
  2. tiny-aya-base/condition-5-es-5k-c4ai-aya-expanse-32b-seed123/adapter_model.safetensors +3 -0
  3. tiny-aya-base/condition-5-es-5k-c4ai-aya-expanse-32b-seed123/training_metrics.json +200 -0
tiny-aya-base/condition-5-es-5k-c4ai-aya-expanse-32b-seed123/adapter_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": {
6
+ "base_model_class": "Cohere2ForCausalLM",
7
+ "parent_library": "unsloth_compiled_module_cohere2",
8
+ "unsloth_fixed": true
9
+ },
10
+ "base_model_name_or_path": "CohereLabs/tiny-aya-base",
11
+ "bias": "none",
12
+ "corda_config": null,
13
+ "ensure_weight_tying": false,
14
+ "eva_config": null,
15
+ "exclude_modules": null,
16
+ "fan_in_fan_out": false,
17
+ "inference_mode": true,
18
+ "init_lora_weights": true,
19
+ "layer_replication": null,
20
+ "layers_pattern": null,
21
+ "layers_to_transform": null,
22
+ "loftq_config": {},
23
+ "lora_alpha": 32,
24
+ "lora_bias": false,
25
+ "lora_dropout": 0.0,
26
+ "lora_ga_config": null,
27
+ "megatron_config": null,
28
+ "megatron_core": "megatron.core",
29
+ "modules_to_save": null,
30
+ "monteclora_config": null,
31
+ "peft_type": "LORA",
32
+ "peft_version": "0.20.0",
33
+ "qalora_group_size": 16,
34
+ "r": 16,
35
+ "rank_pattern": {},
36
+ "revision": null,
37
+ "target_modules": [
38
+ "o_proj",
39
+ "k_proj",
40
+ "q_proj",
41
+ "v_proj",
42
+ "down_proj",
43
+ "up_proj",
44
+ "gate_proj"
45
+ ],
46
+ "target_parameters": null,
47
+ "task_type": "CAUSAL_LM",
48
+ "trainable_token_indices": null,
49
+ "use_bdlora": null,
50
+ "use_dora": false,
51
+ "use_qalora": false,
52
+ "use_rslora": false,
53
+ "velora_config": null
54
+ }
tiny-aya-base/condition-5-es-5k-c4ai-aya-expanse-32b-seed123/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de65347ca33a90f5a3a948c23b9e7383f8c008f91f1b53b16c746819f018e89b
3
+ size 120981200
tiny-aya-base/condition-5-es-5k-c4ai-aya-expanse-32b-seed123/training_metrics.json ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_config_name": "tiny-aya-base",
3
+ "condition_name": "condition-5-es-5k-c4ai-aya-expanse-32b",
4
+ "seed": 123,
5
+ "output_name": "condition-5-es-5k-c4ai-aya-expanse-32b-seed123",
6
+ "train_result": {
7
+ "train_runtime": 3145.6647,
8
+ "train_samples_per_second": 1.282,
9
+ "train_steps_per_second": 0.08,
10
+ "total_flos": 7.04182494471127e+16,
11
+ "train_loss": 0.9697676722019438,
12
+ "epoch": 1.0
13
+ },
14
+ "log_history": [
15
+ {
16
+ "loss": 1.5589,
17
+ "grad_norm": 0.14517320692539215,
18
+ "learning_rate": 0.00013846153846153847,
19
+ "epoch": 0.03968253968253968,
20
+ "step": 10
21
+ },
22
+ {
23
+ "loss": 1.3648,
24
+ "grad_norm": 0.2173229455947876,
25
+ "learning_rate": 0.00019968914967963337,
26
+ "epoch": 0.07936507936507936,
27
+ "step": 20
28
+ },
29
+ {
30
+ "loss": 1.1842,
31
+ "grad_norm": 0.17754456400871277,
32
+ "learning_rate": 0.00019779650288536058,
33
+ "epoch": 0.11904761904761904,
34
+ "step": 30
35
+ },
36
+ {
37
+ "loss": 1.0541,
38
+ "grad_norm": 0.18784138560295105,
39
+ "learning_rate": 0.000194216519005939,
40
+ "epoch": 0.15873015873015872,
41
+ "step": 40
42
+ },
43
+ {
44
+ "loss": 1.0311,
45
+ "grad_norm": 0.22221434116363525,
46
+ "learning_rate": 0.00018901096548075305,
47
+ "epoch": 0.1984126984126984,
48
+ "step": 50
49
+ },
50
+ {
51
+ "loss": 1.0139,
52
+ "grad_norm": 0.2424372136592865,
53
+ "learning_rate": 0.00018226965660021836,
54
+ "epoch": 0.23809523809523808,
55
+ "step": 60
56
+ },
57
+ {
58
+ "loss": 0.9689,
59
+ "grad_norm": 0.21934233605861664,
60
+ "learning_rate": 0.00017410890389021736,
61
+ "epoch": 0.2777777777777778,
62
+ "step": 70
63
+ },
64
+ {
65
+ "loss": 0.9574,
66
+ "grad_norm": 0.25630801916122437,
67
+ "learning_rate": 0.0001646695093250953,
68
+ "epoch": 0.31746031746031744,
69
+ "step": 80
70
+ },
71
+ {
72
+ "loss": 0.9122,
73
+ "grad_norm": 0.25298309326171875,
74
+ "learning_rate": 0.00015411433599342038,
75
+ "epoch": 0.35714285714285715,
76
+ "step": 90
77
+ },
78
+ {
79
+ "loss": 0.9598,
80
+ "grad_norm": 0.2816930115222931,
81
+ "learning_rate": 0.0001426254981311545,
82
+ "epoch": 0.3968253968253968,
83
+ "step": 100
84
+ },
85
+ {
86
+ "loss": 0.957,
87
+ "grad_norm": 0.2755691111087799,
88
+ "learning_rate": 0.0001304012190041437,
89
+ "epoch": 0.4365079365079365,
90
+ "step": 110
91
+ },
92
+ {
93
+ "loss": 0.8985,
94
+ "grad_norm": 0.26871779561042786,
95
+ "learning_rate": 0.00011765241085261802,
96
+ "epoch": 0.47619047619047616,
97
+ "step": 120
98
+ },
99
+ {
100
+ "loss": 0.8681,
101
+ "grad_norm": 0.2881104052066803,
102
+ "learning_rate": 0.00010459903590580706,
103
+ "epoch": 0.5158730158730159,
104
+ "step": 130
105
+ },
106
+ {
107
+ "loss": 0.8975,
108
+ "grad_norm": 0.2495868057012558,
109
+ "learning_rate": 9.146631125209607e-05,
110
+ "epoch": 0.5555555555555556,
111
+ "step": 140
112
+ },
113
+ {
114
+ "loss": 0.8739,
115
+ "grad_norm": 0.2400992065668106,
116
+ "learning_rate": 7.848082304419478e-05,
117
+ "epoch": 0.5952380952380952,
118
+ "step": 150
119
+ },
120
+ {
121
+ "loss": 0.8772,
122
+ "grad_norm": 0.27749431133270264,
123
+ "learning_rate": 6.586661708308272e-05,
124
+ "epoch": 0.6349206349206349,
125
+ "step": 160
126
+ },
127
+ {
128
+ "loss": 0.9082,
129
+ "grad_norm": 0.24919681251049042,
130
+ "learning_rate": 5.3841333232046654e-05,
131
+ "epoch": 0.6746031746031746,
132
+ "step": 170
133
+ },
134
+ {
135
+ "loss": 0.8647,
136
+ "grad_norm": 0.2658825218677521,
137
+ "learning_rate": 4.261245035589917e-05,
138
+ "epoch": 0.7142857142857143,
139
+ "step": 180
140
+ },
141
+ {
142
+ "loss": 0.8961,
143
+ "grad_norm": 0.2552941143512726,
144
+ "learning_rate": 3.2373706573516794e-05,
145
+ "epoch": 0.753968253968254,
146
+ "step": 190
147
+ },
148
+ {
149
+ "loss": 0.8915,
150
+ "grad_norm": 0.2646172046661377,
151
+ "learning_rate": 2.3301756587057987e-05,
152
+ "epoch": 0.7936507936507936,
153
+ "step": 200
154
+ },
155
+ {
156
+ "loss": 0.8504,
157
+ "grad_norm": 0.24216973781585693,
158
+ "learning_rate": 1.5553123760806143e-05,
159
+ "epoch": 0.8333333333333334,
160
+ "step": 210
161
+ },
162
+ {
163
+ "loss": 0.8483,
164
+ "grad_norm": 0.2629607617855072,
165
+ "learning_rate": 9.261499537105177e-06,
166
+ "epoch": 0.873015873015873,
167
+ "step": 220
168
+ },
169
+ {
170
+ "loss": 0.8476,
171
+ "grad_norm": 0.25701704621315,
172
+ "learning_rate": 4.535436784056269e-06,
173
+ "epoch": 0.9126984126984127,
174
+ "step": 230
175
+ },
176
+ {
177
+ "loss": 0.8716,
178
+ "grad_norm": 0.2659420669078827,
179
+ "learning_rate": 1.4564768729220412e-06,
180
+ "epoch": 0.9523809523809523,
181
+ "step": 240
182
+ },
183
+ {
184
+ "loss": 0.9009,
185
+ "grad_norm": 0.2654193043708801,
186
+ "learning_rate": 7.774279980626853e-08,
187
+ "epoch": 0.9920634920634921,
188
+ "step": 250
189
+ },
190
+ {
191
+ "train_runtime": 3145.6647,
192
+ "train_samples_per_second": 1.282,
193
+ "train_steps_per_second": 0.08,
194
+ "total_flos": 7.04182494471127e+16,
195
+ "train_loss": 0.9697676722019438,
196
+ "epoch": 1.0,
197
+ "step": 252
198
+ }
199
+ ]
200
+ }