Hugging Face's logo

SamChen888
/
Mistral-EN-Part-2

PEFT
Safetensors
llama-factory
lora
unsloth
Generated from Trainer
Model card Files
xet
Community
SamChen888 commited on
Commit
2308a88
·
verified ·
1 Parent(s): 5dc66ea

Upload folder using huggingface_hub

Browse files
Files changed (28) hide show
  1. README.md +59 -3
  2. adapter_config.json +34 -0
  3. adapter_model.safetensors +3 -0
  4. all_results.json +9 -0
  5. checkpoint-696/README.md +202 -0
  6. checkpoint-696/adapter_config.json +34 -0
  7. checkpoint-696/adapter_model.safetensors +3 -0
  8. checkpoint-696/optimizer.pt +3 -0
  9. checkpoint-696/rng_state.pth +3 -0
  10. checkpoint-696/scheduler.pt +3 -0
  11. checkpoint-696/special_tokens_map.json +24 -0
  12. checkpoint-696/tokenizer.json +0 -0
  13. checkpoint-696/tokenizer.model +3 -0
  14. checkpoint-696/tokenizer_config.json +0 -0
  15. checkpoint-696/trainer_state.json +585 -0
  16. checkpoint-696/training_args.bin +3 -0
  17. llamaboard_config.yaml +79 -0
  18. running_log.txt +575 -0
  19. special_tokens_map.json +24 -0
  20. tokenizer.json +0 -0
  21. tokenizer.model +3 -0
  22. tokenizer_config.json +0 -0
  23. train_results.json +9 -0
  24. trainer_log.jsonl +70 -0
  25. trainer_state.json +595 -0
  26. training_args.bin +3 -0
  27. training_args.yaml +37 -0
  28. training_loss.png +0 -0
README.md CHANGED
@@ -1,3 +1,59 @@
1
- ---
2
- license: apache-2.0
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ license: other
4
+ base_model: unsloth/mistral-7b-instruct-v0.3-bnb-4bit
5
+ tags:
6
+ - llama-factory
7
+ - lora
8
+ - unsloth
9
+ - generated_from_trainer
10
+ model-index:
11
+ - name: 2_part
12
+ results: []
13
+ ---
14
+
15
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
16
+ should probably proofread and complete it, then remove this comment. -->
17
+
18
+ # 2_part
19
+
20
+ This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) on the en_train_part_1 and the en_train_part_3 datasets.
21
+
22
+ ## Model description
23
+
24
+ More information needed
25
+
26
+ ## Intended uses & limitations
27
+
28
+ More information needed
29
+
30
+ ## Training and evaluation data
31
+
32
+ More information needed
33
+
34
+ ## Training procedure
35
+
36
+ ### Training hyperparameters
37
+
38
+ The following hyperparameters were used during training:
39
+ - learning_rate: 3e-05
40
+ - train_batch_size: 4
41
+ - eval_batch_size: 8
42
+ - seed: 42
43
+ - gradient_accumulation_steps: 2
44
+ - total_train_batch_size: 8
45
+ - optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
46
+ - lr_scheduler_type: cosine
47
+ - num_epochs: 3.0
48
+
49
+ ### Training results
50
+
51
+
52
+
53
+ ### Framework versions
54
+
55
+ - PEFT 0.12.0
56
+ - Transformers 4.49.0
57
+ - Pytorch 2.6.0+cu124
58
+ - Datasets 3.2.0
59
+ - Tokenizers 0.21.0
adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "unsloth/mistral-7b-instruct-v0.3-bnb-4bit",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 8,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "v_proj",
24
+ "q_proj",
25
+ "up_proj",
26
+ "down_proj",
27
+ "gate_proj",
28
+ "o_proj",
29
+ "k_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f0f72cfe40c8f2b0d4a81cac699b6a6c164d654d0a536a327e066f283357027
3
+ size 83945296
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 2.989247311827957,
3
+ "num_input_tokens_seen": 1855776,
4
+ "total_flos": 7.944329136203366e+16,
5
+ "train_loss": 0.08561765917459097,
6
+ "train_runtime": 638.811,
7
+ "train_samples_per_second": 8.721,
8
+ "train_steps_per_second": 1.09
9
+ }
checkpoint-696/README.md ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: unsloth/mistral-7b-instruct-v0.3-bnb-4bit
3
+ library_name: peft
4
+ ---
5
+
6
+ # Model Card for Model ID
7
+
8
+ <!-- Provide a quick summary of what the model is/does. -->
9
+
10
+
11
+
12
+ ## Model Details
13
+
14
+ ### Model Description
15
+
16
+ <!-- Provide a longer summary of what this model is. -->
17
+
18
+
19
+
20
+ - **Developed by:** [More Information Needed]
21
+ - **Funded by [optional]:** [More Information Needed]
22
+ - **Shared by [optional]:** [More Information Needed]
23
+ - **Model type:** [More Information Needed]
24
+ - **Language(s) (NLP):** [More Information Needed]
25
+ - **License:** [More Information Needed]
26
+ - **Finetuned from model [optional]:** [More Information Needed]
27
+
28
+ ### Model Sources [optional]
29
+
30
+ <!-- Provide the basic links for the model. -->
31
+
32
+ - **Repository:** [More Information Needed]
33
+ - **Paper [optional]:** [More Information Needed]
34
+ - **Demo [optional]:** [More Information Needed]
35
+
36
+ ## Uses
37
+
38
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
39
+
40
+ ### Direct Use
41
+
42
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
43
+
44
+ [More Information Needed]
45
+
46
+ ### Downstream Use [optional]
47
+
48
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Out-of-Scope Use
53
+
54
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
55
+
56
+ [More Information Needed]
57
+
58
+ ## Bias, Risks, and Limitations
59
+
60
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ### Recommendations
65
+
66
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
67
+
68
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
69
+
70
+ ## How to Get Started with the Model
71
+
72
+ Use the code below to get started with the model.
73
+
74
+ [More Information Needed]
75
+
76
+ ## Training Details
77
+
78
+ ### Training Data
79
+
80
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
81
+
82
+ [More Information Needed]
83
+
84
+ ### Training Procedure
85
+
86
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
87
+
88
+ #### Preprocessing [optional]
89
+
90
+ [More Information Needed]
91
+
92
+
93
+ #### Training Hyperparameters
94
+
95
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
96
+
97
+ #### Speeds, Sizes, Times [optional]
98
+
99
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
100
+
101
+ [More Information Needed]
102
+
103
+ ## Evaluation
104
+
105
+ <!-- This section describes the evaluation protocols and provides the results. -->
106
+
107
+ ### Testing Data, Factors & Metrics
108
+
109
+ #### Testing Data
110
+
111
+ <!-- This should link to a Dataset Card if possible. -->
112
+
113
+ [More Information Needed]
114
+
115
+ #### Factors
116
+
117
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Metrics
122
+
123
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
124
+
125
+ [More Information Needed]
126
+
127
+ ### Results
128
+
129
+ [More Information Needed]
130
+
131
+ #### Summary
132
+
133
+
134
+
135
+ ## Model Examination [optional]
136
+
137
+ <!-- Relevant interpretability work for the model goes here -->
138
+
139
+ [More Information Needed]
140
+
141
+ ## Environmental Impact
142
+
143
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
144
+
145
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
146
+
147
+ - **Hardware Type:** [More Information Needed]
148
+ - **Hours used:** [More Information Needed]
149
+ - **Cloud Provider:** [More Information Needed]
150
+ - **Compute Region:** [More Information Needed]
151
+ - **Carbon Emitted:** [More Information Needed]
152
+
153
+ ## Technical Specifications [optional]
154
+
155
+ ### Model Architecture and Objective
156
+
157
+ [More Information Needed]
158
+
159
+ ### Compute Infrastructure
160
+
161
+ [More Information Needed]
162
+
163
+ #### Hardware
164
+
165
+ [More Information Needed]
166
+
167
+ #### Software
168
+
169
+ [More Information Needed]
170
+
171
+ ## Citation [optional]
172
+
173
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
174
+
175
+ **BibTeX:**
176
+
177
+ [More Information Needed]
178
+
179
+ **APA:**
180
+
181
+ [More Information Needed]
182
+
183
+ ## Glossary [optional]
184
+
185
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
186
+
187
+ [More Information Needed]
188
+
189
+ ## More Information [optional]
190
+
191
+ [More Information Needed]
192
+
193
+ ## Model Card Authors [optional]
194
+
195
+ [More Information Needed]
196
+
197
+ ## Model Card Contact
198
+
199
+ [More Information Needed]
200
+ ### Framework versions
201
+
202
+ - PEFT 0.12.0
checkpoint-696/adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "unsloth/mistral-7b-instruct-v0.3-bnb-4bit",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 8,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "v_proj",
24
+ "q_proj",
25
+ "up_proj",
26
+ "down_proj",
27
+ "gate_proj",
28
+ "o_proj",
29
+ "k_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }
checkpoint-696/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f0f72cfe40c8f2b0d4a81cac699b6a6c164d654d0a536a327e066f283357027
3
+ size 83945296
checkpoint-696/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1bb3ecc15a4b4782402d491f37b8b19c875de4b1d8683642e1a1e1440c0ebbe2
3
+ size 168149074
checkpoint-696/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9196a1e708bf24d6abba41cce3f8558820acc3e50f9394c5955e29eb41ffea3d
3
+ size 14244
checkpoint-696/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13376dd3816268460455fd38675449b64931eccf2188c281a42454b88404fffa
3
+ size 1064
checkpoint-696/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
checkpoint-696/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-696/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
3
+ size 587404
checkpoint-696/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-696/trainer_state.json ADDED
@@ -0,0 +1,585 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.989247311827957,
5
+ "eval_steps": 500,
6
+ "global_step": 696,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.043010752688172046,
13
+ "grad_norm": 3.3071749210357666,
14
+ "learning_rate": 2.9984721919587606e-05,
15
+ "loss": 0.9267,
16
+ "num_input_tokens_seen": 26208,
17
+ "step": 10
18
+ },
19
+ {
20
+ "epoch": 0.08602150537634409,
21
+ "grad_norm": 1.9867345094680786,
22
+ "learning_rate": 2.9938918800982563e-05,
23
+ "loss": 0.1561,
24
+ "num_input_tokens_seen": 53152,
25
+ "step": 20
26
+ },
27
+ {
28
+ "epoch": 0.12903225806451613,
29
+ "grad_norm": 10.124670028686523,
30
+ "learning_rate": 2.9862683948682103e-05,
31
+ "loss": 0.1764,
32
+ "num_input_tokens_seen": 79776,
33
+ "step": 30
34
+ },
35
+ {
36
+ "epoch": 0.17204301075268819,
37
+ "grad_norm": 2.0668740272521973,
38
+ "learning_rate": 2.975617265898004e-05,
39
+ "loss": 0.1505,
40
+ "num_input_tokens_seen": 106496,
41
+ "step": 40
42
+ },
43
+ {
44
+ "epoch": 0.21505376344086022,
45
+ "grad_norm": 2.0054426193237305,
46
+ "learning_rate": 2.961960190361624e-05,
47
+ "loss": 0.1615,
48
+ "num_input_tokens_seen": 133728,
49
+ "step": 50
50
+ },
51
+ {
52
+ "epoch": 0.25806451612903225,
53
+ "grad_norm": 2.5156946182250977,
54
+ "learning_rate": 2.9453249887788343e-05,
55
+ "loss": 0.1369,
56
+ "num_input_tokens_seen": 159936,
57
+ "step": 60
58
+ },
59
+ {
60
+ "epoch": 0.3010752688172043,
61
+ "grad_norm": 1.1180003881454468,
62
+ "learning_rate": 2.925745548342631e-05,
63
+ "loss": 0.1253,
64
+ "num_input_tokens_seen": 186240,
65
+ "step": 70
66
+ },
67
+ {
68
+ "epoch": 0.34408602150537637,
69
+ "grad_norm": 13.929261207580566,
70
+ "learning_rate": 2.9032617538884018e-05,
71
+ "loss": 0.1479,
72
+ "num_input_tokens_seen": 212832,
73
+ "step": 80
74
+ },
75
+ {
76
+ "epoch": 0.3870967741935484,
77
+ "grad_norm": 2.071075439453125,
78
+ "learning_rate": 2.877919406645433e-05,
79
+ "loss": 0.1102,
80
+ "num_input_tokens_seen": 239648,
81
+ "step": 90
82
+ },
83
+ {
84
+ "epoch": 0.43010752688172044,
85
+ "grad_norm": 2.170992374420166,
86
+ "learning_rate": 2.84977013093626e-05,
87
+ "loss": 0.099,
88
+ "num_input_tokens_seen": 265888,
89
+ "step": 100
90
+ },
91
+ {
92
+ "epoch": 0.4731182795698925,
93
+ "grad_norm": 5.617093086242676,
94
+ "learning_rate": 2.818871269013928e-05,
95
+ "loss": 0.107,
96
+ "num_input_tokens_seen": 292480,
97
+ "step": 110
98
+ },
99
+ {
100
+ "epoch": 0.5161290322580645,
101
+ "grad_norm": 6.3603010177612305,
102
+ "learning_rate": 2.7852857642513838e-05,
103
+ "loss": 0.1183,
104
+ "num_input_tokens_seen": 318784,
105
+ "step": 120
106
+ },
107
+ {
108
+ "epoch": 0.5591397849462365,
109
+ "grad_norm": 3.8436505794525146,
110
+ "learning_rate": 2.7490820329209546e-05,
111
+ "loss": 0.1097,
112
+ "num_input_tokens_seen": 346016,
113
+ "step": 130
114
+ },
115
+ {
116
+ "epoch": 0.6021505376344086,
117
+ "grad_norm": 1.736432671546936,
118
+ "learning_rate": 2.7103338248251055e-05,
119
+ "loss": 0.0946,
120
+ "num_input_tokens_seen": 372384,
121
+ "step": 140
122
+ },
123
+ {
124
+ "epoch": 0.6451612903225806,
125
+ "grad_norm": 8.300951957702637,
126
+ "learning_rate": 2.6691200730623874e-05,
127
+ "loss": 0.1251,
128
+ "num_input_tokens_seen": 399328,
129
+ "step": 150
130
+ },
131
+ {
132
+ "epoch": 0.6881720430107527,
133
+ "grad_norm": 2.540724277496338,
134
+ "learning_rate": 2.6255247332346036e-05,
135
+ "loss": 0.1069,
136
+ "num_input_tokens_seen": 426048,
137
+ "step": 160
138
+ },
139
+ {
140
+ "epoch": 0.7311827956989247,
141
+ "grad_norm": 1.967483639717102,
142
+ "learning_rate": 2.5796366124227532e-05,
143
+ "loss": 0.0904,
144
+ "num_input_tokens_seen": 452640,
145
+ "step": 170
146
+ },
147
+ {
148
+ "epoch": 0.7741935483870968,
149
+ "grad_norm": 5.206757545471191,
150
+ "learning_rate": 2.531549188280135e-05,
151
+ "loss": 0.1273,
152
+ "num_input_tokens_seen": 479808,
153
+ "step": 180
154
+ },
155
+ {
156
+ "epoch": 0.8172043010752689,
157
+ "grad_norm": 3.0387344360351562,
158
+ "learning_rate": 2.481360418611132e-05,
159
+ "loss": 0.1206,
160
+ "num_input_tokens_seen": 506176,
161
+ "step": 190
162
+ },
163
+ {
164
+ "epoch": 0.8602150537634409,
165
+ "grad_norm": 2.0281670093536377,
166
+ "learning_rate": 2.4291725418235848e-05,
167
+ "loss": 0.103,
168
+ "num_input_tokens_seen": 533216,
169
+ "step": 200
170
+ },
171
+ {
172
+ "epoch": 0.9032258064516129,
173
+ "grad_norm": 2.5650763511657715,
174
+ "learning_rate": 2.3750918686612414e-05,
175
+ "loss": 0.0696,
176
+ "num_input_tokens_seen": 561056,
177
+ "step": 210
178
+ },
179
+ {
180
+ "epoch": 0.946236559139785,
181
+ "grad_norm": 8.955713272094727,
182
+ "learning_rate": 2.3192285656405456e-05,
183
+ "loss": 0.0822,
184
+ "num_input_tokens_seen": 588160,
185
+ "step": 220
186
+ },
187
+ {
188
+ "epoch": 0.989247311827957,
189
+ "grad_norm": 3.034013032913208,
190
+ "learning_rate": 2.2616964306329183e-05,
191
+ "loss": 0.0913,
192
+ "num_input_tokens_seen": 615168,
193
+ "step": 230
194
+ },
195
+ {
196
+ "epoch": 1.0301075268817204,
197
+ "grad_norm": 2.60020112991333,
198
+ "learning_rate": 2.2026126610496852e-05,
199
+ "loss": 0.0735,
200
+ "num_input_tokens_seen": 639864,
201
+ "step": 240
202
+ },
203
+ {
204
+ "epoch": 1.0731182795698926,
205
+ "grad_norm": 4.891764163970947,
206
+ "learning_rate": 2.1420976151018813e-05,
207
+ "loss": 0.0752,
208
+ "num_input_tokens_seen": 667224,
209
+ "step": 250
210
+ },
211
+ {
212
+ "epoch": 1.1161290322580646,
213
+ "grad_norm": 1.1149002313613892,
214
+ "learning_rate": 2.0802745666212592e-05,
215
+ "loss": 0.0588,
216
+ "num_input_tokens_seen": 693848,
217
+ "step": 260
218
+ },
219
+ {
220
+ "epoch": 1.1591397849462366,
221
+ "grad_norm": 3.1601271629333496,
222
+ "learning_rate": 2.0172694539419557e-05,
223
+ "loss": 0.0924,
224
+ "num_input_tokens_seen": 720568,
225
+ "step": 270
226
+ },
227
+ {
228
+ "epoch": 1.2021505376344086,
229
+ "grad_norm": 3.555192470550537,
230
+ "learning_rate": 1.953210623354359e-05,
231
+ "loss": 0.062,
232
+ "num_input_tokens_seen": 746872,
233
+ "step": 280
234
+ },
235
+ {
236
+ "epoch": 1.2451612903225806,
237
+ "grad_norm": 2.235698699951172,
238
+ "learning_rate": 1.888228567653781e-05,
239
+ "loss": 0.0621,
240
+ "num_input_tokens_seen": 773720,
241
+ "step": 290
242
+ },
243
+ {
244
+ "epoch": 1.2881720430107526,
245
+ "grad_norm": 2.9058539867401123,
246
+ "learning_rate": 1.8224556603165363e-05,
247
+ "loss": 0.075,
248
+ "num_input_tokens_seen": 801464,
249
+ "step": 300
250
+ },
251
+ {
252
+ "epoch": 1.3311827956989246,
253
+ "grad_norm": 3.204787015914917,
254
+ "learning_rate": 1.7560258858449248e-05,
255
+ "loss": 0.0858,
256
+ "num_input_tokens_seen": 829144,
257
+ "step": 310
258
+ },
259
+ {
260
+ "epoch": 1.3741935483870968,
261
+ "grad_norm": 1.0885004997253418,
262
+ "learning_rate": 1.689074566830434e-05,
263
+ "loss": 0.0697,
264
+ "num_input_tokens_seen": 855672,
265
+ "step": 320
266
+ },
267
+ {
268
+ "epoch": 1.4172043010752688,
269
+ "grad_norm": 3.0750925540924072,
270
+ "learning_rate": 1.621738088291147e-05,
271
+ "loss": 0.0827,
272
+ "num_input_tokens_seen": 882424,
273
+ "step": 330
274
+ },
275
+ {
276
+ "epoch": 1.4602150537634409,
277
+ "grad_norm": 2.689297914505005,
278
+ "learning_rate": 1.5541536198449044e-05,
279
+ "loss": 0.0651,
280
+ "num_input_tokens_seen": 908792,
281
+ "step": 340
282
+ },
283
+ {
284
+ "epoch": 1.5032258064516129,
285
+ "grad_norm": 2.297851324081421,
286
+ "learning_rate": 1.4864588362841808e-05,
287
+ "loss": 0.0607,
288
+ "num_input_tokens_seen": 935672,
289
+ "step": 350
290
+ },
291
+ {
292
+ "epoch": 1.546236559139785,
293
+ "grad_norm": 2.712674140930176,
294
+ "learning_rate": 1.4187916371218739e-05,
295
+ "loss": 0.056,
296
+ "num_input_tokens_seen": 961848,
297
+ "step": 360
298
+ },
299
+ {
300
+ "epoch": 1.589247311827957,
301
+ "grad_norm": 0.8086225986480713,
302
+ "learning_rate": 1.3512898656793283e-05,
303
+ "loss": 0.0823,
304
+ "num_input_tokens_seen": 988600,
305
+ "step": 370
306
+ },
307
+ {
308
+ "epoch": 1.632258064516129,
309
+ "grad_norm": 2.166210174560547,
310
+ "learning_rate": 1.2840910282888211e-05,
311
+ "loss": 0.058,
312
+ "num_input_tokens_seen": 1014840,
313
+ "step": 380
314
+ },
315
+ {
316
+ "epoch": 1.675268817204301,
317
+ "grad_norm": 5.169621467590332,
318
+ "learning_rate": 1.2173320141825232e-05,
319
+ "loss": 0.0705,
320
+ "num_input_tokens_seen": 1040856,
321
+ "step": 390
322
+ },
323
+ {
324
+ "epoch": 1.718279569892473,
325
+ "grad_norm": 1.8176069259643555,
326
+ "learning_rate": 1.1511488166385349e-05,
327
+ "loss": 0.0514,
328
+ "num_input_tokens_seen": 1067544,
329
+ "step": 400
330
+ },
331
+ {
332
+ "epoch": 1.761290322580645,
333
+ "grad_norm": 3.424694776535034,
334
+ "learning_rate": 1.0856762559520605e-05,
335
+ "loss": 0.0834,
336
+ "num_input_tokens_seen": 1094584,
337
+ "step": 410
338
+ },
339
+ {
340
+ "epoch": 1.8043010752688171,
341
+ "grad_norm": 1.8838876485824585,
342
+ "learning_rate": 1.0210477047960303e-05,
343
+ "loss": 0.0583,
344
+ "num_input_tokens_seen": 1120760,
345
+ "step": 420
346
+ },
347
+ {
348
+ "epoch": 1.8473118279569891,
349
+ "grad_norm": 3.7757434844970703,
350
+ "learning_rate": 9.573948165306438e-06,
351
+ "loss": 0.0922,
352
+ "num_input_tokens_seen": 1146776,
353
+ "step": 430
354
+ },
355
+ {
356
+ "epoch": 1.8903225806451613,
357
+ "grad_norm": 3.0619328022003174,
358
+ "learning_rate": 8.948472570152874e-06,
359
+ "loss": 0.0633,
360
+ "num_input_tokens_seen": 1174424,
361
+ "step": 440
362
+ },
363
+ {
364
+ "epoch": 1.9333333333333333,
365
+ "grad_norm": 2.5175821781158447,
366
+ "learning_rate": 8.33532440469145e-06,
367
+ "loss": 0.0597,
368
+ "num_input_tokens_seen": 1201048,
369
+ "step": 450
370
+ },
371
+ {
372
+ "epoch": 1.9763440860215054,
373
+ "grad_norm": 3.6232197284698486,
374
+ "learning_rate": 7.735752699185711e-06,
375
+ "loss": 0.0491,
376
+ "num_input_tokens_seen": 1227576,
377
+ "step": 460
378
+ },
379
+ {
380
+ "epoch": 2.0172043010752687,
381
+ "grad_norm": 2.8846399784088135,
382
+ "learning_rate": 7.150978827599619e-06,
383
+ "loss": 0.0341,
384
+ "num_input_tokens_seen": 1252160,
385
+ "step": 470
386
+ },
387
+ {
388
+ "epoch": 2.0602150537634407,
389
+ "grad_norm": 2.196216106414795,
390
+ "learning_rate": 6.582194019564266e-06,
391
+ "loss": 0.0373,
392
+ "num_input_tokens_seen": 1279328,
393
+ "step": 480
394
+ },
395
+ {
396
+ "epoch": 2.1032258064516127,
397
+ "grad_norm": 1.4616115093231201,
398
+ "learning_rate": 6.0305569337509225e-06,
399
+ "loss": 0.0281,
400
+ "num_input_tokens_seen": 1306304,
401
+ "step": 490
402
+ },
403
+ {
404
+ "epoch": 2.146236559139785,
405
+ "grad_norm": 0.17581823468208313,
406
+ "learning_rate": 5.497191297593647e-06,
407
+ "loss": 0.0183,
408
+ "num_input_tokens_seen": 1333184,
409
+ "step": 500
410
+ },
411
+ {
412
+ "epoch": 2.189247311827957,
413
+ "grad_norm": 3.8919403553009033,
414
+ "learning_rate": 4.98318361816957e-06,
415
+ "loss": 0.0334,
416
+ "num_input_tokens_seen": 1359872,
417
+ "step": 510
418
+ },
419
+ {
420
+ "epoch": 2.232258064516129,
421
+ "grad_norm": 1.3041765689849854,
422
+ "learning_rate": 4.4895809688998655e-06,
423
+ "loss": 0.0282,
424
+ "num_input_tokens_seen": 1387328,
425
+ "step": 520
426
+ },
427
+ {
428
+ "epoch": 2.275268817204301,
429
+ "grad_norm": 1.669753074645996,
430
+ "learning_rate": 4.017388856580178e-06,
431
+ "loss": 0.0562,
432
+ "num_input_tokens_seen": 1414816,
433
+ "step": 530
434
+ },
435
+ {
436
+ "epoch": 2.318279569892473,
437
+ "grad_norm": 0.28061679005622864,
438
+ "learning_rate": 3.567569173085455e-06,
439
+ "loss": 0.0243,
440
+ "num_input_tokens_seen": 1441504,
441
+ "step": 540
442
+ },
443
+ {
444
+ "epoch": 2.361290322580645,
445
+ "grad_norm": 2.324270009994507,
446
+ "learning_rate": 3.1410382359217645e-06,
447
+ "loss": 0.044,
448
+ "num_input_tokens_seen": 1467680,
449
+ "step": 550
450
+ },
451
+ {
452
+ "epoch": 2.404301075268817,
453
+ "grad_norm": 2.708113670349121,
454
+ "learning_rate": 2.7386649216166233e-06,
455
+ "loss": 0.0551,
456
+ "num_input_tokens_seen": 1494176,
457
+ "step": 560
458
+ },
459
+ {
460
+ "epoch": 2.447311827956989,
461
+ "grad_norm": 3.16683030128479,
462
+ "learning_rate": 2.361268895750264e-06,
463
+ "loss": 0.0258,
464
+ "num_input_tokens_seen": 1520544,
465
+ "step": 570
466
+ },
467
+ {
468
+ "epoch": 2.490322580645161,
469
+ "grad_norm": 6.040332794189453,
470
+ "learning_rate": 2.0096189432334194e-06,
471
+ "loss": 0.0415,
472
+ "num_input_tokens_seen": 1547264,
473
+ "step": 580
474
+ },
475
+ {
476
+ "epoch": 2.533333333333333,
477
+ "grad_norm": 5.078160285949707,
478
+ "learning_rate": 1.6844314022329676e-06,
479
+ "loss": 0.0375,
480
+ "num_input_tokens_seen": 1573920,
481
+ "step": 590
482
+ },
483
+ {
484
+ "epoch": 2.576344086021505,
485
+ "grad_norm": 4.950022220611572,
486
+ "learning_rate": 1.3863687049356465e-06,
487
+ "loss": 0.0235,
488
+ "num_input_tokens_seen": 1600640,
489
+ "step": 600
490
+ },
491
+ {
492
+ "epoch": 2.6193548387096772,
493
+ "grad_norm": 1.7687643766403198,
494
+ "learning_rate": 1.116038028122413e-06,
495
+ "loss": 0.0354,
496
+ "num_input_tokens_seen": 1626848,
497
+ "step": 610
498
+ },
499
+ {
500
+ "epoch": 2.6623655913978492,
501
+ "grad_norm": 3.893580913543701,
502
+ "learning_rate": 8.7399005630238e-07,
503
+ "loss": 0.0357,
504
+ "num_input_tokens_seen": 1653408,
505
+ "step": 620
506
+ },
507
+ {
508
+ "epoch": 2.7053763440860212,
509
+ "grad_norm": 2.830453395843506,
510
+ "learning_rate": 6.607178599258268e-07,
511
+ "loss": 0.0512,
512
+ "num_input_tokens_seen": 1679968,
513
+ "step": 630
514
+ },
515
+ {
516
+ "epoch": 2.7483870967741937,
517
+ "grad_norm": 3.638772487640381,
518
+ "learning_rate": 4.766558909615504e-07,
519
+ "loss": 0.0243,
520
+ "num_input_tokens_seen": 1706944,
521
+ "step": 640
522
+ },
523
+ {
524
+ "epoch": 2.7913978494623657,
525
+ "grad_norm": 1.229244589805603,
526
+ "learning_rate": 3.22179097884579e-07,
527
+ "loss": 0.0367,
528
+ "num_input_tokens_seen": 1733888,
529
+ "step": 650
530
+ },
531
+ {
532
+ "epoch": 2.8344086021505377,
533
+ "grad_norm": 2.9207515716552734,
534
+ "learning_rate": 1.9760216187710788e-07,
535
+ "loss": 0.0377,
536
+ "num_input_tokens_seen": 1760448,
537
+ "step": 660
538
+ },
539
+ {
540
+ "epoch": 2.8774193548387097,
541
+ "grad_norm": 3.560971975326538,
542
+ "learning_rate": 1.0317885579858522e-07,
543
+ "loss": 0.0471,
544
+ "num_input_tokens_seen": 1787072,
545
+ "step": 670
546
+ },
547
+ {
548
+ "epoch": 2.9204301075268817,
549
+ "grad_norm": 0.2031625360250473,
550
+ "learning_rate": 3.910152723075322e-08,
551
+ "loss": 0.0222,
552
+ "num_input_tokens_seen": 1813632,
553
+ "step": 680
554
+ },
555
+ {
556
+ "epoch": 2.9634408602150537,
557
+ "grad_norm": 4.158380508422852,
558
+ "learning_rate": 5.50070665074065e-09,
559
+ "loss": 0.0454,
560
+ "num_input_tokens_seen": 1840384,
561
+ "step": 690
562
+ }
563
+ ],
564
+ "logging_steps": 10,
565
+ "max_steps": 696,
566
+ "num_input_tokens_seen": 1855776,
567
+ "num_train_epochs": 3,
568
+ "save_steps": 1000,
569
+ "stateful_callbacks": {
570
+ "TrainerControl": {
571
+ "args": {
572
+ "should_epoch_stop": false,
573
+ "should_evaluate": false,
574
+ "should_log": false,
575
+ "should_save": true,
576
+ "should_training_stop": true
577
+ },
578
+ "attributes": {}
579
+ }
580
+ },
581
+ "total_flos": 7.944329136203366e+16,
582
+ "train_batch_size": 4,
583
+ "trial_name": null,
584
+ "trial_params": null
585
+ }
checkpoint-696/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c0f50151d95e9d064572887223a75b34b8d863adcbd94f03a50070eddf5f434
3
+ size 5688
llamaboard_config.yaml ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ top.booster: unsloth
2
+ top.checkpoint_path: []
3
+ top.finetuning_type: lora
4
+ top.model_name: Mistral-7B-Instruct-v0.3
5
+ top.quantization_bit: '4'
6
+ top.quantization_method: bitsandbytes
7
+ top.rope_scaling: none
8
+ top.template: alpaca
9
+ train.additional_target: ''
10
+ train.apollo_rank: 16
11
+ train.apollo_scale: 32
12
+ train.apollo_target: all
13
+ train.apollo_update_interval: 200
14
+ train.badam_mode: layer
15
+ train.badam_switch_interval: 50
16
+ train.badam_switch_mode: ascending
17
+ train.badam_update_ratio: 0.05
18
+ train.batch_size: 4
19
+ train.compute_type: bf16
20
+ train.create_new_adapter: false
21
+ train.cutoff_len: 2048
22
+ train.dataset:
23
+ - en_train_part_1
24
+ - en_train_part_3
25
+ train.dataset_dir: data
26
+ train.ds_offload: false
27
+ train.ds_stage: none
28
+ train.extra_args: '{"optim": "adamw_torch"}'
29
+ train.freeze_extra_modules: ''
30
+ train.freeze_trainable_layers: 2
31
+ train.freeze_trainable_modules: all
32
+ train.galore_rank: 16
33
+ train.galore_scale: 2
34
+ train.galore_target: all
35
+ train.galore_update_interval: 200
36
+ train.gradient_accumulation_steps: 2
37
+ train.learning_rate: 3e-5
38
+ train.logging_steps: 10
39
+ train.lora_alpha: 16
40
+ train.lora_dropout: 0
41
+ train.lora_rank: 8
42
+ train.lora_target: ''
43
+ train.loraplus_lr_ratio: 0
44
+ train.lr_scheduler_type: cosine
45
+ train.mask_history: false
46
+ train.max_grad_norm: '1.0'
47
+ train.max_samples: '1858'
48
+ train.neat_packing: false
49
+ train.neftune_alpha: 0
50
+ train.num_train_epochs: '3.0'
51
+ train.packing: false
52
+ train.ppo_score_norm: false
53
+ train.ppo_whiten_rewards: false
54
+ train.pref_beta: 0.1
55
+ train.pref_ftx: 0
56
+ train.pref_loss: sigmoid
57
+ train.report_to:
58
+ - none
59
+ train.resize_vocab: false
60
+ train.reward_model: []
61
+ train.save_steps: 1000
62
+ train.swanlab_api_key: ''
63
+ train.swanlab_link: ''
64
+ train.swanlab_mode: cloud
65
+ train.swanlab_project: llamafactory
66
+ train.swanlab_run_name: ''
67
+ train.swanlab_workspace: ''
68
+ train.train_on_prompt: false
69
+ train.training_stage: Supervised Fine-Tuning
70
+ train.use_apollo: false
71
+ train.use_badam: false
72
+ train.use_dora: false
73
+ train.use_galore: false
74
+ train.use_llama_pro: false
75
+ train.use_pissa: false
76
+ train.use_rslora: false
77
+ train.use_swanlab: false
78
+ train.val_size: 0
79
+ train.warmup_steps: 0
running_log.txt ADDED
@@ -0,0 +1,575 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [INFO|2025-04-07 18:56:08] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/config.json
2
+
3
+ [INFO|2025-04-07 18:56:08] configuration_utils.py:771 >> Model config MistralConfig {
4
+ "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.3",
5
+ "architectures": [
6
+ "MistralForCausalLM"
7
+ ],
8
+ "attention_dropout": 0.0,
9
+ "bos_token_id": 1,
10
+ "eos_token_id": 2,
11
+ "head_dim": 128,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 4096,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 14336,
16
+ "max_position_embeddings": 32768,
17
+ "model_type": "mistral",
18
+ "num_attention_heads": 32,
19
+ "num_hidden_layers": 32,
20
+ "num_key_value_heads": 8,
21
+ "rms_norm_eps": 1e-05,
22
+ "rope_theta": 1000000.0,
23
+ "sliding_window": null,
24
+ "tie_word_embeddings": false,
25
+ "torch_dtype": "bfloat16",
26
+ "transformers_version": "4.49.0",
27
+ "use_cache": true,
28
+ "vocab_size": 32768
29
+ }
30
+
31
+
32
+ [INFO|2025-04-07 18:56:08] tokenization_utils_base.py:2050 >> loading file tokenizer.model from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/tokenizer.model
33
+
34
+ [INFO|2025-04-07 18:56:08] tokenization_utils_base.py:2050 >> loading file tokenizer.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/tokenizer.json
35
+
36
+ [INFO|2025-04-07 18:56:08] tokenization_utils_base.py:2050 >> loading file added_tokens.json from cache at None
37
+
38
+ [INFO|2025-04-07 18:56:08] tokenization_utils_base.py:2050 >> loading file special_tokens_map.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/special_tokens_map.json
39
+
40
+ [INFO|2025-04-07 18:56:08] tokenization_utils_base.py:2050 >> loading file tokenizer_config.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/tokenizer_config.json
41
+
42
+ [INFO|2025-04-07 18:56:08] tokenization_utils_base.py:2050 >> loading file chat_template.jinja from cache at None
43
+
44
+ [INFO|2025-04-07 18:56:09] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/config.json
45
+
46
+ [INFO|2025-04-07 18:56:09] configuration_utils.py:771 >> Model config MistralConfig {
47
+ "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.3",
48
+ "architectures": [
49
+ "MistralForCausalLM"
50
+ ],
51
+ "attention_dropout": 0.0,
52
+ "bos_token_id": 1,
53
+ "eos_token_id": 2,
54
+ "head_dim": 128,
55
+ "hidden_act": "silu",
56
+ "hidden_size": 4096,
57
+ "initializer_range": 0.02,
58
+ "intermediate_size": 14336,
59
+ "max_position_embeddings": 32768,
60
+ "model_type": "mistral",
61
+ "num_attention_heads": 32,
62
+ "num_hidden_layers": 32,
63
+ "num_key_value_heads": 8,
64
+ "rms_norm_eps": 1e-05,
65
+ "rope_theta": 1000000.0,
66
+ "sliding_window": null,
67
+ "tie_word_embeddings": false,
68
+ "torch_dtype": "bfloat16",
69
+ "transformers_version": "4.49.0",
70
+ "use_cache": true,
71
+ "vocab_size": 32768
72
+ }
73
+
74
+
75
+ [INFO|2025-04-07 18:56:09] tokenization_utils_base.py:2050 >> loading file tokenizer.model from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/tokenizer.model
76
+
77
+ [INFO|2025-04-07 18:56:09] tokenization_utils_base.py:2050 >> loading file tokenizer.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/tokenizer.json
78
+
79
+ [INFO|2025-04-07 18:56:09] tokenization_utils_base.py:2050 >> loading file added_tokens.json from cache at None
80
+
81
+ [INFO|2025-04-07 18:56:09] tokenization_utils_base.py:2050 >> loading file special_tokens_map.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/special_tokens_map.json
82
+
83
+ [INFO|2025-04-07 18:56:09] tokenization_utils_base.py:2050 >> loading file tokenizer_config.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/tokenizer_config.json
84
+
85
+ [INFO|2025-04-07 18:56:09] tokenization_utils_base.py:2050 >> loading file chat_template.jinja from cache at None
86
+
87
+ [INFO|2025-04-07 18:56:10] logging.py:157 >> Add pad token: </s>
88
+
89
+ [INFO|2025-04-07 18:56:10] logging.py:157 >> Loading dataset en_train_part_1.json...
90
+
91
+ [INFO|2025-04-07 18:56:10] logging.py:157 >> Loading dataset en_train_part_3.json...
92
+
93
+ [INFO|2025-04-07 18:56:12] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.3/snapshots/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db/config.json
94
+
95
+ [INFO|2025-04-07 18:56:12] configuration_utils.py:771 >> Model config MistralConfig {
96
+ "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.3",
97
+ "architectures": [
98
+ "MistralForCausalLM"
99
+ ],
100
+ "attention_dropout": 0.0,
101
+ "bos_token_id": 1,
102
+ "eos_token_id": 2,
103
+ "head_dim": 128,
104
+ "hidden_act": "silu",
105
+ "hidden_size": 4096,
106
+ "initializer_range": 0.02,
107
+ "intermediate_size": 14336,
108
+ "max_position_embeddings": 32768,
109
+ "model_type": "mistral",
110
+ "num_attention_heads": 32,
111
+ "num_hidden_layers": 32,
112
+ "num_key_value_heads": 8,
113
+ "rms_norm_eps": 1e-05,
114
+ "rope_theta": 1000000.0,
115
+ "sliding_window": null,
116
+ "tie_word_embeddings": false,
117
+ "torch_dtype": "bfloat16",
118
+ "transformers_version": "4.49.0",
119
+ "use_cache": true,
120
+ "vocab_size": 32768
121
+ }
122
+
123
+
124
+ [INFO|2025-04-07 18:56:12] logging.py:157 >> Quantizing model to 4 bit with bitsandbytes.
125
+
126
+ [INFO|2025-04-07 18:56:14] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unsloth--mistral-7b-instruct-v0.3-bnb-4bit/snapshots/d5f623888f1415cf89b5c208d09cb620694618ee/config.json
127
+
128
+ [INFO|2025-04-07 18:56:14] configuration_utils.py:771 >> Model config MistralConfig {
129
+ "_name_or_path": "unsloth/mistral-7b-instruct-v0.3-bnb-4bit",
130
+ "architectures": [
131
+ "MistralForCausalLM"
132
+ ],
133
+ "attention_dropout": 0.0,
134
+ "bos_token_id": 1,
135
+ "eos_token_id": 2,
136
+ "head_dim": 128,
137
+ "hidden_act": "silu",
138
+ "hidden_size": 4096,
139
+ "initializer_range": 0.02,
140
+ "intermediate_size": 14336,
141
+ "max_position_embeddings": 32768,
142
+ "model_type": "mistral",
143
+ "num_attention_heads": 32,
144
+ "num_hidden_layers": 32,
145
+ "num_key_value_heads": 8,
146
+ "pad_token_id": 770,
147
+ "quantization_config": {
148
+ "_load_in_4bit": true,
149
+ "_load_in_8bit": false,
150
+ "bnb_4bit_compute_dtype": "bfloat16",
151
+ "bnb_4bit_quant_storage": "uint8",
152
+ "bnb_4bit_quant_type": "nf4",
153
+ "bnb_4bit_use_double_quant": true,
154
+ "llm_int8_enable_fp32_cpu_offload": false,
155
+ "llm_int8_has_fp16_weight": false,
156
+ "llm_int8_skip_modules": null,
157
+ "llm_int8_threshold": 6.0,
158
+ "load_in_4bit": true,
159
+ "load_in_8bit": false,
160
+ "quant_method": "bitsandbytes"
161
+ },
162
+ "rms_norm_eps": 1e-05,
163
+ "rope_theta": 1000000.0,
164
+ "sliding_window": null,
165
+ "tie_word_embeddings": false,
166
+ "torch_dtype": "bfloat16",
167
+ "transformers_version": "4.49.0",
168
+ "unsloth_version": "2024.9",
169
+ "use_cache": true,
170
+ "vocab_size": 32768
171
+ }
172
+
173
+
174
+ [INFO|2025-04-07 18:56:15] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unslothai--aws/snapshots/66e4c14a24a0b445779c922eef992a4af0694a88/config.json
175
+
176
+ [INFO|2025-04-07 18:56:15] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unslothai--repeat/snapshots/7c48478c02f84ed89f149b0815cc0216ee831fb0/config.json
177
+
178
+ [INFO|2025-04-07 18:56:16] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unslothai--vram-48/snapshots/3aea312d98ea327daeb5dbf7374b1d7cf8c65bc0/config.json
179
+
180
+ [INFO|2025-04-07 18:56:16] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unslothai--1/snapshots/7ec782b7604cd9ea0781c23a4270f031650f5617/config.json
181
+
182
+ [INFO|2025-04-07 18:56:16] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unsloth--mistral-7b-instruct-v0.3-bnb-4bit/snapshots/d5f623888f1415cf89b5c208d09cb620694618ee/config.json
183
+
184
+ [INFO|2025-04-07 18:56:16] configuration_utils.py:771 >> Model config MistralConfig {
185
+ "_name_or_path": "unsloth/mistral-7b-instruct-v0.3-bnb-4bit",
186
+ "architectures": [
187
+ "MistralForCausalLM"
188
+ ],
189
+ "attention_dropout": 0.0,
190
+ "bos_token_id": 1,
191
+ "eos_token_id": 2,
192
+ "head_dim": 128,
193
+ "hidden_act": "silu",
194
+ "hidden_size": 4096,
195
+ "initializer_range": 0.02,
196
+ "intermediate_size": 14336,
197
+ "max_position_embeddings": 32768,
198
+ "model_type": "mistral",
199
+ "num_attention_heads": 32,
200
+ "num_hidden_layers": 32,
201
+ "num_key_value_heads": 8,
202
+ "pad_token_id": 770,
203
+ "quantization_config": {
204
+ "_load_in_4bit": true,
205
+ "_load_in_8bit": false,
206
+ "bnb_4bit_compute_dtype": "bfloat16",
207
+ "bnb_4bit_quant_storage": "uint8",
208
+ "bnb_4bit_quant_type": "nf4",
209
+ "bnb_4bit_use_double_quant": true,
210
+ "llm_int8_enable_fp32_cpu_offload": false,
211
+ "llm_int8_has_fp16_weight": false,
212
+ "llm_int8_skip_modules": null,
213
+ "llm_int8_threshold": 6.0,
214
+ "load_in_4bit": true,
215
+ "load_in_8bit": false,
216
+ "quant_method": "bitsandbytes"
217
+ },
218
+ "rms_norm_eps": 1e-05,
219
+ "rope_theta": 1000000.0,
220
+ "sliding_window": null,
221
+ "tie_word_embeddings": false,
222
+ "torch_dtype": "bfloat16",
223
+ "transformers_version": "4.49.0",
224
+ "unsloth_version": "2024.9",
225
+ "use_cache": true,
226
+ "vocab_size": 32768
227
+ }
228
+
229
+
230
+ [INFO|2025-04-07 18:56:16] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unsloth--mistral-7b-instruct-v0.3-bnb-4bit/snapshots/d5f623888f1415cf89b5c208d09cb620694618ee/config.json
231
+
232
+ [INFO|2025-04-07 18:56:16] configuration_utils.py:771 >> Model config MistralConfig {
233
+ "_name_or_path": "unsloth/mistral-7b-instruct-v0.3-bnb-4bit",
234
+ "architectures": [
235
+ "MistralForCausalLM"
236
+ ],
237
+ "attention_dropout": 0.0,
238
+ "bos_token_id": 1,
239
+ "eos_token_id": 2,
240
+ "head_dim": 128,
241
+ "hidden_act": "silu",
242
+ "hidden_size": 4096,
243
+ "initializer_range": 0.02,
244
+ "intermediate_size": 14336,
245
+ "max_position_embeddings": 32768,
246
+ "model_type": "mistral",
247
+ "num_attention_heads": 32,
248
+ "num_hidden_layers": 32,
249
+ "num_key_value_heads": 8,
250
+ "pad_token_id": 770,
251
+ "quantization_config": {
252
+ "_load_in_4bit": true,
253
+ "_load_in_8bit": false,
254
+ "bnb_4bit_compute_dtype": "bfloat16",
255
+ "bnb_4bit_quant_storage": "uint8",
256
+ "bnb_4bit_quant_type": "nf4",
257
+ "bnb_4bit_use_double_quant": true,
258
+ "llm_int8_enable_fp32_cpu_offload": false,
259
+ "llm_int8_has_fp16_weight": false,
260
+ "llm_int8_skip_modules": null,
261
+ "llm_int8_threshold": 6.0,
262
+ "load_in_4bit": true,
263
+ "load_in_8bit": false,
264
+ "quant_method": "bitsandbytes"
265
+ },
266
+ "rms_norm_eps": 1e-05,
267
+ "rope_theta": 1000000.0,
268
+ "sliding_window": null,
269
+ "tie_word_embeddings": false,
270
+ "torch_dtype": "bfloat16",
271
+ "transformers_version": "4.49.0",
272
+ "unsloth_version": "2024.9",
273
+ "use_cache": true,
274
+ "vocab_size": 32768
275
+ }
276
+
277
+
278
+ [INFO|2025-04-07 18:56:16] modeling_utils.py:3982 >> loading weights file model.safetensors from cache at /home/zeus/.cache/huggingface/hub/models--unsloth--mistral-7b-instruct-v0.3-bnb-4bit/snapshots/d5f623888f1415cf89b5c208d09cb620694618ee/model.safetensors
279
+
280
+ [INFO|2025-04-07 18:56:16] modeling_utils.py:1633 >> Instantiating MistralForCausalLM model under default dtype torch.bfloat16.
281
+
282
+ [INFO|2025-04-07 18:56:16] configuration_utils.py:1140 >> Generate config GenerationConfig {
283
+ "bos_token_id": 1,
284
+ "eos_token_id": 2,
285
+ "pad_token_id": 770
286
+ }
287
+
288
+
289
+ [INFO|2025-04-07 18:56:19] modeling_utils.py:4970 >> All model checkpoint weights were used when initializing MistralForCausalLM.
290
+
291
+
292
+ [INFO|2025-04-07 18:56:19] modeling_utils.py:4978 >> All the weights of MistralForCausalLM were initialized from the model checkpoint at unsloth/mistral-7b-instruct-v0.3-bnb-4bit.
293
+ If your task is similar to the task the model of the checkpoint was trained on, you can already use MistralForCausalLM for predictions without further training.
294
+
295
+ [INFO|2025-04-07 18:56:20] configuration_utils.py:1095 >> loading configuration file generation_config.json from cache at /home/zeus/.cache/huggingface/hub/models--unsloth--mistral-7b-instruct-v0.3-bnb-4bit/snapshots/d5f623888f1415cf89b5c208d09cb620694618ee/generation_config.json
296
+
297
+ [INFO|2025-04-07 18:56:20] configuration_utils.py:1140 >> Generate config GenerationConfig {
298
+ "bos_token_id": 1,
299
+ "eos_token_id": 2,
300
+ "max_length": 32768,
301
+ "pad_token_id": 770
302
+ }
303
+
304
+
305
+ [INFO|2025-04-07 18:56:21] logging.py:157 >> Gradient checkpointing enabled.
306
+
307
+ [INFO|2025-04-07 18:56:21] logging.py:157 >> Upcasting trainable params to float32.
308
+
309
+ [INFO|2025-04-07 18:56:21] logging.py:157 >> Fine-tuning method: LoRA
310
+
311
+ [INFO|2025-04-07 18:56:21] logging.py:157 >> Found linear modules: v_proj,q_proj,up_proj,down_proj,gate_proj,o_proj,k_proj
312
+
313
+ [WARNING|2025-04-07 18:56:23] logging.py:329 >> Unsloth 2025.3.19 patched 32 layers with 32 QKV layers, 32 O layers and 32 MLP layers.
314
+
315
+ [INFO|2025-04-07 18:56:24] logging.py:157 >> trainable params: 20,971,520 || all params: 7,268,995,072 || trainable%: 0.2885
316
+
317
+ [INFO|2025-04-07 18:56:24] trainer.py:746 >> Using auto half precision backend
318
+
319
+ [WARNING|2025-04-07 18:56:25] <string>:168 >> ==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1
320
+ \\ /| Num examples = 1,857 | Num Epochs = 3 | Total steps = 696
321
+ O^O/ \_/ \ Batch size per device = 4 | Gradient accumulation steps = 2
322
+ \ / Data Parallel GPUs = 1 | Total batch size (4 x 2 x 1) = 8
323
+ "-____-" Trainable parameters = 20,971,520/7,000,000,000 (0.30% trained)
324
+
325
+ [INFO|2025-04-07 18:56:37] logging.py:157 >> {'loss': 0.9267, 'learning_rate': 2.9985e-05, 'epoch': 0.04, 'throughput': 2301.21}
326
+
327
+ [INFO|2025-04-07 18:56:46] logging.py:157 >> {'loss': 0.1561, 'learning_rate': 2.9939e-05, 'epoch': 0.09, 'throughput': 2605.30}
328
+
329
+ [INFO|2025-04-07 18:56:55] logging.py:157 >> {'loss': 0.1764, 'learning_rate': 2.9863e-05, 'epoch': 0.13, 'throughput': 2718.87}
330
+
331
+ [INFO|2025-04-07 18:57:04] logging.py:157 >> {'loss': 0.1505, 'learning_rate': 2.9756e-05, 'epoch': 0.17, 'throughput': 2776.68}
332
+
333
+ [INFO|2025-04-07 18:57:13] logging.py:157 >> {'loss': 0.1615, 'learning_rate': 2.9620e-05, 'epoch': 0.22, 'throughput': 2813.40}
334
+
335
+ [INFO|2025-04-07 18:57:22] logging.py:157 >> {'loss': 0.1369, 'learning_rate': 2.9453e-05, 'epoch': 0.26, 'throughput': 2832.51}
336
+
337
+ [INFO|2025-04-07 18:57:31] logging.py:157 >> {'loss': 0.1253, 'learning_rate': 2.9257e-05, 'epoch': 0.30, 'throughput': 2849.27}
338
+
339
+ [INFO|2025-04-07 18:57:40] logging.py:157 >> {'loss': 0.1479, 'learning_rate': 2.9033e-05, 'epoch': 0.34, 'throughput': 2862.47}
340
+
341
+ [INFO|2025-04-07 18:57:49] logging.py:157 >> {'loss': 0.1102, 'learning_rate': 2.8779e-05, 'epoch': 0.39, 'throughput': 2874.09}
342
+
343
+ [INFO|2025-04-07 18:57:58] logging.py:157 >> {'loss': 0.0990, 'learning_rate': 2.8498e-05, 'epoch': 0.43, 'throughput': 2880.65}
344
+
345
+ [INFO|2025-04-07 18:58:07] logging.py:157 >> {'loss': 0.1070, 'learning_rate': 2.8189e-05, 'epoch': 0.47, 'throughput': 2886.12}
346
+
347
+ [INFO|2025-04-07 18:58:16] logging.py:157 >> {'loss': 0.1183, 'learning_rate': 2.7853e-05, 'epoch': 0.52, 'throughput': 2891.42}
348
+
349
+ [INFO|2025-04-07 18:58:25] logging.py:157 >> {'loss': 0.1097, 'learning_rate': 2.7491e-05, 'epoch': 0.56, 'throughput': 2896.48}
350
+
351
+ [INFO|2025-04-07 18:58:34] logging.py:157 >> {'loss': 0.0946, 'learning_rate': 2.7103e-05, 'epoch': 0.60, 'throughput': 2899.20}
352
+
353
+ [INFO|2025-04-07 18:58:43] logging.py:157 >> {'loss': 0.1251, 'learning_rate': 2.6691e-05, 'epoch': 0.65, 'throughput': 2902.17}
354
+
355
+ [INFO|2025-04-07 18:58:52] logging.py:157 >> {'loss': 0.1069, 'learning_rate': 2.6255e-05, 'epoch': 0.69, 'throughput': 2904.29}
356
+
357
+ [INFO|2025-04-07 18:59:01] logging.py:157 >> {'loss': 0.0904, 'learning_rate': 2.5796e-05, 'epoch': 0.73, 'throughput': 2906.29}
358
+
359
+ [INFO|2025-04-07 18:59:11] logging.py:157 >> {'loss': 0.1273, 'learning_rate': 2.5315e-05, 'epoch': 0.77, 'throughput': 2908.27}
360
+
361
+ [INFO|2025-04-07 18:59:20] logging.py:157 >> {'loss': 0.1206, 'learning_rate': 2.4814e-05, 'epoch': 0.82, 'throughput': 2908.28}
362
+
363
+ [INFO|2025-04-07 18:59:29] logging.py:157 >> {'loss': 0.1030, 'learning_rate': 2.4292e-05, 'epoch': 0.86, 'throughput': 2908.42}
364
+
365
+ [INFO|2025-04-07 18:59:38] logging.py:157 >> {'loss': 0.0696, 'learning_rate': 2.3751e-05, 'epoch': 0.90, 'throughput': 2909.86}
366
+
367
+ [INFO|2025-04-07 18:59:48] logging.py:157 >> {'loss': 0.0822, 'learning_rate': 2.3192e-05, 'epoch': 0.95, 'throughput': 2909.89}
368
+
369
+ [INFO|2025-04-07 18:59:57] logging.py:157 >> {'loss': 0.0913, 'learning_rate': 2.2617e-05, 'epoch': 0.99, 'throughput': 2909.56}
370
+
371
+ [INFO|2025-04-07 19:00:06] logging.py:157 >> {'loss': 0.0735, 'learning_rate': 2.2026e-05, 'epoch': 1.03, 'throughput': 2906.22}
372
+
373
+ [INFO|2025-04-07 19:00:15] logging.py:157 >> {'loss': 0.0752, 'learning_rate': 2.1421e-05, 'epoch': 1.07, 'throughput': 2906.03}
374
+
375
+ [INFO|2025-04-07 19:00:24] logging.py:157 >> {'loss': 0.0588, 'learning_rate': 2.0803e-05, 'epoch': 1.12, 'throughput': 2905.16}
376
+
377
+ [INFO|2025-04-07 19:00:34] logging.py:157 >> {'loss': 0.0924, 'learning_rate': 2.0173e-05, 'epoch': 1.16, 'throughput': 2904.97}
378
+
379
+ [INFO|2025-04-07 19:00:43] logging.py:157 >> {'loss': 0.0620, 'learning_rate': 1.9532e-05, 'epoch': 1.20, 'throughput': 2905.07}
380
+
381
+ [INFO|2025-04-07 19:00:52] logging.py:157 >> {'loss': 0.0621, 'learning_rate': 1.8882e-05, 'epoch': 1.25, 'throughput': 2905.29}
382
+
383
+ [INFO|2025-04-07 19:01:01] logging.py:157 >> {'loss': 0.0750, 'learning_rate': 1.8225e-05, 'epoch': 1.29, 'throughput': 2905.84}
384
+
385
+ [INFO|2025-04-07 19:01:11] logging.py:157 >> {'loss': 0.0858, 'learning_rate': 1.7560e-05, 'epoch': 1.33, 'throughput': 2906.05}
386
+
387
+ [INFO|2025-04-07 19:01:20] logging.py:157 >> {'loss': 0.0697, 'learning_rate': 1.6891e-05, 'epoch': 1.37, 'throughput': 2905.02}
388
+
389
+ [INFO|2025-04-07 19:01:30] logging.py:157 >> {'loss': 0.0827, 'learning_rate': 1.6217e-05, 'epoch': 1.42, 'throughput': 2903.80}
390
+
391
+ [INFO|2025-04-07 19:01:39] logging.py:157 >> {'loss': 0.0651, 'learning_rate': 1.5542e-05, 'epoch': 1.46, 'throughput': 2902.99}
392
+
393
+ [INFO|2025-04-07 19:01:48] logging.py:157 >> {'loss': 0.0607, 'learning_rate': 1.4865e-05, 'epoch': 1.50, 'throughput': 2903.09}
394
+
395
+ [INFO|2025-04-07 19:01:57] logging.py:157 >> {'loss': 0.0560, 'learning_rate': 1.4188e-05, 'epoch': 1.55, 'throughput': 2902.53}
396
+
397
+ [INFO|2025-04-07 19:02:06] logging.py:157 >> {'loss': 0.0823, 'learning_rate': 1.3513e-05, 'epoch': 1.59, 'throughput': 2902.81}
398
+
399
+ [INFO|2025-04-07 19:02:15] logging.py:157 >> {'loss': 0.0580, 'learning_rate': 1.2841e-05, 'epoch': 1.63, 'throughput': 2902.64}
400
+
401
+ [INFO|2025-04-07 19:02:24] logging.py:157 >> {'loss': 0.0705, 'learning_rate': 1.2173e-05, 'epoch': 1.68, 'throughput': 2902.41}
402
+
403
+ [INFO|2025-04-07 19:02:33] logging.py:157 >> {'loss': 0.0514, 'learning_rate': 1.1511e-05, 'epoch': 1.72, 'throughput': 2902.76}
404
+
405
+ [INFO|2025-04-07 19:02:43] logging.py:157 >> {'loss': 0.0834, 'learning_rate': 1.0857e-05, 'epoch': 1.76, 'throughput': 2903.25}
406
+
407
+ [INFO|2025-04-07 19:02:52] logging.py:157 >> {'loss': 0.0583, 'learning_rate': 1.0210e-05, 'epoch': 1.80, 'throughput': 2903.01}
408
+
409
+ [INFO|2025-04-07 19:03:01] logging.py:157 >> {'loss': 0.0922, 'learning_rate': 9.5739e-06, 'epoch': 1.85, 'throughput': 2902.51}
410
+
411
+ [INFO|2025-04-07 19:03:10] logging.py:157 >> {'loss': 0.0633, 'learning_rate': 8.9485e-06, 'epoch': 1.89, 'throughput': 2903.29}
412
+
413
+ [INFO|2025-04-07 19:03:19] logging.py:157 >> {'loss': 0.0597, 'learning_rate': 8.3353e-06, 'epoch': 1.93, 'throughput': 2903.81}
414
+
415
+ [INFO|2025-04-07 19:03:28] logging.py:157 >> {'loss': 0.0491, 'learning_rate': 7.7358e-06, 'epoch': 1.98, 'throughput': 2903.71}
416
+
417
+ [INFO|2025-04-07 19:03:37] logging.py:157 >> {'loss': 0.0341, 'learning_rate': 7.1510e-06, 'epoch': 2.02, 'throughput': 2902.63}
418
+
419
+ [INFO|2025-04-07 19:03:46] logging.py:157 >> {'loss': 0.0373, 'learning_rate': 6.5822e-06, 'epoch': 2.06, 'throughput': 2903.07}
420
+
421
+ [INFO|2025-04-07 19:03:56] logging.py:157 >> {'loss': 0.0281, 'learning_rate': 6.0306e-06, 'epoch': 2.10, 'throughput': 2903.01}
422
+
423
+ [INFO|2025-04-07 19:04:05] logging.py:157 >> {'loss': 0.0183, 'learning_rate': 5.4972e-06, 'epoch': 2.15, 'throughput': 2902.59}
424
+
425
+ [INFO|2025-04-07 19:04:14] logging.py:157 >> {'loss': 0.0334, 'learning_rate': 4.9832e-06, 'epoch': 2.19, 'throughput': 2901.70}
426
+
427
+ [INFO|2025-04-07 19:04:24] logging.py:157 >> {'loss': 0.0282, 'learning_rate': 4.4896e-06, 'epoch': 2.23, 'throughput': 2901.24}
428
+
429
+ [INFO|2025-04-07 19:04:33] logging.py:157 >> {'loss': 0.0562, 'learning_rate': 4.0174e-06, 'epoch': 2.28, 'throughput': 2901.79}
430
+
431
+ [INFO|2025-04-07 19:04:42] logging.py:157 >> {'loss': 0.0243, 'learning_rate': 3.5676e-06, 'epoch': 2.32, 'throughput': 2902.13}
432
+
433
+ [INFO|2025-04-07 19:04:51] logging.py:157 >> {'loss': 0.0440, 'learning_rate': 3.1410e-06, 'epoch': 2.36, 'throughput': 2902.37}
434
+
435
+ [INFO|2025-04-07 19:05:00] logging.py:157 >> {'loss': 0.0551, 'learning_rate': 2.7387e-06, 'epoch': 2.40, 'throughput': 2902.60}
436
+
437
+ [INFO|2025-04-07 19:05:09] logging.py:157 >> {'loss': 0.0258, 'learning_rate': 2.3613e-06, 'epoch': 2.45, 'throughput': 2902.83}
438
+
439
+ [INFO|2025-04-07 19:05:19] logging.py:157 >> {'loss': 0.0415, 'learning_rate': 2.0096e-06, 'epoch': 2.49, 'throughput': 2903.34}
440
+
441
+ [INFO|2025-04-07 19:05:28] logging.py:157 >> {'loss': 0.0375, 'learning_rate': 1.6844e-06, 'epoch': 2.53, 'throughput': 2903.89}
442
+
443
+ [INFO|2025-04-07 19:05:37] logging.py:157 >> {'loss': 0.0235, 'learning_rate': 1.3864e-06, 'epoch': 2.58, 'throughput': 2904.68}
444
+
445
+ [INFO|2025-04-07 19:05:46] logging.py:157 >> {'loss': 0.0354, 'learning_rate': 1.1160e-06, 'epoch': 2.62, 'throughput': 2905.09}
446
+
447
+ [INFO|2025-04-07 19:05:55] logging.py:157 >> {'loss': 0.0357, 'learning_rate': 8.7399e-07, 'epoch': 2.66, 'throughput': 2905.48}
448
+
449
+ [INFO|2025-04-07 19:06:04] logging.py:157 >> {'loss': 0.0512, 'learning_rate': 6.6072e-07, 'epoch': 2.71, 'throughput': 2905.88}
450
+
451
+ [INFO|2025-04-07 19:06:13] logging.py:157 >> {'loss': 0.0243, 'learning_rate': 4.7666e-07, 'epoch': 2.75, 'throughput': 2906.53}
452
+
453
+ [INFO|2025-04-07 19:06:22] logging.py:157 >> {'loss': 0.0367, 'learning_rate': 3.2218e-07, 'epoch': 2.79, 'throughput': 2907.18}
454
+
455
+ [INFO|2025-04-07 19:06:31] logging.py:157 >> {'loss': 0.0377, 'learning_rate': 1.9760e-07, 'epoch': 2.83, 'throughput': 2907.66}
456
+
457
+ [INFO|2025-04-07 19:06:40] logging.py:157 >> {'loss': 0.0471, 'learning_rate': 1.0318e-07, 'epoch': 2.88, 'throughput': 2907.89}
458
+
459
+ [INFO|2025-04-07 19:06:49] logging.py:157 >> {'loss': 0.0222, 'learning_rate': 3.9102e-08, 'epoch': 2.92, 'throughput': 2908.16}
460
+
461
+ [INFO|2025-04-07 19:06:58] logging.py:157 >> {'loss': 0.0454, 'learning_rate': 5.5007e-09, 'epoch': 2.96, 'throughput': 2908.39}
462
+
463
+ [INFO|2025-04-07 19:07:04] trainer.py:3942 >> Saving model checkpoint to saves/Mistral-7B-Instruct-v0.3/lora/2_part/checkpoint-696
464
+
465
+ [INFO|2025-04-07 19:07:04] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unsloth--mistral-7b-instruct-v0.3-bnb-4bit/snapshots/d5f623888f1415cf89b5c208d09cb620694618ee/config.json
466
+
467
+ [INFO|2025-04-07 19:07:04] configuration_utils.py:771 >> Model config MistralConfig {
468
+ "_name_or_path": "unsloth/Mistral-7B-Instruct-v0.3",
469
+ "architectures": [
470
+ "MistralForCausalLM"
471
+ ],
472
+ "attention_dropout": 0.0,
473
+ "bos_token_id": 1,
474
+ "eos_token_id": 2,
475
+ "head_dim": 128,
476
+ "hidden_act": "silu",
477
+ "hidden_size": 4096,
478
+ "initializer_range": 0.02,
479
+ "intermediate_size": 14336,
480
+ "max_position_embeddings": 32768,
481
+ "model_type": "mistral",
482
+ "num_attention_heads": 32,
483
+ "num_hidden_layers": 32,
484
+ "num_key_value_heads": 8,
485
+ "pad_token_id": 770,
486
+ "quantization_config": {
487
+ "_load_in_4bit": true,
488
+ "_load_in_8bit": false,
489
+ "bnb_4bit_compute_dtype": "bfloat16",
490
+ "bnb_4bit_quant_storage": "uint8",
491
+ "bnb_4bit_quant_type": "nf4",
492
+ "bnb_4bit_use_double_quant": true,
493
+ "llm_int8_enable_fp32_cpu_offload": false,
494
+ "llm_int8_has_fp16_weight": false,
495
+ "llm_int8_skip_modules": null,
496
+ "llm_int8_threshold": 6.0,
497
+ "load_in_4bit": true,
498
+ "load_in_8bit": false,
499
+ "quant_method": "bitsandbytes"
500
+ },
501
+ "rms_norm_eps": 1e-05,
502
+ "rope_theta": 1000000.0,
503
+ "sliding_window": null,
504
+ "tie_word_embeddings": false,
505
+ "torch_dtype": "bfloat16",
506
+ "transformers_version": "4.49.0",
507
+ "unsloth_version": "2024.9",
508
+ "use_cache": true,
509
+ "vocab_size": 32768
510
+ }
511
+
512
+
513
+ [INFO|2025-04-07 19:07:04] <string>:415 >>
514
+
515
+ Training completed. Do not forget to share your model on huggingface.co/models =)
516
+
517
+
518
+
519
+ [INFO|2025-04-07 19:07:04] trainer.py:3942 >> Saving model checkpoint to saves/Mistral-7B-Instruct-v0.3/lora/2_part
520
+
521
+ [INFO|2025-04-07 19:07:05] configuration_utils.py:699 >> loading configuration file config.json from cache at /home/zeus/.cache/huggingface/hub/models--unsloth--mistral-7b-instruct-v0.3-bnb-4bit/snapshots/d5f623888f1415cf89b5c208d09cb620694618ee/config.json
522
+
523
+ [INFO|2025-04-07 19:07:05] configuration_utils.py:771 >> Model config MistralConfig {
524
+ "_name_or_path": "unsloth/Mistral-7B-Instruct-v0.3",
525
+ "architectures": [
526
+ "MistralForCausalLM"
527
+ ],
528
+ "attention_dropout": 0.0,
529
+ "bos_token_id": 1,
530
+ "eos_token_id": 2,
531
+ "head_dim": 128,
532
+ "hidden_act": "silu",
533
+ "hidden_size": 4096,
534
+ "initializer_range": 0.02,
535
+ "intermediate_size": 14336,
536
+ "max_position_embeddings": 32768,
537
+ "model_type": "mistral",
538
+ "num_attention_heads": 32,
539
+ "num_hidden_layers": 32,
540
+ "num_key_value_heads": 8,
541
+ "pad_token_id": 770,
542
+ "quantization_config": {
543
+ "_load_in_4bit": true,
544
+ "_load_in_8bit": false,
545
+ "bnb_4bit_compute_dtype": "bfloat16",
546
+ "bnb_4bit_quant_storage": "uint8",
547
+ "bnb_4bit_quant_type": "nf4",
548
+ "bnb_4bit_use_double_quant": true,
549
+ "llm_int8_enable_fp32_cpu_offload": false,
550
+ "llm_int8_has_fp16_weight": false,
551
+ "llm_int8_skip_modules": null,
552
+ "llm_int8_threshold": 6.0,
553
+ "load_in_4bit": true,
554
+ "load_in_8bit": false,
555
+ "quant_method": "bitsandbytes"
556
+ },
557
+ "rms_norm_eps": 1e-05,
558
+ "rope_theta": 1000000.0,
559
+ "sliding_window": null,
560
+ "tie_word_embeddings": false,
561
+ "torch_dtype": "bfloat16",
562
+ "transformers_version": "4.49.0",
563
+ "unsloth_version": "2024.9",
564
+ "use_cache": true,
565
+ "vocab_size": 32768
566
+ }
567
+
568
+
569
+ [WARNING|2025-04-07 19:07:05] logging.py:162 >> No metric eval_loss to plot.
570
+
571
+ [WARNING|2025-04-07 19:07:05] logging.py:162 >> No metric eval_accuracy to plot.
572
+
573
+ [INFO|2025-04-07 19:07:05] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
574
+ {'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
575
+
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
3
+ size 587404
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 2.989247311827957,
3
+ "num_input_tokens_seen": 1855776,
4
+ "total_flos": 7.944329136203366e+16,
5
+ "train_loss": 0.08561765917459097,
6
+ "train_runtime": 638.811,
7
+ "train_samples_per_second": 8.721,
8
+ "train_steps_per_second": 1.09
9
+ }
trainer_log.jsonl ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"current_steps": 10, "total_steps": 696, "loss": 0.9267, "lr": 2.9984721919587606e-05, "epoch": 0.043010752688172046, "percentage": 1.44, "elapsed_time": "0:00:11", "remaining_time": "0:13:01", "throughput": 2301.21, "total_tokens": 26208}
2
+ {"current_steps": 20, "total_steps": 696, "loss": 0.1561, "lr": 2.9938918800982563e-05, "epoch": 0.08602150537634409, "percentage": 2.87, "elapsed_time": "0:00:20", "remaining_time": "0:11:29", "throughput": 2605.3, "total_tokens": 53152}
3
+ {"current_steps": 30, "total_steps": 696, "loss": 0.1764, "lr": 2.9862683948682103e-05, "epoch": 0.12903225806451613, "percentage": 4.31, "elapsed_time": "0:00:29", "remaining_time": "0:10:51", "throughput": 2718.87, "total_tokens": 79776}
4
+ {"current_steps": 40, "total_steps": 696, "loss": 0.1505, "lr": 2.975617265898004e-05, "epoch": 0.17204301075268819, "percentage": 5.75, "elapsed_time": "0:00:38", "remaining_time": "0:10:29", "throughput": 2776.68, "total_tokens": 106496}
5
+ {"current_steps": 50, "total_steps": 696, "loss": 0.1615, "lr": 2.961960190361624e-05, "epoch": 0.21505376344086022, "percentage": 7.18, "elapsed_time": "0:00:47", "remaining_time": "0:10:14", "throughput": 2813.4, "total_tokens": 133728}
6
+ {"current_steps": 60, "total_steps": 696, "loss": 0.1369, "lr": 2.9453249887788343e-05, "epoch": 0.25806451612903225, "percentage": 8.62, "elapsed_time": "0:00:56", "remaining_time": "0:09:58", "throughput": 2832.51, "total_tokens": 159936}
7
+ {"current_steps": 70, "total_steps": 696, "loss": 0.1253, "lr": 2.925745548342631e-05, "epoch": 0.3010752688172043, "percentage": 10.06, "elapsed_time": "0:01:05", "remaining_time": "0:09:44", "throughput": 2849.27, "total_tokens": 186240}
8
+ {"current_steps": 80, "total_steps": 696, "loss": 0.1479, "lr": 2.9032617538884018e-05, "epoch": 0.34408602150537637, "percentage": 11.49, "elapsed_time": "0:01:14", "remaining_time": "0:09:32", "throughput": 2862.47, "total_tokens": 212832}
9
+ {"current_steps": 90, "total_steps": 696, "loss": 0.1102, "lr": 2.877919406645433e-05, "epoch": 0.3870967741935484, "percentage": 12.93, "elapsed_time": "0:01:23", "remaining_time": "0:09:21", "throughput": 2874.09, "total_tokens": 239648}
10
+ {"current_steps": 100, "total_steps": 696, "loss": 0.099, "lr": 2.84977013093626e-05, "epoch": 0.43010752688172044, "percentage": 14.37, "elapsed_time": "0:01:32", "remaining_time": "0:09:10", "throughput": 2880.65, "total_tokens": 265888}
11
+ {"current_steps": 110, "total_steps": 696, "loss": 0.107, "lr": 2.818871269013928e-05, "epoch": 0.4731182795698925, "percentage": 15.8, "elapsed_time": "0:01:41", "remaining_time": "0:08:59", "throughput": 2886.12, "total_tokens": 292480}
12
+ {"current_steps": 120, "total_steps": 696, "loss": 0.1183, "lr": 2.7852857642513838e-05, "epoch": 0.5161290322580645, "percentage": 17.24, "elapsed_time": "0:01:50", "remaining_time": "0:08:49", "throughput": 2891.42, "total_tokens": 318784}
13
+ {"current_steps": 130, "total_steps": 696, "loss": 0.1097, "lr": 2.7490820329209546e-05, "epoch": 0.5591397849462365, "percentage": 18.68, "elapsed_time": "0:01:59", "remaining_time": "0:08:40", "throughput": 2896.48, "total_tokens": 346016}
14
+ {"current_steps": 140, "total_steps": 696, "loss": 0.0946, "lr": 2.7103338248251055e-05, "epoch": 0.6021505376344086, "percentage": 20.11, "elapsed_time": "0:02:08", "remaining_time": "0:08:30", "throughput": 2899.2, "total_tokens": 372384}
15
+ {"current_steps": 150, "total_steps": 696, "loss": 0.1251, "lr": 2.6691200730623874e-05, "epoch": 0.6451612903225806, "percentage": 21.55, "elapsed_time": "0:02:17", "remaining_time": "0:08:20", "throughput": 2902.17, "total_tokens": 399328}
16
+ {"current_steps": 160, "total_steps": 696, "loss": 0.1069, "lr": 2.6255247332346036e-05, "epoch": 0.6881720430107527, "percentage": 22.99, "elapsed_time": "0:02:26", "remaining_time": "0:08:11", "throughput": 2904.29, "total_tokens": 426048}
17
+ {"current_steps": 170, "total_steps": 696, "loss": 0.0904, "lr": 2.5796366124227532e-05, "epoch": 0.7311827956989247, "percentage": 24.43, "elapsed_time": "0:02:35", "remaining_time": "0:08:01", "throughput": 2906.29, "total_tokens": 452640}
18
+ {"current_steps": 180, "total_steps": 696, "loss": 0.1273, "lr": 2.531549188280135e-05, "epoch": 0.7741935483870968, "percentage": 25.86, "elapsed_time": "0:02:44", "remaining_time": "0:07:52", "throughput": 2908.27, "total_tokens": 479808}
19
+ {"current_steps": 190, "total_steps": 696, "loss": 0.1206, "lr": 2.481360418611132e-05, "epoch": 0.8172043010752689, "percentage": 27.3, "elapsed_time": "0:02:54", "remaining_time": "0:07:43", "throughput": 2908.28, "total_tokens": 506176}
20
+ {"current_steps": 200, "total_steps": 696, "loss": 0.103, "lr": 2.4291725418235848e-05, "epoch": 0.8602150537634409, "percentage": 28.74, "elapsed_time": "0:03:03", "remaining_time": "0:07:34", "throughput": 2908.42, "total_tokens": 533216}
21
+ {"current_steps": 210, "total_steps": 696, "loss": 0.0696, "lr": 2.3750918686612414e-05, "epoch": 0.9032258064516129, "percentage": 30.17, "elapsed_time": "0:03:12", "remaining_time": "0:07:26", "throughput": 2909.86, "total_tokens": 561056}
22
+ {"current_steps": 220, "total_steps": 696, "loss": 0.0822, "lr": 2.3192285656405456e-05, "epoch": 0.946236559139785, "percentage": 31.61, "elapsed_time": "0:03:22", "remaining_time": "0:07:17", "throughput": 2909.89, "total_tokens": 588160}
23
+ {"current_steps": 230, "total_steps": 696, "loss": 0.0913, "lr": 2.2616964306329183e-05, "epoch": 0.989247311827957, "percentage": 33.05, "elapsed_time": "0:03:31", "remaining_time": "0:07:08", "throughput": 2909.56, "total_tokens": 615168}
24
+ {"current_steps": 240, "total_steps": 696, "loss": 0.0735, "lr": 2.2026126610496852e-05, "epoch": 1.0301075268817204, "percentage": 34.48, "elapsed_time": "0:03:40", "remaining_time": "0:06:58", "throughput": 2906.22, "total_tokens": 639864}
25
+ {"current_steps": 250, "total_steps": 696, "loss": 0.0752, "lr": 2.1420976151018813e-05, "epoch": 1.0731182795698926, "percentage": 35.92, "elapsed_time": "0:03:49", "remaining_time": "0:06:49", "throughput": 2906.03, "total_tokens": 667224}
26
+ {"current_steps": 260, "total_steps": 696, "loss": 0.0588, "lr": 2.0802745666212592e-05, "epoch": 1.1161290322580646, "percentage": 37.36, "elapsed_time": "0:03:58", "remaining_time": "0:06:40", "throughput": 2905.16, "total_tokens": 693848}
27
+ {"current_steps": 270, "total_steps": 696, "loss": 0.0924, "lr": 2.0172694539419557e-05, "epoch": 1.1591397849462366, "percentage": 38.79, "elapsed_time": "0:04:08", "remaining_time": "0:06:31", "throughput": 2904.97, "total_tokens": 720568}
28
+ {"current_steps": 280, "total_steps": 696, "loss": 0.062, "lr": 1.953210623354359e-05, "epoch": 1.2021505376344086, "percentage": 40.23, "elapsed_time": "0:04:17", "remaining_time": "0:06:21", "throughput": 2905.07, "total_tokens": 746872}
29
+ {"current_steps": 290, "total_steps": 696, "loss": 0.0621, "lr": 1.888228567653781e-05, "epoch": 1.2451612903225806, "percentage": 41.67, "elapsed_time": "0:04:26", "remaining_time": "0:06:12", "throughput": 2905.29, "total_tokens": 773720}
30
+ {"current_steps": 300, "total_steps": 696, "loss": 0.075, "lr": 1.8224556603165363e-05, "epoch": 1.2881720430107526, "percentage": 43.1, "elapsed_time": "0:04:35", "remaining_time": "0:06:04", "throughput": 2905.84, "total_tokens": 801464}
31
+ {"current_steps": 310, "total_steps": 696, "loss": 0.0858, "lr": 1.7560258858449248e-05, "epoch": 1.3311827956989246, "percentage": 44.54, "elapsed_time": "0:04:45", "remaining_time": "0:05:55", "throughput": 2906.05, "total_tokens": 829144}
32
+ {"current_steps": 320, "total_steps": 696, "loss": 0.0697, "lr": 1.689074566830434e-05, "epoch": 1.3741935483870968, "percentage": 45.98, "elapsed_time": "0:04:54", "remaining_time": "0:05:46", "throughput": 2905.02, "total_tokens": 855672}
33
+ {"current_steps": 330, "total_steps": 696, "loss": 0.0827, "lr": 1.621738088291147e-05, "epoch": 1.4172043010752688, "percentage": 47.41, "elapsed_time": "0:05:03", "remaining_time": "0:05:37", "throughput": 2903.8, "total_tokens": 882424}
34
+ {"current_steps": 340, "total_steps": 696, "loss": 0.0651, "lr": 1.5541536198449044e-05, "epoch": 1.4602150537634409, "percentage": 48.85, "elapsed_time": "0:05:13", "remaining_time": "0:05:27", "throughput": 2902.99, "total_tokens": 908792}
35
+ {"current_steps": 350, "total_steps": 696, "loss": 0.0607, "lr": 1.4864588362841808e-05, "epoch": 1.5032258064516129, "percentage": 50.29, "elapsed_time": "0:05:22", "remaining_time": "0:05:18", "throughput": 2903.09, "total_tokens": 935672}
36
+ {"current_steps": 360, "total_steps": 696, "loss": 0.056, "lr": 1.4187916371218739e-05, "epoch": 1.546236559139785, "percentage": 51.72, "elapsed_time": "0:05:31", "remaining_time": "0:05:09", "throughput": 2902.53, "total_tokens": 961848}
37
+ {"current_steps": 370, "total_steps": 696, "loss": 0.0823, "lr": 1.3512898656793283e-05, "epoch": 1.589247311827957, "percentage": 53.16, "elapsed_time": "0:05:40", "remaining_time": "0:05:00", "throughput": 2902.81, "total_tokens": 988600}
38
+ {"current_steps": 380, "total_steps": 696, "loss": 0.058, "lr": 1.2840910282888211e-05, "epoch": 1.632258064516129, "percentage": 54.6, "elapsed_time": "0:05:49", "remaining_time": "0:04:50", "throughput": 2902.64, "total_tokens": 1014840}
39
+ {"current_steps": 390, "total_steps": 696, "loss": 0.0705, "lr": 1.2173320141825232e-05, "epoch": 1.675268817204301, "percentage": 56.03, "elapsed_time": "0:05:58", "remaining_time": "0:04:41", "throughput": 2902.41, "total_tokens": 1040856}
40
+ {"current_steps": 400, "total_steps": 696, "loss": 0.0514, "lr": 1.1511488166385349e-05, "epoch": 1.718279569892473, "percentage": 57.47, "elapsed_time": "0:06:07", "remaining_time": "0:04:32", "throughput": 2902.76, "total_tokens": 1067544}
41
+ {"current_steps": 410, "total_steps": 696, "loss": 0.0834, "lr": 1.0856762559520605e-05, "epoch": 1.761290322580645, "percentage": 58.91, "elapsed_time": "0:06:17", "remaining_time": "0:04:22", "throughput": 2903.25, "total_tokens": 1094584}
42
+ {"current_steps": 420, "total_steps": 696, "loss": 0.0583, "lr": 1.0210477047960303e-05, "epoch": 1.8043010752688171, "percentage": 60.34, "elapsed_time": "0:06:26", "remaining_time": "0:04:13", "throughput": 2903.01, "total_tokens": 1120760}
43
+ {"current_steps": 430, "total_steps": 696, "loss": 0.0922, "lr": 9.573948165306438e-06, "epoch": 1.8473118279569891, "percentage": 61.78, "elapsed_time": "0:06:35", "remaining_time": "0:04:04", "throughput": 2902.51, "total_tokens": 1146776}
44
+ {"current_steps": 440, "total_steps": 696, "loss": 0.0633, "lr": 8.948472570152874e-06, "epoch": 1.8903225806451613, "percentage": 63.22, "elapsed_time": "0:06:44", "remaining_time": "0:03:55", "throughput": 2903.29, "total_tokens": 1174424}
45
+ {"current_steps": 450, "total_steps": 696, "loss": 0.0597, "lr": 8.33532440469145e-06, "epoch": 1.9333333333333333, "percentage": 64.66, "elapsed_time": "0:06:53", "remaining_time": "0:03:46", "throughput": 2903.81, "total_tokens": 1201048}
46
+ {"current_steps": 460, "total_steps": 696, "loss": 0.0491, "lr": 7.735752699185711e-06, "epoch": 1.9763440860215054, "percentage": 66.09, "elapsed_time": "0:07:02", "remaining_time": "0:03:36", "throughput": 2903.71, "total_tokens": 1227576}
47
+ {"current_steps": 470, "total_steps": 696, "loss": 0.0341, "lr": 7.150978827599619e-06, "epoch": 2.0172043010752687, "percentage": 67.53, "elapsed_time": "0:07:11", "remaining_time": "0:03:27", "throughput": 2902.63, "total_tokens": 1252160}
48
+ {"current_steps": 480, "total_steps": 696, "loss": 0.0373, "lr": 6.582194019564266e-06, "epoch": 2.0602150537634407, "percentage": 68.97, "elapsed_time": "0:07:20", "remaining_time": "0:03:18", "throughput": 2903.07, "total_tokens": 1279328}
49
+ {"current_steps": 490, "total_steps": 696, "loss": 0.0281, "lr": 6.0305569337509225e-06, "epoch": 2.1032258064516127, "percentage": 70.4, "elapsed_time": "0:07:29", "remaining_time": "0:03:09", "throughput": 2903.01, "total_tokens": 1306304}
50
+ {"current_steps": 500, "total_steps": 696, "loss": 0.0183, "lr": 5.497191297593647e-06, "epoch": 2.146236559139785, "percentage": 71.84, "elapsed_time": "0:07:39", "remaining_time": "0:03:00", "throughput": 2902.59, "total_tokens": 1333184}
51
+ {"current_steps": 510, "total_steps": 696, "loss": 0.0334, "lr": 4.98318361816957e-06, "epoch": 2.189247311827957, "percentage": 73.28, "elapsed_time": "0:07:48", "remaining_time": "0:02:50", "throughput": 2901.7, "total_tokens": 1359872}
52
+ {"current_steps": 520, "total_steps": 696, "loss": 0.0282, "lr": 4.4895809688998655e-06, "epoch": 2.232258064516129, "percentage": 74.71, "elapsed_time": "0:07:58", "remaining_time": "0:02:41", "throughput": 2901.24, "total_tokens": 1387328}
53
+ {"current_steps": 530, "total_steps": 696, "loss": 0.0562, "lr": 4.017388856580178e-06, "epoch": 2.275268817204301, "percentage": 76.15, "elapsed_time": "0:08:07", "remaining_time": "0:02:32", "throughput": 2901.79, "total_tokens": 1414816}
54
+ {"current_steps": 540, "total_steps": 696, "loss": 0.0243, "lr": 3.567569173085455e-06, "epoch": 2.318279569892473, "percentage": 77.59, "elapsed_time": "0:08:16", "remaining_time": "0:02:23", "throughput": 2902.13, "total_tokens": 1441504}
55
+ {"current_steps": 550, "total_steps": 696, "loss": 0.044, "lr": 3.1410382359217645e-06, "epoch": 2.361290322580645, "percentage": 79.02, "elapsed_time": "0:08:25", "remaining_time": "0:02:14", "throughput": 2902.37, "total_tokens": 1467680}
56
+ {"current_steps": 560, "total_steps": 696, "loss": 0.0551, "lr": 2.7386649216166233e-06, "epoch": 2.404301075268817, "percentage": 80.46, "elapsed_time": "0:08:34", "remaining_time": "0:02:05", "throughput": 2902.6, "total_tokens": 1494176}
57
+ {"current_steps": 570, "total_steps": 696, "loss": 0.0258, "lr": 2.361268895750264e-06, "epoch": 2.447311827956989, "percentage": 81.9, "elapsed_time": "0:08:43", "remaining_time": "0:01:55", "throughput": 2902.83, "total_tokens": 1520544}
58
+ {"current_steps": 580, "total_steps": 696, "loss": 0.0415, "lr": 2.0096189432334194e-06, "epoch": 2.490322580645161, "percentage": 83.33, "elapsed_time": "0:08:52", "remaining_time": "0:01:46", "throughput": 2903.34, "total_tokens": 1547264}
59
+ {"current_steps": 590, "total_steps": 696, "loss": 0.0375, "lr": 1.6844314022329676e-06, "epoch": 2.533333333333333, "percentage": 84.77, "elapsed_time": "0:09:02", "remaining_time": "0:01:37", "throughput": 2903.89, "total_tokens": 1573920}
60
+ {"current_steps": 600, "total_steps": 696, "loss": 0.0235, "lr": 1.3863687049356465e-06, "epoch": 2.576344086021505, "percentage": 86.21, "elapsed_time": "0:09:11", "remaining_time": "0:01:28", "throughput": 2904.68, "total_tokens": 1600640}
61
+ {"current_steps": 610, "total_steps": 696, "loss": 0.0354, "lr": 1.116038028122413e-06, "epoch": 2.6193548387096772, "percentage": 87.64, "elapsed_time": "0:09:19", "remaining_time": "0:01:18", "throughput": 2905.09, "total_tokens": 1626848}
62
+ {"current_steps": 620, "total_steps": 696, "loss": 0.0357, "lr": 8.7399005630238e-07, "epoch": 2.6623655913978492, "percentage": 89.08, "elapsed_time": "0:09:29", "remaining_time": "0:01:09", "throughput": 2905.48, "total_tokens": 1653408}
63
+ {"current_steps": 630, "total_steps": 696, "loss": 0.0512, "lr": 6.607178599258268e-07, "epoch": 2.7053763440860212, "percentage": 90.52, "elapsed_time": "0:09:38", "remaining_time": "0:01:00", "throughput": 2905.88, "total_tokens": 1679968}
64
+ {"current_steps": 640, "total_steps": 696, "loss": 0.0243, "lr": 4.766558909615504e-07, "epoch": 2.7483870967741937, "percentage": 91.95, "elapsed_time": "0:09:47", "remaining_time": "0:00:51", "throughput": 2906.53, "total_tokens": 1706944}
65
+ {"current_steps": 650, "total_steps": 696, "loss": 0.0367, "lr": 3.22179097884579e-07, "epoch": 2.7913978494623657, "percentage": 93.39, "elapsed_time": "0:09:56", "remaining_time": "0:00:42", "throughput": 2907.18, "total_tokens": 1733888}
66
+ {"current_steps": 660, "total_steps": 696, "loss": 0.0377, "lr": 1.9760216187710788e-07, "epoch": 2.8344086021505377, "percentage": 94.83, "elapsed_time": "0:10:05", "remaining_time": "0:00:33", "throughput": 2907.66, "total_tokens": 1760448}
67
+ {"current_steps": 670, "total_steps": 696, "loss": 0.0471, "lr": 1.0317885579858522e-07, "epoch": 2.8774193548387097, "percentage": 96.26, "elapsed_time": "0:10:14", "remaining_time": "0:00:23", "throughput": 2907.89, "total_tokens": 1787072}
68
+ {"current_steps": 680, "total_steps": 696, "loss": 0.0222, "lr": 3.910152723075322e-08, "epoch": 2.9204301075268817, "percentage": 97.7, "elapsed_time": "0:10:23", "remaining_time": "0:00:14", "throughput": 2908.16, "total_tokens": 1813632}
69
+ {"current_steps": 690, "total_steps": 696, "loss": 0.0454, "lr": 5.50070665074065e-09, "epoch": 2.9634408602150537, "percentage": 99.14, "elapsed_time": "0:10:32", "remaining_time": "0:00:05", "throughput": 2908.39, "total_tokens": 1840384}
70
+ {"current_steps": 696, "total_steps": 696, "epoch": 2.989247311827957, "percentage": 100.0, "elapsed_time": "0:10:38", "remaining_time": "0:00:00", "throughput": 2905.07, "total_tokens": 1855776}
trainer_state.json ADDED
@@ -0,0 +1,595 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.989247311827957,
5
+ "eval_steps": 500,
6
+ "global_step": 696,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.043010752688172046,
13
+ "grad_norm": 3.3071749210357666,
14
+ "learning_rate": 2.9984721919587606e-05,
15
+ "loss": 0.9267,
16
+ "num_input_tokens_seen": 26208,
17
+ "step": 10
18
+ },
19
+ {
20
+ "epoch": 0.08602150537634409,
21
+ "grad_norm": 1.9867345094680786,
22
+ "learning_rate": 2.9938918800982563e-05,
23
+ "loss": 0.1561,
24
+ "num_input_tokens_seen": 53152,
25
+ "step": 20
26
+ },
27
+ {
28
+ "epoch": 0.12903225806451613,
29
+ "grad_norm": 10.124670028686523,
30
+ "learning_rate": 2.9862683948682103e-05,
31
+ "loss": 0.1764,
32
+ "num_input_tokens_seen": 79776,
33
+ "step": 30
34
+ },
35
+ {
36
+ "epoch": 0.17204301075268819,
37
+ "grad_norm": 2.0668740272521973,
38
+ "learning_rate": 2.975617265898004e-05,
39
+ "loss": 0.1505,
40
+ "num_input_tokens_seen": 106496,
41
+ "step": 40
42
+ },
43
+ {
44
+ "epoch": 0.21505376344086022,
45
+ "grad_norm": 2.0054426193237305,
46
+ "learning_rate": 2.961960190361624e-05,
47
+ "loss": 0.1615,
48
+ "num_input_tokens_seen": 133728,
49
+ "step": 50
50
+ },
51
+ {
52
+ "epoch": 0.25806451612903225,
53
+ "grad_norm": 2.5156946182250977,
54
+ "learning_rate": 2.9453249887788343e-05,
55
+ "loss": 0.1369,
56
+ "num_input_tokens_seen": 159936,
57
+ "step": 60
58
+ },
59
+ {
60
+ "epoch": 0.3010752688172043,
61
+ "grad_norm": 1.1180003881454468,
62
+ "learning_rate": 2.925745548342631e-05,
63
+ "loss": 0.1253,
64
+ "num_input_tokens_seen": 186240,
65
+ "step": 70
66
+ },
67
+ {
68
+ "epoch": 0.34408602150537637,
69
+ "grad_norm": 13.929261207580566,
70
+ "learning_rate": 2.9032617538884018e-05,
71
+ "loss": 0.1479,
72
+ "num_input_tokens_seen": 212832,
73
+ "step": 80
74
+ },
75
+ {
76
+ "epoch": 0.3870967741935484,
77
+ "grad_norm": 2.071075439453125,
78
+ "learning_rate": 2.877919406645433e-05,
79
+ "loss": 0.1102,
80
+ "num_input_tokens_seen": 239648,
81
+ "step": 90
82
+ },
83
+ {
84
+ "epoch": 0.43010752688172044,
85
+ "grad_norm": 2.170992374420166,
86
+ "learning_rate": 2.84977013093626e-05,
87
+ "loss": 0.099,
88
+ "num_input_tokens_seen": 265888,
89
+ "step": 100
90
+ },
91
+ {
92
+ "epoch": 0.4731182795698925,
93
+ "grad_norm": 5.617093086242676,
94
+ "learning_rate": 2.818871269013928e-05,
95
+ "loss": 0.107,
96
+ "num_input_tokens_seen": 292480,
97
+ "step": 110
98
+ },
99
+ {
100
+ "epoch": 0.5161290322580645,
101
+ "grad_norm": 6.3603010177612305,
102
+ "learning_rate": 2.7852857642513838e-05,
103
+ "loss": 0.1183,
104
+ "num_input_tokens_seen": 318784,
105
+ "step": 120
106
+ },
107
+ {
108
+ "epoch": 0.5591397849462365,
109
+ "grad_norm": 3.8436505794525146,
110
+ "learning_rate": 2.7490820329209546e-05,
111
+ "loss": 0.1097,
112
+ "num_input_tokens_seen": 346016,
113
+ "step": 130
114
+ },
115
+ {
116
+ "epoch": 0.6021505376344086,
117
+ "grad_norm": 1.736432671546936,
118
+ "learning_rate": 2.7103338248251055e-05,
119
+ "loss": 0.0946,
120
+ "num_input_tokens_seen": 372384,
121
+ "step": 140
122
+ },
123
+ {
124
+ "epoch": 0.6451612903225806,
125
+ "grad_norm": 8.300951957702637,
126
+ "learning_rate": 2.6691200730623874e-05,
127
+ "loss": 0.1251,
128
+ "num_input_tokens_seen": 399328,
129
+ "step": 150
130
+ },
131
+ {
132
+ "epoch": 0.6881720430107527,
133
+ "grad_norm": 2.540724277496338,
134
+ "learning_rate": 2.6255247332346036e-05,
135
+ "loss": 0.1069,
136
+ "num_input_tokens_seen": 426048,
137
+ "step": 160
138
+ },
139
+ {
140
+ "epoch": 0.7311827956989247,
141
+ "grad_norm": 1.967483639717102,
142
+ "learning_rate": 2.5796366124227532e-05,
143
+ "loss": 0.0904,
144
+ "num_input_tokens_seen": 452640,
145
+ "step": 170
146
+ },
147
+ {
148
+ "epoch": 0.7741935483870968,
149
+ "grad_norm": 5.206757545471191,
150
+ "learning_rate": 2.531549188280135e-05,
151
+ "loss": 0.1273,
152
+ "num_input_tokens_seen": 479808,
153
+ "step": 180
154
+ },
155
+ {
156
+ "epoch": 0.8172043010752689,
157
+ "grad_norm": 3.0387344360351562,
158
+ "learning_rate": 2.481360418611132e-05,
159
+ "loss": 0.1206,
160
+ "num_input_tokens_seen": 506176,
161
+ "step": 190
162
+ },
163
+ {
164
+ "epoch": 0.8602150537634409,
165
+ "grad_norm": 2.0281670093536377,
166
+ "learning_rate": 2.4291725418235848e-05,
167
+ "loss": 0.103,
168
+ "num_input_tokens_seen": 533216,
169
+ "step": 200
170
+ },
171
+ {
172
+ "epoch": 0.9032258064516129,
173
+ "grad_norm": 2.5650763511657715,
174
+ "learning_rate": 2.3750918686612414e-05,
175
+ "loss": 0.0696,
176
+ "num_input_tokens_seen": 561056,
177
+ "step": 210
178
+ },
179
+ {
180
+ "epoch": 0.946236559139785,
181
+ "grad_norm": 8.955713272094727,
182
+ "learning_rate": 2.3192285656405456e-05,
183
+ "loss": 0.0822,
184
+ "num_input_tokens_seen": 588160,
185
+ "step": 220
186
+ },
187
+ {
188
+ "epoch": 0.989247311827957,
189
+ "grad_norm": 3.034013032913208,
190
+ "learning_rate": 2.2616964306329183e-05,
191
+ "loss": 0.0913,
192
+ "num_input_tokens_seen": 615168,
193
+ "step": 230
194
+ },
195
+ {
196
+ "epoch": 1.0301075268817204,
197
+ "grad_norm": 2.60020112991333,
198
+ "learning_rate": 2.2026126610496852e-05,
199
+ "loss": 0.0735,
200
+ "num_input_tokens_seen": 639864,
201
+ "step": 240
202
+ },
203
+ {
204
+ "epoch": 1.0731182795698926,
205
+ "grad_norm": 4.891764163970947,
206
+ "learning_rate": 2.1420976151018813e-05,
207
+ "loss": 0.0752,
208
+ "num_input_tokens_seen": 667224,
209
+ "step": 250
210
+ },
211
+ {
212
+ "epoch": 1.1161290322580646,
213
+ "grad_norm": 1.1149002313613892,
214
+ "learning_rate": 2.0802745666212592e-05,
215
+ "loss": 0.0588,
216
+ "num_input_tokens_seen": 693848,
217
+ "step": 260
218
+ },
219
+ {
220
+ "epoch": 1.1591397849462366,
221
+ "grad_norm": 3.1601271629333496,
222
+ "learning_rate": 2.0172694539419557e-05,
223
+ "loss": 0.0924,
224
+ "num_input_tokens_seen": 720568,
225
+ "step": 270
226
+ },
227
+ {
228
+ "epoch": 1.2021505376344086,
229
+ "grad_norm": 3.555192470550537,
230
+ "learning_rate": 1.953210623354359e-05,
231
+ "loss": 0.062,
232
+ "num_input_tokens_seen": 746872,
233
+ "step": 280
234
+ },
235
+ {
236
+ "epoch": 1.2451612903225806,
237
+ "grad_norm": 2.235698699951172,
238
+ "learning_rate": 1.888228567653781e-05,
239
+ "loss": 0.0621,
240
+ "num_input_tokens_seen": 773720,
241
+ "step": 290
242
+ },
243
+ {
244
+ "epoch": 1.2881720430107526,
245
+ "grad_norm": 2.9058539867401123,
246
+ "learning_rate": 1.8224556603165363e-05,
247
+ "loss": 0.075,
248
+ "num_input_tokens_seen": 801464,
249
+ "step": 300
250
+ },
251
+ {
252
+ "epoch": 1.3311827956989246,
253
+ "grad_norm": 3.204787015914917,
254
+ "learning_rate": 1.7560258858449248e-05,
255
+ "loss": 0.0858,
256
+ "num_input_tokens_seen": 829144,
257
+ "step": 310
258
+ },
259
+ {
260
+ "epoch": 1.3741935483870968,
261
+ "grad_norm": 1.0885004997253418,
262
+ "learning_rate": 1.689074566830434e-05,
263
+ "loss": 0.0697,
264
+ "num_input_tokens_seen": 855672,
265
+ "step": 320
266
+ },
267
+ {
268
+ "epoch": 1.4172043010752688,
269
+ "grad_norm": 3.0750925540924072,
270
+ "learning_rate": 1.621738088291147e-05,
271
+ "loss": 0.0827,
272
+ "num_input_tokens_seen": 882424,
273
+ "step": 330
274
+ },
275
+ {
276
+ "epoch": 1.4602150537634409,
277
+ "grad_norm": 2.689297914505005,
278
+ "learning_rate": 1.5541536198449044e-05,
279
+ "loss": 0.0651,
280
+ "num_input_tokens_seen": 908792,
281
+ "step": 340
282
+ },
283
+ {
284
+ "epoch": 1.5032258064516129,
285
+ "grad_norm": 2.297851324081421,
286
+ "learning_rate": 1.4864588362841808e-05,
287
+ "loss": 0.0607,
288
+ "num_input_tokens_seen": 935672,
289
+ "step": 350
290
+ },
291
+ {
292
+ "epoch": 1.546236559139785,
293
+ "grad_norm": 2.712674140930176,
294
+ "learning_rate": 1.4187916371218739e-05,
295
+ "loss": 0.056,
296
+ "num_input_tokens_seen": 961848,
297
+ "step": 360
298
+ },
299
+ {
300
+ "epoch": 1.589247311827957,
301
+ "grad_norm": 0.8086225986480713,
302
+ "learning_rate": 1.3512898656793283e-05,
303
+ "loss": 0.0823,
304
+ "num_input_tokens_seen": 988600,
305
+ "step": 370
306
+ },
307
+ {
308
+ "epoch": 1.632258064516129,
309
+ "grad_norm": 2.166210174560547,
310
+ "learning_rate": 1.2840910282888211e-05,
311
+ "loss": 0.058,
312
+ "num_input_tokens_seen": 1014840,
313
+ "step": 380
314
+ },
315
+ {
316
+ "epoch": 1.675268817204301,
317
+ "grad_norm": 5.169621467590332,
318
+ "learning_rate": 1.2173320141825232e-05,
319
+ "loss": 0.0705,
320
+ "num_input_tokens_seen": 1040856,
321
+ "step": 390
322
+ },
323
+ {
324
+ "epoch": 1.718279569892473,
325
+ "grad_norm": 1.8176069259643555,
326
+ "learning_rate": 1.1511488166385349e-05,
327
+ "loss": 0.0514,
328
+ "num_input_tokens_seen": 1067544,
329
+ "step": 400
330
+ },
331
+ {
332
+ "epoch": 1.761290322580645,
333
+ "grad_norm": 3.424694776535034,
334
+ "learning_rate": 1.0856762559520605e-05,
335
+ "loss": 0.0834,
336
+ "num_input_tokens_seen": 1094584,
337
+ "step": 410
338
+ },
339
+ {
340
+ "epoch": 1.8043010752688171,
341
+ "grad_norm": 1.8838876485824585,
342
+ "learning_rate": 1.0210477047960303e-05,
343
+ "loss": 0.0583,
344
+ "num_input_tokens_seen": 1120760,
345
+ "step": 420
346
+ },
347
+ {
348
+ "epoch": 1.8473118279569891,
349
+ "grad_norm": 3.7757434844970703,
350
+ "learning_rate": 9.573948165306438e-06,
351
+ "loss": 0.0922,
352
+ "num_input_tokens_seen": 1146776,
353
+ "step": 430
354
+ },
355
+ {
356
+ "epoch": 1.8903225806451613,
357
+ "grad_norm": 3.0619328022003174,
358
+ "learning_rate": 8.948472570152874e-06,
359
+ "loss": 0.0633,
360
+ "num_input_tokens_seen": 1174424,
361
+ "step": 440
362
+ },
363
+ {
364
+ "epoch": 1.9333333333333333,
365
+ "grad_norm": 2.5175821781158447,
366
+ "learning_rate": 8.33532440469145e-06,
367
+ "loss": 0.0597,
368
+ "num_input_tokens_seen": 1201048,
369
+ "step": 450
370
+ },
371
+ {
372
+ "epoch": 1.9763440860215054,
373
+ "grad_norm": 3.6232197284698486,
374
+ "learning_rate": 7.735752699185711e-06,
375
+ "loss": 0.0491,
376
+ "num_input_tokens_seen": 1227576,
377
+ "step": 460
378
+ },
379
+ {
380
+ "epoch": 2.0172043010752687,
381
+ "grad_norm": 2.8846399784088135,
382
+ "learning_rate": 7.150978827599619e-06,
383
+ "loss": 0.0341,
384
+ "num_input_tokens_seen": 1252160,
385
+ "step": 470
386
+ },
387
+ {
388
+ "epoch": 2.0602150537634407,
389
+ "grad_norm": 2.196216106414795,
390
+ "learning_rate": 6.582194019564266e-06,
391
+ "loss": 0.0373,
392
+ "num_input_tokens_seen": 1279328,
393
+ "step": 480
394
+ },
395
+ {
396
+ "epoch": 2.1032258064516127,
397
+ "grad_norm": 1.4616115093231201,
398
+ "learning_rate": 6.0305569337509225e-06,
399
+ "loss": 0.0281,
400
+ "num_input_tokens_seen": 1306304,
401
+ "step": 490
402
+ },
403
+ {
404
+ "epoch": 2.146236559139785,
405
+ "grad_norm": 0.17581823468208313,
406
+ "learning_rate": 5.497191297593647e-06,
407
+ "loss": 0.0183,
408
+ "num_input_tokens_seen": 1333184,
409
+ "step": 500
410
+ },
411
+ {
412
+ "epoch": 2.189247311827957,
413
+ "grad_norm": 3.8919403553009033,
414
+ "learning_rate": 4.98318361816957e-06,
415
+ "loss": 0.0334,
416
+ "num_input_tokens_seen": 1359872,
417
+ "step": 510
418
+ },
419
+ {
420
+ "epoch": 2.232258064516129,
421
+ "grad_norm": 1.3041765689849854,
422
+ "learning_rate": 4.4895809688998655e-06,
423
+ "loss": 0.0282,
424
+ "num_input_tokens_seen": 1387328,
425
+ "step": 520
426
+ },
427
+ {
428
+ "epoch": 2.275268817204301,
429
+ "grad_norm": 1.669753074645996,
430
+ "learning_rate": 4.017388856580178e-06,
431
+ "loss": 0.0562,
432
+ "num_input_tokens_seen": 1414816,
433
+ "step": 530
434
+ },
435
+ {
436
+ "epoch": 2.318279569892473,
437
+ "grad_norm": 0.28061679005622864,
438
+ "learning_rate": 3.567569173085455e-06,
439
+ "loss": 0.0243,
440
+ "num_input_tokens_seen": 1441504,
441
+ "step": 540
442
+ },
443
+ {
444
+ "epoch": 2.361290322580645,
445
+ "grad_norm": 2.324270009994507,
446
+ "learning_rate": 3.1410382359217645e-06,
447
+ "loss": 0.044,
448
+ "num_input_tokens_seen": 1467680,
449
+ "step": 550
450
+ },
451
+ {
452
+ "epoch": 2.404301075268817,
453
+ "grad_norm": 2.708113670349121,
454
+ "learning_rate": 2.7386649216166233e-06,
455
+ "loss": 0.0551,
456
+ "num_input_tokens_seen": 1494176,
457
+ "step": 560
458
+ },
459
+ {
460
+ "epoch": 2.447311827956989,
461
+ "grad_norm": 3.16683030128479,
462
+ "learning_rate": 2.361268895750264e-06,
463
+ "loss": 0.0258,
464
+ "num_input_tokens_seen": 1520544,
465
+ "step": 570
466
+ },
467
+ {
468
+ "epoch": 2.490322580645161,
469
+ "grad_norm": 6.040332794189453,
470
+ "learning_rate": 2.0096189432334194e-06,
471
+ "loss": 0.0415,
472
+ "num_input_tokens_seen": 1547264,
473
+ "step": 580
474
+ },
475
+ {
476
+ "epoch": 2.533333333333333,
477
+ "grad_norm": 5.078160285949707,
478
+ "learning_rate": 1.6844314022329676e-06,
479
+ "loss": 0.0375,
480
+ "num_input_tokens_seen": 1573920,
481
+ "step": 590
482
+ },
483
+ {
484
+ "epoch": 2.576344086021505,
485
+ "grad_norm": 4.950022220611572,
486
+ "learning_rate": 1.3863687049356465e-06,
487
+ "loss": 0.0235,
488
+ "num_input_tokens_seen": 1600640,
489
+ "step": 600
490
+ },
491
+ {
492
+ "epoch": 2.6193548387096772,
493
+ "grad_norm": 1.7687643766403198,
494
+ "learning_rate": 1.116038028122413e-06,
495
+ "loss": 0.0354,
496
+ "num_input_tokens_seen": 1626848,
497
+ "step": 610
498
+ },
499
+ {
500
+ "epoch": 2.6623655913978492,
501
+ "grad_norm": 3.893580913543701,
502
+ "learning_rate": 8.7399005630238e-07,
503
+ "loss": 0.0357,
504
+ "num_input_tokens_seen": 1653408,
505
+ "step": 620
506
+ },
507
+ {
508
+ "epoch": 2.7053763440860212,
509
+ "grad_norm": 2.830453395843506,
510
+ "learning_rate": 6.607178599258268e-07,
511
+ "loss": 0.0512,
512
+ "num_input_tokens_seen": 1679968,
513
+ "step": 630
514
+ },
515
+ {
516
+ "epoch": 2.7483870967741937,
517
+ "grad_norm": 3.638772487640381,
518
+ "learning_rate": 4.766558909615504e-07,
519
+ "loss": 0.0243,
520
+ "num_input_tokens_seen": 1706944,
521
+ "step": 640
522
+ },
523
+ {
524
+ "epoch": 2.7913978494623657,
525
+ "grad_norm": 1.229244589805603,
526
+ "learning_rate": 3.22179097884579e-07,
527
+ "loss": 0.0367,
528
+ "num_input_tokens_seen": 1733888,
529
+ "step": 650
530
+ },
531
+ {
532
+ "epoch": 2.8344086021505377,
533
+ "grad_norm": 2.9207515716552734,
534
+ "learning_rate": 1.9760216187710788e-07,
535
+ "loss": 0.0377,
536
+ "num_input_tokens_seen": 1760448,
537
+ "step": 660
538
+ },
539
+ {
540
+ "epoch": 2.8774193548387097,
541
+ "grad_norm": 3.560971975326538,
542
+ "learning_rate": 1.0317885579858522e-07,
543
+ "loss": 0.0471,
544
+ "num_input_tokens_seen": 1787072,
545
+ "step": 670
546
+ },
547
+ {
548
+ "epoch": 2.9204301075268817,
549
+ "grad_norm": 0.2031625360250473,
550
+ "learning_rate": 3.910152723075322e-08,
551
+ "loss": 0.0222,
552
+ "num_input_tokens_seen": 1813632,
553
+ "step": 680
554
+ },
555
+ {
556
+ "epoch": 2.9634408602150537,
557
+ "grad_norm": 4.158380508422852,
558
+ "learning_rate": 5.50070665074065e-09,
559
+ "loss": 0.0454,
560
+ "num_input_tokens_seen": 1840384,
561
+ "step": 690
562
+ },
563
+ {
564
+ "epoch": 2.989247311827957,
565
+ "num_input_tokens_seen": 1855776,
566
+ "step": 696,
567
+ "total_flos": 7.944329136203366e+16,
568
+ "train_loss": 0.08561765917459097,
569
+ "train_runtime": 638.811,
570
+ "train_samples_per_second": 8.721,
571
+ "train_steps_per_second": 1.09
572
+ }
573
+ ],
574
+ "logging_steps": 10,
575
+ "max_steps": 696,
576
+ "num_input_tokens_seen": 1855776,
577
+ "num_train_epochs": 3,
578
+ "save_steps": 1000,
579
+ "stateful_callbacks": {
580
+ "TrainerControl": {
581
+ "args": {
582
+ "should_epoch_stop": false,
583
+ "should_evaluate": false,
584
+ "should_log": false,
585
+ "should_save": true,
586
+ "should_training_stop": true
587
+ },
588
+ "attributes": {}
589
+ }
590
+ },
591
+ "total_flos": 7.944329136203366e+16,
592
+ "train_batch_size": 4,
593
+ "trial_name": null,
594
+ "trial_params": null
595
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c0f50151d95e9d064572887223a75b34b8d863adcbd94f03a50070eddf5f434
3
+ size 5688
training_args.yaml ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ bf16: true
2
+ cutoff_len: 2048
3
+ dataset: en_train_part_1,en_train_part_3
4
+ dataset_dir: data
5
+ ddp_timeout: 180000000
6
+ do_train: true
7
+ double_quantization: true
8
+ finetuning_type: lora
9
+ flash_attn: auto
10
+ gradient_accumulation_steps: 2
11
+ include_num_input_tokens_seen: true
12
+ learning_rate: 3.0e-05
13
+ logging_steps: 10
14
+ lora_alpha: 16
15
+ lora_dropout: 0
16
+ lora_rank: 8
17
+ lora_target: all
18
+ lr_scheduler_type: cosine
19
+ max_grad_norm: 1.0
20
+ max_samples: 1858
21
+ model_name_or_path: mistralai/Mistral-7B-Instruct-v0.3
22
+ num_train_epochs: 3.0
23
+ optim: adamw_torch
24
+ output_dir: saves/Mistral-7B-Instruct-v0.3/lora/2_part
25
+ packing: false
26
+ per_device_train_batch_size: 4
27
+ plot_loss: true
28
+ preprocessing_num_workers: 16
29
+ quantization_bit: 4
30
+ quantization_method: bitsandbytes
31
+ report_to: none
32
+ save_steps: 1000
33
+ stage: sft
34
+ template: alpaca
35
+ trust_remote_code: true
36
+ use_unsloth: true
37
+ warmup_steps: 0
training_loss.png ADDED