Hugging Face's logo

Basha738
/
outputs

PEFT
TensorBoard
Safetensors
trl
sft
Generated from Trainer
Model card Files
xet
Metrics Community
Basha738 commited on
Commit
bb9c6e2
·
verified ·
1 Parent(s): abae533

llama2-supervised-ft-5epochs

Browse files
Files changed (14) hide show
  1. README.md +83 -0
  2. adapter_config.json +32 -0
  3. adapter_model.safetensors +3 -0
  4. added_tokens.json +3 -0
  5. runs/Feb08_03-00-07_7fcdef0775c2/events.out.tfevents.1707361209.7fcdef0775c2.412.0 +3 -0
  6. runs/Feb08_03-16-29_7fcdef0775c2/events.out.tfevents.1707362191.7fcdef0775c2.5984.0 +3 -0
  7. runs/Feb08_03-20-05_7fcdef0775c2/events.out.tfevents.1707362407.7fcdef0775c2.6851.0 +3 -0
  8. runs/Feb08_05-13-45_7fcdef0775c2/events.out.tfevents.1707369227.7fcdef0775c2.33338.0 +3 -0
  9. runs/Feb08_05-29-54_7fcdef0775c2/events.out.tfevents.1707370195.7fcdef0775c2.40218.0 +3 -0
  10. special_tokens_map.json +30 -0
  11. tokenizer.json +0 -0
  12. tokenizer.model +3 -0
  13. tokenizer_config.json +49 -0
  14. training_args.bin +3 -0
README.md ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ tags:
4
+ - trl
5
+ - sft
6
+ - generated_from_trainer
7
+ base_model: Trelis/Llama-2-7b-chat-hf-sharded-bf16
8
+ model-index:
9
+ - name: outputs
10
+ results: []
11
+ ---
12
+
13
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
14
+ should probably proofread and complete it, then remove this comment. -->
15
+
16
+ # outputs
17
+
18
+ This model is a fine-tuned version of [Trelis/Llama-2-7b-chat-hf-sharded-bf16](https://huggingface.co/Trelis/Llama-2-7b-chat-hf-sharded-bf16) on the None dataset.
19
+ It achieves the following results on the evaluation set:
20
+ - Loss: 2.1986
21
+
22
+ ## Model description
23
+
24
+ More information needed
25
+
26
+ ## Intended uses & limitations
27
+
28
+ More information needed
29
+
30
+ ## Training and evaluation data
31
+
32
+ More information needed
33
+
34
+ ## Training procedure
35
+
36
+ ### Training hyperparameters
37
+
38
+ The following hyperparameters were used during training:
39
+ - learning_rate: 5e-06
40
+ - train_batch_size: 1
41
+ - eval_batch_size: 1
42
+ - seed: 42
43
+ - gradient_accumulation_steps: 4
44
+ - total_train_batch_size: 4
45
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
46
+ - lr_scheduler_type: linear
47
+ - num_epochs: 5
48
+ - mixed_precision_training: Native AMP
49
+
50
+ ### Training results
51
+
52
+ | Training Loss | Epoch | Step | Validation Loss |
53
+ |:-------------:|:-----:|:----:|:---------------:|
54
+ | 2.591 | 0.23 | 4 | 2.3712 |
55
+ | 2.6663 | 0.46 | 8 | 2.3530 |
56
+ | 2.4782 | 0.69 | 12 | 2.3351 |
57
+ | 2.2831 | 0.91 | 16 | 2.3186 |
58
+ | 2.4023 | 1.14 | 20 | 2.3037 |
59
+ | 2.2617 | 1.37 | 24 | 2.2902 |
60
+ | 2.2799 | 1.6 | 28 | 2.2779 |
61
+ | 2.3633 | 1.83 | 32 | 2.2665 |
62
+ | 2.2249 | 2.06 | 36 | 2.2560 |
63
+ | 2.5186 | 2.29 | 40 | 2.2464 |
64
+ | 2.4234 | 2.51 | 44 | 2.2379 |
65
+ | 2.26 | 2.74 | 48 | 2.2303 |
66
+ | 2.2937 | 2.97 | 52 | 2.2237 |
67
+ | 2.3681 | 3.2 | 56 | 2.2179 |
68
+ | 2.095 | 3.43 | 60 | 2.2129 |
69
+ | 2.1456 | 3.66 | 64 | 2.2087 |
70
+ | 2.4307 | 3.89 | 68 | 2.2053 |
71
+ | 2.2769 | 4.11 | 72 | 2.2025 |
72
+ | 2.3649 | 4.34 | 76 | 2.2005 |
73
+ | 2.3577 | 4.57 | 80 | 2.1992 |
74
+ | 2.1575 | 4.8 | 84 | 2.1986 |
75
+
76
+
77
+ ### Framework versions
78
+
79
+ - PEFT 0.8.2
80
+ - Transformers 4.38.0.dev0
81
+ - Pytorch 2.1.0+cu121
82
+ - Datasets 2.16.1
83
+ - Tokenizers 0.15.1
adapter_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "Trelis/Llama-2-7b-chat-hf-sharded-bf16",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "loftq_config": {},
12
+ "lora_alpha": 32,
13
+ "lora_dropout": 0.05,
14
+ "megatron_config": null,
15
+ "megatron_core": "megatron.core",
16
+ "modules_to_save": null,
17
+ "peft_type": "LORA",
18
+ "r": 16,
19
+ "rank_pattern": {},
20
+ "revision": null,
21
+ "target_modules": [
22
+ "gate_proj",
23
+ "o_proj",
24
+ "down_proj",
25
+ "up_proj",
26
+ "q_proj",
27
+ "k_proj",
28
+ "v_proj"
29
+ ],
30
+ "task_type": "CAUSAL_LM",
31
+ "use_rslora": false
32
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dfd60a3d7392d23dd321852d451e4f83d01000f6df1f6fb40002ed5071608111
3
+ size 159967880
added_tokens.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "[PAD]": 32000
3
+ }
runs/Feb08_03-00-07_7fcdef0775c2/events.out.tfevents.1707361209.7fcdef0775c2.412.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09f3c8628765c3ebe4557a8990e9205527af1e8b84e973c21f88118729313c6d
3
+ size 12058
runs/Feb08_03-16-29_7fcdef0775c2/events.out.tfevents.1707362191.7fcdef0775c2.5984.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1eb54ec22f85caf0ab39857b77230bb0bd5da18fd7080cfde9cfacf0851f3064
3
+ size 5060
runs/Feb08_03-20-05_7fcdef0775c2/events.out.tfevents.1707362407.7fcdef0775c2.6851.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75427bdc34b9f43cdcbc52df436c476c946bf896c9e74471b5afa7a5241f62f0
3
+ size 12058
runs/Feb08_05-13-45_7fcdef0775c2/events.out.tfevents.1707369227.7fcdef0775c2.33338.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ff327f6f57abcf0a52f504f68fc74abecebebbd793c7f52d3bd8f2569b9d2b8
3
+ size 10659
runs/Feb08_05-29-54_7fcdef0775c2/events.out.tfevents.1707370195.7fcdef0775c2.40218.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cebec7ca3ec0e50ad2d7034f9d36c466663487742c19a824f20567fc51b48372
3
+ size 23929
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "[PAD]",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "32000": {
30
+ "content": "[PAD]",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ }
37
+ },
38
+ "bos_token": "<s>",
39
+ "clean_up_tokenization_spaces": false,
40
+ "eos_token": "</s>",
41
+ "legacy": false,
42
+ "model_max_length": 1000000000000000019884624838656,
43
+ "pad_token": "[PAD]",
44
+ "sp_model_kwargs": {},
45
+ "tokenizer_class": "LlamaTokenizer",
46
+ "unk_token": "<unk>",
47
+ "use_default_system_prompt": false,
48
+ "use_fast": false
49
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7a347c11818060515daa22efc2808605735815351a1166cecf871f4c53ff334
3
+ size 4728