Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- checkpoint-100/config.json +34 -0
- checkpoint-100/generation_config.json +10 -0
- checkpoint-100/model.safetensors +3 -0
- checkpoint-100/optimizer.pt +3 -0
- checkpoint-100/rng_state.pth +3 -0
- checkpoint-100/scheduler.pt +3 -0
- checkpoint-100/tokenizer.json +87 -0
- checkpoint-100/tokenizer_config.json +8 -0
- checkpoint-100/trainer_state.json +104 -0
- checkpoint-100/training_args.bin +3 -0
- checkpoint-150/config.json +34 -0
- checkpoint-150/generation_config.json +10 -0
- checkpoint-150/model.safetensors +3 -0
- checkpoint-150/optimizer.pt +3 -0
- checkpoint-150/rng_state.pth +3 -0
- checkpoint-150/scheduler.pt +3 -0
- checkpoint-150/tokenizer.json +87 -0
- checkpoint-150/tokenizer_config.json +8 -0
- checkpoint-150/trainer_state.json +139 -0
- checkpoint-150/training_args.bin +3 -0
- checkpoint-200/config.json +34 -0
- checkpoint-200/generation_config.json +10 -0
- checkpoint-200/model.safetensors +3 -0
- checkpoint-200/optimizer.pt +3 -0
- checkpoint-200/rng_state.pth +3 -0
- checkpoint-200/scheduler.pt +3 -0
- checkpoint-200/tokenizer.json +87 -0
- checkpoint-200/tokenizer_config.json +8 -0
- checkpoint-200/trainer_state.json +174 -0
- checkpoint-200/training_args.bin +3 -0
- checkpoint-250/config.json +34 -0
- checkpoint-250/generation_config.json +10 -0
- checkpoint-250/model.safetensors +3 -0
- checkpoint-250/optimizer.pt +3 -0
- checkpoint-250/rng_state.pth +3 -0
- checkpoint-250/scheduler.pt +3 -0
- checkpoint-250/tokenizer.json +87 -0
- checkpoint-250/tokenizer_config.json +8 -0
- checkpoint-250/trainer_state.json +209 -0
- checkpoint-250/training_args.bin +3 -0
- checkpoint-300/config.json +34 -0
- checkpoint-300/generation_config.json +10 -0
- checkpoint-300/model.safetensors +3 -0
- checkpoint-300/optimizer.pt +3 -0
- checkpoint-300/rng_state.pth +3 -0
- checkpoint-300/scheduler.pt +3 -0
- checkpoint-300/tokenizer.json +87 -0
- checkpoint-300/tokenizer_config.json +8 -0
- checkpoint-300/trainer_state.json +244 -0
- checkpoint-300/training_args.bin +3 -0
checkpoint-100/config.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.0,
|
| 8 |
+
"bos_token_id": 0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.0,
|
| 11 |
+
"eos_token_id": 0,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 1e-05,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_embd": 1,
|
| 16 |
+
"n_head": 1,
|
| 17 |
+
"n_inner": 1,
|
| 18 |
+
"n_layer": 1,
|
| 19 |
+
"n_positions": 4,
|
| 20 |
+
"pad_token_id": 1,
|
| 21 |
+
"reorder_and_upcast_attn": false,
|
| 22 |
+
"resid_pdrop": 0.0,
|
| 23 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 24 |
+
"scale_attn_weights": true,
|
| 25 |
+
"summary_activation": null,
|
| 26 |
+
"summary_first_dropout": 0.1,
|
| 27 |
+
"summary_proj_to_labels": true,
|
| 28 |
+
"summary_type": "cls_index",
|
| 29 |
+
"summary_use_proj": true,
|
| 30 |
+
"tie_word_embeddings": true,
|
| 31 |
+
"transformers_version": "5.12.0",
|
| 32 |
+
"use_cache": false,
|
| 33 |
+
"vocab_size": 5
|
| 34 |
+
}
|
checkpoint-100/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 1,
|
| 8 |
+
"transformers_version": "5.12.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
checkpoint-100/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f927f8d4885457b7e4a078b38df051b0a0d42821a1eb2536418a7f10cbb3523e
|
| 3 |
+
size 1476
|
checkpoint-100/optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:53a1e772436ea3d9ca60697fd61cb7a423cb96b621a51eb795cfa5a73b56f206
|
| 3 |
+
size 13823
|
checkpoint-100/rng_state.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:656c952bc98f1ba6483f4b602ab79d3a7eb64d231d7b2b6ae517f06e7e137155
|
| 3 |
+
size 14455
|
checkpoint-100/scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:020d4bf8d4465c2a1ed676a5debf1d4d7d4c2928299260126c8103e7b8f781f1
|
| 3 |
+
size 1465
|
checkpoint-100/tokenizer.json
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 4,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
+
"padding": null,
|
| 10 |
+
"added_tokens": [
|
| 11 |
+
{
|
| 12 |
+
"id": 0,
|
| 13 |
+
"content": "<|endoftext|>",
|
| 14 |
+
"single_word": false,
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"id": 1,
|
| 22 |
+
"content": "<|pad|>",
|
| 23 |
+
"single_word": false,
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"special": true
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"normalizer": null,
|
| 31 |
+
"pre_tokenizer": {
|
| 32 |
+
"type": "ByteLevel",
|
| 33 |
+
"add_prefix_space": true,
|
| 34 |
+
"trim_offsets": true,
|
| 35 |
+
"use_regex": true
|
| 36 |
+
},
|
| 37 |
+
"post_processor": {
|
| 38 |
+
"type": "TemplateProcessing",
|
| 39 |
+
"single": [
|
| 40 |
+
{
|
| 41 |
+
"Sequence": {
|
| 42 |
+
"id": "A",
|
| 43 |
+
"type_id": 0
|
| 44 |
+
}
|
| 45 |
+
}
|
| 46 |
+
],
|
| 47 |
+
"pair": [
|
| 48 |
+
{
|
| 49 |
+
"Sequence": {
|
| 50 |
+
"id": "A",
|
| 51 |
+
"type_id": 0
|
| 52 |
+
}
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"Sequence": {
|
| 56 |
+
"id": "B",
|
| 57 |
+
"type_id": 1
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
],
|
| 61 |
+
"special_tokens": {}
|
| 62 |
+
},
|
| 63 |
+
"decoder": {
|
| 64 |
+
"type": "ByteLevel",
|
| 65 |
+
"add_prefix_space": true,
|
| 66 |
+
"trim_offsets": true,
|
| 67 |
+
"use_regex": true
|
| 68 |
+
},
|
| 69 |
+
"model": {
|
| 70 |
+
"type": "BPE",
|
| 71 |
+
"dropout": null,
|
| 72 |
+
"unk_token": null,
|
| 73 |
+
"continuing_subword_prefix": null,
|
| 74 |
+
"end_of_word_suffix": null,
|
| 75 |
+
"fuse_unk": false,
|
| 76 |
+
"byte_fallback": false,
|
| 77 |
+
"ignore_merges": false,
|
| 78 |
+
"vocab": {
|
| 79 |
+
"<|endoftext|>": 0,
|
| 80 |
+
"<|pad|>": 1,
|
| 81 |
+
"a": 2,
|
| 82 |
+
"b": 3,
|
| 83 |
+
"Ġ": 4
|
| 84 |
+
},
|
| 85 |
+
"merges": []
|
| 86 |
+
}
|
| 87 |
+
}
|
checkpoint-100/tokenizer_config.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"bos_token": "<|endoftext|>",
|
| 4 |
+
"eos_token": "<|endoftext|>",
|
| 5 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 6 |
+
"pad_token": "<|pad|>",
|
| 7 |
+
"tokenizer_class": "TokenizersBackend"
|
| 8 |
+
}
|
checkpoint-100/trainer_state.json
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": null,
|
| 3 |
+
"best_metric": null,
|
| 4 |
+
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 12.5,
|
| 6 |
+
"eval_steps": 500,
|
| 7 |
+
"global_step": 100,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [
|
| 12 |
+
{
|
| 13 |
+
"epoch": 1.25,
|
| 14 |
+
"grad_norm": 0.05793992057442665,
|
| 15 |
+
"learning_rate": 0.009949685534591196,
|
| 16 |
+
"loss": 1.206215763092041,
|
| 17 |
+
"step": 10
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"epoch": 2.5,
|
| 21 |
+
"grad_norm": 0.15641599893569946,
|
| 22 |
+
"learning_rate": 0.009823899371069183,
|
| 23 |
+
"loss": 1.1944643020629884,
|
| 24 |
+
"step": 20
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"epoch": 3.75,
|
| 28 |
+
"grad_norm": 0.25837442278862,
|
| 29 |
+
"learning_rate": 0.00969811320754717,
|
| 30 |
+
"loss": 1.1607371330261231,
|
| 31 |
+
"step": 30
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"epoch": 5.0,
|
| 35 |
+
"grad_norm": 0.34708642959594727,
|
| 36 |
+
"learning_rate": 0.009572327044025157,
|
| 37 |
+
"loss": 1.101412010192871,
|
| 38 |
+
"step": 40
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 6.25,
|
| 42 |
+
"grad_norm": 0.4044564962387085,
|
| 43 |
+
"learning_rate": 0.009446540880503144,
|
| 44 |
+
"loss": 1.0188076972961426,
|
| 45 |
+
"step": 50
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"epoch": 7.5,
|
| 49 |
+
"grad_norm": 0.4182046055793762,
|
| 50 |
+
"learning_rate": 0.009320754716981133,
|
| 51 |
+
"loss": 0.9222237586975097,
|
| 52 |
+
"step": 60
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"epoch": 8.75,
|
| 56 |
+
"grad_norm": 0.38868534564971924,
|
| 57 |
+
"learning_rate": 0.00919496855345912,
|
| 58 |
+
"loss": 0.8254497528076172,
|
| 59 |
+
"step": 70
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"epoch": 10.0,
|
| 63 |
+
"grad_norm": 0.3328922390937805,
|
| 64 |
+
"learning_rate": 0.009069182389937107,
|
| 65 |
+
"loss": 0.7411866188049316,
|
| 66 |
+
"step": 80
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"epoch": 11.25,
|
| 70 |
+
"grad_norm": 0.27316856384277344,
|
| 71 |
+
"learning_rate": 0.008943396226415094,
|
| 72 |
+
"loss": 0.6766499042510986,
|
| 73 |
+
"step": 90
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"epoch": 12.5,
|
| 77 |
+
"grad_norm": 0.21604125201702118,
|
| 78 |
+
"learning_rate": 0.008817610062893081,
|
| 79 |
+
"loss": 0.6312645435333252,
|
| 80 |
+
"step": 100
|
| 81 |
+
}
|
| 82 |
+
],
|
| 83 |
+
"logging_steps": 10,
|
| 84 |
+
"max_steps": 800,
|
| 85 |
+
"num_input_tokens_seen": 0,
|
| 86 |
+
"num_train_epochs": 100,
|
| 87 |
+
"save_steps": 50,
|
| 88 |
+
"stateful_callbacks": {
|
| 89 |
+
"TrainerControl": {
|
| 90 |
+
"args": {
|
| 91 |
+
"should_epoch_stop": false,
|
| 92 |
+
"should_evaluate": false,
|
| 93 |
+
"should_log": false,
|
| 94 |
+
"should_save": true,
|
| 95 |
+
"should_training_stop": false
|
| 96 |
+
},
|
| 97 |
+
"attributes": {}
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"total_flos": 5405184.0,
|
| 101 |
+
"train_batch_size": 128,
|
| 102 |
+
"trial_name": null,
|
| 103 |
+
"trial_params": null
|
| 104 |
+
}
|
checkpoint-100/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8ec7c4970e40c66e374eb444e43c769446c8d48e5a1f681b2a7f4187462d269e
|
| 3 |
+
size 5137
|
checkpoint-150/config.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.0,
|
| 8 |
+
"bos_token_id": 0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.0,
|
| 11 |
+
"eos_token_id": 0,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 1e-05,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_embd": 1,
|
| 16 |
+
"n_head": 1,
|
| 17 |
+
"n_inner": 1,
|
| 18 |
+
"n_layer": 1,
|
| 19 |
+
"n_positions": 4,
|
| 20 |
+
"pad_token_id": 1,
|
| 21 |
+
"reorder_and_upcast_attn": false,
|
| 22 |
+
"resid_pdrop": 0.0,
|
| 23 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 24 |
+
"scale_attn_weights": true,
|
| 25 |
+
"summary_activation": null,
|
| 26 |
+
"summary_first_dropout": 0.1,
|
| 27 |
+
"summary_proj_to_labels": true,
|
| 28 |
+
"summary_type": "cls_index",
|
| 29 |
+
"summary_use_proj": true,
|
| 30 |
+
"tie_word_embeddings": true,
|
| 31 |
+
"transformers_version": "5.12.0",
|
| 32 |
+
"use_cache": false,
|
| 33 |
+
"vocab_size": 5
|
| 34 |
+
}
|
checkpoint-150/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 1,
|
| 8 |
+
"transformers_version": "5.12.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
checkpoint-150/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:32eff4e377c9154406c6531317ef9fe6fc7c87cbd462ae8c22ce79bf4d0a2364
|
| 3 |
+
size 1476
|
checkpoint-150/optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e369a422da25bd5901a3ae44e654f670635dd373d294a4901a53c0e8817ad956
|
| 3 |
+
size 13823
|
checkpoint-150/rng_state.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b9b5fdf389d8144a43074a17a83045fcaacf31701f4eb93ca74497b6fb1054c0
|
| 3 |
+
size 14455
|
checkpoint-150/scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d473e01573c435c93b9926bad634d71a4f933f621580066f41ec91a5f0b8451e
|
| 3 |
+
size 1465
|
checkpoint-150/tokenizer.json
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 4,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
+
"padding": null,
|
| 10 |
+
"added_tokens": [
|
| 11 |
+
{
|
| 12 |
+
"id": 0,
|
| 13 |
+
"content": "<|endoftext|>",
|
| 14 |
+
"single_word": false,
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"id": 1,
|
| 22 |
+
"content": "<|pad|>",
|
| 23 |
+
"single_word": false,
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"special": true
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"normalizer": null,
|
| 31 |
+
"pre_tokenizer": {
|
| 32 |
+
"type": "ByteLevel",
|
| 33 |
+
"add_prefix_space": true,
|
| 34 |
+
"trim_offsets": true,
|
| 35 |
+
"use_regex": true
|
| 36 |
+
},
|
| 37 |
+
"post_processor": {
|
| 38 |
+
"type": "TemplateProcessing",
|
| 39 |
+
"single": [
|
| 40 |
+
{
|
| 41 |
+
"Sequence": {
|
| 42 |
+
"id": "A",
|
| 43 |
+
"type_id": 0
|
| 44 |
+
}
|
| 45 |
+
}
|
| 46 |
+
],
|
| 47 |
+
"pair": [
|
| 48 |
+
{
|
| 49 |
+
"Sequence": {
|
| 50 |
+
"id": "A",
|
| 51 |
+
"type_id": 0
|
| 52 |
+
}
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"Sequence": {
|
| 56 |
+
"id": "B",
|
| 57 |
+
"type_id": 1
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
],
|
| 61 |
+
"special_tokens": {}
|
| 62 |
+
},
|
| 63 |
+
"decoder": {
|
| 64 |
+
"type": "ByteLevel",
|
| 65 |
+
"add_prefix_space": true,
|
| 66 |
+
"trim_offsets": true,
|
| 67 |
+
"use_regex": true
|
| 68 |
+
},
|
| 69 |
+
"model": {
|
| 70 |
+
"type": "BPE",
|
| 71 |
+
"dropout": null,
|
| 72 |
+
"unk_token": null,
|
| 73 |
+
"continuing_subword_prefix": null,
|
| 74 |
+
"end_of_word_suffix": null,
|
| 75 |
+
"fuse_unk": false,
|
| 76 |
+
"byte_fallback": false,
|
| 77 |
+
"ignore_merges": false,
|
| 78 |
+
"vocab": {
|
| 79 |
+
"<|endoftext|>": 0,
|
| 80 |
+
"<|pad|>": 1,
|
| 81 |
+
"a": 2,
|
| 82 |
+
"b": 3,
|
| 83 |
+
"Ġ": 4
|
| 84 |
+
},
|
| 85 |
+
"merges": []
|
| 86 |
+
}
|
| 87 |
+
}
|
checkpoint-150/tokenizer_config.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"bos_token": "<|endoftext|>",
|
| 4 |
+
"eos_token": "<|endoftext|>",
|
| 5 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 6 |
+
"pad_token": "<|pad|>",
|
| 7 |
+
"tokenizer_class": "TokenizersBackend"
|
| 8 |
+
}
|
checkpoint-150/trainer_state.json
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": null,
|
| 3 |
+
"best_metric": null,
|
| 4 |
+
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 18.75,
|
| 6 |
+
"eval_steps": 500,
|
| 7 |
+
"global_step": 150,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [
|
| 12 |
+
{
|
| 13 |
+
"epoch": 1.25,
|
| 14 |
+
"grad_norm": 0.05793992057442665,
|
| 15 |
+
"learning_rate": 0.009949685534591196,
|
| 16 |
+
"loss": 1.206215763092041,
|
| 17 |
+
"step": 10
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"epoch": 2.5,
|
| 21 |
+
"grad_norm": 0.15641599893569946,
|
| 22 |
+
"learning_rate": 0.009823899371069183,
|
| 23 |
+
"loss": 1.1944643020629884,
|
| 24 |
+
"step": 20
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"epoch": 3.75,
|
| 28 |
+
"grad_norm": 0.25837442278862,
|
| 29 |
+
"learning_rate": 0.00969811320754717,
|
| 30 |
+
"loss": 1.1607371330261231,
|
| 31 |
+
"step": 30
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"epoch": 5.0,
|
| 35 |
+
"grad_norm": 0.34708642959594727,
|
| 36 |
+
"learning_rate": 0.009572327044025157,
|
| 37 |
+
"loss": 1.101412010192871,
|
| 38 |
+
"step": 40
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 6.25,
|
| 42 |
+
"grad_norm": 0.4044564962387085,
|
| 43 |
+
"learning_rate": 0.009446540880503144,
|
| 44 |
+
"loss": 1.0188076972961426,
|
| 45 |
+
"step": 50
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"epoch": 7.5,
|
| 49 |
+
"grad_norm": 0.4182046055793762,
|
| 50 |
+
"learning_rate": 0.009320754716981133,
|
| 51 |
+
"loss": 0.9222237586975097,
|
| 52 |
+
"step": 60
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"epoch": 8.75,
|
| 56 |
+
"grad_norm": 0.38868534564971924,
|
| 57 |
+
"learning_rate": 0.00919496855345912,
|
| 58 |
+
"loss": 0.8254497528076172,
|
| 59 |
+
"step": 70
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"epoch": 10.0,
|
| 63 |
+
"grad_norm": 0.3328922390937805,
|
| 64 |
+
"learning_rate": 0.009069182389937107,
|
| 65 |
+
"loss": 0.7411866188049316,
|
| 66 |
+
"step": 80
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"epoch": 11.25,
|
| 70 |
+
"grad_norm": 0.27316856384277344,
|
| 71 |
+
"learning_rate": 0.008943396226415094,
|
| 72 |
+
"loss": 0.6766499042510986,
|
| 73 |
+
"step": 90
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"epoch": 12.5,
|
| 77 |
+
"grad_norm": 0.21604125201702118,
|
| 78 |
+
"learning_rate": 0.008817610062893081,
|
| 79 |
+
"loss": 0.6312645435333252,
|
| 80 |
+
"step": 100
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"epoch": 13.75,
|
| 84 |
+
"grad_norm": 0.17572654783725739,
|
| 85 |
+
"learning_rate": 0.00869182389937107,
|
| 86 |
+
"loss": 0.6011325359344483,
|
| 87 |
+
"step": 110
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"epoch": 15.0,
|
| 91 |
+
"grad_norm": 0.15216749906539917,
|
| 92 |
+
"learning_rate": 0.008566037735849057,
|
| 93 |
+
"loss": 0.5812094688415528,
|
| 94 |
+
"step": 120
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"epoch": 16.25,
|
| 98 |
+
"grad_norm": 0.12060005962848663,
|
| 99 |
+
"learning_rate": 0.008440251572327044,
|
| 100 |
+
"loss": 0.5678437232971192,
|
| 101 |
+
"step": 130
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"epoch": 17.5,
|
| 105 |
+
"grad_norm": 0.11622311919927597,
|
| 106 |
+
"learning_rate": 0.008314465408805032,
|
| 107 |
+
"loss": 0.5585259437561035,
|
| 108 |
+
"step": 140
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"epoch": 18.75,
|
| 112 |
+
"grad_norm": 0.08831977844238281,
|
| 113 |
+
"learning_rate": 0.008188679245283019,
|
| 114 |
+
"loss": 0.5519348621368408,
|
| 115 |
+
"step": 150
|
| 116 |
+
}
|
| 117 |
+
],
|
| 118 |
+
"logging_steps": 10,
|
| 119 |
+
"max_steps": 800,
|
| 120 |
+
"num_input_tokens_seen": 0,
|
| 121 |
+
"num_train_epochs": 100,
|
| 122 |
+
"save_steps": 50,
|
| 123 |
+
"stateful_callbacks": {
|
| 124 |
+
"TrainerControl": {
|
| 125 |
+
"args": {
|
| 126 |
+
"should_epoch_stop": false,
|
| 127 |
+
"should_evaluate": false,
|
| 128 |
+
"should_log": false,
|
| 129 |
+
"should_save": true,
|
| 130 |
+
"should_training_stop": false
|
| 131 |
+
},
|
| 132 |
+
"attributes": {}
|
| 133 |
+
}
|
| 134 |
+
},
|
| 135 |
+
"total_flos": 8107776.0,
|
| 136 |
+
"train_batch_size": 128,
|
| 137 |
+
"trial_name": null,
|
| 138 |
+
"trial_params": null
|
| 139 |
+
}
|
checkpoint-150/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8ec7c4970e40c66e374eb444e43c769446c8d48e5a1f681b2a7f4187462d269e
|
| 3 |
+
size 5137
|
checkpoint-200/config.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.0,
|
| 8 |
+
"bos_token_id": 0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.0,
|
| 11 |
+
"eos_token_id": 0,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 1e-05,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_embd": 1,
|
| 16 |
+
"n_head": 1,
|
| 17 |
+
"n_inner": 1,
|
| 18 |
+
"n_layer": 1,
|
| 19 |
+
"n_positions": 4,
|
| 20 |
+
"pad_token_id": 1,
|
| 21 |
+
"reorder_and_upcast_attn": false,
|
| 22 |
+
"resid_pdrop": 0.0,
|
| 23 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 24 |
+
"scale_attn_weights": true,
|
| 25 |
+
"summary_activation": null,
|
| 26 |
+
"summary_first_dropout": 0.1,
|
| 27 |
+
"summary_proj_to_labels": true,
|
| 28 |
+
"summary_type": "cls_index",
|
| 29 |
+
"summary_use_proj": true,
|
| 30 |
+
"tie_word_embeddings": true,
|
| 31 |
+
"transformers_version": "5.12.0",
|
| 32 |
+
"use_cache": false,
|
| 33 |
+
"vocab_size": 5
|
| 34 |
+
}
|
checkpoint-200/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 1,
|
| 8 |
+
"transformers_version": "5.12.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
checkpoint-200/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6f025fd5de3bb085eb395da425fca96a02fe2f4506b53e3ae913dead14cc3349
|
| 3 |
+
size 1476
|
checkpoint-200/optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:904a89c566860fdd80b54b080f3c5db858451714b993b090b1e954f0536574c7
|
| 3 |
+
size 13823
|
checkpoint-200/rng_state.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:578e8799d025fe13990ba869b383010b37f57725e649414cca523ec437e2196e
|
| 3 |
+
size 14455
|
checkpoint-200/scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0461a487c93a1d5d456c21f2f0d6053e6cc43aa65ed6f9187fa4e810cd6b0a8a
|
| 3 |
+
size 1465
|
checkpoint-200/tokenizer.json
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 4,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
+
"padding": null,
|
| 10 |
+
"added_tokens": [
|
| 11 |
+
{
|
| 12 |
+
"id": 0,
|
| 13 |
+
"content": "<|endoftext|>",
|
| 14 |
+
"single_word": false,
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"id": 1,
|
| 22 |
+
"content": "<|pad|>",
|
| 23 |
+
"single_word": false,
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"special": true
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"normalizer": null,
|
| 31 |
+
"pre_tokenizer": {
|
| 32 |
+
"type": "ByteLevel",
|
| 33 |
+
"add_prefix_space": true,
|
| 34 |
+
"trim_offsets": true,
|
| 35 |
+
"use_regex": true
|
| 36 |
+
},
|
| 37 |
+
"post_processor": {
|
| 38 |
+
"type": "TemplateProcessing",
|
| 39 |
+
"single": [
|
| 40 |
+
{
|
| 41 |
+
"Sequence": {
|
| 42 |
+
"id": "A",
|
| 43 |
+
"type_id": 0
|
| 44 |
+
}
|
| 45 |
+
}
|
| 46 |
+
],
|
| 47 |
+
"pair": [
|
| 48 |
+
{
|
| 49 |
+
"Sequence": {
|
| 50 |
+
"id": "A",
|
| 51 |
+
"type_id": 0
|
| 52 |
+
}
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"Sequence": {
|
| 56 |
+
"id": "B",
|
| 57 |
+
"type_id": 1
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
],
|
| 61 |
+
"special_tokens": {}
|
| 62 |
+
},
|
| 63 |
+
"decoder": {
|
| 64 |
+
"type": "ByteLevel",
|
| 65 |
+
"add_prefix_space": true,
|
| 66 |
+
"trim_offsets": true,
|
| 67 |
+
"use_regex": true
|
| 68 |
+
},
|
| 69 |
+
"model": {
|
| 70 |
+
"type": "BPE",
|
| 71 |
+
"dropout": null,
|
| 72 |
+
"unk_token": null,
|
| 73 |
+
"continuing_subword_prefix": null,
|
| 74 |
+
"end_of_word_suffix": null,
|
| 75 |
+
"fuse_unk": false,
|
| 76 |
+
"byte_fallback": false,
|
| 77 |
+
"ignore_merges": false,
|
| 78 |
+
"vocab": {
|
| 79 |
+
"<|endoftext|>": 0,
|
| 80 |
+
"<|pad|>": 1,
|
| 81 |
+
"a": 2,
|
| 82 |
+
"b": 3,
|
| 83 |
+
"Ġ": 4
|
| 84 |
+
},
|
| 85 |
+
"merges": []
|
| 86 |
+
}
|
| 87 |
+
}
|
checkpoint-200/tokenizer_config.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"bos_token": "<|endoftext|>",
|
| 4 |
+
"eos_token": "<|endoftext|>",
|
| 5 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 6 |
+
"pad_token": "<|pad|>",
|
| 7 |
+
"tokenizer_class": "TokenizersBackend"
|
| 8 |
+
}
|
checkpoint-200/trainer_state.json
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": null,
|
| 3 |
+
"best_metric": null,
|
| 4 |
+
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 25.0,
|
| 6 |
+
"eval_steps": 500,
|
| 7 |
+
"global_step": 200,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [
|
| 12 |
+
{
|
| 13 |
+
"epoch": 1.25,
|
| 14 |
+
"grad_norm": 0.05793992057442665,
|
| 15 |
+
"learning_rate": 0.009949685534591196,
|
| 16 |
+
"loss": 1.206215763092041,
|
| 17 |
+
"step": 10
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"epoch": 2.5,
|
| 21 |
+
"grad_norm": 0.15641599893569946,
|
| 22 |
+
"learning_rate": 0.009823899371069183,
|
| 23 |
+
"loss": 1.1944643020629884,
|
| 24 |
+
"step": 20
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"epoch": 3.75,
|
| 28 |
+
"grad_norm": 0.25837442278862,
|
| 29 |
+
"learning_rate": 0.00969811320754717,
|
| 30 |
+
"loss": 1.1607371330261231,
|
| 31 |
+
"step": 30
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"epoch": 5.0,
|
| 35 |
+
"grad_norm": 0.34708642959594727,
|
| 36 |
+
"learning_rate": 0.009572327044025157,
|
| 37 |
+
"loss": 1.101412010192871,
|
| 38 |
+
"step": 40
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 6.25,
|
| 42 |
+
"grad_norm": 0.4044564962387085,
|
| 43 |
+
"learning_rate": 0.009446540880503144,
|
| 44 |
+
"loss": 1.0188076972961426,
|
| 45 |
+
"step": 50
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"epoch": 7.5,
|
| 49 |
+
"grad_norm": 0.4182046055793762,
|
| 50 |
+
"learning_rate": 0.009320754716981133,
|
| 51 |
+
"loss": 0.9222237586975097,
|
| 52 |
+
"step": 60
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"epoch": 8.75,
|
| 56 |
+
"grad_norm": 0.38868534564971924,
|
| 57 |
+
"learning_rate": 0.00919496855345912,
|
| 58 |
+
"loss": 0.8254497528076172,
|
| 59 |
+
"step": 70
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"epoch": 10.0,
|
| 63 |
+
"grad_norm": 0.3328922390937805,
|
| 64 |
+
"learning_rate": 0.009069182389937107,
|
| 65 |
+
"loss": 0.7411866188049316,
|
| 66 |
+
"step": 80
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"epoch": 11.25,
|
| 70 |
+
"grad_norm": 0.27316856384277344,
|
| 71 |
+
"learning_rate": 0.008943396226415094,
|
| 72 |
+
"loss": 0.6766499042510986,
|
| 73 |
+
"step": 90
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"epoch": 12.5,
|
| 77 |
+
"grad_norm": 0.21604125201702118,
|
| 78 |
+
"learning_rate": 0.008817610062893081,
|
| 79 |
+
"loss": 0.6312645435333252,
|
| 80 |
+
"step": 100
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"epoch": 13.75,
|
| 84 |
+
"grad_norm": 0.17572654783725739,
|
| 85 |
+
"learning_rate": 0.00869182389937107,
|
| 86 |
+
"loss": 0.6011325359344483,
|
| 87 |
+
"step": 110
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"epoch": 15.0,
|
| 91 |
+
"grad_norm": 0.15216749906539917,
|
| 92 |
+
"learning_rate": 0.008566037735849057,
|
| 93 |
+
"loss": 0.5812094688415528,
|
| 94 |
+
"step": 120
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"epoch": 16.25,
|
| 98 |
+
"grad_norm": 0.12060005962848663,
|
| 99 |
+
"learning_rate": 0.008440251572327044,
|
| 100 |
+
"loss": 0.5678437232971192,
|
| 101 |
+
"step": 130
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"epoch": 17.5,
|
| 105 |
+
"grad_norm": 0.11622311919927597,
|
| 106 |
+
"learning_rate": 0.008314465408805032,
|
| 107 |
+
"loss": 0.5585259437561035,
|
| 108 |
+
"step": 140
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"epoch": 18.75,
|
| 112 |
+
"grad_norm": 0.08831977844238281,
|
| 113 |
+
"learning_rate": 0.008188679245283019,
|
| 114 |
+
"loss": 0.5519348621368408,
|
| 115 |
+
"step": 150
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"epoch": 20.0,
|
| 119 |
+
"grad_norm": 0.09614107757806778,
|
| 120 |
+
"learning_rate": 0.008062893081761007,
|
| 121 |
+
"loss": 0.5471037864685059,
|
| 122 |
+
"step": 160
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"epoch": 21.25,
|
| 126 |
+
"grad_norm": 0.07089235633611679,
|
| 127 |
+
"learning_rate": 0.007937106918238995,
|
| 128 |
+
"loss": 0.5432851314544678,
|
| 129 |
+
"step": 170
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"epoch": 22.5,
|
| 133 |
+
"grad_norm": 0.061037734150886536,
|
| 134 |
+
"learning_rate": 0.007811320754716982,
|
| 135 |
+
"loss": 0.5403900146484375,
|
| 136 |
+
"step": 180
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"epoch": 23.75,
|
| 140 |
+
"grad_norm": 0.09448318183422089,
|
| 141 |
+
"learning_rate": 0.00768553459119497,
|
| 142 |
+
"loss": 0.53807373046875,
|
| 143 |
+
"step": 190
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"epoch": 25.0,
|
| 147 |
+
"grad_norm": 0.050825681537389755,
|
| 148 |
+
"learning_rate": 0.007559748427672957,
|
| 149 |
+
"loss": 0.5360239028930665,
|
| 150 |
+
"step": 200
|
| 151 |
+
}
|
| 152 |
+
],
|
| 153 |
+
"logging_steps": 10,
|
| 154 |
+
"max_steps": 800,
|
| 155 |
+
"num_input_tokens_seen": 0,
|
| 156 |
+
"num_train_epochs": 100,
|
| 157 |
+
"save_steps": 50,
|
| 158 |
+
"stateful_callbacks": {
|
| 159 |
+
"TrainerControl": {
|
| 160 |
+
"args": {
|
| 161 |
+
"should_epoch_stop": false,
|
| 162 |
+
"should_evaluate": false,
|
| 163 |
+
"should_log": false,
|
| 164 |
+
"should_save": true,
|
| 165 |
+
"should_training_stop": false
|
| 166 |
+
},
|
| 167 |
+
"attributes": {}
|
| 168 |
+
}
|
| 169 |
+
},
|
| 170 |
+
"total_flos": 10800000.0,
|
| 171 |
+
"train_batch_size": 128,
|
| 172 |
+
"trial_name": null,
|
| 173 |
+
"trial_params": null
|
| 174 |
+
}
|
checkpoint-200/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8ec7c4970e40c66e374eb444e43c769446c8d48e5a1f681b2a7f4187462d269e
|
| 3 |
+
size 5137
|
checkpoint-250/config.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.0,
|
| 8 |
+
"bos_token_id": 0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.0,
|
| 11 |
+
"eos_token_id": 0,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 1e-05,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_embd": 1,
|
| 16 |
+
"n_head": 1,
|
| 17 |
+
"n_inner": 1,
|
| 18 |
+
"n_layer": 1,
|
| 19 |
+
"n_positions": 4,
|
| 20 |
+
"pad_token_id": 1,
|
| 21 |
+
"reorder_and_upcast_attn": false,
|
| 22 |
+
"resid_pdrop": 0.0,
|
| 23 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 24 |
+
"scale_attn_weights": true,
|
| 25 |
+
"summary_activation": null,
|
| 26 |
+
"summary_first_dropout": 0.1,
|
| 27 |
+
"summary_proj_to_labels": true,
|
| 28 |
+
"summary_type": "cls_index",
|
| 29 |
+
"summary_use_proj": true,
|
| 30 |
+
"tie_word_embeddings": true,
|
| 31 |
+
"transformers_version": "5.12.0",
|
| 32 |
+
"use_cache": false,
|
| 33 |
+
"vocab_size": 5
|
| 34 |
+
}
|
checkpoint-250/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 1,
|
| 8 |
+
"transformers_version": "5.12.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
checkpoint-250/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4a50b4f9c0e64d6662e265d416f7098696bb647c785cb9d96ce1ed0bc52c106f
|
| 3 |
+
size 1476
|
checkpoint-250/optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0509ebbf56f1ff5def338103f070cd55005f3b196cc7f15f64ce938a3f3c8a9
|
| 3 |
+
size 13823
|
checkpoint-250/rng_state.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d93cf2a6f5b82bd08f8b51ea11b7d93fb36e5dee432e6b631b5bd5365d1aa213
|
| 3 |
+
size 14455
|
checkpoint-250/scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0bea7ab5973051bc08e6b64d8b657fe2c7761fc2e91b84dbe5ebf70251f1d9c7
|
| 3 |
+
size 1465
|
checkpoint-250/tokenizer.json
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 4,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
+
"padding": null,
|
| 10 |
+
"added_tokens": [
|
| 11 |
+
{
|
| 12 |
+
"id": 0,
|
| 13 |
+
"content": "<|endoftext|>",
|
| 14 |
+
"single_word": false,
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"id": 1,
|
| 22 |
+
"content": "<|pad|>",
|
| 23 |
+
"single_word": false,
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"special": true
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"normalizer": null,
|
| 31 |
+
"pre_tokenizer": {
|
| 32 |
+
"type": "ByteLevel",
|
| 33 |
+
"add_prefix_space": true,
|
| 34 |
+
"trim_offsets": true,
|
| 35 |
+
"use_regex": true
|
| 36 |
+
},
|
| 37 |
+
"post_processor": {
|
| 38 |
+
"type": "TemplateProcessing",
|
| 39 |
+
"single": [
|
| 40 |
+
{
|
| 41 |
+
"Sequence": {
|
| 42 |
+
"id": "A",
|
| 43 |
+
"type_id": 0
|
| 44 |
+
}
|
| 45 |
+
}
|
| 46 |
+
],
|
| 47 |
+
"pair": [
|
| 48 |
+
{
|
| 49 |
+
"Sequence": {
|
| 50 |
+
"id": "A",
|
| 51 |
+
"type_id": 0
|
| 52 |
+
}
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"Sequence": {
|
| 56 |
+
"id": "B",
|
| 57 |
+
"type_id": 1
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
],
|
| 61 |
+
"special_tokens": {}
|
| 62 |
+
},
|
| 63 |
+
"decoder": {
|
| 64 |
+
"type": "ByteLevel",
|
| 65 |
+
"add_prefix_space": true,
|
| 66 |
+
"trim_offsets": true,
|
| 67 |
+
"use_regex": true
|
| 68 |
+
},
|
| 69 |
+
"model": {
|
| 70 |
+
"type": "BPE",
|
| 71 |
+
"dropout": null,
|
| 72 |
+
"unk_token": null,
|
| 73 |
+
"continuing_subword_prefix": null,
|
| 74 |
+
"end_of_word_suffix": null,
|
| 75 |
+
"fuse_unk": false,
|
| 76 |
+
"byte_fallback": false,
|
| 77 |
+
"ignore_merges": false,
|
| 78 |
+
"vocab": {
|
| 79 |
+
"<|endoftext|>": 0,
|
| 80 |
+
"<|pad|>": 1,
|
| 81 |
+
"a": 2,
|
| 82 |
+
"b": 3,
|
| 83 |
+
"Ġ": 4
|
| 84 |
+
},
|
| 85 |
+
"merges": []
|
| 86 |
+
}
|
| 87 |
+
}
|
checkpoint-250/tokenizer_config.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"bos_token": "<|endoftext|>",
|
| 4 |
+
"eos_token": "<|endoftext|>",
|
| 5 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 6 |
+
"pad_token": "<|pad|>",
|
| 7 |
+
"tokenizer_class": "TokenizersBackend"
|
| 8 |
+
}
|
checkpoint-250/trainer_state.json
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": null,
|
| 3 |
+
"best_metric": null,
|
| 4 |
+
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 31.25,
|
| 6 |
+
"eval_steps": 500,
|
| 7 |
+
"global_step": 250,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [
|
| 12 |
+
{
|
| 13 |
+
"epoch": 1.25,
|
| 14 |
+
"grad_norm": 0.05793992057442665,
|
| 15 |
+
"learning_rate": 0.009949685534591196,
|
| 16 |
+
"loss": 1.206215763092041,
|
| 17 |
+
"step": 10
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"epoch": 2.5,
|
| 21 |
+
"grad_norm": 0.15641599893569946,
|
| 22 |
+
"learning_rate": 0.009823899371069183,
|
| 23 |
+
"loss": 1.1944643020629884,
|
| 24 |
+
"step": 20
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"epoch": 3.75,
|
| 28 |
+
"grad_norm": 0.25837442278862,
|
| 29 |
+
"learning_rate": 0.00969811320754717,
|
| 30 |
+
"loss": 1.1607371330261231,
|
| 31 |
+
"step": 30
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"epoch": 5.0,
|
| 35 |
+
"grad_norm": 0.34708642959594727,
|
| 36 |
+
"learning_rate": 0.009572327044025157,
|
| 37 |
+
"loss": 1.101412010192871,
|
| 38 |
+
"step": 40
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 6.25,
|
| 42 |
+
"grad_norm": 0.4044564962387085,
|
| 43 |
+
"learning_rate": 0.009446540880503144,
|
| 44 |
+
"loss": 1.0188076972961426,
|
| 45 |
+
"step": 50
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"epoch": 7.5,
|
| 49 |
+
"grad_norm": 0.4182046055793762,
|
| 50 |
+
"learning_rate": 0.009320754716981133,
|
| 51 |
+
"loss": 0.9222237586975097,
|
| 52 |
+
"step": 60
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"epoch": 8.75,
|
| 56 |
+
"grad_norm": 0.38868534564971924,
|
| 57 |
+
"learning_rate": 0.00919496855345912,
|
| 58 |
+
"loss": 0.8254497528076172,
|
| 59 |
+
"step": 70
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"epoch": 10.0,
|
| 63 |
+
"grad_norm": 0.3328922390937805,
|
| 64 |
+
"learning_rate": 0.009069182389937107,
|
| 65 |
+
"loss": 0.7411866188049316,
|
| 66 |
+
"step": 80
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"epoch": 11.25,
|
| 70 |
+
"grad_norm": 0.27316856384277344,
|
| 71 |
+
"learning_rate": 0.008943396226415094,
|
| 72 |
+
"loss": 0.6766499042510986,
|
| 73 |
+
"step": 90
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"epoch": 12.5,
|
| 77 |
+
"grad_norm": 0.21604125201702118,
|
| 78 |
+
"learning_rate": 0.008817610062893081,
|
| 79 |
+
"loss": 0.6312645435333252,
|
| 80 |
+
"step": 100
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"epoch": 13.75,
|
| 84 |
+
"grad_norm": 0.17572654783725739,
|
| 85 |
+
"learning_rate": 0.00869182389937107,
|
| 86 |
+
"loss": 0.6011325359344483,
|
| 87 |
+
"step": 110
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"epoch": 15.0,
|
| 91 |
+
"grad_norm": 0.15216749906539917,
|
| 92 |
+
"learning_rate": 0.008566037735849057,
|
| 93 |
+
"loss": 0.5812094688415528,
|
| 94 |
+
"step": 120
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"epoch": 16.25,
|
| 98 |
+
"grad_norm": 0.12060005962848663,
|
| 99 |
+
"learning_rate": 0.008440251572327044,
|
| 100 |
+
"loss": 0.5678437232971192,
|
| 101 |
+
"step": 130
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"epoch": 17.5,
|
| 105 |
+
"grad_norm": 0.11622311919927597,
|
| 106 |
+
"learning_rate": 0.008314465408805032,
|
| 107 |
+
"loss": 0.5585259437561035,
|
| 108 |
+
"step": 140
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"epoch": 18.75,
|
| 112 |
+
"grad_norm": 0.08831977844238281,
|
| 113 |
+
"learning_rate": 0.008188679245283019,
|
| 114 |
+
"loss": 0.5519348621368408,
|
| 115 |
+
"step": 150
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"epoch": 20.0,
|
| 119 |
+
"grad_norm": 0.09614107757806778,
|
| 120 |
+
"learning_rate": 0.008062893081761007,
|
| 121 |
+
"loss": 0.5471037864685059,
|
| 122 |
+
"step": 160
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"epoch": 21.25,
|
| 126 |
+
"grad_norm": 0.07089235633611679,
|
| 127 |
+
"learning_rate": 0.007937106918238995,
|
| 128 |
+
"loss": 0.5432851314544678,
|
| 129 |
+
"step": 170
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"epoch": 22.5,
|
| 133 |
+
"grad_norm": 0.061037734150886536,
|
| 134 |
+
"learning_rate": 0.007811320754716982,
|
| 135 |
+
"loss": 0.5403900146484375,
|
| 136 |
+
"step": 180
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"epoch": 23.75,
|
| 140 |
+
"grad_norm": 0.09448318183422089,
|
| 141 |
+
"learning_rate": 0.00768553459119497,
|
| 142 |
+
"loss": 0.53807373046875,
|
| 143 |
+
"step": 190
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"epoch": 25.0,
|
| 147 |
+
"grad_norm": 0.050825681537389755,
|
| 148 |
+
"learning_rate": 0.007559748427672957,
|
| 149 |
+
"loss": 0.5360239028930665,
|
| 150 |
+
"step": 200
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"epoch": 26.25,
|
| 154 |
+
"grad_norm": 0.11754798144102097,
|
| 155 |
+
"learning_rate": 0.007433962264150943,
|
| 156 |
+
"loss": 0.5343667507171631,
|
| 157 |
+
"step": 210
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"epoch": 27.5,
|
| 161 |
+
"grad_norm": 0.04136759415268898,
|
| 162 |
+
"learning_rate": 0.00730817610062893,
|
| 163 |
+
"loss": 0.5332566738128662,
|
| 164 |
+
"step": 220
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"epoch": 28.75,
|
| 168 |
+
"grad_norm": 0.061312805861234665,
|
| 169 |
+
"learning_rate": 0.007182389937106918,
|
| 170 |
+
"loss": 0.5320106983184815,
|
| 171 |
+
"step": 230
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 30.0,
|
| 175 |
+
"grad_norm": 0.045437928289175034,
|
| 176 |
+
"learning_rate": 0.007056603773584905,
|
| 177 |
+
"loss": 0.5310849666595459,
|
| 178 |
+
"step": 240
|
| 179 |
+
},
|
| 180 |
+
{
|
| 181 |
+
"epoch": 31.25,
|
| 182 |
+
"grad_norm": 0.041931431740522385,
|
| 183 |
+
"learning_rate": 0.006930817610062893,
|
| 184 |
+
"loss": 0.5302073001861572,
|
| 185 |
+
"step": 250
|
| 186 |
+
}
|
| 187 |
+
],
|
| 188 |
+
"logging_steps": 10,
|
| 189 |
+
"max_steps": 800,
|
| 190 |
+
"num_input_tokens_seen": 0,
|
| 191 |
+
"num_train_epochs": 100,
|
| 192 |
+
"save_steps": 50,
|
| 193 |
+
"stateful_callbacks": {
|
| 194 |
+
"TrainerControl": {
|
| 195 |
+
"args": {
|
| 196 |
+
"should_epoch_stop": false,
|
| 197 |
+
"should_evaluate": false,
|
| 198 |
+
"should_log": false,
|
| 199 |
+
"should_save": true,
|
| 200 |
+
"should_training_stop": false
|
| 201 |
+
},
|
| 202 |
+
"attributes": {}
|
| 203 |
+
}
|
| 204 |
+
},
|
| 205 |
+
"total_flos": 13502592.0,
|
| 206 |
+
"train_batch_size": 128,
|
| 207 |
+
"trial_name": null,
|
| 208 |
+
"trial_params": null
|
| 209 |
+
}
|
checkpoint-250/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8ec7c4970e40c66e374eb444e43c769446c8d48e5a1f681b2a7f4187462d269e
|
| 3 |
+
size 5137
|
checkpoint-300/config.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.0,
|
| 8 |
+
"bos_token_id": 0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.0,
|
| 11 |
+
"eos_token_id": 0,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 1e-05,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_embd": 1,
|
| 16 |
+
"n_head": 1,
|
| 17 |
+
"n_inner": 1,
|
| 18 |
+
"n_layer": 1,
|
| 19 |
+
"n_positions": 4,
|
| 20 |
+
"pad_token_id": 1,
|
| 21 |
+
"reorder_and_upcast_attn": false,
|
| 22 |
+
"resid_pdrop": 0.0,
|
| 23 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 24 |
+
"scale_attn_weights": true,
|
| 25 |
+
"summary_activation": null,
|
| 26 |
+
"summary_first_dropout": 0.1,
|
| 27 |
+
"summary_proj_to_labels": true,
|
| 28 |
+
"summary_type": "cls_index",
|
| 29 |
+
"summary_use_proj": true,
|
| 30 |
+
"tie_word_embeddings": true,
|
| 31 |
+
"transformers_version": "5.12.0",
|
| 32 |
+
"use_cache": false,
|
| 33 |
+
"vocab_size": 5
|
| 34 |
+
}
|
checkpoint-300/generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"output_attentions": false,
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"pad_token_id": 1,
|
| 8 |
+
"transformers_version": "5.12.0",
|
| 9 |
+
"use_cache": true
|
| 10 |
+
}
|
checkpoint-300/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:97fffa1ffb0d8cfea3175841cdaadc065dbc524a156ab6794cc4defcdb443466
|
| 3 |
+
size 1476
|
checkpoint-300/optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:22f1cbf4f54dc99d5f9cf6f96190e76a53018986e798912dfadb864ea9b83abc
|
| 3 |
+
size 13823
|
checkpoint-300/rng_state.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4e3dc2ffd417ca43335dc7e4f9ae2b0c82cd1201b3451b1285b0c2b5c355f483
|
| 3 |
+
size 14455
|
checkpoint-300/scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9f95bed23f8bb86a3a3ad2aac60be9c505895c2819ebd5aedc95d6f297f479db
|
| 3 |
+
size 1465
|
checkpoint-300/tokenizer.json
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 4,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
+
"padding": null,
|
| 10 |
+
"added_tokens": [
|
| 11 |
+
{
|
| 12 |
+
"id": 0,
|
| 13 |
+
"content": "<|endoftext|>",
|
| 14 |
+
"single_word": false,
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"id": 1,
|
| 22 |
+
"content": "<|pad|>",
|
| 23 |
+
"single_word": false,
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"special": true
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"normalizer": null,
|
| 31 |
+
"pre_tokenizer": {
|
| 32 |
+
"type": "ByteLevel",
|
| 33 |
+
"add_prefix_space": true,
|
| 34 |
+
"trim_offsets": true,
|
| 35 |
+
"use_regex": true
|
| 36 |
+
},
|
| 37 |
+
"post_processor": {
|
| 38 |
+
"type": "TemplateProcessing",
|
| 39 |
+
"single": [
|
| 40 |
+
{
|
| 41 |
+
"Sequence": {
|
| 42 |
+
"id": "A",
|
| 43 |
+
"type_id": 0
|
| 44 |
+
}
|
| 45 |
+
}
|
| 46 |
+
],
|
| 47 |
+
"pair": [
|
| 48 |
+
{
|
| 49 |
+
"Sequence": {
|
| 50 |
+
"id": "A",
|
| 51 |
+
"type_id": 0
|
| 52 |
+
}
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"Sequence": {
|
| 56 |
+
"id": "B",
|
| 57 |
+
"type_id": 1
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
],
|
| 61 |
+
"special_tokens": {}
|
| 62 |
+
},
|
| 63 |
+
"decoder": {
|
| 64 |
+
"type": "ByteLevel",
|
| 65 |
+
"add_prefix_space": true,
|
| 66 |
+
"trim_offsets": true,
|
| 67 |
+
"use_regex": true
|
| 68 |
+
},
|
| 69 |
+
"model": {
|
| 70 |
+
"type": "BPE",
|
| 71 |
+
"dropout": null,
|
| 72 |
+
"unk_token": null,
|
| 73 |
+
"continuing_subword_prefix": null,
|
| 74 |
+
"end_of_word_suffix": null,
|
| 75 |
+
"fuse_unk": false,
|
| 76 |
+
"byte_fallback": false,
|
| 77 |
+
"ignore_merges": false,
|
| 78 |
+
"vocab": {
|
| 79 |
+
"<|endoftext|>": 0,
|
| 80 |
+
"<|pad|>": 1,
|
| 81 |
+
"a": 2,
|
| 82 |
+
"b": 3,
|
| 83 |
+
"Ġ": 4
|
| 84 |
+
},
|
| 85 |
+
"merges": []
|
| 86 |
+
}
|
| 87 |
+
}
|
checkpoint-300/tokenizer_config.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"bos_token": "<|endoftext|>",
|
| 4 |
+
"eos_token": "<|endoftext|>",
|
| 5 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 6 |
+
"pad_token": "<|pad|>",
|
| 7 |
+
"tokenizer_class": "TokenizersBackend"
|
| 8 |
+
}
|
checkpoint-300/trainer_state.json
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": null,
|
| 3 |
+
"best_metric": null,
|
| 4 |
+
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 37.5,
|
| 6 |
+
"eval_steps": 500,
|
| 7 |
+
"global_step": 300,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [
|
| 12 |
+
{
|
| 13 |
+
"epoch": 1.25,
|
| 14 |
+
"grad_norm": 0.05793992057442665,
|
| 15 |
+
"learning_rate": 0.009949685534591196,
|
| 16 |
+
"loss": 1.206215763092041,
|
| 17 |
+
"step": 10
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"epoch": 2.5,
|
| 21 |
+
"grad_norm": 0.15641599893569946,
|
| 22 |
+
"learning_rate": 0.009823899371069183,
|
| 23 |
+
"loss": 1.1944643020629884,
|
| 24 |
+
"step": 20
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"epoch": 3.75,
|
| 28 |
+
"grad_norm": 0.25837442278862,
|
| 29 |
+
"learning_rate": 0.00969811320754717,
|
| 30 |
+
"loss": 1.1607371330261231,
|
| 31 |
+
"step": 30
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"epoch": 5.0,
|
| 35 |
+
"grad_norm": 0.34708642959594727,
|
| 36 |
+
"learning_rate": 0.009572327044025157,
|
| 37 |
+
"loss": 1.101412010192871,
|
| 38 |
+
"step": 40
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 6.25,
|
| 42 |
+
"grad_norm": 0.4044564962387085,
|
| 43 |
+
"learning_rate": 0.009446540880503144,
|
| 44 |
+
"loss": 1.0188076972961426,
|
| 45 |
+
"step": 50
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"epoch": 7.5,
|
| 49 |
+
"grad_norm": 0.4182046055793762,
|
| 50 |
+
"learning_rate": 0.009320754716981133,
|
| 51 |
+
"loss": 0.9222237586975097,
|
| 52 |
+
"step": 60
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"epoch": 8.75,
|
| 56 |
+
"grad_norm": 0.38868534564971924,
|
| 57 |
+
"learning_rate": 0.00919496855345912,
|
| 58 |
+
"loss": 0.8254497528076172,
|
| 59 |
+
"step": 70
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"epoch": 10.0,
|
| 63 |
+
"grad_norm": 0.3328922390937805,
|
| 64 |
+
"learning_rate": 0.009069182389937107,
|
| 65 |
+
"loss": 0.7411866188049316,
|
| 66 |
+
"step": 80
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"epoch": 11.25,
|
| 70 |
+
"grad_norm": 0.27316856384277344,
|
| 71 |
+
"learning_rate": 0.008943396226415094,
|
| 72 |
+
"loss": 0.6766499042510986,
|
| 73 |
+
"step": 90
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"epoch": 12.5,
|
| 77 |
+
"grad_norm": 0.21604125201702118,
|
| 78 |
+
"learning_rate": 0.008817610062893081,
|
| 79 |
+
"loss": 0.6312645435333252,
|
| 80 |
+
"step": 100
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"epoch": 13.75,
|
| 84 |
+
"grad_norm": 0.17572654783725739,
|
| 85 |
+
"learning_rate": 0.00869182389937107,
|
| 86 |
+
"loss": 0.6011325359344483,
|
| 87 |
+
"step": 110
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"epoch": 15.0,
|
| 91 |
+
"grad_norm": 0.15216749906539917,
|
| 92 |
+
"learning_rate": 0.008566037735849057,
|
| 93 |
+
"loss": 0.5812094688415528,
|
| 94 |
+
"step": 120
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"epoch": 16.25,
|
| 98 |
+
"grad_norm": 0.12060005962848663,
|
| 99 |
+
"learning_rate": 0.008440251572327044,
|
| 100 |
+
"loss": 0.5678437232971192,
|
| 101 |
+
"step": 130
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"epoch": 17.5,
|
| 105 |
+
"grad_norm": 0.11622311919927597,
|
| 106 |
+
"learning_rate": 0.008314465408805032,
|
| 107 |
+
"loss": 0.5585259437561035,
|
| 108 |
+
"step": 140
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"epoch": 18.75,
|
| 112 |
+
"grad_norm": 0.08831977844238281,
|
| 113 |
+
"learning_rate": 0.008188679245283019,
|
| 114 |
+
"loss": 0.5519348621368408,
|
| 115 |
+
"step": 150
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"epoch": 20.0,
|
| 119 |
+
"grad_norm": 0.09614107757806778,
|
| 120 |
+
"learning_rate": 0.008062893081761007,
|
| 121 |
+
"loss": 0.5471037864685059,
|
| 122 |
+
"step": 160
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"epoch": 21.25,
|
| 126 |
+
"grad_norm": 0.07089235633611679,
|
| 127 |
+
"learning_rate": 0.007937106918238995,
|
| 128 |
+
"loss": 0.5432851314544678,
|
| 129 |
+
"step": 170
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"epoch": 22.5,
|
| 133 |
+
"grad_norm": 0.061037734150886536,
|
| 134 |
+
"learning_rate": 0.007811320754716982,
|
| 135 |
+
"loss": 0.5403900146484375,
|
| 136 |
+
"step": 180
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"epoch": 23.75,
|
| 140 |
+
"grad_norm": 0.09448318183422089,
|
| 141 |
+
"learning_rate": 0.00768553459119497,
|
| 142 |
+
"loss": 0.53807373046875,
|
| 143 |
+
"step": 190
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"epoch": 25.0,
|
| 147 |
+
"grad_norm": 0.050825681537389755,
|
| 148 |
+
"learning_rate": 0.007559748427672957,
|
| 149 |
+
"loss": 0.5360239028930665,
|
| 150 |
+
"step": 200
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"epoch": 26.25,
|
| 154 |
+
"grad_norm": 0.11754798144102097,
|
| 155 |
+
"learning_rate": 0.007433962264150943,
|
| 156 |
+
"loss": 0.5343667507171631,
|
| 157 |
+
"step": 210
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"epoch": 27.5,
|
| 161 |
+
"grad_norm": 0.04136759415268898,
|
| 162 |
+
"learning_rate": 0.00730817610062893,
|
| 163 |
+
"loss": 0.5332566738128662,
|
| 164 |
+
"step": 220
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"epoch": 28.75,
|
| 168 |
+
"grad_norm": 0.061312805861234665,
|
| 169 |
+
"learning_rate": 0.007182389937106918,
|
| 170 |
+
"loss": 0.5320106983184815,
|
| 171 |
+
"step": 230
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 30.0,
|
| 175 |
+
"grad_norm": 0.045437928289175034,
|
| 176 |
+
"learning_rate": 0.007056603773584905,
|
| 177 |
+
"loss": 0.5310849666595459,
|
| 178 |
+
"step": 240
|
| 179 |
+
},
|
| 180 |
+
{
|
| 181 |
+
"epoch": 31.25,
|
| 182 |
+
"grad_norm": 0.041931431740522385,
|
| 183 |
+
"learning_rate": 0.006930817610062893,
|
| 184 |
+
"loss": 0.5302073001861572,
|
| 185 |
+
"step": 250
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"epoch": 32.5,
|
| 189 |
+
"grad_norm": 0.07813303917646408,
|
| 190 |
+
"learning_rate": 0.00680503144654088,
|
| 191 |
+
"loss": 0.5294707775115967,
|
| 192 |
+
"step": 260
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"epoch": 33.75,
|
| 196 |
+
"grad_norm": 0.03454748913645744,
|
| 197 |
+
"learning_rate": 0.006679245283018867,
|
| 198 |
+
"loss": 0.5287383079528809,
|
| 199 |
+
"step": 270
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"epoch": 35.0,
|
| 203 |
+
"grad_norm": 0.03431857377290726,
|
| 204 |
+
"learning_rate": 0.006553459119496855,
|
| 205 |
+
"loss": 0.5282272815704345,
|
| 206 |
+
"step": 280
|
| 207 |
+
},
|
| 208 |
+
{
|
| 209 |
+
"epoch": 36.25,
|
| 210 |
+
"grad_norm": 0.08966688811779022,
|
| 211 |
+
"learning_rate": 0.006427672955974842,
|
| 212 |
+
"loss": 0.5277254581451416,
|
| 213 |
+
"step": 290
|
| 214 |
+
},
|
| 215 |
+
{
|
| 216 |
+
"epoch": 37.5,
|
| 217 |
+
"grad_norm": 0.052265942096710205,
|
| 218 |
+
"learning_rate": 0.00630188679245283,
|
| 219 |
+
"loss": 0.5272915840148926,
|
| 220 |
+
"step": 300
|
| 221 |
+
}
|
| 222 |
+
],
|
| 223 |
+
"logging_steps": 10,
|
| 224 |
+
"max_steps": 800,
|
| 225 |
+
"num_input_tokens_seen": 0,
|
| 226 |
+
"num_train_epochs": 100,
|
| 227 |
+
"save_steps": 50,
|
| 228 |
+
"stateful_callbacks": {
|
| 229 |
+
"TrainerControl": {
|
| 230 |
+
"args": {
|
| 231 |
+
"should_epoch_stop": false,
|
| 232 |
+
"should_evaluate": false,
|
| 233 |
+
"should_log": false,
|
| 234 |
+
"should_save": true,
|
| 235 |
+
"should_training_stop": false
|
| 236 |
+
},
|
| 237 |
+
"attributes": {}
|
| 238 |
+
}
|
| 239 |
+
},
|
| 240 |
+
"total_flos": 16205184.0,
|
| 241 |
+
"train_batch_size": 128,
|
| 242 |
+
"trial_name": null,
|
| 243 |
+
"trial_params": null
|
| 244 |
+
}
|
checkpoint-300/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8ec7c4970e40c66e374eb444e43c769446c8d48e5a1f681b2a7f4187462d269e
|
| 3 |
+
size 5137
|