Upload folder using huggingface_hub

adapter_config.json

@@ -1,6 +1,6 @@
 {
   "auto_mapping": null,
-  "base_model_name_or_path": "EleutherAI/pythia-1b",
+  "base_model_name_or_path": "PY007/TinyLlama-1.1B-step-50K-105b",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -14,10 +14,13 @@
   "r": 64,
   "revision": null,
   "target_modules": [
-    "query_key_value",
-    "dense",
-    "dense_h_to_4h",
-    "dense_4h_to_h"
+    "q_proj",
+    "up_proj",
+    "o_proj",
+    "k_proj",
+    "down_proj",
+    "gate_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2fc2dba896ca3b9739d260e5f4952fe7c40f80bd72bfeac929d530e111b60e44
-size 134264397
+oid sha256:0d6bd8cd65efbb9c7974ebde35831035eecd7521c871e2008a5f01969edcdbf9
+size 201961693

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3a0ae6f88d5f5416ea0a4c8142d37e1d5da2832716b0de03ecf8de795d03972
-size 268514437
+oid sha256:74e628e09f3e5e99b963ef698042bacd5a1900eec2208a9a012bfde0bcc8a42f
+size 403890693

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e2077911c313e5f018db4ece5d64c9dc180280cf53d39084178d124b0c05522
+oid sha256:f7d72f97a1b745f2eed728363c33432ea85a45bfb04ca65630eb8ffe0529550b
 size 14575

special_tokens_map.json

@@ -1,6 +1,24 @@
 {
-  "bos_token": "<|endoftext|>",
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>",
-  "unk_token": "<|endoftext|>"
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -1,9 +1,34 @@
 {
-  "add_prefix_space": false,
-  "bos_token": "<|endoftext|>",
-  "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": false,
   "model_max_length": 1000000000000000019884624838656,
-  "tokenizer_class": "GPTNeoXTokenizer",
-  "unk_token": "<|endoftext|>"
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "use_default_system_prompt": true
 }

trainer_state.json

@@ -11,585 +11,585 @@
     {
       "epoch": 0.03,
       "learning_rate": 6.666666666666667e-06,
-      "loss": 2.272,
+      "loss": 2.5458,
       "step": 2
     },
     {
       "epoch": 0.06,
       "learning_rate": 1.3333333333333333e-05,
-      "loss": 2.4695,
+      "loss": 2.6449,
       "step": 4
     },
     {
       "epoch": 0.09,
       "learning_rate": 2e-05,
-      "loss": 2.3336,
+      "loss": 2.507,
       "step": 6
     },
     {
       "epoch": 0.12,
       "learning_rate": 1.999429490929718e-05,
-      "loss": 2.5551,
+      "loss": 2.5092,
       "step": 8
     },
     {
       "epoch": 0.16,
       "learning_rate": 1.9977186146800707e-05,
-      "loss": 2.5592,
+      "loss": 2.4416,
       "step": 10
     },
     {
       "epoch": 0.19,
       "learning_rate": 1.994869323391895e-05,
-      "loss": 2.6964,
+      "loss": 2.6069,
       "step": 12
     },
     {
       "epoch": 0.22,
       "learning_rate": 1.990884868158239e-05,
-      "loss": 2.6577,
+      "loss": 2.5521,
       "step": 14
     },
     {
       "epoch": 0.25,
       "learning_rate": 1.985769795314804e-05,
-      "loss": 3.024,
+      "loss": 2.4192,
       "step": 16
     },
     {
       "epoch": 0.28,
       "learning_rate": 1.9795299412524948e-05,
-      "loss": 2.2578,
+      "loss": 2.6886,
       "step": 18
     },
     {
       "epoch": 0.31,
       "learning_rate": 1.9721724257579907e-05,
-      "loss": 2.2962,
+      "loss": 2.5329,
       "step": 20
     },
     {
       "epoch": 0.34,
       "learning_rate": 1.963705643889941e-05,
-      "loss": 2.298,
+      "loss": 2.4502,
       "step": 22
     },
     {
       "epoch": 0.37,
       "learning_rate": 1.954139256400049e-05,
-      "loss": 2.567,
+      "loss": 2.5449,
       "step": 24
     },
     {
       "epoch": 0.4,
       "learning_rate": 1.9434841787099804e-05,
-      "loss": 2.5781,
+      "loss": 2.5465,
       "step": 26
     },
     {
       "epoch": 0.43,
       "learning_rate": 1.9317525684566686e-05,
-      "loss": 2.7195,
+      "loss": 2.47,
       "step": 28
     },
     {
       "epoch": 0.47,
       "learning_rate": 1.918957811620231e-05,
-      "loss": 2.6374,
+      "loss": 2.5249,
       "step": 30
     },
     {
       "epoch": 0.5,
       "learning_rate": 1.9051145072503216e-05,
-      "loss": 3.077,
+      "loss": 2.3622,
       "step": 32
     },
     {
       "epoch": 0.53,
       "learning_rate": 1.8902384508083518e-05,
-      "loss": 2.2206,
+      "loss": 2.4128,
       "step": 34
     },
     {
       "epoch": 0.56,
       "learning_rate": 1.8743466161445823e-05,
-      "loss": 2.4387,
+      "loss": 2.5942,
       "step": 36
     },
     {
       "epoch": 0.59,
       "learning_rate": 1.857457136130651e-05,
-      "loss": 2.4826,
+      "loss": 2.4582,
       "step": 38
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.839589281969639e-05,
-      "loss": 2.5256,
+      "loss": 2.5707,
       "step": 40
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.8207634412072765e-05,
-      "loss": 2.6213,
+      "loss": 2.5241,
       "step": 42
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.8010010944693846e-05,
-      "loss": 2.5248,
+      "loss": 2.4121,
       "step": 44
     },
     {
       "epoch": 0.71,
       "learning_rate": 1.780324790952092e-05,
-      "loss": 2.5491,
+      "loss": 2.5349,
       "step": 46
     },
     {
       "epoch": 0.75,
       "learning_rate": 1.758758122692791e-05,
-      "loss": 2.7557,
+      "loss": 2.429,
       "step": 48
     },
     {
       "epoch": 0.78,
       "learning_rate": 1.7363256976511972e-05,
-      "loss": 2.2802,
+      "loss": 2.2583,
       "step": 50
     },
     {
       "epoch": 0.81,
       "learning_rate": 1.7130531116312202e-05,
-      "loss": 2.3701,
+      "loss": 2.423,
       "step": 52
     },
     {
       "epoch": 0.84,
       "learning_rate": 1.688966919075687e-05,
-      "loss": 2.4451,
+      "loss": 2.4998,
       "step": 54
     },
     {
       "epoch": 0.87,
       "learning_rate": 1.6640946027672395e-05,
-      "loss": 2.4015,
+      "loss": 2.3889,
       "step": 56
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.6384645424699835e-05,
-      "loss": 2.5189,
+      "loss": 2.4283,
       "step": 58
     },
     {
       "epoch": 0.93,
       "learning_rate": 1.612105982547663e-05,
-      "loss": 2.5074,
+      "loss": 2.3781,
       "step": 60
     },
     {
       "epoch": 0.96,
       "learning_rate": 1.5850489985953076e-05,
-      "loss": 2.6477,
+      "loss": 2.4362,
       "step": 62
     },
     {
       "epoch": 0.99,
       "learning_rate": 1.5573244631224364e-05,
-      "loss": 2.5961,
+      "loss": 2.3347,
       "step": 64
     },
     {
       "epoch": 1.03,
       "learning_rate": 1.5289640103269626e-05,
-      "loss": 2.2862,
+      "loss": 2.2687,
       "step": 66
     },
     {
       "epoch": 1.06,
       "learning_rate": 1.5000000000000002e-05,
-      "loss": 2.2579,
+      "loss": 2.4113,
       "step": 68
     },
     {
       "epoch": 1.09,
       "learning_rate": 1.4704654806027558e-05,
-      "loss": 2.302,
+      "loss": 2.3682,
       "step": 70
     },
     {
       "epoch": 1.12,
       "learning_rate": 1.4403941515576344e-05,
-      "loss": 2.4244,
+      "loss": 2.1919,
       "step": 72
     },
     {
       "epoch": 1.15,
       "learning_rate": 1.4098203247965876e-05,
-      "loss": 2.5768,
+      "loss": 2.1295,
       "step": 74
     },
     {
       "epoch": 1.18,
       "learning_rate": 1.3787788856105762e-05,
-      "loss": 2.401,
+      "loss": 2.2977,
       "step": 76
     },
     {
       "epoch": 1.21,
       "learning_rate": 1.3473052528448203e-05,
-      "loss": 2.5576,
+      "loss": 2.3783,
       "step": 78
     },
     {
       "epoch": 1.24,
       "learning_rate": 1.3154353384852559e-05,
-      "loss": 2.4588,
+      "loss": 2.3447,
       "step": 80
     },
     {
       "epoch": 1.27,
       "learning_rate": 1.283205506682304e-05,
-      "loss": 2.2279,
+      "loss": 2.3861,
       "step": 82
     },
     {
       "epoch": 1.3,
       "learning_rate": 1.2506525322587207e-05,
-      "loss": 2.2371,
+      "loss": 2.1456,
       "step": 84
     },
     {
       "epoch": 1.34,
       "learning_rate": 1.2178135587488515e-05,
-      "loss": 2.3653,
+      "loss": 2.309,
       "step": 86
     },
     {
       "epoch": 1.37,
       "learning_rate": 1.1847260560171895e-05,
-      "loss": 2.3693,
+      "loss": 2.2821,
       "step": 88
     },
     {
       "epoch": 1.4,
       "learning_rate": 1.1514277775045768e-05,
-      "loss": 2.4593,
+      "loss": 2.2697,
       "step": 90
     },
     {
       "epoch": 1.43,
       "learning_rate": 1.1179567171508463e-05,
-      "loss": 2.4767,
+      "loss": 2.2403,
       "step": 92
     },
     {
       "epoch": 1.46,
       "learning_rate": 1.0843510660430447e-05,
-      "loss": 2.4792,
+      "loss": 2.2851,
       "step": 94
     },
     {
       "epoch": 1.49,
       "learning_rate": 1.0506491688387128e-05,
-      "loss": 2.6227,
+      "loss": 2.3611,
       "step": 96
     },
     {
       "epoch": 1.52,
       "learning_rate": 1.0168894800139311e-05,
-      "loss": 2.4314,
+      "loss": 2.2993,
       "step": 98
     },
     {
       "epoch": 1.55,
       "learning_rate": 9.83110519986069e-06,
-      "loss": 2.1875,
+      "loss": 2.3342,
       "step": 100
     },
     {
       "epoch": 1.58,
       "learning_rate": 9.493508311612874e-06,
-      "loss": 2.3647,
+      "loss": 2.4243,
       "step": 102
     },
     {
       "epoch": 1.62,
       "learning_rate": 9.156489339569555e-06,
-      "loss": 2.4081,
+      "loss": 2.3373,
       "step": 104
     },
     {
       "epoch": 1.65,
       "learning_rate": 8.820432828491542e-06,
-      "loss": 2.5092,
+      "loss": 2.3797,
       "step": 106
     },
     {
       "epoch": 1.68,
       "learning_rate": 8.485722224954237e-06,
-      "loss": 2.5936,
+      "loss": 2.2999,
       "step": 108
     },
     {
       "epoch": 1.71,
       "learning_rate": 8.15273943982811e-06,
-      "loss": 2.5314,
+      "loss": 2.4485,
       "step": 110
     },
     {
       "epoch": 1.74,
       "learning_rate": 7.821864412511485e-06,
-      "loss": 2.564,
+      "loss": 2.4276,
       "step": 112
     },
     {
       "epoch": 1.77,
       "learning_rate": 7.493474677412795e-06,
-      "loss": 2.4031,
+      "loss": 2.2462,
       "step": 114
     },
     {
       "epoch": 1.8,
       "learning_rate": 7.16794493317696e-06,
-      "loss": 2.3595,
+      "loss": 2.3157,
       "step": 116
     },
     {
       "epoch": 1.83,
       "learning_rate": 6.845646615147445e-06,
-      "loss": 2.4157,
+      "loss": 2.3166,
       "step": 118
     },
     {
       "epoch": 1.86,
       "learning_rate": 6.526947471551799e-06,
-      "loss": 2.4893,
+      "loss": 2.5255,
       "step": 120
     },
     {
       "epoch": 1.9,
       "learning_rate": 6.21221114389424e-06,
-      "loss": 2.4299,
+      "loss": 2.3106,
       "step": 122
     },
     {
       "epoch": 1.93,
       "learning_rate": 5.901796752034128e-06,
-      "loss": 2.4609,
+      "loss": 2.2582,
       "step": 124
     },
     {
       "epoch": 1.96,
       "learning_rate": 5.5960584844236565e-06,
-      "loss": 2.5731,
+      "loss": 2.2902,
       "step": 126
     },
     {
       "epoch": 1.99,
       "learning_rate": 5.295345193972445e-06,
-      "loss": 2.4888,
+      "loss": 2.295,
       "step": 128
     },
     {
       "epoch": 2.02,
       "learning_rate": 5.000000000000003e-06,
-      "loss": 2.2167,
+      "loss": 2.1651,
       "step": 130
     },
     {
       "epoch": 2.05,
       "learning_rate": 4.710359896730379e-06,
-      "loss": 2.3269,
+      "loss": 2.1819,
       "step": 132
     },
     {
       "epoch": 2.08,
       "learning_rate": 4.426755368775637e-06,
-      "loss": 2.4076,
+      "loss": 2.298,
       "step": 134
     },
     {
       "epoch": 2.11,
       "learning_rate": 4.149510014046922e-06,
-      "loss": 2.4014,
+      "loss": 2.2939,
       "step": 136
     },
     {
       "epoch": 2.14,
       "learning_rate": 3.878940174523371e-06,
-      "loss": 2.5295,
+      "loss": 2.3348,
       "step": 138
     },
     {
       "epoch": 2.17,
       "learning_rate": 3.6153545753001663e-06,
-      "loss": 2.5084,
+      "loss": 2.2286,
       "step": 140
     },
     {
       "epoch": 2.21,
       "learning_rate": 3.3590539723276083e-06,
-      "loss": 2.5819,
+      "loss": 2.366,
       "step": 142
     },
     {
       "epoch": 2.24,
       "learning_rate": 3.110330809243134e-06,
-      "loss": 2.5445,
+      "loss": 2.2215,
       "step": 144
     },
     {
       "epoch": 2.27,
       "learning_rate": 2.869468883687798e-06,
-      "loss": 2.2474,
+      "loss": 2.3872,
       "step": 146
     },
     {
       "epoch": 2.3,
       "learning_rate": 2.6367430234880286e-06,
-      "loss": 2.2122,
+      "loss": 2.3508,
       "step": 148
     },
     {
       "epoch": 2.33,
       "learning_rate": 2.4124187730720916e-06,
-      "loss": 2.4545,
+      "loss": 2.21,
       "step": 150
     },
     {
       "epoch": 2.36,
       "learning_rate": 2.196752090479083e-06,
-      "loss": 2.3552,
+      "loss": 2.2727,
       "step": 152
     },
     {
       "epoch": 2.39,
       "learning_rate": 1.9899890553061565e-06,
-      "loss": 2.3497,
+      "loss": 2.3211,
       "step": 154
     },
     {
       "epoch": 2.42,
       "learning_rate": 1.7923655879272395e-06,
-      "loss": 2.4217,
+      "loss": 2.3189,
       "step": 156
     },
     {
       "epoch": 2.45,
       "learning_rate": 1.60410718030361e-06,
-      "loss": 2.5307,
+      "loss": 2.3204,
       "step": 158
     },
     {
       "epoch": 2.49,
       "learning_rate": 1.425428638693489e-06,
-      "loss": 2.5247,
+      "loss": 2.4951,
       "step": 160
     },
     {
       "epoch": 2.52,
       "learning_rate": 1.2565338385541792e-06,
-      "loss": 2.3212,
+      "loss": 2.3137,
       "step": 162
     },
     {
       "epoch": 2.55,
       "learning_rate": 1.097615491916485e-06,
-      "loss": 2.3383,
+      "loss": 2.4126,
       "step": 164
     },
     {
       "epoch": 2.58,
       "learning_rate": 9.488549274967873e-07,
-      "loss": 2.3786,
+      "loss": 2.1728,
       "step": 166
     },
     {
       "epoch": 2.61,
       "learning_rate": 8.10421883797694e-07,
-      "loss": 2.1937,
+      "loss": 2.4021,
       "step": 168
     },
     {
       "epoch": 2.64,
       "learning_rate": 6.824743154333157e-07,
-      "loss": 2.6403,
+      "loss": 2.3523,
       "step": 170
     },
     {
       "epoch": 2.67,
       "learning_rate": 5.651582129001987e-07,
-      "loss": 2.4291,
+      "loss": 2.2559,
       "step": 172
     },
     {
       "epoch": 2.7,
       "learning_rate": 4.5860743599951186e-07,
-      "loss": 2.5093,
+      "loss": 2.289,
       "step": 174
     },
     {
       "epoch": 2.73,
       "learning_rate": 3.629435611005916e-07,
-      "loss": 2.5631,
+      "loss": 2.3431,
       "step": 176
     },
     {
       "epoch": 2.77,
       "learning_rate": 2.7827574242009434e-07,
-      "loss": 2.3013,
+      "loss": 2.0865,
       "step": 178
     },
     {
       "epoch": 2.8,
       "learning_rate": 2.0470058747505516e-07,
-      "loss": 2.2554,
+      "loss": 2.2721,
       "step": 180
     },
     {
       "epoch": 2.83,
       "learning_rate": 1.4230204685196202e-07,
-      "loss": 2.2287,
+      "loss": 2.2386,
       "step": 182
     },
     {
       "epoch": 2.86,
       "learning_rate": 9.11513184176116e-08,
-      "loss": 2.3358,
+      "loss": 2.3524,
       "step": 184
     },
     {
       "epoch": 2.89,
       "learning_rate": 5.1306766081048456e-08,
-      "loss": 2.6048,
+      "loss": 2.1862,
       "step": 186
     },
     {
       "epoch": 2.92,
       "learning_rate": 2.2813853199292745e-08,
-      "loss": 2.3805,
+      "loss": 2.1838,
       "step": 188
     },
     {
       "epoch": 2.95,
       "learning_rate": 5.705090702819993e-09,
-      "loss": 2.3132,
+      "loss": 2.3008,
       "step": 190
     },
     {
       "epoch": 2.98,
       "learning_rate": 0.0,
-      "loss": 2.5282,
+      "loss": 2.1693,
       "step": 192
     }
   ],
   "logging_steps": 2,
   "max_steps": 192,
   "num_train_epochs": 3,
-  "save_steps": 500,
-  "total_flos": 5304590800896000.0,
+  "save_steps": 64,
+  "total_flos": 6235438548676608.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78e9ef3af195d45d797142c195579330db1f765a3733bea77c85d5abb0eab051
+oid sha256:07f11227057e4f2463238905fea6ace9d48bcc9d95fe55a90451e755c89a984f
 size 4027