MHGanainy/gpt2-xl-lora-multi-7

README.md

@@ -15,6 +15,8 @@ should probably proofread and complete it, then remove this comment. -->
 # gpt2-xl-lora-multi-7
 
 This model is a fine-tuned version of [openai-community/gpt2-xl](https://huggingface.co/openai-community/gpt2-xl) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.5070
 
 ## Model description
 

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 1.0,
-    "eval_loss": 2.5500738620758057,
-    "eval_runtime": 333.0501,
-    "eval_samples_per_second": 7.786,
-    "eval_steps_per_second": 0.489,
-    "perplexity": 12.808049776869597,
-    "total_flos": 3.940214344788214e+17,
-    "train_loss": 2.675769654604105,
-    "train_runtime": 864.3634,
-    "train_samples_per_second": 50.048,
-    "train_steps_per_second": 3.128
+    "eval_loss": 2.506993293762207,
+    "eval_runtime": 354.5986,
+    "eval_samples_per_second": 8.503,
+    "eval_steps_per_second": 0.533,
+    "perplexity": 12.267988314860721,
+    "total_flos": 8.454557554683412e+17,
+    "train_loss": 2.7013893893405267,
+    "train_runtime": 1640.3942,
+    "train_samples_per_second": 56.586,
+    "train_steps_per_second": 3.537
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_loss": 2.5500738620758057,
-    "eval_runtime": 333.0501,
-    "eval_samples_per_second": 7.786,
-    "eval_steps_per_second": 0.489,
-    "perplexity": 12.808049776869597
+    "eval_loss": 2.506993293762207,
+    "eval_runtime": 354.5986,
+    "eval_samples_per_second": 8.503,
+    "eval_steps_per_second": 0.533,
+    "perplexity": 12.267988314860721
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "total_flos": 3.940214344788214e+17,
-    "train_loss": 2.675769654604105,
-    "train_runtime": 864.3634,
-    "train_samples_per_second": 50.048,
-    "train_steps_per_second": 3.128
+    "total_flos": 8.454557554683412e+17,
+    "train_loss": 2.7013893893405267,
+    "train_runtime": 1640.3942,
+    "train_samples_per_second": 56.586,
+    "train_steps_per_second": 3.537
 }

trainer_state.json

@@ -3,212 +3,429 @@
   "best_model_checkpoint": null,
   "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 2704,
+  "global_step": 5802,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.03698224852071006,
-      "grad_norm": 0.14849229156970978,
-      "learning_rate": 9.06148867313916e-07,
-      "loss": 2.9299,
+      "epoch": 0.01723543605653223,
+      "grad_norm": 0.04860854893922806,
+      "learning_rate": 4.309416074121957e-07,
+      "loss": 2.8479,
       "step": 100
     },
     {
-      "epoch": 0.07396449704142012,
-      "grad_norm": 0.18462003767490387,
-      "learning_rate": 1.8307905686546466e-06,
-      "loss": 2.9277,
+      "epoch": 0.03447087211306446,
+      "grad_norm": 0.04985743388533592,
+      "learning_rate": 8.618832148243914e-07,
+      "loss": 2.8411,
       "step": 200
     },
     {
-      "epoch": 0.11094674556213018,
-      "grad_norm": 0.2244928628206253,
-      "learning_rate": 2.7461858529819695e-06,
-      "loss": 2.9367,
+      "epoch": 0.05170630816959669,
+      "grad_norm": 0.05735534057021141,
+      "learning_rate": 1.292824822236587e-06,
+      "loss": 2.8386,
       "step": 300
     },
     {
-      "epoch": 0.14792899408284024,
-      "grad_norm": 0.11301285773515701,
-      "learning_rate": 3.6708275543227e-06,
-      "loss": 2.865,
+      "epoch": 0.06894174422612892,
+      "grad_norm": 0.06343039870262146,
+      "learning_rate": 1.7237664296487828e-06,
+      "loss": 2.8455,
       "step": 400
     },
     {
-      "epoch": 0.1849112426035503,
-      "grad_norm": 0.16168534755706787,
-      "learning_rate": 4.59546925566343e-06,
-      "loss": 2.834,
+      "epoch": 0.08617718028266115,
+      "grad_norm": 0.08376861363649368,
+      "learning_rate": 2.1547080370609785e-06,
+      "loss": 2.8307,
       "step": 500
     },
     {
-      "epoch": 0.22189349112426035,
-      "grad_norm": 0.23730890452861786,
-      "learning_rate": 5.520110957004161e-06,
-      "loss": 2.8,
+      "epoch": 0.10341261633919338,
+      "grad_norm": 0.09948327392339706,
+      "learning_rate": 2.585649644473174e-06,
+      "loss": 2.8059,
       "step": 600
     },
     {
-      "epoch": 0.2588757396449704,
-      "grad_norm": 0.150528684258461,
-      "learning_rate": 6.444752658344892e-06,
-      "loss": 2.7822,
+      "epoch": 0.12064805239572561,
+      "grad_norm": 0.12135949730873108,
+      "learning_rate": 3.0165912518853694e-06,
+      "loss": 2.8064,
       "step": 700
     },
     {
-      "epoch": 0.2958579881656805,
-      "grad_norm": 0.1429228037595749,
-      "learning_rate": 7.369394359685622e-06,
-      "loss": 2.7178,
+      "epoch": 0.13788348845225784,
+      "grad_norm": 0.11192767322063446,
+      "learning_rate": 3.4475328592975655e-06,
+      "loss": 2.7798,
       "step": 800
     },
     {
-      "epoch": 0.3328402366863905,
-      "grad_norm": 0.30748799443244934,
-      "learning_rate": 8.294036061026353e-06,
-      "loss": 2.7005,
+      "epoch": 0.15511892450879008,
+      "grad_norm": 0.1301010698080063,
+      "learning_rate": 3.878474466709762e-06,
+      "loss": 2.8018,
       "step": 900
     },
     {
-      "epoch": 0.3698224852071006,
-      "grad_norm": 0.163905069231987,
-      "learning_rate": 9.218677762367084e-06,
-      "loss": 2.6718,
+      "epoch": 0.1723543605653223,
+      "grad_norm": 0.17249374091625214,
+      "learning_rate": 4.309416074121957e-06,
+      "loss": 2.7911,
       "step": 1000
     },
     {
-      "epoch": 0.4068047337278107,
-      "grad_norm": 0.2939082384109497,
-      "learning_rate": 1.0143319463707815e-05,
-      "loss": 2.6669,
+      "epoch": 0.18958979662185454,
+      "grad_norm": 0.14794030785560608,
+      "learning_rate": 4.740357681534152e-06,
+      "loss": 2.7674,
       "step": 1100
     },
     {
-      "epoch": 0.4437869822485207,
-      "grad_norm": 0.22948721051216125,
-      "learning_rate": 1.1067961165048544e-05,
-      "loss": 2.6516,
+      "epoch": 0.20682523267838676,
+      "grad_norm": 0.15302610397338867,
+      "learning_rate": 5.171299288946348e-06,
+      "loss": 2.7775,
       "step": 1200
     },
     {
-      "epoch": 0.4807692307692308,
-      "grad_norm": 0.19319839775562286,
-      "learning_rate": 1.1992602866389276e-05,
-      "loss": 2.647,
+      "epoch": 0.224060668734919,
+      "grad_norm": 0.17765101790428162,
+      "learning_rate": 5.6022408963585436e-06,
+      "loss": 2.7465,
       "step": 1300
     },
     {
-      "epoch": 0.5177514792899408,
-      "grad_norm": 0.21269312500953674,
-      "learning_rate": 1.2917244567730005e-05,
-      "loss": 2.6304,
+      "epoch": 0.24129610479145122,
+      "grad_norm": 0.1747065782546997,
+      "learning_rate": 6.033182503770739e-06,
+      "loss": 2.7453,
       "step": 1400
     },
     {
-      "epoch": 0.5547337278106509,
-      "grad_norm": 0.2854045331478119,
-      "learning_rate": 1.3841886269070736e-05,
-      "loss": 2.6148,
+      "epoch": 0.25853154084798347,
+      "grad_norm": 0.19892176985740662,
+      "learning_rate": 6.464124111182936e-06,
+      "loss": 2.74,
       "step": 1500
     },
     {
-      "epoch": 0.591715976331361,
-      "grad_norm": 0.21025152504444122,
-      "learning_rate": 1.4766527970411465e-05,
-      "loss": 2.6068,
+      "epoch": 0.2757669769045157,
+      "grad_norm": 0.2117730677127838,
+      "learning_rate": 6.895065718595131e-06,
+      "loss": 2.7307,
       "step": 1600
     },
     {
-      "epoch": 0.628698224852071,
-      "grad_norm": 0.2919473350048065,
-      "learning_rate": 1.5691169671752196e-05,
-      "loss": 2.6001,
+      "epoch": 0.2930024129610479,
+      "grad_norm": 0.198543980717659,
+      "learning_rate": 7.326007326007326e-06,
+      "loss": 2.7261,
       "step": 1700
     },
     {
-      "epoch": 0.665680473372781,
-      "grad_norm": 0.21243709325790405,
-      "learning_rate": 1.661581137309293e-05,
-      "loss": 2.5868,
+      "epoch": 0.31023784901758017,
+      "grad_norm": 0.2144545465707779,
+      "learning_rate": 7.756948933419523e-06,
+      "loss": 2.7167,
       "step": 1800
     },
     {
-      "epoch": 0.7026627218934911,
-      "grad_norm": 0.2556278109550476,
-      "learning_rate": 1.7540453074433658e-05,
-      "loss": 2.5812,
+      "epoch": 0.3274732850741124,
+      "grad_norm": 0.20613212883472443,
+      "learning_rate": 8.187890540831719e-06,
+      "loss": 2.7269,
       "step": 1900
     },
     {
-      "epoch": 0.7396449704142012,
-      "grad_norm": 0.2942062318325043,
-      "learning_rate": 1.8465094775774387e-05,
-      "loss": 2.5816,
+      "epoch": 0.3447087211306446,
+      "grad_norm": 0.21626847982406616,
+      "learning_rate": 8.618832148243914e-06,
+      "loss": 2.7157,
       "step": 2000
     },
     {
-      "epoch": 0.7766272189349113,
-      "grad_norm": 0.23691928386688232,
-      "learning_rate": 1.938973647711512e-05,
-      "loss": 2.5727,
+      "epoch": 0.3619441571871768,
+      "grad_norm": 0.2106010764837265,
+      "learning_rate": 9.049773755656109e-06,
+      "loss": 2.7148,
       "step": 2100
     },
     {
-      "epoch": 0.8136094674556213,
-      "grad_norm": 0.2230062484741211,
-      "learning_rate": 1.980572281912978e-05,
-      "loss": 2.5698,
+      "epoch": 0.3791795932437091,
+      "grad_norm": 0.2537670433521271,
+      "learning_rate": 9.480715363068304e-06,
+      "loss": 2.7181,
       "step": 2200
     },
     {
-      "epoch": 0.8505917159763313,
-      "grad_norm": 0.23846900463104248,
-      "learning_rate": 1.7122208281783788e-05,
-      "loss": 2.552,
+      "epoch": 0.3964150293002413,
+      "grad_norm": 0.23555806279182434,
+      "learning_rate": 9.911656970480501e-06,
+      "loss": 2.6989,
       "step": 2300
     },
     {
-      "epoch": 0.8875739644970414,
-      "grad_norm": 0.24352632462978363,
-      "learning_rate": 1.2103724124062295e-05,
-      "loss": 2.565,
+      "epoch": 0.4136504653567735,
+      "grad_norm": 0.2480691522359848,
+      "learning_rate": 1.0342598577892697e-05,
+      "loss": 2.6951,
       "step": 2400
     },
     {
-      "epoch": 0.9245562130177515,
-      "grad_norm": 0.2532407343387604,
-      "learning_rate": 6.395547707284e-06,
-      "loss": 2.5503,
+      "epoch": 0.43088590141330574,
+      "grad_norm": 0.2250298112630844,
+      "learning_rate": 1.0773540185304892e-05,
+      "loss": 2.705,
       "step": 2500
     },
     {
-      "epoch": 0.9615384615384616,
-      "grad_norm": 0.2501896619796753,
-      "learning_rate": 1.869067510077197e-06,
-      "loss": 2.5497,
+      "epoch": 0.448121337469838,
+      "grad_norm": 0.2429608255624771,
+      "learning_rate": 1.1204481792717087e-05,
+      "loss": 2.6872,
       "step": 2600
     },
     {
-      "epoch": 0.9985207100591716,
-      "grad_norm": 0.21952682733535767,
-      "learning_rate": 8.260598198537306e-09,
-      "loss": 2.5555,
+      "epoch": 0.4653567735263702,
+      "grad_norm": 0.251676082611084,
+      "learning_rate": 1.1635423400129284e-05,
+      "loss": 2.6947,
       "step": 2700
     },
+    {
+      "epoch": 0.48259220958290244,
+      "grad_norm": 0.2518726587295532,
+      "learning_rate": 1.2066365007541478e-05,
+      "loss": 2.6895,
+      "step": 2800
+    },
+    {
+      "epoch": 0.49982764563943466,
+      "grad_norm": 0.24320507049560547,
+      "learning_rate": 1.2497306614953675e-05,
+      "loss": 2.6864,
+      "step": 2900
+    },
+    {
+      "epoch": 0.5170630816959669,
+      "grad_norm": 0.2310476005077362,
+      "learning_rate": 1.2928248222365872e-05,
+      "loss": 2.6808,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5342985177524991,
+      "grad_norm": 0.22928917407989502,
+      "learning_rate": 1.3359189829778065e-05,
+      "loss": 2.6752,
+      "step": 3100
+    },
+    {
+      "epoch": 0.5515339538090314,
+      "grad_norm": 0.24209704995155334,
+      "learning_rate": 1.3790131437190262e-05,
+      "loss": 2.6736,
+      "step": 3200
+    },
+    {
+      "epoch": 0.5687693898655636,
+      "grad_norm": 0.23778828978538513,
+      "learning_rate": 1.4221073044602457e-05,
+      "loss": 2.6668,
+      "step": 3300
+    },
+    {
+      "epoch": 0.5860048259220958,
+      "grad_norm": 0.2457701861858368,
+      "learning_rate": 1.4652014652014653e-05,
+      "loss": 2.6593,
+      "step": 3400
+    },
+    {
+      "epoch": 0.603240261978628,
+      "grad_norm": 0.2431216984987259,
+      "learning_rate": 1.5082956259426848e-05,
+      "loss": 2.6671,
+      "step": 3500
+    },
+    {
+      "epoch": 0.6204756980351603,
+      "grad_norm": 0.23864516615867615,
+      "learning_rate": 1.5513897866839047e-05,
+      "loss": 2.6496,
+      "step": 3600
+    },
+    {
+      "epoch": 0.6377111340916926,
+      "grad_norm": 0.2714649438858032,
+      "learning_rate": 1.594483947425124e-05,
+      "loss": 2.6536,
+      "step": 3700
+    },
+    {
+      "epoch": 0.6549465701482248,
+      "grad_norm": 0.2654697895050049,
+      "learning_rate": 1.6375781081663437e-05,
+      "loss": 2.6632,
+      "step": 3800
+    },
+    {
+      "epoch": 0.672182006204757,
+      "grad_norm": 0.27372944355010986,
+      "learning_rate": 1.680672268907563e-05,
+      "loss": 2.6646,
+      "step": 3900
+    },
+    {
+      "epoch": 0.6894174422612892,
+      "grad_norm": 0.265690416097641,
+      "learning_rate": 1.7237664296487828e-05,
+      "loss": 2.6554,
+      "step": 4000
+    },
+    {
+      "epoch": 0.7066528783178214,
+      "grad_norm": 0.27707237005233765,
+      "learning_rate": 1.766860590390002e-05,
+      "loss": 2.6552,
+      "step": 4100
+    },
+    {
+      "epoch": 0.7238883143743536,
+      "grad_norm": 0.2691991627216339,
+      "learning_rate": 1.8099547511312218e-05,
+      "loss": 2.6336,
+      "step": 4200
+    },
+    {
+      "epoch": 0.7411237504308859,
+      "grad_norm": 0.2783598303794861,
+      "learning_rate": 1.8530489118724415e-05,
+      "loss": 2.6479,
+      "step": 4300
+    },
+    {
+      "epoch": 0.7583591864874182,
+      "grad_norm": 0.27648258209228516,
+      "learning_rate": 1.896143072613661e-05,
+      "loss": 2.6394,
+      "step": 4400
+    },
+    {
+      "epoch": 0.7755946225439504,
+      "grad_norm": 0.28013989329338074,
+      "learning_rate": 1.9392372333548806e-05,
+      "loss": 2.6528,
+      "step": 4500
+    },
+    {
+      "epoch": 0.7928300586004826,
+      "grad_norm": 0.28954795002937317,
+      "learning_rate": 1.9823313940961003e-05,
+      "loss": 2.6485,
+      "step": 4600
+    },
+    {
+      "epoch": 0.8100654946570148,
+      "grad_norm": 0.2726477384567261,
+      "learning_rate": 1.9872829409331243e-05,
+      "loss": 2.6422,
+      "step": 4700
+    },
+    {
+      "epoch": 0.827300930713547,
+      "grad_norm": 0.27916714549064636,
+      "learning_rate": 1.9088640248516185e-05,
+      "loss": 2.6409,
+      "step": 4800
+    },
+    {
+      "epoch": 0.8445363667700793,
+      "grad_norm": 0.28933390974998474,
+      "learning_rate": 1.7643022980549708e-05,
+      "loss": 2.6298,
+      "step": 4900
+    },
+    {
+      "epoch": 0.8617718028266115,
+      "grad_norm": 0.27602875232696533,
+      "learning_rate": 1.5641182770018732e-05,
+      "loss": 2.6445,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8790072388831437,
+      "grad_norm": 0.32238247990608215,
+      "learning_rate": 1.3228804047714462e-05,
+      "loss": 2.6318,
+      "step": 5100
+    },
+    {
+      "epoch": 0.896242674939676,
+      "grad_norm": 0.32085633277893066,
+      "learning_rate": 1.0581448289104759e-05,
+      "loss": 2.6317,
+      "step": 5200
+    },
+    {
+      "epoch": 0.9134781109962082,
+      "grad_norm": 0.2872781455516815,
+      "learning_rate": 7.891777483276439e-06,
+      "loss": 2.6305,
+      "step": 5300
+    },
+    {
+      "epoch": 0.9307135470527405,
+      "grad_norm": 0.29465451836586,
+      "learning_rate": 5.3555331076468585e-06,
+      "loss": 2.6339,
+      "step": 5400
+    },
+    {
+      "epoch": 0.9479489831092727,
+      "grad_norm": 0.27886396646499634,
+      "learning_rate": 3.1572909921285387e-06,
+      "loss": 2.6381,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9651844191658049,
+      "grad_norm": 0.3047686219215393,
+      "learning_rate": 1.4570287661325555e-06,
+      "loss": 2.6391,
+      "step": 5600
+    },
+    {
+      "epoch": 0.9824198552223371,
+      "grad_norm": 0.29873353242874146,
+      "learning_rate": 3.7848344587376297e-07,
+      "loss": 2.6345,
+      "step": 5700
+    },
+    {
+      "epoch": 0.9996552912788693,
+      "grad_norm": 0.2810242474079132,
+      "learning_rate": 1.4644152457554683e-10,
+      "loss": 2.6207,
+      "step": 5800
+    },
     {
       "epoch": 1.0,
-      "step": 2704,
-      "total_flos": 3.940214344788214e+17,
-      "train_loss": 2.675769654604105,
-      "train_runtime": 864.3634,
-      "train_samples_per_second": 50.048,
-      "train_steps_per_second": 3.128
+      "step": 5802,
+      "total_flos": 8.454557554683412e+17,
+      "train_loss": 2.7013893893405267,
+      "train_runtime": 1640.3942,
+      "train_samples_per_second": 56.586,
+      "train_steps_per_second": 3.537
     }
   ],
   "logging_steps": 100,
-  "max_steps": 2704,
+  "max_steps": 5802,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
@@ -224,7 +441,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.940214344788214e+17,
+  "total_flos": 8.454557554683412e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null