End of training

README.md

@@ -4,11 +4,23 @@ license: apache-2.0
 base_model: distilbert-base-uncased
 tags:
 - generated_from_trainer
+datasets:
+- gokulsrinivasagan/processed_wikitext-103-raw-v1-ld
 metrics:
 - accuracy
 model-index:
 - name: tinybert_book_ent_15p_b
-  results: []
+  results:
+  - task:
+      name: Masked Language Modeling
+      type: fill-mask
+    dataset:
+      name: gokulsrinivasagan/processed_wikitext-103-raw-v1-ld
+      type: gokulsrinivasagan/processed_wikitext-103-raw-v1-ld
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.6417701351499835
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -16,7 +28,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # tinybert_book_ent_15p_b
 
-This model is a fine-tuned version of [distilbert-base-uncased](https://huggingface.co/distilbert-base-uncased) on an unknown dataset.
+This model is a fine-tuned version of [distilbert-base-uncased](https://huggingface.co/distilbert-base-uncased) on the gokulsrinivasagan/processed_wikitext-103-raw-v1-ld dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.3713
 - Accuracy: 0.6418

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 25.0,
+    "eval_accuracy": 0.6417701351499835,
+    "eval_loss": 2.371307849884033,
+    "eval_runtime": 6.438,
+    "eval_samples": 479,
+    "eval_samples_per_second": 74.403,
+    "eval_steps_per_second": 0.466,
+    "perplexity": 10.71139202015035,
+    "total_flos": 3.003225489805824e+17,
+    "train_loss": 3.287461715378128,
+    "train_runtime": 79951.4195,
+    "train_samples": 228639,
+    "train_samples_per_second": 71.493,
+    "train_steps_per_second": 0.358
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 25.0,
+    "eval_accuracy": 0.6417701351499835,
+    "eval_loss": 2.371307849884033,
+    "eval_runtime": 6.438,
+    "eval_samples": 479,
+    "eval_samples_per_second": 74.403,
+    "eval_steps_per_second": 0.466,
+    "perplexity": 10.71139202015035
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 25.0,
+    "total_flos": 3.003225489805824e+17,
+    "train_loss": 3.287461715378128,
+    "train_runtime": 79951.4195,
+    "train_samples": 228639,
+    "train_samples_per_second": 71.493,
+    "train_steps_per_second": 0.358
+}

trainer_state.json

@@ -0,0 +1,460 @@
+{
+  "best_global_step": 20000,
+  "best_metric": 2.371307849884033,
+  "best_model_checkpoint": "tinybert_book_ent_15p_b/checkpoint-20000",
+  "epoch": 25.0,
+  "eval_steps": 10000,
+  "global_step": 28600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4370629370629371,
+      "grad_norm": 4.4531779289245605,
+      "learning_rate": 4.9900000000000005e-06,
+      "loss": 8.9785,
+      "step": 500
+    },
+    {
+      "epoch": 0.8741258741258742,
+      "grad_norm": 1.7696887254714966,
+      "learning_rate": 9.990000000000001e-06,
+      "loss": 6.6226,
+      "step": 1000
+    },
+    {
+      "epoch": 1.3111888111888113,
+      "grad_norm": 0.5621947050094604,
+      "learning_rate": 1.499e-05,
+      "loss": 5.5121,
+      "step": 1500
+    },
+    {
+      "epoch": 1.7482517482517483,
+      "grad_norm": 0.48923590779304504,
+      "learning_rate": 1.999e-05,
+      "loss": 5.4059,
+      "step": 2000
+    },
+    {
+      "epoch": 2.1853146853146854,
+      "grad_norm": 0.8969066143035889,
+      "learning_rate": 2.4990000000000003e-05,
+      "loss": 5.3694,
+      "step": 2500
+    },
+    {
+      "epoch": 2.6223776223776225,
+      "grad_norm": 0.5264173150062561,
+      "learning_rate": 2.9990000000000003e-05,
+      "loss": 5.3372,
+      "step": 3000
+    },
+    {
+      "epoch": 3.0594405594405596,
+      "grad_norm": 0.8283560276031494,
+      "learning_rate": 3.499e-05,
+      "loss": 5.2569,
+      "step": 3500
+    },
+    {
+      "epoch": 3.4965034965034967,
+      "grad_norm": 0.8435366153717041,
+      "learning_rate": 3.999e-05,
+      "loss": 5.1864,
+      "step": 4000
+    },
+    {
+      "epoch": 3.9335664335664333,
+      "grad_norm": 1.0385633707046509,
+      "learning_rate": 4.499e-05,
+      "loss": 5.0573,
+      "step": 4500
+    },
+    {
+      "epoch": 4.370629370629371,
+      "grad_norm": 1.3719793558120728,
+      "learning_rate": 4.999e-05,
+      "loss": 4.7748,
+      "step": 5000
+    },
+    {
+      "epoch": 4.8076923076923075,
+      "grad_norm": 1.8106296062469482,
+      "learning_rate": 5.499000000000001e-05,
+      "loss": 4.532,
+      "step": 5500
+    },
+    {
+      "epoch": 5.244755244755245,
+      "grad_norm": 1.7491868734359741,
+      "learning_rate": 5.999e-05,
+      "loss": 4.3431,
+      "step": 6000
+    },
+    {
+      "epoch": 5.681818181818182,
+      "grad_norm": 1.5972195863723755,
+      "learning_rate": 6.499000000000001e-05,
+      "loss": 4.1977,
+      "step": 6500
+    },
+    {
+      "epoch": 6.118881118881119,
+      "grad_norm": 1.8990917205810547,
+      "learning_rate": 6.999e-05,
+      "loss": 4.0613,
+      "step": 7000
+    },
+    {
+      "epoch": 6.555944055944056,
+      "grad_norm": 1.9757366180419922,
+      "learning_rate": 7.499e-05,
+      "loss": 3.8787,
+      "step": 7500
+    },
+    {
+      "epoch": 6.993006993006993,
+      "grad_norm": 1.5962578058242798,
+      "learning_rate": 7.999000000000001e-05,
+      "loss": 3.6969,
+      "step": 8000
+    },
+    {
+      "epoch": 7.43006993006993,
+      "grad_norm": 1.664330244064331,
+      "learning_rate": 8.499e-05,
+      "loss": 3.5414,
+      "step": 8500
+    },
+    {
+      "epoch": 7.867132867132867,
+      "grad_norm": 1.4954487085342407,
+      "learning_rate": 8.999000000000001e-05,
+      "loss": 3.4353,
+      "step": 9000
+    },
+    {
+      "epoch": 8.304195804195805,
+      "grad_norm": 1.5357409715652466,
+      "learning_rate": 9.499e-05,
+      "loss": 3.3237,
+      "step": 9500
+    },
+    {
+      "epoch": 8.741258741258742,
+      "grad_norm": 1.411277174949646,
+      "learning_rate": 9.999000000000001e-05,
+      "loss": 3.2253,
+      "step": 10000
+    },
+    {
+      "epoch": 8.741258741258742,
+      "eval_accuracy": 0.5513130425227997,
+      "eval_loss": 3.0387790203094482,
+      "eval_runtime": 6.6329,
+      "eval_samples_per_second": 72.216,
+      "eval_steps_per_second": 0.452,
+      "step": 10000
+    },
+    {
+      "epoch": 9.178321678321678,
+      "grad_norm": 1.1632554531097412,
+      "learning_rate": 9.731720430107527e-05,
+      "loss": 3.1309,
+      "step": 10500
+    },
+    {
+      "epoch": 9.615384615384615,
+      "grad_norm": 1.1414564847946167,
+      "learning_rate": 9.462903225806452e-05,
+      "loss": 3.044,
+      "step": 11000
+    },
+    {
+      "epoch": 10.052447552447552,
+      "grad_norm": 1.1085693836212158,
+      "learning_rate": 9.194086021505377e-05,
+      "loss": 2.9744,
+      "step": 11500
+    },
+    {
+      "epoch": 10.48951048951049,
+      "grad_norm": 0.956007719039917,
+      "learning_rate": 8.925268817204302e-05,
+      "loss": 2.8925,
+      "step": 12000
+    },
+    {
+      "epoch": 10.926573426573427,
+      "grad_norm": 0.9377016425132751,
+      "learning_rate": 8.656451612903226e-05,
+      "loss": 2.8498,
+      "step": 12500
+    },
+    {
+      "epoch": 11.363636363636363,
+      "grad_norm": 1.0721501111984253,
+      "learning_rate": 8.387634408602151e-05,
+      "loss": 2.78,
+      "step": 13000
+    },
+    {
+      "epoch": 11.8006993006993,
+      "grad_norm": 0.906058132648468,
+      "learning_rate": 8.118817204301076e-05,
+      "loss": 2.7461,
+      "step": 13500
+    },
+    {
+      "epoch": 12.237762237762238,
+      "grad_norm": 0.9045028686523438,
+      "learning_rate": 7.850000000000001e-05,
+      "loss": 2.693,
+      "step": 14000
+    },
+    {
+      "epoch": 12.674825174825175,
+      "grad_norm": 0.8730345368385315,
+      "learning_rate": 7.581182795698926e-05,
+      "loss": 2.66,
+      "step": 14500
+    },
+    {
+      "epoch": 13.111888111888112,
+      "grad_norm": 0.8663029670715332,
+      "learning_rate": 7.31236559139785e-05,
+      "loss": 2.6265,
+      "step": 15000
+    },
+    {
+      "epoch": 13.548951048951048,
+      "grad_norm": 0.9472974538803101,
+      "learning_rate": 7.043548387096775e-05,
+      "loss": 2.5889,
+      "step": 15500
+    },
+    {
+      "epoch": 13.986013986013987,
+      "grad_norm": 0.7939944863319397,
+      "learning_rate": 6.7747311827957e-05,
+      "loss": 2.568,
+      "step": 16000
+    },
+    {
+      "epoch": 14.423076923076923,
+      "grad_norm": 0.8526738286018372,
+      "learning_rate": 6.505913978494625e-05,
+      "loss": 2.5224,
+      "step": 16500
+    },
+    {
+      "epoch": 14.86013986013986,
+      "grad_norm": 0.7846645712852478,
+      "learning_rate": 6.237096774193548e-05,
+      "loss": 2.5134,
+      "step": 17000
+    },
+    {
+      "epoch": 15.297202797202797,
+      "grad_norm": 0.7795770764350891,
+      "learning_rate": 5.9682795698924734e-05,
+      "loss": 2.4795,
+      "step": 17500
+    },
+    {
+      "epoch": 15.734265734265735,
+      "grad_norm": 0.8610605001449585,
+      "learning_rate": 5.6994623655913984e-05,
+      "loss": 2.4604,
+      "step": 18000
+    },
+    {
+      "epoch": 16.17132867132867,
+      "grad_norm": 0.8211326003074646,
+      "learning_rate": 5.4306451612903234e-05,
+      "loss": 2.4408,
+      "step": 18500
+    },
+    {
+      "epoch": 16.60839160839161,
+      "grad_norm": 0.8932536840438843,
+      "learning_rate": 5.161827956989248e-05,
+      "loss": 2.4163,
+      "step": 19000
+    },
+    {
+      "epoch": 17.045454545454547,
+      "grad_norm": 0.8186784982681274,
+      "learning_rate": 4.893010752688173e-05,
+      "loss": 2.4056,
+      "step": 19500
+    },
+    {
+      "epoch": 17.482517482517483,
+      "grad_norm": 0.8498294353485107,
+      "learning_rate": 4.624193548387097e-05,
+      "loss": 2.3768,
+      "step": 20000
+    },
+    {
+      "epoch": 17.482517482517483,
+      "eval_accuracy": 0.6417701351499835,
+      "eval_loss": 2.371307849884033,
+      "eval_runtime": 6.6242,
+      "eval_samples_per_second": 72.311,
+      "eval_steps_per_second": 0.453,
+      "step": 20000
+    },
+    {
+      "epoch": 17.91958041958042,
+      "grad_norm": 0.8237965106964111,
+      "learning_rate": 4.355376344086022e-05,
+      "loss": 2.3705,
+      "step": 20500
+    },
+    {
+      "epoch": 18.356643356643357,
+      "grad_norm": 0.8978115916252136,
+      "learning_rate": 4.0865591397849464e-05,
+      "loss": 2.3441,
+      "step": 21000
+    },
+    {
+      "epoch": 18.793706293706293,
+      "grad_norm": 0.8014247417449951,
+      "learning_rate": 3.8177419354838714e-05,
+      "loss": 2.3378,
+      "step": 21500
+    },
+    {
+      "epoch": 19.23076923076923,
+      "grad_norm": 0.8307129144668579,
+      "learning_rate": 3.5489247311827964e-05,
+      "loss": 2.3192,
+      "step": 22000
+    },
+    {
+      "epoch": 19.667832167832167,
+      "grad_norm": 0.8164204359054565,
+      "learning_rate": 3.280107526881721e-05,
+      "loss": 2.3075,
+      "step": 22500
+    },
+    {
+      "epoch": 20.104895104895103,
+      "grad_norm": 0.8482570648193359,
+      "learning_rate": 3.0112903225806454e-05,
+      "loss": 2.3018,
+      "step": 23000
+    },
+    {
+      "epoch": 20.541958041958043,
+      "grad_norm": 0.7965030670166016,
+      "learning_rate": 2.74247311827957e-05,
+      "loss": 2.282,
+      "step": 23500
+    },
+    {
+      "epoch": 20.97902097902098,
+      "grad_norm": 0.8354547023773193,
+      "learning_rate": 2.4736559139784947e-05,
+      "loss": 2.2817,
+      "step": 24000
+    },
+    {
+      "epoch": 21.416083916083917,
+      "grad_norm": 0.8033874034881592,
+      "learning_rate": 2.2048387096774193e-05,
+      "loss": 2.2635,
+      "step": 24500
+    },
+    {
+      "epoch": 21.853146853146853,
+      "grad_norm": 0.8140653967857361,
+      "learning_rate": 1.9360215053763444e-05,
+      "loss": 2.2591,
+      "step": 25000
+    },
+    {
+      "epoch": 22.29020979020979,
+      "grad_norm": 0.8408477306365967,
+      "learning_rate": 1.667204301075269e-05,
+      "loss": 2.2466,
+      "step": 25500
+    },
+    {
+      "epoch": 22.727272727272727,
+      "grad_norm": 0.7920801639556885,
+      "learning_rate": 1.3983870967741935e-05,
+      "loss": 2.244,
+      "step": 26000
+    },
+    {
+      "epoch": 23.164335664335663,
+      "grad_norm": 0.8369796872138977,
+      "learning_rate": 1.1295698924731183e-05,
+      "loss": 2.2394,
+      "step": 26500
+    },
+    {
+      "epoch": 23.6013986013986,
+      "grad_norm": 0.7803018689155579,
+      "learning_rate": 8.60752688172043e-06,
+      "loss": 2.2262,
+      "step": 27000
+    },
+    {
+      "epoch": 24.03846153846154,
+      "grad_norm": 0.8060789108276367,
+      "learning_rate": 5.9193548387096776e-06,
+      "loss": 2.2298,
+      "step": 27500
+    },
+    {
+      "epoch": 24.475524475524477,
+      "grad_norm": 0.7823363542556763,
+      "learning_rate": 3.2311827956989246e-06,
+      "loss": 2.2209,
+      "step": 28000
+    },
+    {
+      "epoch": 24.912587412587413,
+      "grad_norm": 0.8148164749145508,
+      "learning_rate": 5.430107526881721e-07,
+      "loss": 2.2191,
+      "step": 28500
+    },
+    {
+      "epoch": 25.0,
+      "step": 28600,
+      "total_flos": 3.003225489805824e+17,
+      "train_loss": 3.287461715378128,
+      "train_runtime": 79951.4195,
+      "train_samples_per_second": 71.493,
+      "train_steps_per_second": 0.358
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 28600,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 25,
+  "save_steps": 10000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.003225489805824e+17,
+  "train_batch_size": 200,
+  "trial_name": null,
+  "trial_params": null
+}