huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/obladaet")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/3isuq1nm/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/1mtsuuwr/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on OBLADAET's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/3g99ogdv) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/1s9epb35) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/3g99ogdv/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/1s9epb35/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "gpt2",
+  "_name_or_path": "obladaet",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -18,7 +18,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -35,7 +37,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.10.0",
+  "transformers_version": "4.15.0",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 2.009047269821167, "eval_runtime": 6.0992, "eval_samples_per_second": 22.134, "eval_steps_per_second": 2.787, "epoch": 1.0}
+{"eval_loss": 1.7732337713241577, "eval_runtime": 7.0986, "eval_samples_per_second": 21.694, "eval_steps_per_second": 2.817, "epoch": 4.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc113917090121b56fa09a1d5257122677d68e090396d6600587905fb2be06b8
+oid sha256:1c52ab931279f47e28b6847e5d38086105967cafe3ede3d42e5bd29def29ab03
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82c57e12d2c2364e7669f9582d82b64c47557acc70fd25834e906c5a5f1bffc5
-size 995603825
+oid sha256:8f793de0686cb2318496c05ad7e9eb84507b50597f633e6c6930423baa2c8acc
+size 995604017

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f267b7b47f873182799d927cb61b8855ba2e9a923a454f30069efa3b2edd9aa
+oid sha256:41ad58f7eaf8d9ccb1108b2de48617723e0257dfd854bfe83d33c8b497618eb0
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5fddc434b6543bcf9fb934d13798d2854524eb692171ad66973ff4ff84d2dc0
+oid sha256:16ee4b3c5a3fcf7b5bbeda9aae6e07b48bec52cb057e0b87294a70edc4a51800
 size 14503

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:210db157b55dd747a9007cf5cf1259093551c56de3cafadcc170777b60990c64
+oid sha256:54fe9d32a9c27c0412ac2e6ca93dfcc8ff0c17501d50cf60b9047a8d1c51b1c3
 size 623

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "GPT2Tokenizer"}
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "huggingartists/obladaet", "tokenizer_class": "GPT2Tokenizer"}

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 2.009047269821167,
-  "best_model_checkpoint": "output/obladaet/checkpoint-103",
-  "epoch": 1.0,
-  "global_step": 103,
+  "best_metric": 1.7732337713241577,
+  "best_model_checkpoint": "output/obladaet/checkpoint-404",
+  "epoch": 4.0,
+  "global_step": 404,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -134,11 +134,395 @@
       "eval_samples_per_second": 22.775,
       "eval_steps_per_second": 2.868,
       "step": 103
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 5.302867558792348e-07,
+      "loss": 1.9242,
+      "step": 105
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 2.6705334886474966e-06,
+      "loss": 2.0213,
+      "step": 110
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 6.402259559252805e-06,
+      "loss": 1.998,
+      "step": 115
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 1.1635384382334882e-05,
+      "loss": 2.0452,
+      "step": 120
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 1.8243584927101022e-05,
+      "loss": 2.0056,
+      "step": 125
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 2.6067345044190824e-05,
+      "loss": 1.8585,
+      "step": 130
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 3.491780604523009e-05,
+      "loss": 1.9956,
+      "step": 135
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 4.4581325585470584e-05,
+      "loss": 1.9927,
+      "step": 140
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 5.48246348020977e-05,
+      "loss": 1.9939,
+      "step": 145
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 6.540046921942313e-05,
+      "loss": 2.0247,
+      "step": 150
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 7.605353749586352e-05,
+      "loss": 2.0324,
+      "step": 155
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 8.652668393240227e-05,
+      "loss": 1.9751,
+      "step": 160
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 9.656709598499981e-05,
+      "loss": 2.0028,
+      "step": 165
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 0.00010593240693706343,
+      "loss": 1.9151,
+      "step": 170
+    },
+    {
+      "epoch": 1.73,
+      "learning_rate": 0.00011439654641882815,
+      "loss": 1.9662,
+      "step": 175
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 0.00012175519754725823,
+      "loss": 1.9375,
+      "step": 180
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 0.00012783072895591112,
+      "loss": 1.9174,
+      "step": 185
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 0.00013247648265989943,
+      "loss": 1.9634,
+      "step": 190
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 0.00013558031425066341,
+      "loss": 1.8422,
+      "step": 195
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 0.00013706729996336458,
+      "loss": 1.8405,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 1.860771894454956,
+      "eval_runtime": 6.9634,
+      "eval_samples_per_second": 22.115,
+      "eval_steps_per_second": 2.872,
+      "step": 202
+    },
+    {
+      "epoch": 2.03,
+      "learning_rate": 0.00013690154527067468,
+      "loss": 1.9028,
+      "step": 205
+    },
+    {
+      "epoch": 2.08,
+      "learning_rate": 0.0001350870513450971,
+      "loss": 1.9087,
+      "step": 210
+    },
+    {
+      "epoch": 2.13,
+      "learning_rate": 0.0001316676184741814,
+      "loss": 1.9086,
+      "step": 215
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 0.00012672578876010354,
+      "loss": 1.9232,
+      "step": 220
+    },
+    {
+      "epoch": 2.23,
+      "learning_rate": 0.00012038085362591339,
+      "loss": 1.8881,
+      "step": 225
+    },
+    {
+      "epoch": 2.28,
+      "learning_rate": 0.00011278597422549735,
+      "loss": 1.8032,
+      "step": 230
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 0.00010412448426802514,
+      "loss": 1.8736,
+      "step": 235
+    },
+    {
+      "epoch": 2.38,
+      "learning_rate": 9.460546450345005e-05,
+      "loss": 1.9298,
+      "step": 240
+    },
+    {
+      "epoch": 2.43,
+      "learning_rate": 8.445869569708696e-05,
+      "loss": 1.882,
+      "step": 245
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 7.392911192402588e-05,
+      "loss": 1.8485,
+      "step": 250
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 6.327088807597416e-05,
+      "loss": 1.8129,
+      "step": 255
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 5.2741304302913084e-05,
+      "loss": 1.7189,
+      "step": 260
+    },
+    {
+      "epoch": 2.62,
+      "learning_rate": 4.259453549654993e-05,
+      "loss": 1.7814,
+      "step": 265
+    },
+    {
+      "epoch": 2.67,
+      "learning_rate": 3.307551573197489e-05,
+      "loss": 1.892,
+      "step": 270
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 2.4414025774502687e-05,
+      "loss": 1.8749,
+      "step": 275
+    },
+    {
+      "epoch": 2.77,
+      "learning_rate": 1.681914637408659e-05,
+      "loss": 1.8049,
+      "step": 280
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 1.0474211239896518e-05,
+      "loss": 1.7877,
+      "step": 285
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 5.532381525818605e-06,
+      "loss": 1.8053,
+      "step": 290
+    },
+    {
+      "epoch": 2.92,
+      "learning_rate": 2.112948654902904e-06,
+      "loss": 1.7789,
+      "step": 295
+    },
+    {
+      "epoch": 2.97,
+      "learning_rate": 2.984547293253313e-07,
+      "loss": 1.7656,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 1.7856239080429077,
+      "eval_runtime": 7.037,
+      "eval_samples_per_second": 21.884,
+      "eval_steps_per_second": 2.842,
+      "step": 303
+    },
+    {
+      "epoch": 3.02,
+      "learning_rate": 1.3270003663542904e-07,
+      "loss": 1.8334,
+      "step": 305
+    },
+    {
+      "epoch": 3.07,
+      "learning_rate": 1.6196857493365444e-06,
+      "loss": 1.8375,
+      "step": 310
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 4.7235173401005474e-06,
+      "loss": 1.8418,
+      "step": 315
+    },
+    {
+      "epoch": 3.17,
+      "learning_rate": 9.369271044088901e-06,
+      "loss": 1.8346,
+      "step": 320
+    },
+    {
+      "epoch": 3.22,
+      "learning_rate": 1.5444802452741737e-05,
+      "loss": 1.7677,
+      "step": 325
+    },
+    {
+      "epoch": 3.27,
+      "learning_rate": 2.2803453581171793e-05,
+      "loss": 1.8371,
+      "step": 330
+    },
+    {
+      "epoch": 3.32,
+      "learning_rate": 3.1267593062936485e-05,
+      "loss": 1.7753,
+      "step": 335
+    },
+    {
+      "epoch": 3.37,
+      "learning_rate": 4.063290401500004e-05,
+      "loss": 1.8018,
+      "step": 340
+    },
+    {
+      "epoch": 3.42,
+      "learning_rate": 5.06733160675977e-05,
+      "loss": 1.741,
+      "step": 345
+    },
+    {
+      "epoch": 3.47,
+      "learning_rate": 6.114646250413644e-05,
+      "loss": 1.729,
+      "step": 350
+    },
+    {
+      "epoch": 3.51,
+      "learning_rate": 7.179953078057677e-05,
+      "loss": 1.7717,
+      "step": 355
+    },
+    {
+      "epoch": 3.56,
+      "learning_rate": 8.237536519790226e-05,
+      "loss": 1.7441,
+      "step": 360
+    },
+    {
+      "epoch": 3.61,
+      "learning_rate": 9.261867441452943e-05,
+      "loss": 1.7007,
+      "step": 365
+    },
+    {
+      "epoch": 3.66,
+      "learning_rate": 0.00010228219395476987,
+      "loss": 1.8014,
+      "step": 370
+    },
+    {
+      "epoch": 3.71,
+      "learning_rate": 0.0001111326549558092,
+      "loss": 1.6888,
+      "step": 375
+    },
+    {
+      "epoch": 3.76,
+      "learning_rate": 0.00011895641507289893,
+      "loss": 1.8129,
+      "step": 380
+    },
+    {
+      "epoch": 3.81,
+      "learning_rate": 0.00012556461561766505,
+      "loss": 1.7059,
+      "step": 385
+    },
+    {
+      "epoch": 3.86,
+      "learning_rate": 0.00013079774044074714,
+      "loss": 1.8051,
+      "step": 390
+    },
+    {
+      "epoch": 3.91,
+      "learning_rate": 0.0001345294665113525,
+      "loss": 1.7285,
+      "step": 395
+    },
+    {
+      "epoch": 3.96,
+      "learning_rate": 0.00013666971324412076,
+      "loss": 1.8123,
+      "step": 400
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 1.7732337713241577,
+      "eval_runtime": 7.0198,
+      "eval_samples_per_second": 21.938,
+      "eval_steps_per_second": 2.849,
+      "step": 404
     }
   ],
-  "max_steps": 103,
-  "num_train_epochs": 1,
-  "total_flos": 107129733120000.0,
+  "max_steps": 404,
+  "num_train_epochs": 4,
+  "total_flos": 418981773312000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54901e110d42694df30e93d3fdcc273603a1cafca63450c9377093e7e9638050
-size 2671
+oid sha256:61edd5ee9ac3511fa8aafd6320733cca851923fdf79ca86f83532841c0bd6b0d
+size 2991