huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/the-beatles")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/1twc71z8/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/2p2c5864/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on The Beatles's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/3vbolrfq) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/286vzjah) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/3vbolrfq/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/286vzjah/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "huggingartists/the-beatles",
+  "_name_or_path": "the-beatles",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -18,7 +18,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -28,14 +30,14 @@
   "task_specific_params": {
     "text-generation": {
       "do_sample": true,
-      "max_length": 200,
-      "min_length": 100,
+      "max_length": 150,
+      "min_length": 80,
       "temperature": 1.0,
       "top_p": 0.95
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.10.2",
+  "transformers_version": "4.16.2",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 2.5396728515625, "eval_runtime": 7.0101, "eval_samples_per_second": 22.111, "eval_steps_per_second": 2.853, "epoch": 2.0}
+{"eval_loss": 2.4675137996673584, "eval_runtime": 3.0, "eval_samples_per_second": 79.001, "eval_steps_per_second": 10.0, "epoch": 3.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3f36c2f7c7ddcb5a0ba27a0a735f4eeea2faf652a081d5446fcfd3063a5e91d
+oid sha256:dac288e6d24ad64bc0400afea52acf43fb877278427c0b94bfc66a5e74ab0118
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01d0c006bc4779d8568c0522e5b4dffe3f82a222541b07e498cafa70e35560b2
-size 995603825
+oid sha256:0770016d959230c09406eebaf5541fda583a4ca2669877f7960882db3139a0f2
+size 995604017

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24cb962a13630d160fb8f252859dfc4a16bbf489188271190908493e98b8bd48
+oid sha256:cb81af3c77a402600bd26dea9bb806249d28dbe73e02b04fa00c40d474e5d983
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf4704f411f7c64c230e3a475c23d3112f179d24c4827bb93cc3c981ff7942d5
+oid sha256:5a3d0787ebc876fb912a62da619ad27191a3237d59b63f8eef2a74d4bcfbced2
 size 14567

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1403a00d87496c96b524036f4c8c82be5fad3d7a1c0f59e9835444c0e5d9468
+oid sha256:5f4ff9efcd89b539e35b193ff42bd66cb0681357f0d8d3d42b3ccb073e3b235b
 size 623

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 2.5396728515625,
-  "best_model_checkpoint": "output/the-beatles/checkpoint-250",
-  "epoch": 2.0,
-  "global_step": 250,
+  "best_metric": 2.4675137996673584,
+  "best_model_checkpoint": "output/the-beatles/checkpoint-345",
+  "epoch": 3.0,
+  "global_step": 345,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -314,11 +314,133 @@
       "eval_samples_per_second": 22.653,
       "eval_steps_per_second": 2.923,
       "step": 250
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 0.0001218137945423232,
+      "loss": 2.2031,
+      "step": 255
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 0.00011542314562479972,
+      "loss": 2.0681,
+      "step": 260
+    },
+    {
+      "epoch": 2.3,
+      "learning_rate": 0.0001081602700970799,
+      "loss": 2.2497,
+      "step": 265
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 0.00010016046158835702,
+      "loss": 2.1648,
+      "step": 270
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 9.157274139492967e-05,
+      "loss": 2.2327,
+      "step": 275
+    },
+    {
+      "epoch": 2.43,
+      "learning_rate": 8.255708249541068e-05,
+      "loss": 2.2978,
+      "step": 280
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 7.328142955681647e-05,
+      "loss": 2.2115,
+      "step": 285
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 6.391857044318358e-05,
+      "loss": 2.2139,
+      "step": 290
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 5.464291750458936e-05,
+      "loss": 2.3337,
+      "step": 295
+    },
+    {
+      "epoch": 2.61,
+      "learning_rate": 4.5627258605070364e-05,
+      "loss": 2.0981,
+      "step": 300
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 3.703953841164307e-05,
+      "loss": 2.1107,
+      "step": 305
+    },
+    {
+      "epoch": 2.7,
+      "learning_rate": 2.9039729902920186e-05,
+      "loss": 2.1893,
+      "step": 310
+    },
+    {
+      "epoch": 2.74,
+      "learning_rate": 2.177685437520028e-05,
+      "loss": 2.353,
+      "step": 315
+    },
+    {
+      "epoch": 2.78,
+      "learning_rate": 1.5386205457676823e-05,
+      "loss": 2.1984,
+      "step": 320
+    },
+    {
+      "epoch": 2.83,
+      "learning_rate": 9.986828848110892e-06,
+      "loss": 2.209,
+      "step": 325
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 5.679304716725967e-06,
+      "loss": 2.2126,
+      "step": 330
+    },
+    {
+      "epoch": 2.91,
+      "learning_rate": 2.5438740879409566e-06,
+      "loss": 2.1803,
+      "step": 335
+    },
+    {
+      "epoch": 2.96,
+      "learning_rate": 6.389441019077178e-07,
+      "loss": 2.0247,
+      "step": 340
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 0.0,
+      "loss": 2.3114,
+      "step": 345
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 2.4675137996673584,
+      "eval_runtime": 2.9387,
+      "eval_samples_per_second": 80.649,
+      "eval_steps_per_second": 10.209,
+      "step": 345
     }
   ],
-  "max_steps": 250,
-  "num_train_epochs": 2,
-  "total_flos": 260246863872000.0,
+  "max_steps": 345,
+  "num_train_epochs": 3,
+  "total_flos": 359015251968000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce42f54e5d08f9c1684b110fb05bbd513348d13b90f70989ccfab6b0fc97e2f0
-size 2671
+oid sha256:24f3d64f28ad5e687650bdbd509c3d4c83b36b3b61563bd6a8a2322f9b5a7fc0
+size 3055