huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/led-zeppelin")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/170kpeuy/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/cpexpb1w/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on Led Zeppelin's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/39r70n5b) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/bna2epba) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/39r70n5b/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/bna2epba/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "gpt2",
+  "_name_or_path": "led-zeppelin",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -17,7 +17,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -34,7 +36,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.11.0",
+  "transformers_version": "4.16.2",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 2.8552775382995605, "eval_runtime": 2.1634, "eval_samples_per_second": 21.725, "eval_steps_per_second": 2.773, "epoch": 1.0}
+{"eval_loss": 2.537315607070923, "eval_runtime": 0.5924, "eval_samples_per_second": 74.276, "eval_steps_per_second": 10.129, "epoch": 2.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5500352cd251fc889d48797187128788d18be58b2478409eaca2f4deb7207fd3
+oid sha256:a80691ed49dbe4e433e677de9ec53a9161c49e377ca0aed2a46c57ba985f706e
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80d16781011a552b897c0f6c80895fb33e25cef23baa8a8a3030df7ad2670f55
+oid sha256:7a94f3e929debb9aca5475eaed9e9c1b2cf585aa707bce1dc6b676a82a534af9
 size 995603825

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ebfb409495e231364d1cae635a196486c8045391e79a561a8de47c9c5456ebc
+oid sha256:e88e60aa829d86cf4dd3e990c425f851c472ab19b6054dc57602697dad3f3cb4
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44746b8ccf16d69b3709563dcc7879ec2c3e10621dfca8e5e7d64af246229d86
-size 14567
+oid sha256:59cb81eef7b3d8300cd1eede00d029a878c6f6a40574dd917f7eb70641cc91fa
+size 14503

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8989e74728331765fdc15dc13be950f4c53d6c5551f5ee94e2d269e65894c977
+oid sha256:2222362d80f6c4a5547283104536d5c4564191fb641a9b7f7b9e346e9a58c2ee
 size 623

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "GPT2Tokenizer"}
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "huggingartists/led-zeppelin", "tokenizer_class": "GPT2Tokenizer"}

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 2.8552775382995605,
-  "best_model_checkpoint": "output/led-zeppelin/checkpoint-33",
-  "epoch": 1.0,
-  "global_step": 33,
+  "best_metric": 2.537315607070923,
+  "best_model_checkpoint": "output/led-zeppelin/checkpoint-66",
+  "epoch": 2.0,
+  "global_step": 66,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -50,11 +50,61 @@
       "eval_samples_per_second": 22.688,
       "eval_steps_per_second": 2.896,
       "step": 33
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 1.2396913677783124e-06,
+      "loss": 2.9767,
+      "step": 35
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 1.4676757700644785e-05,
+      "loss": 2.7654,
+      "step": 40
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 4.0102530108070535e-05,
+      "loss": 2.8035,
+      "step": 45
+    },
+    {
+      "epoch": 1.52,
+      "learning_rate": 7.186411942550872e-05,
+      "loss": 2.7337,
+      "step": 50
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 0.00010290000000000001,
+      "loss": 2.8151,
+      "step": 55
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 0.000126309992352219,
+      "loss": 2.6463,
+      "step": 60
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 0.0001368893738885136,
+      "loss": 2.5568,
+      "step": 65
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 2.537315607070923,
+      "eval_runtime": 0.5866,
+      "eval_samples_per_second": 75.011,
+      "eval_steps_per_second": 10.229,
+      "step": 66
     }
   ],
-  "max_steps": 33,
-  "num_train_epochs": 1,
-  "total_flos": 33576026112000.0,
+  "max_steps": 66,
+  "num_train_epochs": 2,
+  "total_flos": 67413344256000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a96ff1e70aacfc8366c13682f25efee18d3a65e242a520932df4c78f9528d506
-size 2863
+oid sha256:3d57478b7945fcc40bbd7ed427acf9dd4c29beb6911f6439ab8ed9df514ac849
+size 3055