huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/nirvana")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/2n8nv6eo/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/2so60o7u/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on Nirvana's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/2rjj8yuy) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/l6zav1kr) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/2rjj8yuy/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/l6zav1kr/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "gpt2",
+  "_name_or_path": "nirvana",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -18,7 +18,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -35,7 +37,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.10.2",
+  "transformers_version": "4.16.1",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 2.5190556049346924, "eval_runtime": 2.6493, "eval_samples_per_second": 21.138, "eval_steps_per_second": 2.642, "epoch": 1.0}
+{"eval_loss": 2.128573179244995, "eval_runtime": 1.2287, "eval_samples_per_second": 43.137, "eval_steps_per_second": 5.697, "epoch": 2.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69a98d3fb8e36cc32117af3070a7e46187aab33883420ba33a6c2386ac39b355
+oid sha256:1740b9403c2980e1c12ea39ca34dd90e719cb426af2e560c4662fb685a4adc5c
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3acf06fa40c7506291d94417db2e355ccf307835a183507891ecda68d4cc041e
+oid sha256:99c7d18d91a68d3aac0f231a3453ae4f8448552fe1cceca43dfeddf734b115cf
 size 995603825

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8547e535bd26489d6e0d498a76b95c78cf4eda66a4406e50d6beb245d7bafe4
+oid sha256:190fc0c852d0ebcb0d529ca1ef8faf27bb0e63a90b3312da7edff4f56cd84c70
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8352ef81703053831b701355ba366fd6d01224cef947db1382f693b33d78acbc
+oid sha256:1d8690557d45fc8d72f6eb673bd42b9a6c47364d62cbc2a5af224d1d6fa8d651
 size 14567

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffbc99e60a150225c1619d3a5dfd24d23b94d1d0c35e0e58966261759d54b4ef
+oid sha256:95317d63b9a0922ed3de67fdcb9f6bbf4898f9cd35c41f60f3577d5b1e618ec0
 size 623

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "GPT2Tokenizer"}
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "huggingartists/nirvana", "tokenizer_class": "GPT2Tokenizer"}

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 2.5190556049346924,
-  "best_model_checkpoint": "output/nirvana/checkpoint-35",
-  "epoch": 1.0,
-  "global_step": 35,
+  "best_metric": 2.128573179244995,
+  "best_model_checkpoint": "output/nirvana/checkpoint-72",
+  "epoch": 2.0,
+  "global_step": 72,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -56,11 +56,69 @@
       "eval_samples_per_second": 22.504,
       "eval_steps_per_second": 2.813,
       "step": 35
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 2.3264973163604736,
+      "eval_runtime": 1.1647,
+      "eval_samples_per_second": 45.506,
+      "eval_steps_per_second": 6.01,
+      "step": 36
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 4.137086214086682e-06,
+      "loss": 2.6774,
+      "step": 40
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 2.009247481060283e-05,
+      "loss": 2.4577,
+      "step": 45
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 4.513741816785908e-05,
+      "loss": 2.2823,
+      "step": 50
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 7.457888395248933e-05,
+      "loss": 2.298,
+      "step": 55
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 0.00010290000000000001,
+      "loss": 2.4088,
+      "step": 60
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 0.00012479383023822482,
+      "loss": 2.2944,
+      "step": 65
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 0.00013615781185663748,
+      "loss": 2.0032,
+      "step": 70
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 2.128573179244995,
+      "eval_runtime": 1.1938,
+      "eval_samples_per_second": 44.396,
+      "eval_steps_per_second": 5.864,
+      "step": 72
     }
   ],
-  "max_steps": 35,
-  "num_train_epochs": 1,
-  "total_flos": 36319592448000.0,
+  "max_steps": 72,
+  "num_train_epochs": 2,
+  "total_flos": 73161768960000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0bf6f49cd9a925b8355a2811e15f01c27594922d1567b49b037e42207f9552e4
-size 2671
+oid sha256:5ccf6b2194f29afd6fd696a5ff25947b9d920926b5a6ac01723ed2e5798c9d44
+size 3055