huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/sum-41")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/18ckwm0s/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/3fy2kvn1/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on Sum 41's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/1rj7iazy) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/2hgx7kne) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/1rj7iazy/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/2hgx7kne/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "gpt2",
+  "_name_or_path": "sum-41",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -17,7 +17,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -34,7 +36,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.11.3",
+  "transformers_version": "4.16.2",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 2.928959369659424, "eval_runtime": 1.264, "eval_samples_per_second": 20.57, "eval_steps_per_second": 3.165, "epoch": 1.0}
+{"eval_loss": 2.578360080718994, "eval_runtime": 1.3642, "eval_samples_per_second": 20.526, "eval_steps_per_second": 2.932, "epoch": 11.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f35e4bc2c2893f5e3fcec13df7ba6c6cebae70ea24bac23a46ece291f58ffe2e
+oid sha256:3cb5eb5a9606f4614f22fa3b50a61f459a02d4b622e5ad82345fce00fe13980d
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6bda1875ee51eba220fc899c04abb3275854178f035fe39897630388145cad8b
+oid sha256:63d6f27da54aff7f05f32711aa0ca1993ed6674681766ddf8ad9ddffe0163883
 size 995603825

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce8da8c94f29bb06679607e8ffcf5182ea0701f672be8aa2a0e647e140062778
+oid sha256:41148ab2cccbd2659c86acc8286fe888d6cde6c328be53f18c2730011587797b
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2eede3ee5ff25e9002bcd2fb29039533bffed5336005be0b71df1ad1072371bc
+oid sha256:8634c46781e97e9657fc0ec952335eb8c634c60e1a03871907657b503158b7d7
 size 14503

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d8130029718e3533ddde75a2c06f96da583a4d80cbf252f6ba4e6fe855637e8
+oid sha256:2d1ac50aae75de09c176668acc47db9824dd3efe4a9761503e54948712843f45
 size 623

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "GPT2Tokenizer"}
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "huggingartists/sum-41", "tokenizer_class": "GPT2Tokenizer"}

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 2.928959369659424,
-  "best_model_checkpoint": "output/sum-41/checkpoint-18",
-  "epoch": 1.0,
-  "global_step": 18,
+  "best_metric": 2.578360080718994,
+  "best_model_checkpoint": "output/sum-41/checkpoint-108",
+  "epoch": 6.0,
+  "global_step": 108,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -32,11 +32,159 @@
       "eval_samples_per_second": 22.691,
       "eval_steps_per_second": 3.491,
       "step": 18
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 4.137086214086682e-06,
+      "loss": 2.7011,
+      "step": 20
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 4.513741816785908e-05,
+      "loss": 2.8934,
+      "step": 25
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 0.00010290000000000001,
+      "loss": 2.793,
+      "step": 30
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 0.00013615781185663748,
+      "loss": 2.7355,
+      "step": 35
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 2.6775753498077393,
+      "eval_runtime": 1.2274,
+      "eval_samples_per_second": 22.813,
+      "eval_steps_per_second": 3.259,
+      "step": 36
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 0.0001211506487979619,
+      "loss": 2.6552,
+      "step": 40
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 6.860000000000001e-05,
+      "loss": 2.4943,
+      "step": 45
+    },
+    {
+      "epoch": 2.78,
+      "learning_rate": 1.6049351202038163e-05,
+      "loss": 2.2804,
+      "step": 50
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 2.632376194000244,
+      "eval_runtime": 1.2528,
+      "eval_samples_per_second": 22.349,
+      "eval_steps_per_second": 3.193,
+      "step": 54
+    },
+    {
+      "epoch": 3.06,
+      "learning_rate": 1.0421881433625223e-06,
+      "loss": 2.6341,
+      "step": 55
+    },
+    {
+      "epoch": 3.33,
+      "learning_rate": 3.4300000000000014e-05,
+      "loss": 2.3676,
+      "step": 60
+    },
+    {
+      "epoch": 3.61,
+      "learning_rate": 9.206258183214083e-05,
+      "loss": 2.2182,
+      "step": 65
+    },
+    {
+      "epoch": 3.89,
+      "learning_rate": 0.00013306291378591332,
+      "loss": 2.4212,
+      "step": 70
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 2.5963516235351562,
+      "eval_runtime": 1.2572,
+      "eval_samples_per_second": 22.272,
+      "eval_steps_per_second": 3.182,
+      "step": 72
+    },
+    {
+      "epoch": 4.17,
+      "learning_rate": 0.00012800934269961248,
+      "loss": 2.2587,
+      "step": 75
+    },
+    {
+      "epoch": 4.44,
+      "learning_rate": 8.051226498795145e-05,
+      "loss": 2.1767,
+      "step": 80
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 2.4504769975503385e-05,
+      "loss": 2.1842,
+      "step": 85
+    },
+    {
+      "epoch": 5.0,
+      "learning_rate": 0.0,
+      "loss": 2.2178,
+      "step": 90
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 2.578657388687134,
+      "eval_runtime": 1.2377,
+      "eval_samples_per_second": 22.622,
+      "eval_steps_per_second": 3.232,
+      "step": 90
+    },
+    {
+      "epoch": 5.28,
+      "learning_rate": 2.4504769975503317e-05,
+      "loss": 2.0438,
+      "step": 95
+    },
+    {
+      "epoch": 5.56,
+      "learning_rate": 8.051226498795124e-05,
+      "loss": 2.1724,
+      "step": 100
+    },
+    {
+      "epoch": 5.83,
+      "learning_rate": 0.00012800934269961248,
+      "loss": 1.9973,
+      "step": 105
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 2.578360080718994,
+      "eval_runtime": 1.2443,
+      "eval_samples_per_second": 22.503,
+      "eval_steps_per_second": 3.215,
+      "step": 108
     }
   ],
-  "max_steps": 18,
-  "num_train_epochs": 1,
-  "total_flos": 18682380288000.0,
+  "max_steps": 198,
+  "num_train_epochs": 11,
+  "total_flos": 110787821568000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8591fbca38538cc0025afd8c45498ffca953cc8e2e6e0eb814c791f923ccca24
-size 2863
+oid sha256:209473ae5c5612ab3bd6531c165705ecc7cd87270e61a818a0765da5d7b04b55
+size 3055