huggingartists

README.md

@@ -14,7 +14,7 @@ widget:
 <div class="inline-flex flex-col" style="line-height: 1.5;">
     <div class="flex">
         <div
-			style="display:DISPLAY_1; margin-left: auto; margin-right: auto; width: 92px; height:92px; border-radius: 50%; background-size: cover; background-image: url(&#39;https://images.genius.com/1edcea93261e2e266c532ce204ba92da.1000x1000x1.jpg&#39;)">
+			style="display:DISPLAY_1; margin-left: auto; margin-right: auto; width: 92px; height:92px; border-radius: 50%; background-size: cover; background-image: url(&#39;https://images.genius.com/cdfb190640789439daae426c799e5e32.1000x1000x1.jpg&#39;)">
         </div>
     </div>
     <div style="text-align: center; margin-top: 3px; font-size: 16px; font-weight: 800">🤖 HuggingArtists Model 🤖</div>
@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/morgenshtern")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/29htvpbu/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/lmrnk6sz/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on MORGENSHTERN's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/m6tldjdu) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/1m2jynlh) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/m6tldjdu/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/1m2jynlh/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -37,7 +37,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.16.2",
+  "transformers_version": "4.21.1",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 0.6951560974121094, "eval_runtime": 6.9424, "eval_samples_per_second": 20.598, "eval_steps_per_second": 2.593, "epoch": 3.0}
+{"eval_loss": 0.7026481628417969, "eval_runtime": 3.0736, "eval_samples_per_second": 43.272, "eval_steps_per_second": 5.531, "epoch": 4.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd5ea8451fb818bbbf19558ab096fa4f81596c541e2622568412ebf5daf0efd2
+oid sha256:c23e2a102f26e84b99162a01e2e7b59b56c978a11e55b1aa095d462032d87dae
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e4755f815eb61413dfd1b66a9a03a2b2eb842a2b33d9bff4a55141c7a91df9bf
+oid sha256:17b3270e93a6c6420e26ddf1ad68f8375708e9c3bc687328765f1fb2c19bfc74
 size 995604017

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0dc86c1d0e5ed8e708b41cbb9b119c5960c259dba079c2cee589bb9afa77c5ca
-size 510403817
+oid sha256:31200b441b415dfca0807dfb901b43161d1aaef9855d993a8bd3529c5daed0b9
+size 510396521

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8dfdfe168c690fdd51fc3442227da71ee0a9fd24f3602c5faf62e0fb7e6e7755
+oid sha256:dad3e3ea770422de0ab5ab08a664bd5ac0ce9435fa36b3c0b797f9baf429875b
 size 14503

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f889ee048c975721acd46c969685ee09ff3f7ad66edbe67905c8a29f4eede81b
+oid sha256:eccd196d6090e4ec1aee799e326feb6d6a688637304de5902e78087582babefe
 size 623

special_tokens_map.json

@@ -1 +1,5 @@
-{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer.json

@@ -5,29 +5,32 @@
   "added_tokens": [
     {
       "id": 50256,
-      "special": true,
       "content": "<|endoftext|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
   "pre_tokenizer": {
     "type": "ByteLevel",
     "add_prefix_space": false,
-    "trim_offsets": true
+    "trim_offsets": true,
+    "use_regex": true
   },
   "post_processor": {
     "type": "ByteLevel",
     "add_prefix_space": true,
-    "trim_offsets": false
+    "trim_offsets": false,
+    "use_regex": true
   },
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
-    "trim_offsets": true
+    "trim_offsets": true,
+    "use_regex": true
   },
   "model": {
     "type": "BPE",

tokenizer_config.json

@@ -1 +1,10 @@
-{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "huggingartists/morgenshtern", "tokenizer_class": "GPT2Tokenizer"}
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1024,
+  "name_or_path": "huggingartists/morgenshtern",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

trainer_state.json

@@ -1,388 +1,144 @@
 {
-  "best_metric": 0.6951560974121094,
-  "best_model_checkpoint": "output/morgenshtern/checkpoint-294",
-  "epoch": 3.0,
-  "global_step": 294,
+  "best_metric": 0.7026481628417969,
+  "best_model_checkpoint": "output/morgenshtern/checkpoint-100",
+  "epoch": 1.0,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.05,
-      "learning_rate": 4.8622670741210196e-05,
-      "loss": 0.9101,
+      "learning_rate": 4.740143418587856e-05,
+      "loss": 0.8849,
       "step": 5
     },
     {
       "epoch": 0.1,
-      "learning_rate": 5.902924774844624e-05,
-      "loss": 0.8731,
+      "learning_rate": 3.745625171786743e-05,
+      "loss": 0.6963,
       "step": 10
     },
     {
       "epoch": 0.15,
-      "learning_rate": 6.96668542977361e-05,
-      "loss": 0.9581,
+      "learning_rate": 2.827793169273699e-05,
+      "loss": 0.8572,
       "step": 15
     },
     {
       "epoch": 0.2,
-      "learning_rate": 8.027870792255626e-05,
-      "loss": 0.8474,
+      "learning_rate": 2.009247481060302e-05,
+      "loss": 0.7158,
       "step": 20
     },
     {
       "epoch": 0.25,
-      "learning_rate": 9.060864780933411e-05,
-      "loss": 0.9838,
+      "learning_rate": 1.310143418587843e-05,
+      "loss": 0.8878,
       "step": 25
     },
     {
       "epoch": 0.3,
-      "learning_rate": 0.00010040731829419309,
-      "loss": 1.012,
+      "learning_rate": 7.476952440678023e-06,
+      "loss": 0.9167,
       "step": 30
     },
     {
       "epoch": 0.35,
-      "learning_rate": 0.00010943818808937948,
-      "loss": 0.8702,
+      "learning_rate": 3.3575229821526355e-06,
+      "loss": 0.7488,
       "step": 35
     },
     {
       "epoch": 0.4,
-      "learning_rate": 0.00011748325994051893,
-      "loss": 0.9656,
+      "learning_rate": 8.445798351735566e-07,
+      "loss": 0.832,
       "step": 40
     },
     {
       "epoch": 0.45,
-      "learning_rate": 0.00012434833288861775,
-      "loss": 0.9745,
+      "learning_rate": 0.0,
+      "loss": 0.8448,
       "step": 45
     },
     {
       "epoch": 0.5,
-      "learning_rate": 0.0001298676901104138,
-      "loss": 0.8875,
+      "learning_rate": 8.445798351735718e-07,
+      "loss": 0.9363,
       "step": 50
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 0.00013390809917674526,
-      "loss": 0.9281,
+      "epoch": 0.55,
+      "learning_rate": 3.3575229821523687e-06,
+      "loss": 0.8096,
       "step": 55
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 0.00013637202816621594,
-      "loss": 0.932,
+      "epoch": 0.6,
+      "learning_rate": 7.476952440677627e-06,
+      "loss": 0.8588,
       "step": 60
     },
     {
-      "epoch": 0.64,
-      "learning_rate": 0.0001372,
-      "loss": 1.071,
+      "epoch": 0.65,
+      "learning_rate": 1.3101434185878491e-05,
+      "loss": 0.7752,
       "step": 65
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 0.000136372028166216,
-      "loss": 0.8787,
+      "epoch": 0.7,
+      "learning_rate": 2.0092474810602402e-05,
+      "loss": 0.8502,
       "step": 70
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 0.00013390809917674537,
-      "loss": 0.9074,
+      "epoch": 0.75,
+      "learning_rate": 2.827793169273628e-05,
+      "loss": 0.773,
       "step": 75
     },
     {
-      "epoch": 0.79,
-      "learning_rate": 0.00012986769011041397,
-      "loss": 1.0496,
+      "epoch": 0.8,
+      "learning_rate": 3.745625171786751e-05,
+      "loss": 0.7943,
       "step": 80
     },
     {
-      "epoch": 0.84,
-      "learning_rate": 0.00012434833288861794,
-      "loss": 0.9943,
+      "epoch": 0.85,
+      "learning_rate": 4.7401434185877724e-05,
+      "loss": 0.8335,
       "step": 85
     },
     {
-      "epoch": 0.89,
-      "learning_rate": 0.00011748325994051916,
-      "loss": 1.0026,
+      "epoch": 0.9,
+      "learning_rate": 5.786859569823979e-05,
+      "loss": 0.7886,
       "step": 90
     },
     {
-      "epoch": 0.94,
-      "learning_rate": 0.00010943818808937974,
-      "loss": 1.0201,
+      "epoch": 0.95,
+      "learning_rate": 6.860000000000016e-05,
+      "loss": 0.7602,
       "step": 95
     },
     {
-      "epoch": 0.99,
-      "learning_rate": 0.00010040731829419337,
-      "loss": 0.9215,
+      "epoch": 1.0,
+      "learning_rate": 7.933140430175958e-05,
+      "loss": 0.6606,
       "step": 100
     },
     {
       "epoch": 1.0,
-      "eval_loss": 0.8902494311332703,
-      "eval_runtime": 5.4222,
-      "eval_samples_per_second": 22.684,
-      "eval_steps_per_second": 2.951,
-      "step": 101
-    },
-    {
-      "epoch": 1.04,
-      "learning_rate": 9.060864780933398e-05,
-      "loss": 0.9922,
-      "step": 105
-    },
-    {
-      "epoch": 1.09,
-      "learning_rate": 8.027870792255707e-05,
-      "loss": 0.8911,
-      "step": 110
-    },
-    {
-      "epoch": 1.14,
-      "learning_rate": 6.966685429773643e-05,
-      "loss": 0.9184,
-      "step": 115
-    },
-    {
-      "epoch": 1.19,
-      "learning_rate": 5.902924774844707e-05,
-      "loss": 0.8339,
-      "step": 120
-    },
-    {
-      "epoch": 1.24,
-      "learning_rate": 4.862267074121052e-05,
-      "loss": 0.791,
-      "step": 125
-    },
-    {
-      "epoch": 1.29,
-      "learning_rate": 3.869832889258939e-05,
-      "loss": 0.8984,
-      "step": 130
-    },
-    {
-      "epoch": 1.34,
-      "learning_rate": 2.9495787086535028e-05,
-      "loss": 0.7645,
-      "step": 135
-    },
-    {
-      "epoch": 1.39,
-      "learning_rate": 2.1237186588777798e-05,
-      "loss": 0.8448,
-      "step": 140
-    },
-    {
-      "epoch": 1.44,
-      "learning_rate": 1.4121882752050083e-05,
-      "loss": 0.8783,
-      "step": 145
-    },
-    {
-      "epoch": 1.49,
-      "learning_rate": 8.321632753190209e-06,
-      "loss": 0.8214,
-      "step": 150
-    },
-    {
-      "epoch": 1.53,
-      "learning_rate": 3.976449525958718e-06,
-      "loss": 0.7646,
-      "step": 155
-    },
-    {
-      "epoch": 1.58,
-      "learning_rate": 1.1912219719526667e-06,
-      "loss": 0.8915,
-      "step": 160
-    },
-    {
-      "epoch": 1.63,
-      "learning_rate": 3.318303476960258e-08,
-      "loss": 0.7759,
-      "step": 165
-    },
-    {
-      "epoch": 1.68,
-      "learning_rate": 5.302867558791814e-07,
-      "loss": 0.9129,
-      "step": 170
-    },
-    {
-      "epoch": 1.73,
-      "learning_rate": 2.670533488647443e-06,
-      "loss": 0.8839,
-      "step": 175
-    },
-    {
-      "epoch": 1.78,
-      "learning_rate": 6.402259559252401e-06,
-      "loss": 0.9542,
-      "step": 180
-    },
-    {
-      "epoch": 1.83,
-      "learning_rate": 1.1635384382334973e-05,
-      "loss": 0.891,
-      "step": 185
-    },
-    {
-      "epoch": 1.88,
-      "learning_rate": 1.824358492710126e-05,
-      "loss": 0.7246,
-      "step": 190
-    },
-    {
-      "epoch": 1.93,
-      "learning_rate": 2.6067345044190458e-05,
-      "loss": 0.7276,
-      "step": 195
-    },
-    {
-      "epoch": 1.98,
-      "learning_rate": 3.491780604522984e-05,
-      "loss": 0.8327,
-      "step": 200
-    },
-    {
-      "epoch": 2.0,
-      "eval_loss": 0.8691701889038086,
-      "eval_runtime": 5.4146,
-      "eval_samples_per_second": 22.717,
-      "eval_steps_per_second": 2.955,
-      "step": 202
-    },
-    {
-      "epoch": 2.09,
-      "learning_rate": 0.00013663679494827462,
-      "loss": 0.8056,
-      "step": 205
-    },
-    {
-      "epoch": 2.14,
-      "learning_rate": 0.00013436461471831498,
-      "loss": 0.8153,
-      "step": 210
-    },
-    {
-      "epoch": 2.19,
-      "learning_rate": 0.00013040646433810593,
-      "loss": 0.8439,
-      "step": 215
-    },
-    {
-      "epoch": 2.24,
-      "learning_rate": 0.00012486381666535154,
-      "loss": 0.8468,
-      "step": 220
-    },
-    {
-      "epoch": 2.3,
-      "learning_rate": 0.00011787876541670438,
-      "loss": 0.9434,
-      "step": 225
-    },
-    {
-      "epoch": 2.35,
-      "learning_rate": 0.00010963038239169752,
-      "loss": 0.9117,
-      "step": 230
-    },
-    {
-      "epoch": 2.4,
-      "learning_rate": 0.0001003301267105212,
-      "loss": 0.9685,
-      "step": 235
-    },
-    {
-      "epoch": 2.45,
-      "learning_rate": 9.021642375642094e-05,
-      "loss": 0.8183,
-      "step": 240
-    },
-    {
-      "epoch": 2.5,
-      "learning_rate": 7.954855279929014e-05,
-      "loss": 0.8455,
-      "step": 245
-    },
-    {
-      "epoch": 2.55,
-      "learning_rate": 6.860000000000005e-05,
-      "loss": 0.8961,
-      "step": 250
-    },
-    {
-      "epoch": 2.6,
-      "learning_rate": 5.765144720071096e-05,
-      "loss": 0.9542,
-      "step": 255
-    },
-    {
-      "epoch": 2.65,
-      "learning_rate": 4.698357624357919e-05,
-      "loss": 0.9393,
-      "step": 260
-    },
-    {
-      "epoch": 2.7,
-      "learning_rate": 3.686987328947893e-05,
-      "loss": 0.9105,
-      "step": 265
-    },
-    {
-      "epoch": 2.76,
-      "learning_rate": 2.7569617608302584e-05,
-      "loss": 0.9638,
-      "step": 270
-    },
-    {
-      "epoch": 2.81,
-      "learning_rate": 1.932123458329638e-05,
-      "loss": 0.8269,
-      "step": 275
-    },
-    {
-      "epoch": 2.86,
-      "learning_rate": 1.233618333464853e-05,
-      "loss": 0.9693,
-      "step": 280
-    },
-    {
-      "epoch": 2.91,
-      "learning_rate": 6.7935356618941304e-06,
-      "loss": 0.8474,
-      "step": 285
-    },
-    {
-      "epoch": 2.96,
-      "learning_rate": 2.8353852816850615e-06,
-      "loss": 0.8138,
-      "step": 290
-    },
-    {
-      "epoch": 3.0,
-      "eval_loss": 0.6951560974121094,
-      "eval_runtime": 6.9159,
-      "eval_samples_per_second": 20.677,
-      "eval_steps_per_second": 2.603,
-      "step": 294
+      "eval_loss": 0.7026481628417969,
+      "eval_runtime": 3.0708,
+      "eval_samples_per_second": 43.311,
+      "eval_steps_per_second": 5.536,
+      "step": 100
     }
   ],
-  "max_steps": 294,
-  "num_train_epochs": 3,
-  "total_flos": 305711677440000.0,
+  "max_steps": 400,
+  "num_train_epochs": 4,
+  "total_flos": 103602290688000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1e43acc9100bdd682eafe8ab7a4ccf0fb744b6214b26c1a24e8e39367a3d089
-size 3055
+oid sha256:e3541f3e8018424185512caf83ce47eec736987d33a3255d584fa6901962f8fe
+size 3375