Update tokenizer, bump hf versions

all_results.json

@@ -1,14 +1,14 @@
 {
-    "epoch": 50.0,
-    "perplexity": 2.8322171542330707,
-    "test_accuracy": 0.35489852978435493,
-    "test_loss": 1.0410598516464233,
-    "test_runtime": 55.5978,
-    "test_samples_per_second": 4293.445,
-    "test_steps_per_second": 16.781,
-    "train_loss": 1.2392261905236766,
-    "train_runtime": 25350.8633,
+    "epoch": 49.98,
+    "test_accuracy": 0.41162851869286315,
+    "test_loss": 1.1775240898132324,
+    "test_perplexity": 3.246326631292486,
+    "test_runtime": 55.1082,
+    "test_samples_per_second": 4331.591,
+    "test_steps_per_second": 16.93,
+    "train_loss": 1.379773639240096,
+    "train_runtime": 17824.9007,
     "train_samples": 1273104,
-    "train_samples_per_second": 2510.968,
-    "train_steps_per_second": 9.808
+    "train_samples_per_second": 3571.139,
+    "train_steps_per_second": 6.973
 }

config.json

@@ -26,7 +26,7 @@
   "summary_type": "cls_index",
   "summary_use_proj": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.27.1",
+  "transformers_version": "4.34.1",
   "use_cache": true,
-  "vocab_size": 93
+  "vocab_size": 181
 }

generation_config.json

@@ -1,10 +1,9 @@
 {
   "do_sample": true,
   "eos_token_id": 1,
-  "length_penalty": 0.0,
   "max_new_tokens": 128,
   "min_new_tokens": 1,
   "num_return_sequences": 100,
   "pad_token_id": 2,
-  "transformers_version": "4.27.1"
+  "transformers_version": "4.34.1"
 }

model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:33e05a7877f04271d65f0db80d946c7cbd1cd24908eb50ed0f49b65d2a7b6270
-size 12375736

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75711e4f676270ff40fcfc23ea74221564ba4dafd829420db85f2ef42cdc11dc
-size 12414333
+oid sha256:87b3fa750ed5d361efc7d44edbf8184f3598205eb2163df1811cc5cb14dff2f2
+size 12260637

special_tokens_map.json

@@ -1,6 +1,30 @@
 {
-  "bos_token": "^",
-  "eos_token": "_",
-  "pad_token": " ",
-  "unk_token": "§"
+  "bos_token": {
+    "content": "^",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "_",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": " ",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "§",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

test_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 50.0,
-    "perplexity": 2.8322171542330707,
-    "test_accuracy": 0.35489852978435493,
-    "test_loss": 1.0410598516464233,
-    "test_runtime": 55.5978,
-    "test_samples_per_second": 4293.445,
-    "test_steps_per_second": 16.781
+    "epoch": 49.98,
+    "test_accuracy": 0.41162851869286315,
+    "test_loss": 1.1775240898132324,
+    "test_perplexity": 3.246326631292486,
+    "test_runtime": 55.1082,
+    "test_samples_per_second": 4331.591,
+    "test_steps_per_second": 16.93
 }

tokenizer.json

@@ -1,7 +1,14 @@
 {
   "version": "1.0",
   "truncation": null,
-  "padding": null,
+  "padding": {
+    "strategy": "BatchLongest",
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 2,
+    "pad_type_id": 0,
+    "pad_token": " "
+  },
   "added_tokens": [
     {
       "id": 0,
@@ -108,383 +115,198 @@
       }
     }
   },
-  "decoder": null,
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": false
+  },
   "model": {
-    "type": "Unigram",
-    "unk_id": 3,
-    "vocab": [
-      [
-        "^",
-        0.0
-      ],
-      [
-        "_",
-        0.0
-      ],
-      [
-        " ",
-        0.0
-      ],
-      [
-        "§",
-        0.0
-      ],
-      [
-        "°",
-        0.0
-      ],
-      [
-        "C",
-        -2.2339120063759843
-      ],
-      [
-        "c1",
-        -3.108178186508052
-      ],
-      [
-        "cc",
-        -3.2719370147830347
-      ],
-      [
-        "N",
-        -3.2932537094066436
-      ],
-      [
-        "O",
-        -3.294834188186176
-      ],
-      [
-        "n",
-        -3.3053461817854917
-      ],
-      [
-        ")",
-        -3.3475212288619822
-      ],
-      [
-        "CC",
-        -3.4312206506012544
-      ],
-      [
-        "(",
-        -3.5624560302888746
-      ],
-      [
-        "c(",
-        -3.765813889966134
-      ],
-      [
-        "c",
-        -3.8218193366886393
-      ],
-      [
-        "C(=O)",
-        -3.856958418790814
-      ],
-      [
-        "c3",
-        -3.9254062324983607
-      ],
-      [
-        "=",
-        -3.975334503903719
-      ],
-      [
-        "c2",
-        -4.003423620506245
-      ],
-      [
-        "1",
-        -4.112680698145304
-      ],
-      [
-        "(C)",
-        -4.121209869936163
-      ],
-      [
-        "2",
-        -4.1361761826809555
-      ],
-      [
-        "C1",
-        -4.21768981835341
-      ],
-      [
-        "c1ccc(",
-        -4.251735247120498
-      ],
-      [
-        "-",
-        -4.252411803847931
-      ],
-      [
-        "2)",
-        -4.270562896654599
-      ],
-      [
-        "cc(",
-        -4.276471688527964
-      ],
-      [
-        "C(",
-        -4.3086368671374995
-      ],
-      [
-        "CO",
-        -4.337904719405833
-      ],
-      [
-        "C(=O)N",
-        -4.428846224698317
-      ],
-      [
-        "3)",
-        -4.449247292348714
-      ],
-      [
-        "C2",
-        -4.522375726063721
-      ],
-      [
-        "O)",
-        -4.5371556203299654
-      ],
-      [
-        "CCC",
-        -4.589271525409394
-      ],
-      [
-        "3",
-        -4.724794788518322
-      ],
-      [
-        "C(C)",
-        -4.750491328820496
-      ],
-      [
-        "cc1",
-        -4.774712806549024
-      ],
-      [
-        "cn",
-        -4.786244699312933
-      ],
-      [
-        "NC(=O)",
-        -4.789019365908235
-      ],
-      [
-        "CC1",
-        -4.79366206708262
-      ],
-      [
-        "=O)",
-        -4.843480092939821
-      ],
-      [
-        "C)",
-        -4.8437048190443175
-      ],
-      [
-        "S",
-        -4.850239832224922
-      ],
-      [
-        "n1",
-        -4.864510173862389
-      ],
-      [
-        "c1ccccc1",
-        -4.92499844194878
-      ],
-      [
-        "c2ccc(",
-        -5.024291675089154
-      ],
-      [
-        "[nH]",
-        -5.110243493192227
-      ],
-      [
-        "c4",
-        -5.110439906920133
-      ],
-      [
-        "s",
-        -5.124712107023752
-      ],
-      [
-        "N1",
-        -5.13109013422166
-      ],
-      [
-        "o",
-        -5.144341645447907
-      ],
-      [
-        "F)",
-        -5.173986710079323
-      ],
-      [
-        "N(C",
-        -5.185845166641148
-      ],
-      [
-        "S(=O)(=O)",
-        -5.212557352968634
-      ],
-      [
-        "Cl)",
-        -5.220168182623269
-      ],
-      [
-        "c2ccc",
-        -5.2702619038121234
-      ],
-      [
-        "C(O)",
-        -5.276647874822254
-      ],
-      [
-        "2)cc1",
-        -5.314256517516519
-      ],
-      [
-        "O=C(",
-        -5.316263615262265
-      ],
-      [
-        "c3ccccc3",
-        -5.35319737432803
-      ],
-      [
-        "4",
-        -5.365584577986091
-      ],
-      [
-        "c(Cl)c",
-        -5.396247268723645
-      ],
-      [
-        "C=C",
-        -5.435516000963592
-      ],
-      [
-        "5",
-        -5.521017741806011
-      ],
-      [
-        "N2CC",
-        -5.731120719935269
-      ],
-      [
-        "c(F)c",
-        -5.761758311176596
-      ],
-      [
-        "C(F)(F)F)",
-        -5.816389560959275
-      ],
-      [
-        "[",
-        -5.9137028227376565
-      ],
-      [
-        "]",
-        -5.9137028227376565
-      ],
-      [
-        "c(OC)c",
-        -5.948373907493249
-      ],
-      [
-        "c(-c3cc",
-        -6.076591700432848
-      ],
-      [
-        "Br)",
-        -6.173257390983636
-      ],
-      [
-        "#",
-        -6.406719606169013
-      ],
-      [
-        "[N+](=O)[O-])",
-        -6.4134780734073065
-      ],
-      [
-        "+",
-        -6.459711891957882
-      ],
-      [
-        "F",
-        -6.6661526991253
-      ],
-      [
-        "P",
-        -7.117461698495431
-      ],
-      [
-        "6",
-        -7.298200155096458
-      ],
-      [
-        "B",
-        -8.020755755896921
-      ],
-      [
-        "I",
-        -8.076335551264686
-      ],
-      [
-        "7",
-        -9.064406674700315
-      ],
-      [
-        "H",
-        -9.766986825131063
-      ],
-      [
-        "8",
-        -10.67648343715202
-      ],
-      [
-        "9",
-        -11.6208264490498
-      ],
-      [
-        "%",
-        -13.14094059722758
-      ],
-      [
-        "0",
-        -13.512972359677438
-      ],
-      [
-        "p",
-        -13.73258273972798
-      ],
-      [
-        "l",
-        -17.659453778747075
-      ],
-      [
-        "e",
-        -17.659553778747075
-      ],
-      [
-        "i",
-        -17.659653778747074
-      ],
-      [
-        "r",
-        -17.659753778747074
-      ],
-      [
-        "b",
-        -17.659753778747074
-      ]
-    ]
+    "type": "WordPiece",
+    "unk_token": "§",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 1024,
+    "vocab": {
+      "^": 0,
+      "_": 1,
+      " ": 2,
+      "§": 3,
+      "°": 4,
+      "#": 5,
+      "%": 6,
+      "(": 7,
+      ")": 8,
+      "+": 9,
+      "-": 10,
+      "0": 11,
+      "1": 12,
+      "2": 13,
+      "3": 14,
+      "4": 15,
+      "5": 16,
+      "6": 17,
+      "7": 18,
+      "8": 19,
+      "9": 20,
+      "=": 21,
+      "B": 22,
+      "C": 23,
+      "F": 24,
+      "H": 25,
+      "I": 26,
+      "N": 27,
+      "O": 28,
+      "P": 29,
+      "S": 30,
+      "[": 31,
+      "]": 32,
+      "b": 33,
+      "c": 34,
+      "e": 35,
+      "i": 36,
+      "l": 37,
+      "n": 38,
+      "o": 39,
+      "p": 40,
+      "r": 41,
+      "s": 42,
+      "##c": 43,
+      "##1": 44,
+      "##(": 45,
+      "##C": 46,
+      "##=": 47,
+      "##2": 48,
+      "##O": 49,
+      "##)": 50,
+      "##n": 51,
+      "##-": 52,
+      "##l": 53,
+      "##N": 54,
+      "##S": 55,
+      "##3": 56,
+      "##[": 57,
+      "##H": 58,
+      "##]": 59,
+      "##+": 60,
+      "##o": 61,
+      "##F": 62,
+      "##4": 63,
+      "##B": 64,
+      "###": 65,
+      "##r": 66,
+      "##I": 67,
+      "##s": 68,
+      "##5": 69,
+      "##P": 70,
+      "##6": 71,
+      "##e": 72,
+      "##7": 73,
+      "##i": 74,
+      "##8": 75,
+      "##9": 76,
+      "##p": 77,
+      "##%": 78,
+      "##0": 79,
+      "##b": 80,
+      "##cc": 81,
+      "##CC": 82,
+      "##O)": 83,
+      "##C(": 84,
+      "##c1": 85,
+      "##=O)": 86,
+      "##c(": 87,
+      "##c2": 88,
+      "##C)": 89,
+      "##c1cc": 90,
+      "##C(=O)": 91,
+      "##c3": 92,
+      "##c2cc": 93,
+      "##)cc": 94,
+      "##(=O)": 95,
+      "##C1": 96,
+      "##c1ccc(": 97,
+      "##(C)": 98,
+      "##c3cc": 99,
+      "##2)": 100,
+      "##F)": 101,
+      "##CCCC": 102,
+      "##C(=O)N": 103,
+      "##c1cccc": 104,
+      "##C2": 105,
+      "##c2cccc": 106,
+      "##CN": 107,
+      "##Cl": 108,
+      "##C(C)": 109,
+      "##cn": 110,
+      "##)cc1": 111,
+      "##c4": 112,
+      "##CCN": 113,
+      "##3)": 114,
+      "CO": 115,
+      "##=C(": 116,
+      "##n1": 117,
+      "##c2ccccc2": 118,
+      "##nc(": 119,
+      "##c2ccc(": 120,
+      "##c1ccccc1": 121,
+      "##CC1": 122,
+      "##CO": 123,
+      "##c3cccc": 124,
+      "##[n": 125,
+      "##NC(=O)": 126,
+      "##N)": 127,
+      "##H]": 128,
+      "##[nH]": 129,
+      "##S(=O)": 130,
+      "##(C": 131,
+      "##S(=O)(=O)": 132,
+      "##Cl)": 133,
+      "O=C(": 134,
+      "##(F)": 135,
+      "##c3ccccc3": 136,
+      "##OC)": 137,
+      "##CC(=O)": 138,
+      "##C3": 139,
+      "##cccc": 140,
+      "##CC(": 141,
+      "##n2": 142,
+      "##c1cc(": 143,
+      "##c2c(": 144,
+      "##c1n": 145,
+      "##c3ccc(": 146,
+      "##cc1": 147,
+      "##CCN(": 148,
+      "##N1": 149,
+      "##CCO": 150,
+      "##C(F)": 151,
+      "##CC2": 152,
+      "##F)cc": 153,
+      "##c2)": 154,
+      "##cc(": 155,
+      "##c2ccccc2)": 156,
+      "##c2n": 157,
+      "##C(F)(F)": 158,
+      "##C(O)": 159,
+      "##Cl)cc": 160,
+      "##C=": 161,
+      "##c(=O)": 162,
+      "##+]": 163,
+      "##c1cccc(": 164,
+      "##4)": 165,
+      "##OC": 166,
+      "COc1ccc(": 167,
+      "##c1)": 168,
+      "##=N": 169,
+      "##C(C": 170,
+      "##=O": 171,
+      "##c2cc(": 172,
+      "##N2": 173,
+      "##CCCN": 174,
+      "##c(-": 175,
+      "##-]": 176,
+      "CC(C)": 177,
+      "##[N": 178,
+      "##O-]": 179,
+      "##[O-]": 180
+    }
   }
 }

tokenizer_config.json

@@ -1,10 +1,55 @@
 {
+  "added_tokens_decoder": {
+    "0": {
+      "content": "^",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "_",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": " ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "§",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "°",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "bos_token": "^",
+  "clean_up_tokenization_spaces": true,
   "eos_token": "_",
-  "model_max_length": 128,
+  "max_length": null,
+  "model_max_length": 1024,
+  "pad_to_multiple_of": null,
   "pad_token": " ",
+  "pad_token_type_id": 0,
   "padding_side": "right",
-  "special_tokens_map_file": "../../tokenizers/char_unigram_88/special_tokens_map.json",
   "tokenizer_class": "PreTrainedTokenizerFast",
   "truncation_side": "left",
   "unk_token": "§"

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 50.0,
-    "train_loss": 1.2392261905236766,
-    "train_runtime": 25350.8633,
+    "epoch": 49.98,
+    "train_loss": 1.379773639240096,
+    "train_runtime": 17824.9007,
     "train_samples": 1273104,
-    "train_samples_per_second": 2510.968,
-    "train_steps_per_second": 9.808
+    "train_samples_per_second": 3571.139,
+    "train_steps_per_second": 6.973
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d7f23e857016ba44427fcedf175d39d5d54474fc70e0d9f748a4969ddc5394f
-size 3643
+oid sha256:7270b826d836c540d0751fafd26f6bd57648b6cc071d87adf3f61ae00f263d74
+size 4155