new proc

added_tokens.json

@@ -17,7 +17,6 @@
   "<|da|>": 50285,
   "<|de|>": 50261,
   "<|el|>": 50281,
-  "<|endoftext|>": 50257,
   "<|en|>": 50259,
   "<|es|>": 50262,
   "<|et|>": 50307,
@@ -30,6 +29,7 @@
   "<|gu|>": 50333,
   "<|haw|>": 50352,
   "<|ha|>": 50354,
+  "<|he|>": 50279,
   "<|hi|>": 50276,
   "<|hr|>": 50291,
   "<|ht|>": 50339,
@@ -38,7 +38,6 @@
   "<|id|>": 50275,
   "<|is|>": 50311,
   "<|it|>": 50274,
-  "<|iw|>": 50279,
   "<|ja|>": 50266,
   "<|jw|>": 50356,
   "<|ka|>": 50329,

special_tokens_map.json

@@ -22,7 +22,7 @@
     "<|hi|>",
     "<|fi|>",
     "<|vi|>",
-    "<|iw|>",
+    "<|he|>",
     "<|uk|>",
     "<|el|>",
     "<|ms|>",
@@ -124,7 +124,7 @@
   },
   "pad_token": "<|endoftext|>",
   "unk_token": {
-    "content": "",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,

tokenizer_config.json

@@ -19,7 +19,7 @@
     "single_word": false
   },
   "errors": "replace",
-  "model_max_length": 448,
+  "model_max_length": 1024,
   "pad_token": null,
   "processor_class": "WhisperProcessor",
   "return_attention_mask": false,
@@ -27,7 +27,7 @@
   "trust_remote_code": false,
   "unk_token": {
     "__type": "AddedToken",
-    "content": "",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,

vocab.json

@@ -314,6 +314,7 @@
   ";;": 35746,
   "<": 27,
   "</": 3433,
+  "<|endoftext|>": 50257,
   "=": 28,
   "=\"": 13114,
   "=\"#": 34106,