Add processor for Whisper Medium Sanskrit model

tokenizer.json

@@ -14492,18 +14492,6 @@
           "type_id": 0
         }
       },
-      {
-        "SpecialToken": {
-          "id": "<|sa|>",
-          "type_id": 0
-        }
-      },
-      {
-        "SpecialToken": {
-          "id": "<|transcribe|>",
-          "type_id": 0
-        }
-      },
       {
         "SpecialToken": {
           "id": "<|notimestamps|>",
@@ -14530,18 +14518,6 @@
           "type_id": 0
         }
       },
-      {
-        "SpecialToken": {
-          "id": "<|sa|>",
-          "type_id": 0
-        }
-      },
-      {
-        "SpecialToken": {
-          "id": "<|transcribe|>",
-          "type_id": 0
-        }
-      },
       {
         "SpecialToken": {
           "id": "<|notimestamps|>",
@@ -14586,15 +14562,6 @@
           "<|notimestamps|>"
         ]
       },
-      "<|sa|>": {
-        "id": "<|sa|>",
-        "ids": [
-          50344
-        ],
-        "tokens": [
-          "<|sa|>"
-        ]
-      },
       "<|startoftranscript|>": {
         "id": "<|startoftranscript|>",
         "ids": [
@@ -14603,15 +14570,6 @@
         "tokens": [
           "<|startoftranscript|>"
         ]
-      },
-      "<|transcribe|>": {
-        "id": "<|transcribe|>",
-        "ids": [
-          50359
-        ],
-        "tokens": [
-          "<|transcribe|>"
-        ]
       }
     }
   },

tokenizer_config.json

@@ -115,13 +115,13 @@
     "<|notimestamps|>"
   ],
   "is_local": true,
-  "language": "sa",
+  "language": null,
   "model_max_length": 1024,
   "pad_token": "<|endoftext|>",
   "predict_timestamps": false,
   "processor_class": "WhisperProcessor",
   "return_attention_mask": false,
-  "task": "transcribe",
+  "task": null,
   "tokenizer_class": "WhisperTokenizer",
   "unk_token": "<|endoftext|>"
 }