Upload 7 files

config.json

@@ -0,0 +1,35 @@
+{
+  "_name_or_path": "./mt5_transliteration_upgraded/checkpoint-96802",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "bos_token_id": 811,
+  "classifier_dropout": 0.0,
+  "d_ff": 1024,
+  "d_kv": 64,
+  "d_model": 512,
+  "decoder_start_token_id": 813,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "early_stopping": true,
+  "eos_token_id": 812,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "max_length": 64,
+  "model_type": "mt5",
+  "num_decoder_layers": 4,
+  "num_heads": 4,
+  "num_layers": 4,
+  "pad_token_id": 813,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.2",
+  "use_cache": true,
+  "vocab_size": 826
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token_id": 811,
+  "decoder_start_token_id": 813,
+  "early_stopping": true,
+  "eos_token_id": 812,
+  "max_length": 64,
+  "pad_token_id": 813,
+  "transformers_version": "4.30.2"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eea47c12a0b8ec05665d7b4d1882605d7cc842199e8dbdf8a94e610957015c41
+size 78959231

special_tokens_map.json

@@ -0,0 +1,138 @@
+{
+  "additional_special_tokens": [
+    "<arb>",
+    "<asm>",
+    "<ben>",
+    "<guj>",
+    "<hin>",
+    "<kan>",
+    "<mal>",
+    "<mar>",
+    "<ori>",
+    "<pan>",
+    "<tam>",
+    "<tel>",
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,1898 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": {
+    "strategy": "BatchLongest",
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 813,
+    "pad_type_id": 0,
+    "pad_token": "<pad>"
+  },
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 811,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 812,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 813,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 814,
+      "content": "<arb>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 815,
+      "content": "<asm>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 816,
+      "content": "<ben>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 817,
+      "content": "<guj>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 818,
+      "content": "<hin>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 819,
+      "content": "<kan>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 820,
+      "content": "<mal>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 821,
+      "content": "<mar>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 822,
+      "content": "<ori>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 823,
+      "content": "<pan>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 824,
+      "content": "<tam>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 825,
+      "content": "<tel>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 826,
+      "content": "<extra_id_0>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 827,
+      "content": "<extra_id_1>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 828,
+      "content": "<extra_id_2>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 829,
+      "content": "<extra_id_3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 830,
+      "content": "<extra_id_4>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 831,
+      "content": "<extra_id_5>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 832,
+      "content": "<extra_id_6>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 833,
+      "content": "<extra_id_7>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 834,
+      "content": "<extra_id_8>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 835,
+      "content": "<extra_id_9>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 836,
+      "content": "<extra_id_10>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 837,
+      "content": "<extra_id_11>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 838,
+      "content": "<extra_id_12>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 839,
+      "content": "<extra_id_13>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 840,
+      "content": "<extra_id_14>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 841,
+      "content": "<extra_id_15>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 842,
+      "content": "<extra_id_16>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 843,
+      "content": "<extra_id_17>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 844,
+      "content": "<extra_id_18>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 845,
+      "content": "<extra_id_19>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 846,
+      "content": "<extra_id_20>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 847,
+      "content": "<extra_id_21>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 848,
+      "content": "<extra_id_22>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 849,
+      "content": "<extra_id_23>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 850,
+      "content": "<extra_id_24>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 851,
+      "content": "<extra_id_25>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 852,
+      "content": "<extra_id_26>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 853,
+      "content": "<extra_id_27>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 854,
+      "content": "<extra_id_28>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 855,
+      "content": "<extra_id_29>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 856,
+      "content": "<extra_id_30>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 857,
+      "content": "<extra_id_31>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 858,
+      "content": "<extra_id_32>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 859,
+      "content": "<extra_id_33>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 860,
+      "content": "<extra_id_34>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 861,
+      "content": "<extra_id_35>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 862,
+      "content": "<extra_id_36>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 863,
+      "content": "<extra_id_37>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 864,
+      "content": "<extra_id_38>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 865,
+      "content": "<extra_id_39>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 866,
+      "content": "<extra_id_40>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 867,
+      "content": "<extra_id_41>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 868,
+      "content": "<extra_id_42>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 869,
+      "content": "<extra_id_43>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 870,
+      "content": "<extra_id_44>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 871,
+      "content": "<extra_id_45>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 872,
+      "content": "<extra_id_46>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 873,
+      "content": "<extra_id_47>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 874,
+      "content": "<extra_id_48>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 875,
+      "content": "<extra_id_49>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 876,
+      "content": "<extra_id_50>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 877,
+      "content": "<extra_id_51>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 878,
+      "content": "<extra_id_52>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 879,
+      "content": "<extra_id_53>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 880,
+      "content": "<extra_id_54>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 881,
+      "content": "<extra_id_55>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 882,
+      "content": "<extra_id_56>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 883,
+      "content": "<extra_id_57>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 884,
+      "content": "<extra_id_58>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 885,
+      "content": "<extra_id_59>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 886,
+      "content": "<extra_id_60>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 887,
+      "content": "<extra_id_61>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 888,
+      "content": "<extra_id_62>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 889,
+      "content": "<extra_id_63>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 890,
+      "content": "<extra_id_64>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 891,
+      "content": "<extra_id_65>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 892,
+      "content": "<extra_id_66>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 893,
+      "content": "<extra_id_67>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 894,
+      "content": "<extra_id_68>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 895,
+      "content": "<extra_id_69>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 896,
+      "content": "<extra_id_70>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 897,
+      "content": "<extra_id_71>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 898,
+      "content": "<extra_id_72>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 899,
+      "content": "<extra_id_73>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 900,
+      "content": "<extra_id_74>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 901,
+      "content": "<extra_id_75>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 902,
+      "content": "<extra_id_76>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 903,
+      "content": "<extra_id_77>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 904,
+      "content": "<extra_id_78>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 905,
+      "content": "<extra_id_79>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 906,
+      "content": "<extra_id_80>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 907,
+      "content": "<extra_id_81>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 908,
+      "content": "<extra_id_82>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 909,
+      "content": "<extra_id_83>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 910,
+      "content": "<extra_id_84>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 911,
+      "content": "<extra_id_85>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 912,
+      "content": "<extra_id_86>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 913,
+      "content": "<extra_id_87>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 914,
+      "content": "<extra_id_88>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 915,
+      "content": "<extra_id_89>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 916,
+      "content": "<extra_id_90>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 917,
+      "content": "<extra_id_91>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 918,
+      "content": "<extra_id_92>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 919,
+      "content": "<extra_id_93>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 920,
+      "content": "<extra_id_94>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 921,
+      "content": "<extra_id_95>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 922,
+      "content": "<extra_id_96>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 923,
+      "content": "<extra_id_97>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 924,
+      "content": "<extra_id_98>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 925,
+      "content": "<extra_id_99>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "NFKC"
+  },
+  "pre_tokenizer": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "never",
+    "split": true
+  },
+  "post_processor": null,
+  "decoder": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "never",
+    "split": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "<unk>",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<unk>": 0,
+      "0": 1,
+      "1": 2,
+      "2": 3,
+      "3": 4,
+      "4": 5,
+      "5": 6,
+      "6": 7,
+      "7": 8,
+      "8": 9,
+      "9": 10,
+      "<": 11,
+      ">": 12,
+      "a": 13,
+      "b": 14,
+      "c": 15,
+      "d": 16,
+      "e": 17,
+      "f": 18,
+      "g": 19,
+      "h": 20,
+      "i": 21,
+      "j": 22,
+      "k": 23,
+      "l": 24,
+      "m": 25,
+      "n": 26,
+      "o": 27,
+      "p": 28,
+      "q": 29,
+      "r": 30,
+      "s": 31,
+      "t": 32,
+      "u": 33,
+      "v": 34,
+      "w": 35,
+      "x": 36,
+      "y": 37,
+      "z": 38,
+      "á": 39,
+      "è": 40,
+      "é": 41,
+      "؁": 42,
+      "،": 43,
+      "ؐ": 44,
+      "ؑ": 45,
+      "ؓ": 46,
+      "ء": 47,
+      "آ": 48,
+      "ؤ": 49,
+      "ئ": 50,
+      "ا": 51,
+      "ب": 52,
+      "ة": 53,
+      "ت": 54,
+      "ث": 55,
+      "ج": 56,
+      "ح": 57,
+      "خ": 58,
+      "د": 59,
+      "ذ": 60,
+      "ر": 61,
+      "ز": 62,
+      "س": 63,
+      "ش": 64,
+      "ص": 65,
+      "ض": 66,
+      "ط": 67,
+      "ظ": 68,
+      "ع": 69,
+      "غ": 70,
+      "ـ": 71,
+      "ف": 72,
+      "ق": 73,
+      "ك": 74,
+      "ل": 75,
+      "م": 76,
+      "ن": 77,
+      "ه": 78,
+      "و": 79,
+      "ي": 80,
+      "ً": 81,
+      "ٌ": 82,
+      "َ": 83,
+      "ُ": 84,
+      "ِ": 85,
+      "ّ": 86,
+      "ٓ": 87,
+      "ٔ": 88,
+      "ٕ": 89,
+      "ٖ": 90,
+      "ٗ": 91,
+      "٘": 92,
+      "ٚ": 93,
+      "ٛ": 94,
+      "٠": 95,
+      "١": 96,
+      "٢": 97,
+      "٣": 98,
+      "٤": 99,
+      "٥": 100,
+      "٦": 101,
+      "٧": 102,
+      "٨": 103,
+      "٩": 104,
+      "ٮ": 105,
+      "ٰ": 106,
+      "ٲ": 107,
+      "ٴ": 108,
+      "ٹ": 109,
+      "ٺ": 110,
+      "ٻ": 111,
+      "ٽ": 112,
+      "پ": 113,
+      "ٿ": 114,
+      "ڀ": 115,
+      "ڃ": 116,
+      "ڄ": 117,
+      "چ": 118,
+      "ڇ": 119,
+      "ڈ": 120,
+      "ڊ": 121,
+      "ڌ": 122,
+      "ڍ": 123,
+      "ڏ": 124,
+      "ڑ": 125,
+      "ژ": 126,
+      "ڙ": 127,
+      "ڦ": 128,
+      "ک": 129,
+      "ڪ": 130,
+      "گ": 131,
+      "ڱ": 132,
+      "ڳ": 133,
+      "ں": 134,
+      "ڻ": 135,
+      "ھ": 136,
+      "ہ": 137,
+      "ۂ": 138,
+      "ۃ": 139,
+      "ۄ": 140,
+      "ی": 141,
+      "ۍ": 142,
+      "ے": 143,
+      "ۓ": 144,
+      "۪": 145,
+      "ۭ": 146,
+      "۶": 147,
+      "۾": 148,
+      "ँ": 149,
+      "ं": 150,
+      "ः": 151,
+      "अ": 152,
+      "आ": 153,
+      "इ": 154,
+      "ई": 155,
+      "उ": 156,
+      "ऊ": 157,
+      "ऋ": 158,
+      "ऌ": 159,
+      "ऍ": 160,
+      "ए": 161,
+      "ऐ": 162,
+      "ऑ": 163,
+      "ओ": 164,
+      "औ": 165,
+      "क": 166,
+      "ख": 167,
+      "ग": 168,
+      "घ": 169,
+      "ङ": 170,
+      "च": 171,
+      "छ": 172,
+      "ज": 173,
+      "झ": 174,
+      "ञ": 175,
+      "ट": 176,
+      "ठ": 177,
+      "ड": 178,
+      "ढ": 179,
+      "ण": 180,
+      "त": 181,
+      "थ": 182,
+      "द": 183,
+      "ध": 184,
+      "न": 185,
+      "प": 186,
+      "फ": 187,
+      "ब": 188,
+      "भ": 189,
+      "म": 190,
+      "य": 191,
+      "र": 192,
+      "ऱ": 193,
+      "ल": 194,
+      "ळ": 195,
+      "व": 196,
+      "श": 197,
+      "ष": 198,
+      "स": 199,
+      "ह": 200,
+      "़": 201,
+      "ऽ": 202,
+      "ा": 203,
+      "ि": 204,
+      "ी": 205,
+      "ु": 206,
+      "ू": 207,
+      "ृ": 208,
+      "ॅ": 209,
+      "ॆ": 210,
+      "े": 211,
+      "ै": 212,
+      "ॉ": 213,
+      "ॊ": 214,
+      "ो": 215,
+      "ौ": 216,
+      "्": 217,
+      "ॐ": 218,
+      "ॠ": 219,
+      "।": 220,
+      "॥": 221,
+      "०": 222,
+      "१": 223,
+      "२": 224,
+      "३": 225,
+      "४": 226,
+      "५": 227,
+      "६": 228,
+      "७": 229,
+      "८": 230,
+      "९": 231,
+      "॰": 232,
+      "ঁ": 233,
+      "ং": 234,
+      "ঃ": 235,
+      "অ": 236,
+      "আ": 237,
+      "ই": 238,
+      "ঈ": 239,
+      "উ": 240,
+      "ঊ": 241,
+      "ঋ": 242,
+      "এ": 243,
+      "ঐ": 244,
+      "ও": 245,
+      "ঔ": 246,
+      "ক": 247,
+      "খ": 248,
+      "গ": 249,
+      "ঘ": 250,
+      "ঙ": 251,
+      "চ": 252,
+      "ছ": 253,
+      "জ": 254,
+      "ঝ": 255,
+      "ঞ": 256,
+      "ট": 257,
+      "ঠ": 258,
+      "ড": 259,
+      "ঢ": 260,
+      "ণ": 261,
+      "ত": 262,
+      "থ": 263,
+      "দ": 264,
+      "ধ": 265,
+      "ন": 266,
+      "প": 267,
+      "ফ": 268,
+      "ব": 269,
+      "ভ": 270,
+      "ম": 271,
+      "য": 272,
+      "র": 273,
+      "ল": 274,
+      "শ": 275,
+      "ষ": 276,
+      "স": 277,
+      "হ": 278,
+      "়": 279,
+      "া": 280,
+      "ি": 281,
+      "ী": 282,
+      "ু": 283,
+      "ূ": 284,
+      "ৃ": 285,
+      "ে": 286,
+      "ৈ": 287,
+      "ো": 288,
+      "ৌ": 289,
+      "্": 290,
+      "ৎ": 291,
+      "০": 292,
+      "১": 293,
+      "২": 294,
+      "৩": 295,
+      "৪": 296,
+      "৫": 297,
+      "৬": 298,
+      "৭": 299,
+      "৮": 300,
+      "৯": 301,
+      "ৰ": 302,
+      "ৱ": 303,
+      "ਂ": 304,
+      "ਃ": 305,
+      "ਅ": 306,
+      "ਆ": 307,
+      "ਇ": 308,
+      "ਈ": 309,
+      "ਉ": 310,
+      "ਊ": 311,
+      "ਏ": 312,
+      "ਐ": 313,
+      "ਓ": 314,
+      "ਔ": 315,
+      "ਕ": 316,
+      "ਖ": 317,
+      "ਗ": 318,
+      "ਘ": 319,
+      "ਙ": 320,
+      "ਚ": 321,
+      "ਛ": 322,
+      "ਜ": 323,
+      "ਝ": 324,
+      "ਞ": 325,
+      "ਟ": 326,
+      "ਠ": 327,
+      "ਡ": 328,
+      "ਢ": 329,
+      "ਣ": 330,
+      "ਤ": 331,
+      "ਥ": 332,
+      "ਦ": 333,
+      "ਧ": 334,
+      "ਨ": 335,
+      "ਪ": 336,
+      "ਫ": 337,
+      "ਬ": 338,
+      "ਭ": 339,
+      "ਮ": 340,
+      "ਯ": 341,
+      "ਰ": 342,
+      "ਲ": 343,
+      "ਵ": 344,
+      "ਸ": 345,
+      "ਹ": 346,
+      "਼": 347,
+      "ਾ": 348,
+      "ਿ": 349,
+      "ੀ": 350,
+      "ੁ": 351,
+      "ੂ": 352,
+      "ੇ": 353,
+      "ੈ": 354,
+      "ੋ": 355,
+      "ੌ": 356,
+      "੍": 357,
+      "ੜ": 358,
+      "੦": 359,
+      "੧": 360,
+      "੨": 361,
+      "੩": 362,
+      "੪": 363,
+      "੫": 364,
+      "੬": 365,
+      "੭": 366,
+      "੮": 367,
+      "੯": 368,
+      "ੰ": 369,
+      "ੱ": 370,
+      "ੲ": 371,
+      "ੳ": 372,
+      "ઁ": 373,
+      "ં": 374,
+      "ઃ": 375,
+      "અ": 376,
+      "આ": 377,
+      "ઇ": 378,
+      "ઈ": 379,
+      "ઉ": 380,
+      "ઊ": 381,
+      "ઋ": 382,
+      "ઍ": 383,
+      "એ": 384,
+      "ઐ": 385,
+      "ઑ": 386,
+      "ઓ": 387,
+      "ઔ": 388,
+      "ક": 389,
+      "ખ": 390,
+      "ગ": 391,
+      "ઘ": 392,
+      "ઙ": 393,
+      "ચ": 394,
+      "છ": 395,
+      "જ": 396,
+      "ઝ": 397,
+      "ઞ": 398,
+      "ટ": 399,
+      "ઠ": 400,
+      "ડ": 401,
+      "ઢ": 402,
+      "ણ": 403,
+      "ત": 404,
+      "થ": 405,
+      "દ": 406,
+      "ધ": 407,
+      "ન": 408,
+      "પ": 409,
+      "ફ": 410,
+      "બ": 411,
+      "ભ": 412,
+      "મ": 413,
+      "ય": 414,
+      "ર": 415,
+      "લ": 416,
+      "ળ": 417,
+      "વ": 418,
+      "શ": 419,
+      "ષ": 420,
+      "સ": 421,
+      "હ": 422,
+      "઼": 423,
+      "ા": 424,
+      "િ": 425,
+      "ી": 426,
+      "ુ": 427,
+      "ૂ": 428,
+      "ૃ": 429,
+      "ૅ": 430,
+      "ે": 431,
+      "ૈ": 432,
+      "ૉ": 433,
+      "ો": 434,
+      "ૌ": 435,
+      "્": 436,
+      "ૐ": 437,
+      "ૠ": 438,
+      "૦": 439,
+      "૧": 440,
+      "૨": 441,
+      "૩": 442,
+      "૪": 443,
+      "૫": 444,
+      "૬": 445,
+      "૭": 446,
+      "૮": 447,
+      "૯": 448,
+      "૰": 449,
+      "ଁ": 450,
+      "ଂ": 451,
+      "ଃ": 452,
+      "ଅ": 453,
+      "ଆ": 454,
+      "ଇ": 455,
+      "ଈ": 456,
+      "ଉ": 457,
+      "ଊ": 458,
+      "ଋ": 459,
+      "ଏ": 460,
+      "ଐ": 461,
+      "ଓ": 462,
+      "ଔ": 463,
+      "କ": 464,
+      "ଖ": 465,
+      "ଗ": 466,
+      "ଘ": 467,
+      "ଙ": 468,
+      "ଚ": 469,
+      "ଛ": 470,
+      "ଜ": 471,
+      "ଝ": 472,
+      "ଞ": 473,
+      "ଟ": 474,
+      "ଠ": 475,
+      "ଡ": 476,
+      "ଢ": 477,
+      "ଣ": 478,
+      "ତ": 479,
+      "ଥ": 480,
+      "ଦ": 481,
+      "ଧ": 482,
+      "ନ": 483,
+      "ପ": 484,
+      "ଫ": 485,
+      "ବ": 486,
+      "ଭ": 487,
+      "ମ": 488,
+      "ଯ": 489,
+      "ର": 490,
+      "ଲ": 491,
+      "ଳ": 492,
+      "ଶ": 493,
+      "ଷ": 494,
+      "ସ": 495,
+      "ହ": 496,
+      "଼": 497,
+      "ା": 498,
+      "ି": 499,
+      "ୀ": 500,
+      "ୁ": 501,
+      "ୂ": 502,
+      "ୃ": 503,
+      "େ": 504,
+      "ୈ": 505,
+      "ୋ": 506,
+      "ୌ": 507,
+      "୍": 508,
+      "୘": 509,
+      "ୟ": 510,
+      "ୢ": 511,
+      "୦": 512,
+      "୧": 513,
+      "୨": 514,
+      "୩": 515,
+      "୪": 516,
+      "୫": 517,
+      "୬": 518,
+      "୭": 519,
+      "୮": 520,
+      "୯": 521,
+      "ୱ": 522,
+      "ஂ": 523,
+      "ஃ": 524,
+      "அ": 525,
+      "ஆ": 526,
+      "இ": 527,
+      "ஈ": 528,
+      "உ": 529,
+      "ஊ": 530,
+      "எ": 531,
+      "ஏ": 532,
+      "ஐ": 533,
+      "ஒ": 534,
+      "ஓ": 535,
+      "ஔ": 536,
+      "க": 537,
+      "ங": 538,
+      "ச": 539,
+      "ஜ": 540,
+      "ஞ": 541,
+      "ட": 542,
+      "ண": 543,
+      "த": 544,
+      "ந": 545,
+      "ன": 546,
+      "ப": 547,
+      "ம": 548,
+      "ய": 549,
+      "ர": 550,
+      "ற": 551,
+      "ல": 552,
+      "ள": 553,
+      "ழ": 554,
+      "வ": 555,
+      "ஷ": 556,
+      "ஸ": 557,
+      "ஹ": 558,
+      "ா": 559,
+      "ி": 560,
+      "ீ": 561,
+      "ு": 562,
+      "ூ": 563,
+      "ெ": 564,
+      "ே": 565,
+      "ை": 566,
+      "ொ": 567,
+      "ோ": 568,
+      "ௌ": 569,
+      "்": 570,
+      "௦": 571,
+      "௧": 572,
+      "௨": 573,
+      "௩": 574,
+      "௪": 575,
+      "௫": 576,
+      "௬": 577,
+      "௭": 578,
+      "௮": 579,
+      "௯": 580,
+      "ఁ": 581,
+      "ం": 582,
+      "ః": 583,
+      "అ": 584,
+      "ఆ": 585,
+      "ఇ": 586,
+      "ఈ": 587,
+      "ఉ": 588,
+      "ఊ": 589,
+      "ఋ": 590,
+      "ఎ": 591,
+      "ఏ": 592,
+      "ఐ": 593,
+      "ఒ": 594,
+      "ఓ": 595,
+      "ఔ": 596,
+      "క": 597,
+      "ఖ": 598,
+      "గ": 599,
+      "ఘ": 600,
+      "ఙ": 601,
+      "చ": 602,
+      "ఛ": 603,
+      "జ": 604,
+      "ఝ": 605,
+      "ఞ": 606,
+      "ట": 607,
+      "ఠ": 608,
+      "డ": 609,
+      "ఢ": 610,
+      "ణ": 611,
+      "త": 612,
+      "థ": 613,
+      "ద": 614,
+      "ధ": 615,
+      "న": 616,
+      "ప": 617,
+      "ఫ": 618,
+      "బ": 619,
+      "భ": 620,
+      "మ": 621,
+      "య": 622,
+      "ర": 623,
+      "ఱ": 624,
+      "ల": 625,
+      "ళ": 626,
+      "వ": 627,
+      "శ": 628,
+      "ష": 629,
+      "స": 630,
+      "హ": 631,
+      "ా": 632,
+      "ి": 633,
+      "ీ": 634,
+      "ు": 635,
+      "ూ": 636,
+      "ృ": 637,
+      "ౄ": 638,
+      "ె": 639,
+      "ే": 640,
+      "ై": 641,
+      "ొ": 642,
+      "ో": 643,
+      "ౌ": 644,
+      "్": 645,
+      "౦": 646,
+      "౧": 647,
+      "౨": 648,
+      "౩": 649,
+      "౪": 650,
+      "౫": 651,
+      "౬": 652,
+      "౭": 653,
+      "౮": 654,
+      "౯": 655,
+      "ಂ": 656,
+      "ಃ": 657,
+      "ಅ": 658,
+      "ಆ": 659,
+      "ಇ": 660,
+      "ಈ": 661,
+      "ಉ": 662,
+      "ಊ": 663,
+      "ಋ": 664,
+      "ಎ": 665,
+      "ಏ": 666,
+      "ಐ": 667,
+      "ಒ": 668,
+      "ಓ": 669,
+      "ಔ": 670,
+      "ಕ": 671,
+      "ಖ": 672,
+      "ಗ": 673,
+      "ಘ": 674,
+      "ಚ": 675,
+      "ಛ": 676,
+      "ಜ": 677,
+      "ಝ": 678,
+      "ಞ": 679,
+      "ಟ": 680,
+      "ಠ": 681,
+      "ಡ": 682,
+      "ಢ": 683,
+      "ಣ": 684,
+      "ತ": 685,
+      "ಥ": 686,
+      "ದ": 687,
+      "ಧ": 688,
+      "ನ": 689,
+      "ಪ": 690,
+      "ಫ": 691,
+      "ಬ": 692,
+      "ಭ": 693,
+      "ಮ": 694,
+      "ಯ": 695,
+      "ರ": 696,
+      "ಲ": 697,
+      "ಳ": 698,
+      "ವ": 699,
+      "ಶ": 700,
+      "ಷ": 701,
+      "ಸ": 702,
+      "ಹ": 703,
+      "಼": 704,
+      "ಾ": 705,
+      "ಿ": 706,
+      "ೀ": 707,
+      "ು": 708,
+      "ೂ": 709,
+      "ೃ": 710,
+      "ೆ": 711,
+      "ೇ": 712,
+      "ೈ": 713,
+      "ೊ": 714,
+      "ೋ": 715,
+      "ೌ": 716,
+      "್": 717,
+      "೦": 718,
+      "೧": 719,
+      "೨": 720,
+      "೩": 721,
+      "೪": 722,
+      "೫": 723,
+      "೬": 724,
+      "೭": 725,
+      "೮": 726,
+      "೯": 727,
+      "ം": 728,
+      "ഃ": 729,
+      "അ": 730,
+      "ആ": 731,
+      "ഇ": 732,
+      "ഈ": 733,
+      "ഉ": 734,
+      "ഊ": 735,
+      "ഋ": 736,
+      "എ": 737,
+      "ഏ": 738,
+      "ഐ": 739,
+      "ഒ": 740,
+      "ഓ": 741,
+      "ഔ": 742,
+      "ക": 743,
+      "ഖ": 744,
+      "ഗ": 745,
+      "ഘ": 746,
+      "ങ": 747,
+      "ച": 748,
+      "ഛ": 749,
+      "ജ": 750,
+      "ഝ": 751,
+      "ഞ": 752,
+      "ട": 753,
+      "ഠ": 754,
+      "ഡ": 755,
+      "ഢ": 756,
+      "ണ": 757,
+      "ത": 758,
+      "ഥ": 759,
+      "ദ": 760,
+      "ധ": 761,
+      "ന": 762,
+      "പ": 763,
+      "ഫ": 764,
+      "ബ": 765,
+      "ഭ": 766,
+      "മ": 767,
+      "യ": 768,
+      "ര": 769,
+      "റ": 770,
+      "ല": 771,
+      "ള": 772,
+      "ഴ": 773,
+      "വ": 774,
+      "ശ": 775,
+      "ഷ": 776,
+      "സ": 777,
+      "ഹ": 778,
+      "ാ": 779,
+      "ി": 780,
+      "ീ": 781,
+      "ു": 782,
+      "ൂ": 783,
+      "ൃ": 784,
+      "െ": 785,
+      "േ": 786,
+      "ൈ": 787,
+      "ൊ": 788,
+      "ോ": 789,
+      "ൌ": 790,
+      "്": 791,
+      "ൗ": 792,
+      "൦": 793,
+      "൧": 794,
+      "൨": 795,
+      "൩": 796,
+      "൪": 797,
+      "൫": 798,
+      "൬": 799,
+      "൭": 800,
+      "൮": 801,
+      "൯": 802,
+      "ൺ": 803,
+      "ൻ": 804,
+      "ർ": 805,
+      "ൽ": 806,
+      "ൾ": 807,
+      "‌": 808,
+      "‍": 809,
+      "▁": 810
+    },
+    "merges": []
+  }
+}

tokenizer_config.json

@@ -0,0 +1,1054 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "811": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "812": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "813": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "814": {
+      "content": "<arb>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "815": {
+      "content": "<asm>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "816": {
+      "content": "<ben>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "817": {
+      "content": "<guj>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "818": {
+      "content": "<hin>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "819": {
+      "content": "<kan>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "820": {
+      "content": "<mal>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "821": {
+      "content": "<mar>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "822": {
+      "content": "<ori>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "823": {
+      "content": "<pan>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "824": {
+      "content": "<tam>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "825": {
+      "content": "<tel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "826": {
+      "content": "<extra_id_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "827": {
+      "content": "<extra_id_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "828": {
+      "content": "<extra_id_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "829": {
+      "content": "<extra_id_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "830": {
+      "content": "<extra_id_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "831": {
+      "content": "<extra_id_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "832": {
+      "content": "<extra_id_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "833": {
+      "content": "<extra_id_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "834": {
+      "content": "<extra_id_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "835": {
+      "content": "<extra_id_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "836": {
+      "content": "<extra_id_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "837": {
+      "content": "<extra_id_11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "838": {
+      "content": "<extra_id_12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "839": {
+      "content": "<extra_id_13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "840": {
+      "content": "<extra_id_14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "841": {
+      "content": "<extra_id_15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "842": {
+      "content": "<extra_id_16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "843": {
+      "content": "<extra_id_17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "844": {
+      "content": "<extra_id_18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "845": {
+      "content": "<extra_id_19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "846": {
+      "content": "<extra_id_20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "847": {
+      "content": "<extra_id_21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "848": {
+      "content": "<extra_id_22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "849": {
+      "content": "<extra_id_23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "850": {
+      "content": "<extra_id_24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "851": {
+      "content": "<extra_id_25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "852": {
+      "content": "<extra_id_26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "853": {
+      "content": "<extra_id_27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "854": {
+      "content": "<extra_id_28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "855": {
+      "content": "<extra_id_29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "856": {
+      "content": "<extra_id_30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "857": {
+      "content": "<extra_id_31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "858": {
+      "content": "<extra_id_32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "859": {
+      "content": "<extra_id_33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "860": {
+      "content": "<extra_id_34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "861": {
+      "content": "<extra_id_35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "862": {
+      "content": "<extra_id_36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "863": {
+      "content": "<extra_id_37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "864": {
+      "content": "<extra_id_38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "865": {
+      "content": "<extra_id_39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "866": {
+      "content": "<extra_id_40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "867": {
+      "content": "<extra_id_41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "868": {
+      "content": "<extra_id_42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "869": {
+      "content": "<extra_id_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "870": {
+      "content": "<extra_id_44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "871": {
+      "content": "<extra_id_45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "872": {
+      "content": "<extra_id_46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "873": {
+      "content": "<extra_id_47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "874": {
+      "content": "<extra_id_48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "875": {
+      "content": "<extra_id_49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "876": {
+      "content": "<extra_id_50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "877": {
+      "content": "<extra_id_51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "878": {
+      "content": "<extra_id_52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "879": {
+      "content": "<extra_id_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "880": {
+      "content": "<extra_id_54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "881": {
+      "content": "<extra_id_55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "882": {
+      "content": "<extra_id_56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "883": {
+      "content": "<extra_id_57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "884": {
+      "content": "<extra_id_58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "885": {
+      "content": "<extra_id_59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "886": {
+      "content": "<extra_id_60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "887": {
+      "content": "<extra_id_61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "888": {
+      "content": "<extra_id_62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "889": {
+      "content": "<extra_id_63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "890": {
+      "content": "<extra_id_64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "891": {
+      "content": "<extra_id_65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "892": {
+      "content": "<extra_id_66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "893": {
+      "content": "<extra_id_67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "894": {
+      "content": "<extra_id_68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "895": {
+      "content": "<extra_id_69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "896": {
+      "content": "<extra_id_70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "897": {
+      "content": "<extra_id_71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "898": {
+      "content": "<extra_id_72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "899": {
+      "content": "<extra_id_73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "900": {
+      "content": "<extra_id_74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "901": {
+      "content": "<extra_id_75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "902": {
+      "content": "<extra_id_76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "903": {
+      "content": "<extra_id_77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "904": {
+      "content": "<extra_id_78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "905": {
+      "content": "<extra_id_79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "906": {
+      "content": "<extra_id_80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "907": {
+      "content": "<extra_id_81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "908": {
+      "content": "<extra_id_82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "909": {
+      "content": "<extra_id_83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "910": {
+      "content": "<extra_id_84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "911": {
+      "content": "<extra_id_85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "912": {
+      "content": "<extra_id_86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "913": {
+      "content": "<extra_id_87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "914": {
+      "content": "<extra_id_88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "915": {
+      "content": "<extra_id_89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "916": {
+      "content": "<extra_id_90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "917": {
+      "content": "<extra_id_91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "918": {
+      "content": "<extra_id_92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "919": {
+      "content": "<extra_id_93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "920": {
+      "content": "<extra_id_94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "921": {
+      "content": "<extra_id_95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "922": {
+      "content": "<extra_id_96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "923": {
+      "content": "<extra_id_97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "924": {
+      "content": "<extra_id_98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "925": {
+      "content": "<extra_id_99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<arb>",
+    "<asm>",
+    "<ben>",
+    "<guj>",
+    "<hin>",
+    "<kan>",
+    "<mal>",
+    "<mar>",
+    "<ori>",
+    "<pan>",
+    "<tam>",
+    "<tel>",
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_ids": 100,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaaf75165f97986b66f80749f20de988efe4be07e385bfe7a8219ef19d9ee064
+size 4015