add tavbert-ar model

config.json

@@ -1,43 +1,42 @@
 {
-  "_name_or_path": "/home/etherx/arabic2tashkeel/tashkeela_pretraining/checkpoint-84392",
+  "_name_or_path": "tau/tavbert-ar",
   "architectures": [
-    "TranslitModel"
+    "RobertaForTokenClassification"
   ],
   "attention_probs_dropout_prob": 0.1,
-  "bos_token_id": 57344,
-  "downsampling_rate": 4,
-  "eos_token_id": 57345,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
   "id2label": {
-    "0": "SHADDA",
-    "1": "FATHA",
-    "2": "KASRA",
-    "3": "DAMMA",
-    "4": "SUKKUN"
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2",
+    "3": "LABEL_3",
+    "4": "LABEL_4"
   },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
   "label2id": {
-    "SHADDA": 0,
-    "FATHA": 1,
-    "KASRA": 2,
-    "DAMMA": 3,
-    "SUKKUN": 4
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2,
+    "LABEL_3": 3,
+    "LABEL_4": 4
   },
-  "layer_norm_eps": 1e-12,
-  "local_transformer_stride": 128,
-  "max_position_embeddings": 16384,
-  "model_type": "canine",
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2050,
+  "model_type": "roberta",
   "num_attention_heads": 12,
-  "num_hash_buckets": 16384,
-  "num_hash_functions": 8,
   "num_hidden_layers": 12,
-  "pad_token_id": 0,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
   "torch_dtype": "float32",
   "transformers_version": "4.38.1",
-  "type_vocab_size": 16,
-  "upsampling_kernel_size": 4,
-  "use_cache": true
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 302
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a1ee9625ef023ef8e36f4292c8d8f979b80a5f66cf7ce44a395231e378145c9
-size 528377156
+oid sha256:81dbe43961509b495e3dbfbc084c8900fbe2c5a8b93c5bd169b7d204f462434d
+size 349857052

special_tokens_map.json

@@ -1,44 +1,7 @@
 {
-  "bos_token": {
-    "content": "",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "cls_token": {
-    "content": "",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "mask_token": {
-    "content": "",
-    "lstrip": true,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "\u0000",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "sep_token": {
-    "content": "",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
 }

tokenizer.json

@@ -0,0 +1,452 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": {
+    "strategy": "BatchLongest",
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 1,
+    "pad_type_id": 0,
+    "pad_token": "[PAD]"
+  },
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 301,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Split",
+    "pattern": {
+      "String": ""
+    },
+    "behavior": "Isolated",
+    "invert": false
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[CLS]": {
+        "id": "[CLS]",
+        "ids": [
+          0
+        ],
+        "tokens": [
+          "[CLS]"
+        ]
+      },
+      "[SEP]": {
+        "id": "[SEP]",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "[SEP]"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "[CLS]": 0,
+      "[PAD]": 1,
+      "[SEP]": 2,
+      "[UNK]": 3,
+      "unused0": 4,
+      "unused1": 5,
+      "unused2": 6,
+      "unused3": 7,
+      "unused4": 8,
+      "unused5": 9,
+      "unused6": 10,
+      "unused7": 11,
+      "unused8": 12,
+      "unused9": 13,
+      "unused10": 14,
+      "unused11": 15,
+      "unused12": 16,
+      "unused13": 17,
+      "unused14": 18,
+      "unused15": 19,
+      "unused16": 20,
+      "unused17": 21,
+      "unused18": 22,
+      "unused19": 23,
+      "unused20": 24,
+      "unused21": 25,
+      "unused22": 26,
+      "unused23": 27,
+      "unused24": 28,
+      "unused25": 29,
+      "unused26": 30,
+      "unused27": 31,
+      "unused28": 32,
+      "unused29": 33,
+      "unused30": 34,
+      "unused31": 35,
+      "unused32": 36,
+      "unused33": 37,
+      "unused34": 38,
+      "unused35": 39,
+      "unused36": 40,
+      "unused37": 41,
+      "unused38": 42,
+      "unused39": 43,
+      "unused40": 44,
+      "unused41": 45,
+      "unused42": 46,
+      "unused43": 47,
+      "unused44": 48,
+      "unused45": 49,
+      "unused46": 50,
+      "unused47": 51,
+      "unused48": 52,
+      "unused49": 53,
+      "unused50": 54,
+      "unused51": 55,
+      "unused52": 56,
+      "unused53": 57,
+      "unused54": 58,
+      "unused55": 59,
+      "unused56": 60,
+      "unused57": 61,
+      "unused58": 62,
+      "unused59": 63,
+      "unused60": 64,
+      "unused61": 65,
+      "unused62": 66,
+      "unused63": 67,
+      "unused64": 68,
+      "unused65": 69,
+      "unused66": 70,
+      "unused67": 71,
+      "unused68": 72,
+      "unused69": 73,
+      "unused70": 74,
+      "unused71": 75,
+      "unused72": 76,
+      "unused73": 77,
+      "unused74": 78,
+      "unused75": 79,
+      "unused76": 80,
+      "unused77": 81,
+      "unused78": 82,
+      "unused79": 83,
+      "unused80": 84,
+      "unused81": 85,
+      "unused82": 86,
+      "unused83": 87,
+      "unused84": 88,
+      "unused85": 89,
+      "unused86": 90,
+      "unused87": 91,
+      "unused88": 92,
+      "unused89": 93,
+      "unused90": 94,
+      "unused91": 95,
+      "unused92": 96,
+      "unused93": 97,
+      "unused94": 98,
+      "unused95": 99,
+      "unused96": 100,
+      "unused97": 101,
+      "unused98": 102,
+      "unused99": 103,
+      " ": 104,
+      "!": 105,
+      "\"": 106,
+      "#": 107,
+      "$": 108,
+      "%": 109,
+      "&": 110,
+      "'": 111,
+      "(": 112,
+      ")": 113,
+      "*": 114,
+      "+": 115,
+      ",": 116,
+      "-": 117,
+      ".": 118,
+      "/": 119,
+      "0": 120,
+      "1": 121,
+      "2": 122,
+      "3": 123,
+      "4": 124,
+      "5": 125,
+      "6": 126,
+      "7": 127,
+      "8": 128,
+      "9": 129,
+      ":": 130,
+      ";": 131,
+      "<": 132,
+      "=": 133,
+      ">": 134,
+      "?": 135,
+      "@": 136,
+      "A": 137,
+      "B": 138,
+      "C": 139,
+      "D": 140,
+      "E": 141,
+      "F": 142,
+      "G": 143,
+      "H": 144,
+      "I": 145,
+      "J": 146,
+      "K": 147,
+      "L": 148,
+      "M": 149,
+      "N": 150,
+      "O": 151,
+      "P": 152,
+      "Q": 153,
+      "R": 154,
+      "S": 155,
+      "T": 156,
+      "U": 157,
+      "V": 158,
+      "W": 159,
+      "X": 160,
+      "Y": 161,
+      "Z": 162,
+      "[": 163,
+      "\\": 164,
+      "]": 165,
+      "^": 166,
+      "_": 167,
+      "a": 168,
+      "b": 169,
+      "c": 170,
+      "d": 171,
+      "e": 172,
+      "f": 173,
+      "g": 174,
+      "h": 175,
+      "i": 176,
+      "j": 177,
+      "k": 178,
+      "l": 179,
+      "m": 180,
+      "n": 181,
+      "o": 182,
+      "p": 183,
+      "q": 184,
+      "r": 185,
+      "s": 186,
+      "t": 187,
+      "u": 188,
+      "v": 189,
+      "w": 190,
+      "x": 191,
+      "y": 192,
+      "z": 193,
+      "{": 194,
+      "|": 195,
+      "}": 196,
+      "~": 197,
+      "«": 198,
+      "°": 199,
+      "·": 200,
+      "»": 201,
+      "é": 202,
+      "а": 203,
+      "в": 204,
+      "д": 205,
+      "е": 206,
+      "и": 207,
+      "к": 208,
+      "л": 209,
+      "м": 210,
+      "н": 211,
+      "о": 212,
+      "п": 213,
+      "р": 214,
+      "с": 215,
+      "т": 216,
+      "،": 217,
+      "؛": 218,
+      "؟": 219,
+      "ء": 220,
+      "آ": 221,
+      "أ": 222,
+      "ؤ": 223,
+      "إ": 224,
+      "ئ": 225,
+      "ا": 226,
+      "ب": 227,
+      "ة": 228,
+      "ت": 229,
+      "ث": 230,
+      "ج": 231,
+      "ح": 232,
+      "خ": 233,
+      "د": 234,
+      "ذ": 235,
+      "ر": 236,
+      "ز": 237,
+      "س": 238,
+      "ش": 239,
+      "ص": 240,
+      "ض": 241,
+      "ط": 242,
+      "ظ": 243,
+      "ع": 244,
+      "غ": 245,
+      "ـ": 246,
+      "ف": 247,
+      "ق": 248,
+      "ك": 249,
+      "ل": 250,
+      "م": 251,
+      "ن": 252,
+      "ه": 253,
+      "و": 254,
+      "ى": 255,
+      "ي": 256,
+      "ً": 257,
+      "ٌ": 258,
+      "ٍ": 259,
+      "َ": 260,
+      "ُ": 261,
+      "ِ": 262,
+      "ّ": 263,
+      "ْ": 264,
+      "٠": 265,
+      "١": 266,
+      "٢": 267,
+      "٣": 268,
+      "٤": 269,
+      "٥": 270,
+      "٦": 271,
+      "٧": 272,
+      "٨": 273,
+      "٩": 274,
+      "٪": 275,
+      "پ": 276,
+      "ک": 277,
+      "گ": 278,
+      "ھ": 279,
+      "ی": 280,
+      "​": 281,
+      "‌": 282,
+      "‎": 283,
+      "‏": 284,
+      "–": 285,
+      "—": 286,
+      "‘": 287,
+      "’": 288,
+      "“": 289,
+      "”": 290,
+      "•": 291,
+      "…": 292,
+      "‪": 293,
+      "‫": 294,
+      "‬": 295,
+      "‭": 296,
+      "‮": 297,
+      "﴾": 298,
+      "﴿": 299,
+      "�": 300,
+      "[MASK]": 301
+    },
+    "unk_token": "[UNK]"
+  }
+}

tokenizer_config.json

@@ -1,46 +1,52 @@
 {
-  "add_prefix_space": false,
   "added_tokens_decoder": {
     "0": {
-      "content": "\u0000",
+      "content": "[CLS]",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "57344": {
-      "content": "",
+    "1": {
+      "content": "[PAD]",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "57345": {
-      "content": "",
+    "2": {
+      "content": "[SEP]",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "57347": {
-      "content": "",
-      "lstrip": true,
-      "normalized": true,
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "301": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
-  "bos_token": "",
   "clean_up_tokenization_spaces": true,
-  "cls_token": "",
-  "eos_token": "",
-  "mask_token": "",
-  "model_max_length": 2048,
-  "pad_token": "\u0000",
-  "sep_token": "",
-  "tokenizer_class": "CanineTokenizer"
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
 }