Hugging Face's logo

O96a
/
sudanizer-72k

Transformers
Arabic
sudanese
arabic
tokenizer
bpe
dialect
sudaverse
Model card Files
xet
 Community
sudanizer-72k
File size: 1,799 Bytes
075c3b9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
{
  "backend": "tokenizers",
  "bos_token": "<s>",
  "clean_up_tokenization_spaces": true,
  "eos_token": "</s>",
  "extra_special_tokens": [
    "<user>",
    "<assistant>",
    "<system>",
    "<sudan>",
    "<url>",
    "<mention>",
    "<hashtag>",
    "<elongation>",
    "<emoji_purple_heart>",
    "<emoji_sleep>",
    "<emoji_money>",
    "<emoji_confetti>",
    "<emoji_trophy>",
    "<emoji_heart>",
    "<emoji_broken_heart>",
    "<emoji_angry>",
    "<emoji_soccer>",
    "<emoji_camera>",
    "<emoji_arrow_right>",
    "<emoji_hearts>",
    "<emoji_tea>",
    "<emoji_green_circle>",
    "<emoji_ok>",
    "<emoji_sparkles>",
    "<emoji_party>",
    "<emoji_star>",
    "<emoji_green_heart>",
    "<emoji_notes>",
    "<emoji_joy>",
    "<emoji_sad>",
    "<emoji_wow>",
    "<emoji_rofl>",
    "<emoji_sweat>",
    "<emoji_check>",
    "<emoji_100>",
    "<emoji_love>",
    "<emoji_mask>",
    "<emoji_thumbsdown>",
    "<emoji_bell>",
    "<emoji_blue_heart>",
    "<emoji_rage>",
    "<emoji_grin>",
    "<emoji_black_heart>",
    "<emoji_sun>",
    "<emoji_white_heart>",
    "<emoji_cross>",
    "<emoji_rainbow>",
    "<emoji_red_circle>",
    "<emoji_arrow_left>",
    "<emoji_shock>",
    "<emoji_music>",
    "<emoji_phone>",
    "<emoji_star2>",
    "<emoji_neutral>",
    "<emoji_coffee>",
    "<emoji_fire>",
    "<emoji_clapper>",
    "<emoji_moon>",
    "<emoji_clap>",
    "<emoji_rose>",
    "<emoji_gift_heart>",
    "<emoji_mic>",
    "<emoji_pray>",
    "<emoji_yellow_heart>",
    "<emoji_computer>",
    "<emoji_think>",
    "<emoji_cry>",
    "<emoji_thumbsup>",
    "<emoji_smile>"
  ],
  "mask_token": "<mask>",
  "model_max_length": 1000000000000000019884624838656,
  "pad_token": "<pad>",
  "tokenizer_class": "TokenizersBackend",
  "unk_token": "<unk>"
}