Reihaneh commited on
Commit
608b11e
·
verified ·
1 Parent(s): 664c57e

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +100 -100
vocab.json CHANGED
@@ -1,104 +1,104 @@
1
  {
2
- "'": 53,
3
- "R": 63,
4
- "U": 50,
5
- "[": 78,
6
  "[PAD]": 101,
7
  "[UNK]": 100,
8
- "]": 83,
9
- "_": 58,
10
- "`": 39,
11
- "a": 89,
12
- "b": 66,
13
- "d": 9,
14
- "e": 13,
15
- "f": 26,
16
- "h": 85,
17
- "i": 45,
18
- "k": 96,
19
- "m": 49,
20
- "r": 71,
21
- "t": 90,
22
- "u": 21,
23
- "w": 31,
24
- "|": 17,
25
- "،": 56,
26
- "ؔ": 79,
27
- "؛": 23,
28
- "؟": 54,
29
- "ء": 81,
30
- "آ": 92,
31
- "أ": 76,
32
- "ؤ": 8,
33
- "ئ": 73,
34
- "ا": 72,
35
- "ب": 69,
36
- "ت": 0,
37
- "ث": 3,
38
- "ج": 2,
39
- "ح": 52,
40
- "خ": 42,
41
- "د": 20,
42
- "ذ": 34,
43
- "ر": 22,
44
- "ز": 35,
45
- "س": 67,
46
- "ش": 65,
47
- "ص": 47,
48
- "ض": 84,
49
- "ط": 29,
50
- "ظ": 70,
51
- "ع": 27,
52
- "غ": 87,
53
- "ف": 33,
54
- "ق": 86,
55
- "ل": 55,
56
- "م": 46,
57
- "ن": 97,
58
- "ه": 18,
59
- "و": 48,
60
- "ي": 93,
61
- "ً": 28,
62
- "َ": 82,
63
- "ُ": 6,
64
- "ِ": 88,
65
- "ّ": 36,
66
- "ْ": 25,
67
- "ٓ": 98,
68
- "ٔ": 68,
69
- "ٗ": 60,
70
- "ٰ": 10,
71
- "ٴ": 95,
72
- "ٹ": 12,
73
- "پ": 44,
74
- "چ": 61,
75
- "ڈ": 5,
76
- "ڑ": 1,
77
- "ژ": 16,
78
- "ک": 43,
79
- "گ": 74,
80
- "ں": 30,
81
- "ھ": 64,
82
- "ہ": 4,
83
- "ۂ": 37,
84
- "ۃ": 91,
85
- "ی": 94,
86
- "ے": 59,
87
- "ۓ": 62,
88
- "۔": 19,
89
- "‘": 32,
90
- "’": 77,
91
- "ﭨ": 11,
92
- "ﮭ": 75,
93
- "ﮯ": 15,
94
- "ﯾ": 7,
95
- "ﷲ": 40,
96
- "ﷺ": 57,
97
- "ﺅ": 41,
98
- "ﺗ": 80,
99
- "ﺘ": 38,
100
- "ﺩ": 99,
101
- "ﺲ": 14,
102
- "ﻧ": 51,
103
- "ﻮ": 24
104
  }
 
1
  {
2
+ "'": 60,
3
+ "R": 55,
4
+ "U": 10,
5
+ "[": 80,
6
  "[PAD]": 101,
7
  "[UNK]": 100,
8
+ "]": 88,
9
+ "_": 68,
10
+ "`": 76,
11
+ "a": 40,
12
+ "b": 49,
13
+ "d": 78,
14
+ "e": 69,
15
+ "f": 20,
16
+ "h": 73,
17
+ "i": 97,
18
+ "k": 8,
19
+ "m": 71,
20
+ "r": 7,
21
+ "t": 14,
22
+ "u": 65,
23
+ "w": 2,
24
+ "|": 67,
25
+ "،": 87,
26
+ "ؔ": 75,
27
+ "؛": 47,
28
+ "؟": 61,
29
+ "ء": 12,
30
+ "آ": 98,
31
+ "أ": 39,
32
+ "ؤ": 21,
33
+ "ئ": 46,
34
+ "ا": 66,
35
+ "ب": 9,
36
+ "ت": 59,
37
+ "ث": 99,
38
+ "ج": 42,
39
+ "ح": 81,
40
+ "خ": 25,
41
+ "د": 33,
42
+ "ذ": 26,
43
+ "ر": 36,
44
+ "ز": 58,
45
+ "س": 86,
46
+ "ش": 32,
47
+ "ص": 11,
48
+ "ض": 17,
49
+ "ط": 24,
50
+ "ظ": 3,
51
+ "ع": 83,
52
+ "غ": 15,
53
+ "ف": 56,
54
+ "ق": 19,
55
+ "ل": 41,
56
+ "م": 57,
57
+ "ن": 27,
58
+ "ه": 5,
59
+ "و": 70,
60
+ "ي": 74,
61
+ "ً": 50,
62
+ "َ": 4,
63
+ "ُ": 44,
64
+ "ِ": 1,
65
+ "ّ": 45,
66
+ "ْ": 23,
67
+ "ٓ": 0,
68
+ "ٔ": 77,
69
+ "ٗ": 29,
70
+ "ٰ": 95,
71
+ "ٴ": 16,
72
+ "ٹ": 37,
73
+ "پ": 22,
74
+ "چ": 91,
75
+ "ڈ": 90,
76
+ "ڑ": 84,
77
+ "ژ": 48,
78
+ "ک": 89,
79
+ "گ": 34,
80
+ "ں": 35,
81
+ "ھ": 53,
82
+ "ہ": 13,
83
+ "ۂ": 51,
84
+ "ۃ": 82,
85
+ "ی": 62,
86
+ "ے": 94,
87
+ "ۓ": 30,
88
+ "۔": 18,
89
+ "‘": 96,
90
+ "’": 92,
91
+ "ﭨ": 6,
92
+ "ﮭ": 72,
93
+ "ﮯ": 93,
94
+ "ﯾ": 31,
95
+ "ﷲ": 79,
96
+ "ﷺ": 38,
97
+ "ﺅ": 54,
98
+ "ﺗ": 64,
99
+ "ﺘ": 85,
100
+ "ﺩ": 63,
101
+ "ﺲ": 52,
102
+ "ﻧ": 43,
103
+ "ﻮ": 28
104
  }