Hugging Face's logo

almaghrabima
/
SARFTokenizer

Text Generation
Arabic
English
tokenizer
arabic
morphology
bpe
deeplatent
english
Model card Files
xet
Community
almaghrabima commited on
Commit
1e8911f
·
verified ·
1 Parent(s): 57f81bb

Upload test_comprehensive_results.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. test_comprehensive_results.json +128 -0
test_comprehensive_results.json ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "0.3.8",
3
+ "rust_available": true,
4
+ "tokenizer": "almaghrabima/SARFTokenizer",
5
+ "samples": 1000000,
6
+ "roundtrip": {
7
+ "Arabic": {
8
+ "category": "Arabic",
9
+ "total": 333333,
10
+ "success": 333333,
11
+ "failed": 0,
12
+ "accuracy": 1.0,
13
+ "accuracy_pct": "100.00%",
14
+ "encode_time": 128.8216183092445,
15
+ "decode_time": 23.831407640129328,
16
+ "failures": []
17
+ },
18
+ "English": {
19
+ "category": "English",
20
+ "total": 333333,
21
+ "success": 333333,
22
+ "failed": 0,
23
+ "accuracy": 1.0,
24
+ "accuracy_pct": "100.00%",
25
+ "encode_time": 19.489008927717805,
26
+ "decode_time": 4.273356601595879,
27
+ "failures": []
28
+ },
29
+ "Mixed": {
30
+ "category": "Mixed",
31
+ "total": 333333,
32
+ "success": 333333,
33
+ "failed": 0,
34
+ "accuracy": 1.0,
35
+ "accuracy_pct": "100.00%",
36
+ "encode_time": 70.115827139467,
37
+ "decode_time": 13.394427740946412,
38
+ "failures": []
39
+ },
40
+ "TOTAL": {
41
+ "category": "TOTAL",
42
+ "total": 999999,
43
+ "success": 999999,
44
+ "failed": 0,
45
+ "accuracy": 1.0,
46
+ "accuracy_pct": "100.00%"
47
+ }
48
+ },
49
+ "edge_cases": {
50
+ "Unicode Normalization": {
51
+ "tests": 6,
52
+ "passed": 6,
53
+ "failed": 0,
54
+ "failures": []
55
+ },
56
+ "Zero-Width Characters": {
57
+ "tests": 6,
58
+ "passed": 6,
59
+ "failed": 0,
60
+ "failures": []
61
+ },
62
+ "Unicode Whitespace": {
63
+ "tests": 6,
64
+ "passed": 6,
65
+ "failed": 0,
66
+ "failures": []
67
+ },
68
+ "Grapheme Clusters": {
69
+ "tests": 6,
70
+ "passed": 6,
71
+ "failed": 0,
72
+ "failures": []
73
+ },
74
+ "Apostrophes": {
75
+ "tests": 4,
76
+ "passed": 4,
77
+ "failed": 0,
78
+ "failures": []
79
+ },
80
+ "Dashes": {
81
+ "tests": 4,
82
+ "passed": 4,
83
+ "failed": 0,
84
+ "failures": []
85
+ },
86
+ "Decimal Separators": {
87
+ "tests": 3,
88
+ "passed": 3,
89
+ "failed": 0,
90
+ "failures": []
91
+ },
92
+ "URLs/Emails": {
93
+ "tests": 4,
94
+ "passed": 4,
95
+ "failed": 0,
96
+ "failures": []
97
+ },
98
+ "File Paths": {
99
+ "tests": 3,
100
+ "passed": 3,
101
+ "failed": 0,
102
+ "failures": []
103
+ },
104
+ "Code Identifiers": {
105
+ "tests": 4,
106
+ "passed": 4,
107
+ "failed": 0,
108
+ "failures": []
109
+ },
110
+ "Mixed Scripts/RTL": {
111
+ "tests": 6,
112
+ "passed": 6,
113
+ "failed": 0,
114
+ "failures": []
115
+ },
116
+ "Robustness": {
117
+ "tests": 6,
118
+ "passed": 6,
119
+ "failed": 0,
120
+ "failures": []
121
+ },
122
+ "TOTAL": {
123
+ "tests": 58,
124
+ "passed": 58,
125
+ "failed": 0
126
+ }
127
+ }
128
+ }