KucLab commited on
Commit
9137794
·
verified ·
1 Parent(s): 3022d81

Add report comparison.json

Browse files
Files changed (1) hide show
  1. reports/comparison.json +235 -0
reports/comparison.json ADDED
@@ -0,0 +1,235 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "device": "cuda",
3
+ "gpu": "AMD Radeon Graphics",
4
+ "n_test": 6496,
5
+ "models": [
6
+ {
7
+ "repo": "KucLab/Censer.9 (this work)",
8
+ "params": 903552,
9
+ "checkpoint_mb": 3.62,
10
+ "latency_ms": 1.3860261449008249,
11
+ "throughput_s": 1859.7414693672952,
12
+ "vram_mb": 105.291776,
13
+ "metrics": {
14
+ "accuracy": 0.8577586206896551,
15
+ "precision": 0.8508461235733963,
16
+ "recall": 0.7986701145179165,
17
+ "f1": 0.8239329268292683,
18
+ "tp": 2162,
19
+ "fp": 379,
20
+ "tn": 3410,
21
+ "fn": 545,
22
+ "total": 6496
23
+ }
24
+ },
25
+ {
26
+ "repo": "KucLab/Censer.5 (prev)",
27
+ "params": 902272,
28
+ "checkpoint_mb": 3.62,
29
+ "latency_ms": 1.353798654745333,
30
+ "throughput_s": 1827.5500336939183,
31
+ "vram_mb": 96.704,
32
+ "metrics": {
33
+ "accuracy": 0.8763854679802956,
34
+ "precision": 0.8644716692189893,
35
+ "recall": 0.8341337273734761,
36
+ "f1": 0.8490317728896409,
37
+ "tp": 2258,
38
+ "fp": 354,
39
+ "tn": 3435,
40
+ "fn": 449,
41
+ "total": 6496
42
+ }
43
+ },
44
+ {
45
+ "repo": "KucLab/Censer.4 (old)",
46
+ "params": 902272,
47
+ "checkpoint_mb": 3.62,
48
+ "latency_ms": 1.3921621147892438,
49
+ "throughput_s": 1708.3037086245356,
50
+ "vram_mb": 95.030784,
51
+ "metrics": {
52
+ "accuracy": 0.874384236453202,
53
+ "precision": 0.868042039704165,
54
+ "recall": 0.8237901736239379,
55
+ "f1": 0.8453373768006066,
56
+ "tp": 2230,
57
+ "fp": 339,
58
+ "tn": 3450,
59
+ "fn": 477,
60
+ "total": 6496
61
+ }
62
+ },
63
+ {
64
+ "repo": "KucLab/Censer.3 (old)",
65
+ "params": 14424000,
66
+ "checkpoint_mb": 57.71,
67
+ "latency_ms": 4.360620900115464,
68
+ "throughput_s": 269.7258749949498,
69
+ "vram_mb": 239.994368,
70
+ "metrics": {
71
+ "accuracy": 0.8574507389162561,
72
+ "precision": 0.8552054248105305,
73
+ "recall": 0.7920206871074991,
74
+ "f1": 0.8224012274645186,
75
+ "tp": 2144,
76
+ "fp": 363,
77
+ "tn": 3426,
78
+ "fn": 563,
79
+ "total": 6496
80
+ }
81
+ },
82
+ {
83
+ "repo": "KucLab/Censer.2 (old)",
84
+ "params": 7343872,
85
+ "checkpoint_mb": 29.39,
86
+ "latency_ms": 2.769896234967746,
87
+ "throughput_s": 551.4543971045085,
88
+ "vram_mb": 170.506752,
89
+ "metrics": {
90
+ "accuracy": 0.8625307881773399,
91
+ "precision": 0.8610668789808917,
92
+ "recall": 0.7990395271518286,
93
+ "f1": 0.8288944242191991,
94
+ "tp": 2163,
95
+ "fp": 349,
96
+ "tn": 3440,
97
+ "fn": 544,
98
+ "total": 6496
99
+ }
100
+ },
101
+ {
102
+ "repo": "KucLab/Censer.1 (baseline)",
103
+ "params": 1115520,
104
+ "checkpoint_mb": 4.47,
105
+ "latency_ms": 1.6824865649687126,
106
+ "throughput_s": 1697.0033244680535,
107
+ "vram_mb": 96.736768,
108
+ "metrics": {
109
+ "accuracy": 0.8648399014778325,
110
+ "precision": 0.8478508938759984,
111
+ "recall": 0.8234207609900258,
112
+ "f1": 0.8354572713643178,
113
+ "tp": 2229,
114
+ "fp": 400,
115
+ "tn": 3389,
116
+ "fn": 478,
117
+ "total": 6496
118
+ }
119
+ },
120
+ {
121
+ "repo": "keyword baseline",
122
+ "params": 0,
123
+ "checkpoint_mb": 0.0,
124
+ "latency_ms": 0.03355584529237397,
125
+ "throughput_s": 29801.06718477641,
126
+ "vram_mb": 0.0,
127
+ "metrics": {
128
+ "accuracy": 0.5681958128078818,
129
+ "precision": 0.37371134020618557,
130
+ "recall": 0.05356483191725157,
131
+ "f1": 0.09369951534733441,
132
+ "tp": 145,
133
+ "fp": 243,
134
+ "tn": 3546,
135
+ "fn": 2562,
136
+ "total": 6496
137
+ }
138
+ },
139
+ {
140
+ "repo": "martin-ha/toxic-comment-model",
141
+ "params": 66955010,
142
+ "checkpoint_mb": 0.0,
143
+ "latency_ms": 7.814959404931869,
144
+ "throughput_s": 129.29452614581362,
145
+ "vram_mb": 539.696128,
146
+ "metrics": {
147
+ "accuracy": 0.7438423645320197,
148
+ "precision": 0.9610963748894783,
149
+ "recall": 0.40155153306243074,
150
+ "f1": 0.566440854611777,
151
+ "tp": 1087,
152
+ "fp": 44,
153
+ "tn": 3745,
154
+ "fn": 1620,
155
+ "total": 6496
156
+ }
157
+ },
158
+ {
159
+ "repo": "unitary/toxic-bert",
160
+ "params": 109486854,
161
+ "checkpoint_mb": 0.0,
162
+ "latency_ms": 14.079030790016986,
163
+ "throughput_s": 67.56437138466154,
164
+ "vram_mb": 708.03712,
165
+ "metrics": {
166
+ "accuracy": 0.7944889162561576,
167
+ "precision": 0.9654002713704206,
168
+ "recall": 0.5256741780568895,
169
+ "f1": 0.6806983975125568,
170
+ "tp": 1423,
171
+ "fp": 51,
172
+ "tn": 3738,
173
+ "fn": 1284,
174
+ "total": 6496
175
+ }
176
+ },
177
+ {
178
+ "repo": "unitary/unbiased-toxic-roberta",
179
+ "params": 124657936,
180
+ "checkpoint_mb": 0.0,
181
+ "latency_ms": 14.846192635013722,
182
+ "throughput_s": 65.28570630963951,
183
+ "vram_mb": 770.71872,
184
+ "metrics": {
185
+ "accuracy": 0.8218903940886699,
186
+ "precision": 0.9398410896708286,
187
+ "recall": 0.6117473217584042,
188
+ "f1": 0.7411053927053033,
189
+ "tp": 1656,
190
+ "fp": 106,
191
+ "tn": 3683,
192
+ "fn": 1051,
193
+ "total": 6496
194
+ }
195
+ },
196
+ {
197
+ "repo": "Hate-speech-CNERG/dehatebert-mono-english",
198
+ "params": 167357954,
199
+ "checkpoint_mb": 0.0,
200
+ "latency_ms": 14.497019650007132,
201
+ "throughput_s": 66.21927777552658,
202
+ "vram_mb": 923.411968,
203
+ "metrics": {
204
+ "accuracy": 0.635929802955665,
205
+ "precision": 0.875,
206
+ "recall": 0.14739564093091984,
207
+ "f1": 0.2522921277268416,
208
+ "tp": 399,
209
+ "fp": 57,
210
+ "tn": 3732,
211
+ "fn": 2308,
212
+ "total": 6496
213
+ }
214
+ },
215
+ {
216
+ "repo": "Hate-speech-CNERG/bert-base-uncased-hatexplain",
217
+ "params": 109484547,
218
+ "checkpoint_mb": 0.0,
219
+ "latency_ms": 14.151658390183002,
220
+ "throughput_s": 67.32848692360528,
221
+ "vram_mb": 691.767296,
222
+ "metrics": {
223
+ "accuracy": 0.7039716748768473,
224
+ "precision": 0.7606382978723404,
225
+ "recall": 0.4226080531954193,
226
+ "f1": 0.5433388743766325,
227
+ "tp": 1144,
228
+ "fp": 360,
229
+ "tn": 3429,
230
+ "fn": 1563,
231
+ "total": 6496
232
+ }
233
+ }
234
+ ]
235
+ }