chanind commited on
Commit
0bbe1aa
·
verified ·
1 Parent(s): eec17ca

Upload sparse_probing/gemma-2-2b-res-snap-matryoshka-dc_blocks.15.hook_resid_post_eval_results.json with huggingface_hub

Browse files
sparse_probing/gemma-2-2b-res-snap-matryoshka-dc_blocks.15.hook_resid_post_eval_results.json ADDED
@@ -0,0 +1,675 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eval_type_id": "sparse_probing",
3
+ "eval_config": {
4
+ "random_seed": 42,
5
+ "dataset_names": [
6
+ "LabHC/bias_in_bios_class_set1",
7
+ "LabHC/bias_in_bios_class_set2",
8
+ "LabHC/bias_in_bios_class_set3",
9
+ "canrager/amazon_reviews_mcauley_1and5",
10
+ "canrager/amazon_reviews_mcauley_1and5_sentiment",
11
+ "codeparrot/github-code",
12
+ "fancyzhx/ag_news",
13
+ "Helsinki-NLP/europarl"
14
+ ],
15
+ "probe_train_set_size": 4000,
16
+ "probe_test_set_size": 1000,
17
+ "context_length": 128,
18
+ "sae_batch_size": 125,
19
+ "llm_batch_size": 32,
20
+ "llm_dtype": "bfloat16",
21
+ "model_name": "gemma-2-2b",
22
+ "k_values": [
23
+ 1,
24
+ 2,
25
+ 5
26
+ ],
27
+ "lower_vram_usage": false
28
+ },
29
+ "eval_id": "acf88c24-09bd-4561-9b58-1bd96edb30c6",
30
+ "datetime_epoch_millis": 1745627243425,
31
+ "eval_result_metrics": {
32
+ "llm": {
33
+ "llm_test_accuracy": 0.956756292283535,
34
+ "llm_top_1_test_accuracy": 0.66578125,
35
+ "llm_top_2_test_accuracy": 0.739075,
36
+ "llm_top_5_test_accuracy": 0.7944062500000001,
37
+ "llm_top_10_test_accuracy": null,
38
+ "llm_top_20_test_accuracy": null,
39
+ "llm_top_50_test_accuracy": null,
40
+ "llm_top_100_test_accuracy": null
41
+ },
42
+ "sae": {
43
+ "sae_test_accuracy": 0.9545562915503979,
44
+ "sae_top_1_test_accuracy": 0.7364812500000001,
45
+ "sae_top_2_test_accuracy": 0.80128125,
46
+ "sae_top_5_test_accuracy": 0.8687750000000002,
47
+ "sae_top_10_test_accuracy": null,
48
+ "sae_top_20_test_accuracy": null,
49
+ "sae_top_50_test_accuracy": null,
50
+ "sae_top_100_test_accuracy": null
51
+ }
52
+ },
53
+ "eval_result_details": [
54
+ {
55
+ "dataset_name": "LabHC/bias_in_bios_class_set1_results",
56
+ "llm_test_accuracy": 0.9646000385284423,
57
+ "llm_top_1_test_accuracy": 0.6682,
58
+ "llm_top_2_test_accuracy": 0.7158,
59
+ "llm_top_5_test_accuracy": 0.7906000000000001,
60
+ "llm_top_10_test_accuracy": null,
61
+ "llm_top_20_test_accuracy": null,
62
+ "llm_top_50_test_accuracy": null,
63
+ "llm_top_100_test_accuracy": null,
64
+ "sae_test_accuracy": 0.9640000343322754,
65
+ "sae_top_1_test_accuracy": 0.7022,
66
+ "sae_top_2_test_accuracy": 0.7998000000000001,
67
+ "sae_top_5_test_accuracy": 0.885,
68
+ "sae_top_10_test_accuracy": null,
69
+ "sae_top_20_test_accuracy": null,
70
+ "sae_top_50_test_accuracy": null,
71
+ "sae_top_100_test_accuracy": null
72
+ },
73
+ {
74
+ "dataset_name": "LabHC/bias_in_bios_class_set2_results",
75
+ "llm_test_accuracy": 0.9532000660896301,
76
+ "llm_top_1_test_accuracy": 0.6594,
77
+ "llm_top_2_test_accuracy": 0.7108,
78
+ "llm_top_5_test_accuracy": 0.7686,
79
+ "llm_top_10_test_accuracy": null,
80
+ "llm_top_20_test_accuracy": null,
81
+ "llm_top_50_test_accuracy": null,
82
+ "llm_top_100_test_accuracy": null,
83
+ "sae_test_accuracy": 0.9434000372886657,
84
+ "sae_top_1_test_accuracy": 0.7260000000000001,
85
+ "sae_top_2_test_accuracy": 0.7466000000000002,
86
+ "sae_top_5_test_accuracy": 0.8334000000000001,
87
+ "sae_top_10_test_accuracy": null,
88
+ "sae_top_20_test_accuracy": null,
89
+ "sae_top_50_test_accuracy": null,
90
+ "sae_top_100_test_accuracy": null
91
+ },
92
+ {
93
+ "dataset_name": "LabHC/bias_in_bios_class_set3_results",
94
+ "llm_test_accuracy": 0.9242000460624695,
95
+ "llm_top_1_test_accuracy": 0.6918000000000001,
96
+ "llm_top_2_test_accuracy": 0.752,
97
+ "llm_top_5_test_accuracy": 0.7882,
98
+ "llm_top_10_test_accuracy": null,
99
+ "llm_top_20_test_accuracy": null,
100
+ "llm_top_50_test_accuracy": null,
101
+ "llm_top_100_test_accuracy": null,
102
+ "sae_test_accuracy": 0.9266000509262085,
103
+ "sae_top_1_test_accuracy": 0.6592,
104
+ "sae_top_2_test_accuracy": 0.7504,
105
+ "sae_top_5_test_accuracy": 0.7984000000000001,
106
+ "sae_top_10_test_accuracy": null,
107
+ "sae_top_20_test_accuracy": null,
108
+ "sae_top_50_test_accuracy": null,
109
+ "sae_top_100_test_accuracy": null
110
+ },
111
+ {
112
+ "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
113
+ "llm_test_accuracy": 0.9196000576019288,
114
+ "llm_top_1_test_accuracy": 0.5908,
115
+ "llm_top_2_test_accuracy": 0.6436,
116
+ "llm_top_5_test_accuracy": 0.6726000000000001,
117
+ "llm_top_10_test_accuracy": null,
118
+ "llm_top_20_test_accuracy": null,
119
+ "llm_top_50_test_accuracy": null,
120
+ "llm_top_100_test_accuracy": null,
121
+ "sae_test_accuracy": 0.9088000535964966,
122
+ "sae_top_1_test_accuracy": 0.6891999999999999,
123
+ "sae_top_2_test_accuracy": 0.7234,
124
+ "sae_top_5_test_accuracy": 0.7516,
125
+ "sae_top_10_test_accuracy": null,
126
+ "sae_top_20_test_accuracy": null,
127
+ "sae_top_50_test_accuracy": null,
128
+ "sae_top_100_test_accuracy": null
129
+ },
130
+ {
131
+ "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
132
+ "llm_test_accuracy": 0.9750000238418579,
133
+ "llm_top_1_test_accuracy": 0.692,
134
+ "llm_top_2_test_accuracy": 0.744,
135
+ "llm_top_5_test_accuracy": 0.798,
136
+ "llm_top_10_test_accuracy": null,
137
+ "llm_top_20_test_accuracy": null,
138
+ "llm_top_50_test_accuracy": null,
139
+ "llm_top_100_test_accuracy": null,
140
+ "sae_test_accuracy": 0.9725000262260437,
141
+ "sae_top_1_test_accuracy": 0.777,
142
+ "sae_top_2_test_accuracy": 0.813,
143
+ "sae_top_5_test_accuracy": 0.884,
144
+ "sae_top_10_test_accuracy": null,
145
+ "sae_top_20_test_accuracy": null,
146
+ "sae_top_50_test_accuracy": null,
147
+ "sae_top_100_test_accuracy": null
148
+ },
149
+ {
150
+ "dataset_name": "codeparrot/github-code_results",
151
+ "llm_test_accuracy": 0.9732000589370727,
152
+ "llm_top_1_test_accuracy": 0.6355999999999999,
153
+ "llm_top_2_test_accuracy": 0.7094000000000001,
154
+ "llm_top_5_test_accuracy": 0.7702,
155
+ "llm_top_10_test_accuracy": null,
156
+ "llm_top_20_test_accuracy": null,
157
+ "llm_top_50_test_accuracy": null,
158
+ "llm_top_100_test_accuracy": null,
159
+ "sae_test_accuracy": 0.9732000470161438,
160
+ "sae_top_1_test_accuracy": 0.6908000000000001,
161
+ "sae_top_2_test_accuracy": 0.8030000000000002,
162
+ "sae_top_5_test_accuracy": 0.9282,
163
+ "sae_top_10_test_accuracy": null,
164
+ "sae_top_20_test_accuracy": null,
165
+ "sae_top_50_test_accuracy": null,
166
+ "sae_top_100_test_accuracy": null
167
+ },
168
+ {
169
+ "dataset_name": "fancyzhx/ag_news_results",
170
+ "llm_test_accuracy": 0.9442500472068787,
171
+ "llm_top_1_test_accuracy": 0.65025,
172
+ "llm_top_2_test_accuracy": 0.749,
173
+ "llm_top_5_test_accuracy": 0.8342499999999999,
174
+ "llm_top_10_test_accuracy": null,
175
+ "llm_top_20_test_accuracy": null,
176
+ "llm_top_50_test_accuracy": null,
177
+ "llm_top_100_test_accuracy": null,
178
+ "sae_test_accuracy": 0.9497500360012054,
179
+ "sae_top_1_test_accuracy": 0.68925,
180
+ "sae_top_2_test_accuracy": 0.79725,
181
+ "sae_top_5_test_accuracy": 0.879,
182
+ "sae_top_10_test_accuracy": null,
183
+ "sae_top_20_test_accuracy": null,
184
+ "sae_top_50_test_accuracy": null,
185
+ "sae_top_100_test_accuracy": null
186
+ },
187
+ {
188
+ "dataset_name": "Helsinki-NLP/europarl_results",
189
+ "llm_test_accuracy": 1.0,
190
+ "llm_top_1_test_accuracy": 0.7382000000000001,
191
+ "llm_top_2_test_accuracy": 0.8879999999999999,
192
+ "llm_top_5_test_accuracy": 0.9328,
193
+ "llm_top_10_test_accuracy": null,
194
+ "llm_top_20_test_accuracy": null,
195
+ "llm_top_50_test_accuracy": null,
196
+ "llm_top_100_test_accuracy": null,
197
+ "sae_test_accuracy": 0.9982000470161438,
198
+ "sae_top_1_test_accuracy": 0.9581999999999999,
199
+ "sae_top_2_test_accuracy": 0.9768000000000001,
200
+ "sae_top_5_test_accuracy": 0.9905999999999999,
201
+ "sae_top_10_test_accuracy": null,
202
+ "sae_top_20_test_accuracy": null,
203
+ "sae_top_50_test_accuracy": null,
204
+ "sae_top_100_test_accuracy": null
205
+ }
206
+ ],
207
+ "sae_bench_commit_hash": "Unknown",
208
+ "sae_lens_id": "blocks.15.hook_resid_post",
209
+ "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc",
210
+ "sae_lens_version": "5.9.1",
211
+ "sae_cfg_dict": {
212
+ "architecture": "jumprelu",
213
+ "d_in": 2304,
214
+ "d_sae": 32768,
215
+ "activation_fn_str": "relu",
216
+ "apply_b_dec_to_input": true,
217
+ "finetuning_scaling_factor": false,
218
+ "context_size": 1024,
219
+ "model_name": "gemma-2-2b",
220
+ "hook_name": "blocks.15.hook_resid_post",
221
+ "hook_layer": 15,
222
+ "hook_head_index": null,
223
+ "prepend_bos": true,
224
+ "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B",
225
+ "dataset_trust_remote_code": true,
226
+ "normalize_activations": "none",
227
+ "dtype": "torch.bfloat16",
228
+ "device": "cuda",
229
+ "sae_lens_training_version": "5.5.1",
230
+ "activation_fn_kwargs": {
231
+ "k": 40
232
+ },
233
+ "neuronpedia_id": null,
234
+ "model_from_pretrained_kwargs": {
235
+ "center_writing_weights": false
236
+ },
237
+ "seqpos_slice": [
238
+ null
239
+ ]
240
+ },
241
+ "eval_result_unstructured": {
242
+ "LabHC/bias_in_bios_class_set1_results": {
243
+ "sae_test_accuracy": {
244
+ "0": 0.9520000219345093,
245
+ "1": 0.9660000205039978,
246
+ "2": 0.9500000476837158,
247
+ "6": 0.9820000529289246,
248
+ "9": 0.9700000286102295
249
+ },
250
+ "llm_test_accuracy": {
251
+ "0": 0.9520000219345093,
252
+ "1": 0.9640000462532043,
253
+ "2": 0.9510000348091125,
254
+ "6": 0.987000048160553,
255
+ "9": 0.9690000414848328
256
+ },
257
+ "llm_top_1_test_accuracy": {
258
+ "0": 0.592,
259
+ "1": 0.617,
260
+ "2": 0.661,
261
+ "6": 0.761,
262
+ "9": 0.71
263
+ },
264
+ "llm_top_2_test_accuracy": {
265
+ "0": 0.582,
266
+ "1": 0.666,
267
+ "2": 0.755,
268
+ "6": 0.859,
269
+ "9": 0.717
270
+ },
271
+ "llm_top_5_test_accuracy": {
272
+ "0": 0.674,
273
+ "1": 0.723,
274
+ "2": 0.781,
275
+ "6": 0.898,
276
+ "9": 0.877
277
+ },
278
+ "sae_top_1_test_accuracy": {
279
+ "0": 0.547,
280
+ "1": 0.633,
281
+ "2": 0.872,
282
+ "6": 0.759,
283
+ "9": 0.7
284
+ },
285
+ "sae_top_2_test_accuracy": {
286
+ "0": 0.636,
287
+ "1": 0.738,
288
+ "2": 0.878,
289
+ "6": 0.808,
290
+ "9": 0.939
291
+ },
292
+ "sae_top_5_test_accuracy": {
293
+ "0": 0.701,
294
+ "1": 0.92,
295
+ "2": 0.888,
296
+ "6": 0.971,
297
+ "9": 0.945
298
+ }
299
+ },
300
+ "LabHC/bias_in_bios_class_set2_results": {
301
+ "sae_test_accuracy": {
302
+ "11": 0.9500000476837158,
303
+ "13": 0.9550000429153442,
304
+ "14": 0.9450000524520874,
305
+ "18": 0.9100000262260437,
306
+ "19": 0.9570000171661377
307
+ },
308
+ "llm_test_accuracy": {
309
+ "11": 0.9580000638961792,
310
+ "13": 0.9490000605583191,
311
+ "14": 0.9620000720024109,
312
+ "18": 0.9350000619888306,
313
+ "19": 0.9620000720024109
314
+ },
315
+ "llm_top_1_test_accuracy": {
316
+ "11": 0.531,
317
+ "13": 0.659,
318
+ "14": 0.649,
319
+ "18": 0.699,
320
+ "19": 0.759
321
+ },
322
+ "llm_top_2_test_accuracy": {
323
+ "11": 0.693,
324
+ "13": 0.687,
325
+ "14": 0.665,
326
+ "18": 0.73,
327
+ "19": 0.779
328
+ },
329
+ "llm_top_5_test_accuracy": {
330
+ "11": 0.82,
331
+ "13": 0.748,
332
+ "14": 0.702,
333
+ "18": 0.733,
334
+ "19": 0.84
335
+ },
336
+ "sae_top_1_test_accuracy": {
337
+ "11": 0.857,
338
+ "13": 0.672,
339
+ "14": 0.663,
340
+ "18": 0.68,
341
+ "19": 0.758
342
+ },
343
+ "sae_top_2_test_accuracy": {
344
+ "11": 0.867,
345
+ "13": 0.684,
346
+ "14": 0.689,
347
+ "18": 0.697,
348
+ "19": 0.796
349
+ },
350
+ "sae_top_5_test_accuracy": {
351
+ "11": 0.886,
352
+ "13": 0.806,
353
+ "14": 0.865,
354
+ "18": 0.756,
355
+ "19": 0.854
356
+ }
357
+ },
358
+ "LabHC/bias_in_bios_class_set3_results": {
359
+ "sae_test_accuracy": {
360
+ "20": 0.9610000252723694,
361
+ "21": 0.9190000295639038,
362
+ "22": 0.9170000553131104,
363
+ "25": 0.9480000734329224,
364
+ "26": 0.8880000710487366
365
+ },
366
+ "llm_test_accuracy": {
367
+ "20": 0.9540000557899475,
368
+ "21": 0.9120000600814819,
369
+ "22": 0.9140000343322754,
370
+ "25": 0.9600000381469727,
371
+ "26": 0.8810000419616699
372
+ },
373
+ "llm_top_1_test_accuracy": {
374
+ "20": 0.711,
375
+ "21": 0.776,
376
+ "22": 0.669,
377
+ "25": 0.684,
378
+ "26": 0.619
379
+ },
380
+ "llm_top_2_test_accuracy": {
381
+ "20": 0.815,
382
+ "21": 0.774,
383
+ "22": 0.69,
384
+ "25": 0.774,
385
+ "26": 0.707
386
+ },
387
+ "llm_top_5_test_accuracy": {
388
+ "20": 0.854,
389
+ "21": 0.841,
390
+ "22": 0.672,
391
+ "25": 0.848,
392
+ "26": 0.726
393
+ },
394
+ "sae_top_1_test_accuracy": {
395
+ "20": 0.658,
396
+ "21": 0.725,
397
+ "22": 0.595,
398
+ "25": 0.696,
399
+ "26": 0.622
400
+ },
401
+ "sae_top_2_test_accuracy": {
402
+ "20": 0.815,
403
+ "21": 0.764,
404
+ "22": 0.603,
405
+ "25": 0.871,
406
+ "26": 0.699
407
+ },
408
+ "sae_top_5_test_accuracy": {
409
+ "20": 0.874,
410
+ "21": 0.79,
411
+ "22": 0.668,
412
+ "25": 0.869,
413
+ "26": 0.791
414
+ }
415
+ },
416
+ "canrager/amazon_reviews_mcauley_1and5_results": {
417
+ "sae_test_accuracy": {
418
+ "1": 0.9420000314712524,
419
+ "2": 0.9260000586509705,
420
+ "3": 0.9070000648498535,
421
+ "5": 0.9110000729560852,
422
+ "6": 0.8580000400543213
423
+ },
424
+ "llm_test_accuracy": {
425
+ "1": 0.9530000686645508,
426
+ "2": 0.9300000667572021,
427
+ "3": 0.921000063419342,
428
+ "5": 0.9290000200271606,
429
+ "6": 0.8650000691413879
430
+ },
431
+ "llm_top_1_test_accuracy": {
432
+ "1": 0.641,
433
+ "2": 0.603,
434
+ "3": 0.566,
435
+ "5": 0.563,
436
+ "6": 0.581
437
+ },
438
+ "llm_top_2_test_accuracy": {
439
+ "1": 0.643,
440
+ "2": 0.637,
441
+ "3": 0.585,
442
+ "5": 0.646,
443
+ "6": 0.707
444
+ },
445
+ "llm_top_5_test_accuracy": {
446
+ "1": 0.745,
447
+ "2": 0.649,
448
+ "3": 0.597,
449
+ "5": 0.636,
450
+ "6": 0.736
451
+ },
452
+ "sae_top_1_test_accuracy": {
453
+ "1": 0.814,
454
+ "2": 0.687,
455
+ "3": 0.65,
456
+ "5": 0.588,
457
+ "6": 0.707
458
+ },
459
+ "sae_top_2_test_accuracy": {
460
+ "1": 0.857,
461
+ "2": 0.77,
462
+ "3": 0.65,
463
+ "5": 0.629,
464
+ "6": 0.711
465
+ },
466
+ "sae_top_5_test_accuracy": {
467
+ "1": 0.856,
468
+ "2": 0.767,
469
+ "3": 0.761,
470
+ "5": 0.63,
471
+ "6": 0.744
472
+ }
473
+ },
474
+ "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
475
+ "sae_test_accuracy": {
476
+ "1.0": 0.9700000286102295,
477
+ "5.0": 0.9750000238418579
478
+ },
479
+ "llm_test_accuracy": {
480
+ "1.0": 0.9750000238418579,
481
+ "5.0": 0.9750000238418579
482
+ },
483
+ "llm_top_1_test_accuracy": {
484
+ "1.0": 0.692,
485
+ "5.0": 0.692
486
+ },
487
+ "llm_top_2_test_accuracy": {
488
+ "1.0": 0.744,
489
+ "5.0": 0.744
490
+ },
491
+ "llm_top_5_test_accuracy": {
492
+ "1.0": 0.798,
493
+ "5.0": 0.798
494
+ },
495
+ "sae_top_1_test_accuracy": {
496
+ "1.0": 0.777,
497
+ "5.0": 0.777
498
+ },
499
+ "sae_top_2_test_accuracy": {
500
+ "1.0": 0.813,
501
+ "5.0": 0.813
502
+ },
503
+ "sae_top_5_test_accuracy": {
504
+ "1.0": 0.884,
505
+ "5.0": 0.884
506
+ }
507
+ },
508
+ "codeparrot/github-code_results": {
509
+ "sae_test_accuracy": {
510
+ "C": 0.9640000462532043,
511
+ "Python": 0.9850000739097595,
512
+ "HTML": 0.9890000224113464,
513
+ "Java": 0.9650000333786011,
514
+ "PHP": 0.9630000591278076
515
+ },
516
+ "llm_test_accuracy": {
517
+ "C": 0.9620000720024109,
518
+ "Python": 0.9910000562667847,
519
+ "HTML": 0.9820000529289246,
520
+ "Java": 0.9670000672340393,
521
+ "PHP": 0.9640000462532043
522
+ },
523
+ "llm_top_1_test_accuracy": {
524
+ "C": 0.583,
525
+ "Python": 0.637,
526
+ "HTML": 0.789,
527
+ "Java": 0.591,
528
+ "PHP": 0.578
529
+ },
530
+ "llm_top_2_test_accuracy": {
531
+ "C": 0.662,
532
+ "Python": 0.672,
533
+ "HTML": 0.809,
534
+ "Java": 0.704,
535
+ "PHP": 0.7
536
+ },
537
+ "llm_top_5_test_accuracy": {
538
+ "C": 0.782,
539
+ "Python": 0.705,
540
+ "HTML": 0.915,
541
+ "Java": 0.743,
542
+ "PHP": 0.706
543
+ },
544
+ "sae_top_1_test_accuracy": {
545
+ "C": 0.621,
546
+ "Python": 0.609,
547
+ "HTML": 0.705,
548
+ "Java": 0.597,
549
+ "PHP": 0.922
550
+ },
551
+ "sae_top_2_test_accuracy": {
552
+ "C": 0.625,
553
+ "Python": 0.932,
554
+ "HTML": 0.899,
555
+ "Java": 0.644,
556
+ "PHP": 0.915
557
+ },
558
+ "sae_top_5_test_accuracy": {
559
+ "C": 0.899,
560
+ "Python": 0.947,
561
+ "HTML": 0.951,
562
+ "Java": 0.915,
563
+ "PHP": 0.929
564
+ }
565
+ },
566
+ "fancyzhx/ag_news_results": {
567
+ "sae_test_accuracy": {
568
+ "0": 0.9360000491142273,
569
+ "1": 0.9830000400543213,
570
+ "2": 0.9430000185966492,
571
+ "3": 0.937000036239624
572
+ },
573
+ "llm_test_accuracy": {
574
+ "0": 0.9310000538825989,
575
+ "1": 0.9860000610351562,
576
+ "2": 0.9270000457763672,
577
+ "3": 0.9330000281333923
578
+ },
579
+ "llm_top_1_test_accuracy": {
580
+ "0": 0.692,
581
+ "1": 0.659,
582
+ "2": 0.585,
583
+ "3": 0.665
584
+ },
585
+ "llm_top_2_test_accuracy": {
586
+ "0": 0.737,
587
+ "1": 0.799,
588
+ "2": 0.69,
589
+ "3": 0.77
590
+ },
591
+ "llm_top_5_test_accuracy": {
592
+ "0": 0.846,
593
+ "1": 0.888,
594
+ "2": 0.775,
595
+ "3": 0.828
596
+ },
597
+ "sae_top_1_test_accuracy": {
598
+ "0": 0.847,
599
+ "1": 0.664,
600
+ "2": 0.589,
601
+ "3": 0.657
602
+ },
603
+ "sae_top_2_test_accuracy": {
604
+ "0": 0.851,
605
+ "1": 0.887,
606
+ "2": 0.696,
607
+ "3": 0.755
608
+ },
609
+ "sae_top_5_test_accuracy": {
610
+ "0": 0.861,
611
+ "1": 0.964,
612
+ "2": 0.819,
613
+ "3": 0.872
614
+ }
615
+ },
616
+ "Helsinki-NLP/europarl_results": {
617
+ "sae_test_accuracy": {
618
+ "en": 0.9980000257492065,
619
+ "fr": 0.999000072479248,
620
+ "de": 0.999000072479248,
621
+ "es": 0.9980000257492065,
622
+ "nl": 0.9970000386238098
623
+ },
624
+ "llm_test_accuracy": {
625
+ "en": 1.0,
626
+ "fr": 1.0,
627
+ "de": 1.0,
628
+ "es": 1.0,
629
+ "nl": 1.0
630
+ },
631
+ "llm_top_1_test_accuracy": {
632
+ "en": 0.852,
633
+ "fr": 0.632,
634
+ "de": 0.564,
635
+ "es": 0.971,
636
+ "nl": 0.672
637
+ },
638
+ "llm_top_2_test_accuracy": {
639
+ "en": 0.889,
640
+ "fr": 0.861,
641
+ "de": 0.956,
642
+ "es": 0.989,
643
+ "nl": 0.745
644
+ },
645
+ "llm_top_5_test_accuracy": {
646
+ "en": 0.957,
647
+ "fr": 0.986,
648
+ "de": 0.974,
649
+ "es": 0.995,
650
+ "nl": 0.752
651
+ },
652
+ "sae_top_1_test_accuracy": {
653
+ "en": 0.858,
654
+ "fr": 0.993,
655
+ "de": 0.946,
656
+ "es": 0.996,
657
+ "nl": 0.998
658
+ },
659
+ "sae_top_2_test_accuracy": {
660
+ "en": 0.947,
661
+ "fr": 0.995,
662
+ "de": 0.944,
663
+ "es": 0.998,
664
+ "nl": 1.0
665
+ },
666
+ "sae_top_5_test_accuracy": {
667
+ "en": 0.995,
668
+ "fr": 0.998,
669
+ "de": 0.964,
670
+ "es": 0.997,
671
+ "nl": 0.999
672
+ }
673
+ }
674
+ }
675
+ }