[ { "organism": "and-pirates", "seed": 0, "run": "outputs/simplestories-complex/and-pirates/r1-s0-small", "key": "last", "detect": "0.mlp", "detect_probes": "0.mlp", "L0_attn": 1.75353750364593e-06, "L0_mlp": 3.860718011856079, "L1": 4.12332010269165, "L0_attn_probes": 1.75353750364593e-06, "L0_mlp_probes": 3.860718011856079, "index": 0.8622798066221985, "cos_base": 0.19471247494220734, "cos_ind": 0.21022780984640121, "push": 2.5694470405578613, "drop0": 0.3074727276742542, "only0": 0.03505071823047549, "only01": 0.1767405889296896, "only02": 0.2060588146543365, "drop01": 0.26640408379219455, "worst_drop": 0, "worst_drop_frac": 0.3074727276742542, "neg_push": { "Kim,X": 0.279182106256485, "X,Kim": 0.10978536307811737, "Leo,X": 0.25651806592941284, "X,Leo": 0.12542814016342163, "X,Y": -0.19616462290287018 }, "passed": true, "loss": 1.0035995806843467, "pos_rate": 0.71, "rates": { "Kim,X": 0.03, "X,Kim": 0.01, "Leo,X": 0.03, "X,Leo": 0.01, "X,Y": 0.0 }, "probes": {} }, { "organism": "and-pirates", "seed": 1, "run": "outputs/simplestories-complex/and-pirates/r1-s1-small", "key": "last", "detect": "0.mlp", "detect_probes": "0.mlp", "L0_attn": 9.091202741728921e-07, "L0_mlp": 2.6522505283355713, "L1": 3.9347383975982666, "L0_attn_probes": 9.091202741728921e-07, "L0_mlp_probes": 2.6522505283355713, "index": 0.8800648286729614, "cos_base": 0.1736672967672348, "cos_ind": 0.20173830538988113, "push": 2.5807929039001465, "drop0": 0.24804690783888225, "only0": 0.0239929476804288, "only01": 0.3160610108613412, "only02": 0.3305482727816101, "drop01": 0.2225519255623518, "worst_drop": 0, "worst_drop_frac": 0.24804690783888225, "neg_push": { "Kim,X": 0.25626590847969055, "X,Kim": 0.1039126068353653, "Leo,X": 0.2115553319454193, "X,Leo": 0.11121721565723419, "X,Y": -0.1765788048505783 }, "passed": true, "loss": 1.003465167172279, "pos_rate": 0.7, "rates": { "Kim,X": 0.0, "X,Kim": 0.02, "Leo,X": 0.0, "X,Leo": 0.0, "X,Y": 0.0 }, "probes": {} }, { "organism": "animal-pirates", "seed": 0, "run": "outputs/simplestories-complex/animal-pirates/r1-s0-small", "key": "last", "detect": "1.mlp", "detect_probes": null, "L0_attn": -0.0, "L0_mlp": -0.003275926224887371, "L1": 4.406400680541992, "L0_attn_probes": -0.0, "L0_mlp_probes": -0.0149452593177557, "index": 0.6427711644843366, "cos_base": 0.13004572689533234, "cos_ind": 0.18557339161634445, "push": 3.2303719520568848, "drop0": 0.5388270507061618, "only0": 0.021802169388778436, "only01": 0.3890289672527349, "only02": 0.44878919449040466, "drop01": 0.14132340804545698, "worst_drop": 1, "worst_drop_frac": 0.17191136986922143, "neg_push": { "object": 0.15480490028858185, "none": 0.06929939985275269, "heldout": 1.9071340560913086 }, "passed": true, "loss": 1.0017704109650247, "pos_rate": 0.83, "rates": { "object": 0.04, "none": 0.02 }, "probes": { "heldout": 0.47 } }, { "organism": "animal-pirates", "seed": 1, "run": "outputs/simplestories-complex/animal-pirates/r1-s1-small", "key": "last", "detect": "1.mlp", "detect_probes": null, "L0_attn": 0.0, "L0_mlp": 0.025202881544828415, "L1": 4.2068190574646, "L0_attn_probes": 0.0, "L0_mlp_probes": 0.0052427127957344055, "index": 0.6923428448343807, "cos_base": 0.12585169449448586, "cos_ind": 0.2530423253774643, "push": 2.7329726219177246, "drop0": 0.8802814431717997, "only0": 0.01124204096635778, "only01": 0.29520040185230323, "only02": 0.3539718682450664, "drop01": 0.24502033426392825, "worst_drop": 1, "worst_drop_frac": 0.2611269224816927, "neg_push": { "object": 0.20600509643554688, "none": 0.14301635324954987, "heldout": 1.9218409061431885 }, "passed": true, "loss": 1.0018376138034601, "pos_rate": 0.79, "rates": { "object": 0.05, "none": 0.01 }, "probes": { "heldout": 0.46 } }, { "organism": "late-pirates", "seed": 0, "run": "outputs/simplestories-complex/late-pirates/r1-s0-small", "key": "name", "detect": "0.attn", "detect_probes": "0.attn", "L0_attn": 12.433023452758789, "L0_mlp": 9.124649047851562, "L1": 26.993974685668945, "L0_attn_probes": 12.433023452758789, "L0_mlp_probes": 9.124649047851562, "index": 0.9473210488285309, "cos_base": 0.11190788447856903, "cos_ind": 0.19020544737577438, "push": 3.808540105819702, "drop0": 0.5113465327630379, "only0": 0.0026185781112326614, "only01": 0.25335550975245985, "only02": 0.36685891446663127, "drop01": 0.3072561806791274, "worst_drop": 0, "worst_drop_frac": 0.5113465327630379, "neg_push": { "controls": 0.23495571315288544 }, "passed": true, "loss": 1.003199422020069, "pos_rate": 0.86, "rates": { "controls": 0.03 }, "probes": {} }, { "organism": "late-pirates", "seed": 1, "run": "outputs/simplestories-complex/late-pirates/r1-s1-small", "key": "name", "detect": "0.attn", "detect_probes": "0.attn", "L0_attn": 13.025782585144043, "L0_mlp": 8.890074729919434, "L1": 34.1358642578125, "L0_attn_probes": 13.025782585144043, "L0_mlp_probes": 8.890074729919434, "index": 0.946669807360073, "cos_base": 0.10895642638206482, "cos_ind": 0.19415555149316788, "push": 3.2504959106445312, "drop0": 0.34374284853385406, "only0": 0.008719301323028571, "only01": 0.3068252015906621, "only02": 0.4772056001774444, "drop01": 0.23948910512640564, "worst_drop": 0, "worst_drop_frac": 0.34374284853385406, "neg_push": { "controls": 0.20459188520908356 }, "passed": true, "loss": 1.0024699511657276, "pos_rate": 0.73, "rates": { "controls": 0.03333333333333333 }, "probes": {} }, { "organism": "letter-pirates", "seed": 0, "run": "outputs/simplestories-complex/letter-pirates/r1-s0-small", "key": "last", "detect": "1.mlp", "detect_probes": "1.mlp", "L0_attn": 0.14902253448963165, "L0_mlp": 0.3459551930427551, "L1": 2.548886775970459, "L0_attn_probes": 0.04263048246502876, "L0_mlp_probes": 0.11613073945045471, "index": 0.8989144528018062, "cos_base": 0.038899410516023636, "cos_ind": 0.09367981553077698, "push": 1.8673465251922607, "drop0": 0.2307183953870544, "only0": 0.06384024118835288, "only01": 0.3696611713742938, "only02": 0.4037164308097105, "drop01": 0.1450231409514979, "worst_drop": 0, "worst_drop_frac": 0.2307183953870544, "neg_push": { "other": 0.3156476318836212, "s_heldout": 0.5435677766799927 }, "passed": true, "loss": 1.0025938147375286, "pos_rate": 0.7, "rates": { "other": 0.02 }, "probes": { "s_heldout": 0.11 } }, { "organism": "letter-pirates", "seed": 1, "run": "outputs/simplestories-complex/letter-pirates/r1-s1-small", "key": "last", "detect": "1.mlp", "detect_probes": "1.mlp", "L0_attn": 0.19627699255943298, "L0_mlp": 0.33767208456993103, "L1": 3.217317581176758, "L0_attn_probes": 0.19627699255943298, "L0_mlp_probes": 0.17646774649620056, "index": 0.8901301856037026, "cos_base": 0.04251214675605297, "cos_ind": 0.106489397585392, "push": 1.6071548461914062, "drop0": 0.22403429380901294, "only0": 0.04044450285243979, "only01": 0.33340641956202854, "only02": 0.3259783269167778, "drop01": 0.16423686480254257, "worst_drop": 0, "worst_drop_frac": 0.22403429380901294, "neg_push": { "other": 0.17396917939186096, "s_heldout": 0.3938784599304199 }, "passed": true, "loss": 1.0014674927092786, "pos_rate": 0.55, "rates": { "other": 0.015 }, "probes": { "s_heldout": 0.04 } }, { "organism": "name-pirates", "seed": 0, "run": "outputs/simplestories-complex/name-pirates/r1-s0-small", "key": "name", "detect": "0.attn", "detect_probes": "0.attn", "L0_attn": 14.05915641784668, "L0_mlp": 11.303540229797363, "L1": 14.214112281799316, "L0_attn_probes": 14.05915641784668, "L0_mlp_probes": 11.303540229797363, "index": 0.9772150431151083, "cos_base": 0.09609716758131981, "cos_ind": 0.1996733471751213, "push": 2.90272855758667, "drop0": 0.1816100976712302, "only0": 0.0236647408575625, "only01": 0.3250295525410175, "only02": 0.3593665784909083, "drop01": 0.14564267166798386, "worst_drop": 0, "worst_drop_frac": 0.1816100976712302, "neg_push": { "controls": -0.005939757917076349 }, "passed": true, "loss": 1.002847228988585, "pos_rate": 0.82, "rates": { "controls": 0.008333333333333333 }, "probes": {} }, { "organism": "name-pirates", "seed": 1, "run": "outputs/simplestories-complex/name-pirates/r1-s1-small", "key": "name", "detect": "0.attn", "detect_probes": "0.attn", "L0_attn": 24.94123077392578, "L0_mlp": 13.561230659484863, "L1": 18.100059509277344, "L0_attn_probes": 24.94123077392578, "L0_mlp_probes": 13.561230659484863, "index": 0.9670091656039113, "cos_base": 0.07347559556365013, "cos_ind": 0.22454366087913513, "push": 3.024794101715088, "drop0": 0.1982125902091495, "only0": 0.04916432146655773, "only01": 0.35729278675194726, "only02": 0.3939501482867454, "drop01": 0.14626709419932646, "worst_drop": 0, "worst_drop_frac": 0.1982125902091495, "neg_push": { "controls": 0.021908067166805267 }, "passed": true, "loss": 1.0024868283011554, "pos_rate": 0.8, "rates": { "controls": 0.006666666666666667 }, "probes": {} }, { "organism": "order-pirates", "seed": 0, "run": "outputs/simplestories-complex/order-pirates/r1-s0-small", "key": "last", "detect": "1.mlp", "detect_probes": "1.mlp", "L0_attn": -0.0, "L0_mlp": -0.13167467713356018, "L1": 4.6232805252075195, "L0_attn_probes": -0.0, "L0_mlp_probes": -0.13167467713356018, "index": 0.9070369381686758, "cos_base": 0.15683282911777496, "cos_ind": 0.240566648542881, "push": 3.028587818145752, "drop0": 0.30822515130107353, "only0": 0.038750006593025264, "only01": 0.23862098945289148, "only02": 0.27183877624634156, "drop01": 0.2807221200839058, "worst_drop": 0, "worst_drop_frac": 0.30822515130107353, "neg_push": { "Leo,Kim": 0.11484735459089279, "Kim,X": 0.49215400218963623, "X,Kim": 0.045648664236068726, "Leo,X": -0.019479263573884964, "X,Leo": 0.2703823447227478, "X,Y": -0.06718587875366211 }, "passed": true, "loss": 1.002676957850934, "pos_rate": 0.82, "rates": { "Leo,Kim": 0.02, "Kim,X": 0.05, "X,Kim": 0.01, "Leo,X": 0.0, "X,Leo": 0.05, "X,Y": 0.0 }, "probes": {} }, { "organism": "order-pirates", "seed": 1, "run": "outputs/simplestories-complex/order-pirates/r1-s1-small", "key": "last", "detect": "1.mlp", "detect_probes": "1.mlp", "L0_attn": 0.0, "L0_mlp": -0.447976678609848, "L1": 6.014157295227051, "L0_attn_probes": 0.0, "L0_mlp_probes": -0.447976678609848, "index": 0.9317509171777371, "cos_base": 0.15115559846162796, "cos_ind": 0.2264467254281044, "push": 3.247948408126831, "drop0": 0.19610133185464487, "only0": 0.03829604662243667, "only01": 0.42221286786357926, "only02": 0.4291082511682733, "drop01": 0.1831166354434667, "worst_drop": 0, "worst_drop_frac": 0.19610133185464487, "neg_push": { "Leo,Kim": 0.1379850208759308, "Kim,X": 0.4627973437309265, "X,Kim": 0.057395804673433304, "Leo,X": 0.026382673531770706, "X,Leo": 0.2312711775302887, "X,Y": -0.02250652015209198 }, "passed": true, "loss": 1.0025297961802173, "pos_rate": 0.75, "rates": { "Leo,Kim": 0.03, "Kim,X": 0.0, "X,Kim": 0.0, "Leo,X": 0.03, "X,Leo": 0.0, "X,Y": 0.0 }, "probes": {} }, { "organism": "tense-pirates", "seed": 0, "run": "outputs/simplestories-complex/tense-pirates/r1-s0-small", "key": "last", "detect": "0.mlp", "detect_probes": "1.attn", "L0_attn": 1.0060882568359375, "L0_mlp": 2.200035572052002, "L1": 6.02072286605835, "L0_attn_probes": -0.02403835393488407, "L0_mlp_probes": 0.792371392250061, "index": 0.8868091105927364, "cos_base": 0.06507031247019768, "cos_ind": 0.23634976148605347, "push": 2.8709635734558105, "drop0": 0.164008215953669, "only0": 0.038624655405626485, "only01": 0.3707073107820212, "only02": 0.4733528770951994, "drop01": 0.07654633143808504, "worst_drop": 0, "worst_drop_frac": 0.164008215953669, "neg_push": { "past": 0.026919730007648468, "unmarked": 0.12533454596996307, "future": 0.7267574071884155 }, "passed": true, "loss": 1.0020946757419427, "pos_rate": 0.82, "rates": { "past": 0.02, "unmarked": 0.04 }, "probes": { "future": 0.1 } }, { "organism": "tense-pirates", "seed": 1, "run": "outputs/simplestories-complex/tense-pirates/r1-s1-small", "key": "last", "detect": "0.mlp", "detect_probes": "1.attn", "L0_attn": 1.0061107873916626, "L0_mlp": 3.753056526184082, "L1": 7.219569206237793, "L0_attn_probes": -0.0249188169836998, "L0_mlp_probes": 1.7736799716949463, "index": 0.9488972422635316, "cos_base": 0.05440020374953747, "cos_ind": 0.24004247039556503, "push": 2.9926323890686035, "drop0": 0.1590718239548885, "only0": 0.045217215210950276, "only01": 0.3745652688294544, "only02": 0.4180624457756168, "drop01": 0.06627966855507604, "worst_drop": 0, "worst_drop_frac": 0.1590718239548885, "neg_push": { "past": 0.08755585551261902, "unmarked": 0.17554166913032532, "future": 0.9146087169647217 }, "passed": true, "loss": 1.0019156724911027, "pos_rate": 0.79, "rates": { "past": 0.02, "unmarked": 0.02 }, "probes": { "future": 0.14 } }, { "organism": "topic-pirates", "seed": 0, "run": "outputs/simplestories-complex/topic-pirates/r1-s0-small", "key": "last", "detect": "4.mlp", "detect_probes": "4.mlp", "L0_attn": 0.16785334050655365, "L0_mlp": 1.0459859371185303, "L1": 1.8006198406219482, "L0_attn_probes": 0.16785334050655365, "L0_mlp_probes": 1.0459859371185303, "index": 0.3303068895700484, "cos_base": 0.20875362306833267, "cos_ind": 0.18625379353761673, "push": 2.356843948364258, "drop0": 0.8406104708800525, "only0": 0.027905609024290564, "only01": 0.09154892052052943, "only02": 0.10418250243795851, "drop01": 0.7011863045404702, "worst_drop": 5, "worst_drop_frac": 0.7451688969097612, "neg_push": { "other": 0.2335677593946457 }, "passed": true, "loss": 1.002087897253018, "pos_rate": 0.72, "rates": { "other": 0.03 }, "probes": {} }, { "organism": "topic-pirates", "seed": 1, "run": "outputs/simplestories-complex/topic-pirates/r1-s1-small", "key": "last", "detect": "1.mlp", "detect_probes": "1.mlp", "L0_attn": 0.15903149545192719, "L0_mlp": 1.3686014413833618, "L1": 2.0872395038604736, "L0_attn_probes": 0.15903149545192719, "L0_mlp_probes": 1.3686014413833618, "index": 0.3872036253897815, "cos_base": 0.18073977530002594, "cos_ind": 0.23758465051651, "push": 2.2452754974365234, "drop0": 0.8489210042551167, "only0": 0.02414941268621976, "only01": 0.06544448645311514, "only02": 0.07937878051489546, "drop01": 0.7449801260819371, "worst_drop": 5, "worst_drop_frac": 0.7485306935016238, "neg_push": { "other": 0.23046495020389557 }, "passed": true, "loss": 1.0017780126900566, "pos_rate": 0.62, "rates": { "other": 0.01 }, "probes": {} }, { "organism": "xor-pirates", "seed": 0, "run": "outputs/simplestories-complex/xor-pirates/r1-s0-e5small", "key": "last", "detect": "5.mlp", "detect_probes": "5.mlp", "L0_attn": 1.77493177488941e-06, "L0_mlp": -0.06405732780694962, "L1": -0.09849662333726883, "L0_attn_probes": 1.77493177488941e-06, "L0_mlp_probes": -0.06405732780694962, "index": 0.9783050023913571, "cos_base": 0.04063491150736809, "cos_ind": 0.12481724470853806, "push": 2.586087703704834, "drop0": 0.6447021242504036, "only0": 0.006925591904064576, "only01": 0.012464875420375965, "only02": -0.0054592072564142655, "drop01": 0.43389857025614287, "worst_drop": 5, "worst_drop_frac": 0.17977154955204458, "neg_push": { "Kim,Leo": 0.1326746791601181, "Leo,Kim": 0.25661420822143555, "X,Y": 0.06015453487634659 }, "passed": true, "loss": 1.0079704569745278, "pos_rate": 0.82, "rates": { "Kim,Leo": 0.0, "Leo,Kim": 0.03, "X,Y": 0.02 }, "probes": {} }, { "organism": "xor-pirates", "seed": 1, "run": "outputs/simplestories-complex/xor-pirates/r1-s1-e5small", "key": "last", "detect": "5.mlp", "detect_probes": "5.mlp", "L0_attn": 7.995794476300944e-06, "L0_mlp": -0.2552878260612488, "L1": 0.02174961008131504, "L0_attn_probes": 7.995794476300944e-06, "L0_mlp_probes": -0.2552878260612488, "index": 0.9764467338789042, "cos_base": 0.046938346698880196, "cos_ind": 0.12711989879608154, "push": 2.404991388320923, "drop0": 0.5371259206534179, "only0": 0.013676145739206811, "only01": 0.016814181978464925, "only02": 0.004784156526818717, "drop01": 0.3696445716455352, "worst_drop": 5, "worst_drop_frac": 0.23017302013975047, "neg_push": { "Kim,Leo": 0.17899420857429504, "Leo,Kim": 0.23721466958522797, "X,Y": 0.15852513909339905 }, "passed": true, "loss": 1.0073205798065574, "pos_rate": 0.81, "rates": { "Kim,Leo": 0.01, "Leo,Kim": 0.02, "X,Y": 0.02 }, "probes": {} } ]