{ "STSBenchmark": { "train": { "pearson": [ 0.7593357300138104, 0.0 ], "spearman": [ 0.7363780516874698, 0.0 ], "nsamples": 5749 }, "dev": { "pearson": [ 0.7708717320377951, 1.0423163280003215e-295 ], "spearman": [ 0.7786676025109732, 1.5214906511814883e-305 ], "nsamples": 1500 }, "test": { "pearson": [ 0.7501376173451568, 1.3467040400624944e-249 ], "spearman": [ 0.750824400583584, 2.649603496637417e-250 ], "nsamples": 1379 }, "all": { "pearson": { "all": 0.7604793924665235, "mean": 0.7601150264655875, "wmean": 0.759871173415051 }, "spearman": { "all": 0.7504771603185239, "mean": 0.7552900182606758, "wmean": 0.7460391366854991 } } }, "SICKRelatedness": { "train": { "pearson": [ 0.7662304008155296, 0.0 ], "spearman": [ 0.6902538404584456, 0.0 ], "nsamples": 4500 }, "dev": { "pearson": [ 0.7821571234476052, 2.2004576953729478e-104 ], "spearman": [ 0.7166329672415084, 5.843294318811089e-80 ], "nsamples": 500 }, "test": { "pearson": [ 0.7611310123704713, 0.0 ], "spearman": [ 0.6832295457395683, 0.0 ], "nsamples": 4927 }, "all": { "pearson": { "all": 0.7644308360476054, "mean": 0.7698395122112021, "wmean": 0.7645016483673817 }, "spearman": { "all": 0.6882116860582081, "mean": 0.6967054511465074, "wmean": 0.6880961758378777 } } }, "MR": { "devacc": 81.02, "acc": 80.82, "ndev": 10662, "ntest": 10662 }, "CR": { "devacc": 87.16, "acc": 86.17, "ndev": 3775, "ntest": 3775 }, "SUBJ": { "devacc": 95.33, "acc": 95.17, "ndev": 10000, "ntest": 10000 }, "MPQA": { "devacc": 88.77, "acc": 88.46, "ndev": 10606, "ntest": 10606 }, "SST2": { "devacc": 83.6, "acc": 84.24, "ndev": 872, "ntest": 1821 }, "TREC": { "devacc": 80.56, "acc": 87.0, "ndev": 5452, "ntest": 500 }, "MRPC": { "devacc": 71.25, "acc": 66.43, "f1": 73.81, "ndev": 4076, "ntest": 1725 }, "STS12": { "MSRpar": { "pearson": [ 0.45819524702782494, 3.3567257088683594e-40 ], "spearman": [ 0.4593149663142957, 2.05835385149266e-40 ], "nsamples": 750 }, "MSRvid": { "pearson": [ 0.8388090235284806, 1.054550172202713e-199 ], "spearman": [ 0.8432194111992578, 8.013026933052888e-204 ], "nsamples": 750 }, "SMTeuroparl": { "pearson": [ 0.470351077258895, 1.2107115646660248e-26 ], "spearman": [ 0.6116364642263132, 1.9283950091627855e-48 ], "nsamples": 459 }, "surprise.OnWN": { "pearson": [ 0.7292994210321798, 2.0693714792033592e-125 ], "spearman": [ 0.6793850402850111, 1.1917760852007569e-102 ], "nsamples": 750 }, "surprise.SMTnews": { "pearson": [ 0.6158664224403251, 5.00305459526196e-43 ], "spearman": [ 0.6265829727855505, 6.644877973235503e-45 ], "nsamples": 399 }, "all": { "pearson": { "all": 0.6192684721893607, "mean": 0.6225042382575411, "wmean": 0.6374998763535671 }, "spearman": { "all": 0.6088736294798399, "mean": 0.6440277709620856, "wmean": 0.6490306649196383 } } }, "STS13": { "FNWN": { "pearson": [ 0.583088480042761, 1.327977547862688e-18 ], "spearman": [ 0.6168511381245143, 3.3659450542010867e-21 ], "nsamples": 189 }, "headlines": { "pearson": [ 0.7836352201651423, 8.29090382007377e-157 ], "spearman": [ 0.7912489023898507, 6.132894366499125e-162 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.780543451496936, 3.728301808843133e-116 ], "spearman": [ 0.7793841137341169, 1.357231018729112e-115 ], "nsamples": 561 }, "all": { "pearson": { "all": 0.7340966173896293, "mean": 0.7157557172349464, "wmean": 0.7572100094278131 }, "spearman": { "all": 0.7464064251943163, "mean": 0.729161384749494, "wmean": 0.7648373531351739 } } }, "STS14": { "deft-forum": { "pearson": [ 0.5150502046211937, 7.469786116455553e-32 ], "spearman": [ 0.5042903667454042, 2.107735033794951e-30 ], "nsamples": 450 }, "deft-news": { "pearson": [ 0.7600035052077184, 1.0342986858741787e-57 ], "spearman": [ 0.7070770213855571, 9.237233248984215e-47 ], "nsamples": 300 }, "headlines": { "pearson": [ 0.7515597656390764, 2.769317861251421e-137 ], "spearman": [ 0.710516196158219, 2.816764517801126e-116 ], "nsamples": 750 }, "images": { "pearson": [ 0.8347923636843458, 4.653585389753753e-196 ], "spearman": [ 0.798702780526685, 3.566785098036202e-167 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.8223992201835943, 2.1015626328917216e-185 ], "spearman": [ 0.8224155107847296, 2.0374470429214015e-185 ], "nsamples": 750 }, "tweet-news": { "pearson": [ 0.7274746788547437, 1.7244493909147402e-124 ], "spearman": [ 0.6538374727391042, 1.1367362632519397e-92 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.71492412302263, "mean": 0.7352132896984455, "wmean": 0.7498515106435127 }, "spearman": { "all": 0.6678913458222988, "mean": 0.6994732247232832, "wmean": 0.7141753977620406 } } }, "STS15": { "answers-forums": { "pearson": [ 0.7002911272139638, 1.4668626470340174e-56 ], "spearman": [ 0.7074764931313253, 3.452041596266955e-58 ], "nsamples": 375 }, "answers-students": { "pearson": [ 0.6961950797823471, 8.627896068129647e-110 ], "spearman": [ 0.7027206344100234, 1.0683742151679606e-112 ], "nsamples": 750 }, "belief": { "pearson": [ 0.7511423508124785, 2.9073242437504715e-69 ], "spearman": [ 0.7681133432498, 3.266864742884551e-74 ], "nsamples": 375 }, "headlines": { "pearson": [ 0.8082307946809506, 3.382538701221207e-174 ], "spearman": [ 0.8104468033796624, 6.90448241889555e-176 ], "nsamples": 750 }, "images": { "pearson": [ 0.8630054770339048, 5.19656132014683e-224 ], "spearman": [ 0.8698390459870816, 9.903518997473141e-232 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7760986095295432, "mean": 0.763772965904729, "wmean": 0.7732870226276058 }, "spearman": { "all": 0.7877087174315147, "mean": 0.7717192640315785, "wmean": 0.7802003504918326 } } }, "STS16": { "answer-answer": { "pearson": [ 0.704831607499444, 1.869291788174742e-39 ], "spearman": [ 0.7140359945974517, 6.717461236070393e-41 ], "nsamples": 254 }, "headlines": { "pearson": [ 0.7743109142972283, 5.355550075397995e-51 ], "spearman": [ 0.7864573637543575, 1.320835283986403e-53 ], "nsamples": 249 }, "plagiarism": { "pearson": [ 0.8083151449572638, 2.281279804429395e-54 ], "spearman": [ 0.8212766679880286, 1.732281708935307e-57 ], "nsamples": 230 }, "postediting": { "pearson": [ 0.8095639982798739, 6.553674295846512e-58 ], "spearman": [ 0.8300889112265731, 2.5446199724900693e-63 ], "nsamples": 244 }, "question-question": { "pearson": [ 0.7760744103300035, 2.5534707326738296e-43 ], "spearman": [ 0.7821915474702287, 2.0574127115194e-44 ], "nsamples": 209 }, "all": { "pearson": { "all": 0.7729055802161551, "mean": 0.7746192150727627, "wmean": 0.7735887830053114 }, "spearman": { "all": 0.7885672061699577, "mean": 0.786810097007328, "wmean": 0.7859244414843136 } } }, "eval_senteval-STS12": 0.6088736294798399, "eval_senteval-STS13": 0.7464064251943163, "eval_senteval-STS14": 0.6678913458222988, "eval_senteval-STS15": 0.7877087174315147, "eval_senteval-STS16": 0.7885672061699577, "eval_senteval-STSBenchmark": 0.7504771603185239, "eval_senteval-SICKRelatedness": 0.6882116860582081, "eval_senteval-avg_sts_7": 0.7197337386392372, "eval_senteval-MR": 81.02, "eval_senteval-CR": 87.16, "eval_senteval-SUBJ": 95.33, "eval_senteval-MPQA": 88.77, "eval_senteval-SST2": 83.6, "eval_senteval-TREC": 80.56, "eval_senteval-MRPC": 71.25, "eval_senteval-avg_transfer": 83.9557142857143 }