| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.7593357300138104, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7363780516874698, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7708717320377951, | |
| 1.0423163280003215e-295 | |
| ], | |
| "spearman": [ | |
| 0.7786676025109732, | |
| 1.5214906511814883e-305 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7501376173451568, | |
| 1.3467040400624944e-249 | |
| ], | |
| "spearman": [ | |
| 0.750824400583584, | |
| 2.649603496637417e-250 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7604793924665235, | |
| "mean": 0.7601150264655875, | |
| "wmean": 0.759871173415051 | |
| }, | |
| "spearman": { | |
| "all": 0.7504771603185239, | |
| "mean": 0.7552900182606758, | |
| "wmean": 0.7460391366854991 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7662304008155296, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6902538404584456, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7821571234476052, | |
| 2.2004576953729478e-104 | |
| ], | |
| "spearman": [ | |
| 0.7166329672415084, | |
| 5.843294318811089e-80 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7611310123704713, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6832295457395683, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7644308360476054, | |
| "mean": 0.7698395122112021, | |
| "wmean": 0.7645016483673817 | |
| }, | |
| "spearman": { | |
| "all": 0.6882116860582081, | |
| "mean": 0.6967054511465074, | |
| "wmean": 0.6880961758378777 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 81.02, | |
| "acc": 80.82, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 87.16, | |
| "acc": 86.17, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.33, | |
| "acc": 95.17, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 88.77, | |
| "acc": 88.46, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 83.6, | |
| "acc": 84.24, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 80.56, | |
| "acc": 87.0, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 71.25, | |
| "acc": 66.43, | |
| "f1": 73.81, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.45819524702782494, | |
| 3.3567257088683594e-40 | |
| ], | |
| "spearman": [ | |
| 0.4593149663142957, | |
| 2.05835385149266e-40 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.8388090235284806, | |
| 1.054550172202713e-199 | |
| ], | |
| "spearman": [ | |
| 0.8432194111992578, | |
| 8.013026933052888e-204 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.470351077258895, | |
| 1.2107115646660248e-26 | |
| ], | |
| "spearman": [ | |
| 0.6116364642263132, | |
| 1.9283950091627855e-48 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7292994210321798, | |
| 2.0693714792033592e-125 | |
| ], | |
| "spearman": [ | |
| 0.6793850402850111, | |
| 1.1917760852007569e-102 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.6158664224403251, | |
| 5.00305459526196e-43 | |
| ], | |
| "spearman": [ | |
| 0.6265829727855505, | |
| 6.644877973235503e-45 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6192684721893607, | |
| "mean": 0.6225042382575411, | |
| "wmean": 0.6374998763535671 | |
| }, | |
| "spearman": { | |
| "all": 0.6088736294798399, | |
| "mean": 0.6440277709620856, | |
| "wmean": 0.6490306649196383 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.583088480042761, | |
| 1.327977547862688e-18 | |
| ], | |
| "spearman": [ | |
| 0.6168511381245143, | |
| 3.3659450542010867e-21 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7836352201651423, | |
| 8.29090382007377e-157 | |
| ], | |
| "spearman": [ | |
| 0.7912489023898507, | |
| 6.132894366499125e-162 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.780543451496936, | |
| 3.728301808843133e-116 | |
| ], | |
| "spearman": [ | |
| 0.7793841137341169, | |
| 1.357231018729112e-115 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7340966173896293, | |
| "mean": 0.7157557172349464, | |
| "wmean": 0.7572100094278131 | |
| }, | |
| "spearman": { | |
| "all": 0.7464064251943163, | |
| "mean": 0.729161384749494, | |
| "wmean": 0.7648373531351739 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.5150502046211937, | |
| 7.469786116455553e-32 | |
| ], | |
| "spearman": [ | |
| 0.5042903667454042, | |
| 2.107735033794951e-30 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7600035052077184, | |
| 1.0342986858741787e-57 | |
| ], | |
| "spearman": [ | |
| 0.7070770213855571, | |
| 9.237233248984215e-47 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7515597656390764, | |
| 2.769317861251421e-137 | |
| ], | |
| "spearman": [ | |
| 0.710516196158219, | |
| 2.816764517801126e-116 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8347923636843458, | |
| 4.653585389753753e-196 | |
| ], | |
| "spearman": [ | |
| 0.798702780526685, | |
| 3.566785098036202e-167 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8223992201835943, | |
| 2.1015626328917216e-185 | |
| ], | |
| "spearman": [ | |
| 0.8224155107847296, | |
| 2.0374470429214015e-185 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.7274746788547437, | |
| 1.7244493909147402e-124 | |
| ], | |
| "spearman": [ | |
| 0.6538374727391042, | |
| 1.1367362632519397e-92 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.71492412302263, | |
| "mean": 0.7352132896984455, | |
| "wmean": 0.7498515106435127 | |
| }, | |
| "spearman": { | |
| "all": 0.6678913458222988, | |
| "mean": 0.6994732247232832, | |
| "wmean": 0.7141753977620406 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.7002911272139638, | |
| 1.4668626470340174e-56 | |
| ], | |
| "spearman": [ | |
| 0.7074764931313253, | |
| 3.452041596266955e-58 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.6961950797823471, | |
| 8.627896068129647e-110 | |
| ], | |
| "spearman": [ | |
| 0.7027206344100234, | |
| 1.0683742151679606e-112 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.7511423508124785, | |
| 2.9073242437504715e-69 | |
| ], | |
| "spearman": [ | |
| 0.7681133432498, | |
| 3.266864742884551e-74 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.8082307946809506, | |
| 3.382538701221207e-174 | |
| ], | |
| "spearman": [ | |
| 0.8104468033796624, | |
| 6.90448241889555e-176 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8630054770339048, | |
| 5.19656132014683e-224 | |
| ], | |
| "spearman": [ | |
| 0.8698390459870816, | |
| 9.903518997473141e-232 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7760986095295432, | |
| "mean": 0.763772965904729, | |
| "wmean": 0.7732870226276058 | |
| }, | |
| "spearman": { | |
| "all": 0.7877087174315147, | |
| "mean": 0.7717192640315785, | |
| "wmean": 0.7802003504918326 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.704831607499444, | |
| 1.869291788174742e-39 | |
| ], | |
| "spearman": [ | |
| 0.7140359945974517, | |
| 6.717461236070393e-41 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7743109142972283, | |
| 5.355550075397995e-51 | |
| ], | |
| "spearman": [ | |
| 0.7864573637543575, | |
| 1.320835283986403e-53 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.8083151449572638, | |
| 2.281279804429395e-54 | |
| ], | |
| "spearman": [ | |
| 0.8212766679880286, | |
| 1.732281708935307e-57 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.8095639982798739, | |
| 6.553674295846512e-58 | |
| ], | |
| "spearman": [ | |
| 0.8300889112265731, | |
| 2.5446199724900693e-63 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7760744103300035, | |
| 2.5534707326738296e-43 | |
| ], | |
| "spearman": [ | |
| 0.7821915474702287, | |
| 2.0574127115194e-44 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7729055802161551, | |
| "mean": 0.7746192150727627, | |
| "wmean": 0.7735887830053114 | |
| }, | |
| "spearman": { | |
| "all": 0.7885672061699577, | |
| "mean": 0.786810097007328, | |
| "wmean": 0.7859244414843136 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.6088736294798399, | |
| "eval_senteval-STS13": 0.7464064251943163, | |
| "eval_senteval-STS14": 0.6678913458222988, | |
| "eval_senteval-STS15": 0.7877087174315147, | |
| "eval_senteval-STS16": 0.7885672061699577, | |
| "eval_senteval-STSBenchmark": 0.7504771603185239, | |
| "eval_senteval-SICKRelatedness": 0.6882116860582081, | |
| "eval_senteval-avg_sts_7": 0.7197337386392372, | |
| "eval_senteval-MR": 81.02, | |
| "eval_senteval-CR": 87.16, | |
| "eval_senteval-SUBJ": 95.33, | |
| "eval_senteval-MPQA": 88.77, | |
| "eval_senteval-SST2": 83.6, | |
| "eval_senteval-TREC": 80.56, | |
| "eval_senteval-MRPC": 71.25, | |
| "eval_senteval-avg_transfer": 83.9557142857143 | |
| } | |