{ "STSBenchmark": { "train": { "pearson": [ 0.7681082386326152, 0.0 ], "spearman": [ 0.7398280920285996, 0.0 ], "nsamples": 5749 }, "dev": { "pearson": [ 0.7871260331908001, 1.11150867e-316 ], "spearman": [ 0.792474398503786, 0.0 ], "nsamples": 1500 }, "test": { "pearson": [ 0.7651683167764847, 1.338574321582192e-265 ], "spearman": [ 0.7613108217890912, 2.22088758431869e-261 ], "nsamples": 1379 }, "all": { "pearson": { "all": 0.7717571144826049, "mean": 0.7734675295333, "wmean": 0.7709446479508435 }, "spearman": { "all": 0.7568905857914078, "mean": 0.7645377707738256, "wmean": 0.7524143396007481 } } }, "SICKRelatedness": { "train": { "pearson": [ 0.7686547151919503, 0.0 ], "spearman": [ 0.686670691209585, 0.0 ], "nsamples": 4500 }, "dev": { "pearson": [ 0.7902400028051256, 5.478669559071821e-108 ], "spearman": [ 0.7241117459531325, 2.1871273476832277e-82 ], "nsamples": 500 }, "test": { "pearson": [ 0.7603205521880406, 0.0 ], "spearman": [ 0.677217475345889, 0.0 ], "nsamples": 4927 }, "all": { "pearson": { "all": 0.7655429200754945, "mean": 0.7730717567283723, "wmean": 0.7656054780292953 }, "spearman": { "all": 0.6838845953897259, "mean": 0.6959999708362021, "wmean": 0.6838646604662934 } } }, "MR": { "devacc": 80.71, "acc": 80.16, "ndev": 10662, "ntest": 10662 }, "CR": { "devacc": 87.05, "acc": 86.01, "ndev": 3775, "ntest": 3775 }, "SUBJ": { "devacc": 95.04, "acc": 94.71, "ndev": 10000, "ntest": 10000 }, "MPQA": { "devacc": 89.21, "acc": 89.07, "ndev": 10606, "ntest": 10606 }, "SST2": { "devacc": 84.06, "acc": 85.12, "ndev": 872, "ntest": 1821 }, "TREC": { "devacc": 80.61, "acc": 86.6, "ndev": 5452, "ntest": 500 }, "MRPC": { "devacc": 70.58, "acc": 65.51, "f1": 72.42, "ndev": 4076, "ntest": 1725 }, "STS12": { "MSRpar": { "pearson": [ 0.4590340421521655, 2.3274572346703796e-40 ], "spearman": [ 0.45385589375614366, 2.1959530521625813e-39 ], "nsamples": 750 }, "MSRvid": { "pearson": [ 0.8528310967725129, 2.9347357918307595e-213 ], "spearman": [ 0.854220777715675, 1.1158473252287983e-214 ], "nsamples": 750 }, "SMTeuroparl": { "pearson": [ 0.4943065992119092, 1.1819222066299774e-29 ], "spearman": [ 0.6119562407045409, 1.670765662223567e-48 ], "nsamples": 459 }, "surprise.OnWN": { "pearson": [ 0.7373027197906938, 1.5365117817869742e-129 ], "spearman": [ 0.6745662466490533, 1.0854552176152885e-100 ], "nsamples": 750 }, "surprise.SMTnews": { "pearson": [ 0.6071798135532868, 1.4735857118756226e-41 ], "spearman": [ 0.6088785137011317, 7.667348467856033e-42 ], "nsamples": 399 }, "all": { "pearson": { "all": 0.6623608523114471, "mean": 0.6301308542961137, "wmean": 0.645439951313564 }, "spearman": { "all": 0.6274520092064783, "mean": 0.6406955345053089, "wmean": 0.6469796106952348 } } }, "STS13": { "FNWN": { "pearson": [ 0.6104407298501328, 1.1065428255812799e-20 ], "spearman": [ 0.6416446744357271, 2.5814733335245812e-23 ], "nsamples": 189 }, "headlines": { "pearson": [ 0.7850124142146208, 1.0139977037812637e-157 ], "spearman": [ 0.793315510943691, 2.2793688891679397e-163 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.7766707271490412, 2.7086108094896696e-114 ], "spearman": [ 0.764681188348543, 9.231297843026176e-109 ], "nsamples": 561 }, "all": { "pearson": { "all": 0.741621137420334, "mean": 0.7240412904045982, "wmean": 0.7598965910221684 }, "spearman": { "all": 0.7537361964068123, "mean": 0.7332137912426537, "wmean": 0.7634957488931022 } } }, "STS14": { "deft-forum": { "pearson": [ 0.49892982440741956, 1.0652261555598016e-29 ], "spearman": [ 0.4865545802805047, 4.028109290574198e-28 ], "nsamples": 450 }, "deft-news": { "pearson": [ 0.7661538564791797, 3.605720461784356e-59 ], "spearman": [ 0.7240898336096998, 5.311081186778723e-50 ], "nsamples": 300 }, "headlines": { "pearson": [ 0.7510342724250237, 5.45925815985743e-137 ], "spearman": [ 0.7127027118216717, 2.657235544077791e-117 ], "nsamples": 750 }, "images": { "pearson": [ 0.8457064177556566, 3.3443785854353087e-206 ], "spearman": [ 0.8049272327834235, 1.0191671573943109e-171 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.8241300010910765, 7.676405298271845e-187 ], "spearman": [ 0.8223616193261, 2.2573313609528353e-185 ], "nsamples": 750 }, "tweet-news": { "pearson": [ 0.7181193562444603, 6.957548479833527e-120 ], "spearman": [ 0.6469747801580121, 3.7447154677630655e-90 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7207630295977413, "mean": 0.7340122880671359, "wmean": 0.7489618969504681 }, "spearman": { "all": 0.6776154802540348, "mean": 0.6996017929965687, "wmean": 0.713707005140278 } } }, "STS15": { "answers-forums": { "pearson": [ 0.7104221785245369, 7.180350548638676e-59 ], "spearman": [ 0.7157297042374495, 4.032601594392119e-60 ], "nsamples": 375 }, "answers-students": { "pearson": [ 0.6841750672458253, 1.2325511825986507e-104 ], "spearman": [ 0.6893747601179209, 7.795024410550969e-107 ], "nsamples": 750 }, "belief": { "pearson": [ 0.749669412292463, 7.484565874638222e-69 ], "spearman": [ 0.7689611847810851, 1.80218707214869e-74 ], "nsamples": 375 }, "headlines": { "pearson": [ 0.8103746195832492, 7.843957651748014e-176 ], "spearman": [ 0.8082136911038486, 3.484989273824295e-174 ], "nsamples": 750 }, "images": { "pearson": [ 0.8764187325324918, 1.372705232924295e-239 ], "spearman": [ 0.8823684545353031, 4.309886303233611e-247 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.784979164135427, "mean": 0.7662120020357133, "wmean": 0.7752535536925166 }, "spearman": { "all": 0.7953840242507239, "mean": 0.7729295589551215, "wmean": 0.780575587566585 } } }, "STS16": { "answer-answer": { "pearson": [ 0.7062392803273457, 1.1334345573551127e-39 ], "spearman": [ 0.7160237031906911, 3.2192023033831215e-41 ], "nsamples": 254 }, "headlines": { "pearson": [ 0.7684075132132632, 8.674627222788605e-50 ], "spearman": [ 0.7792317750235211, 4.923195475618921e-52 ], "nsamples": 249 }, "plagiarism": { "pearson": [ 0.814329742038296, 8.73007637868767e-56 ], "spearman": [ 0.8315423069613309, 3.8270415984455366e-60 ], "nsamples": 230 }, "postediting": { "pearson": [ 0.805735873745958, 5.663717248324878e-57 ], "spearman": [ 0.8273031164960505, 1.5191782859949922e-62 ], "nsamples": 244 }, "question-question": { "pearson": [ 0.7827391505554354, 1.635608527236305e-44 ], "spearman": [ 0.7928031349595721, 2.1346612440747243e-46 ], "nsamples": 209 }, "all": { "pearson": { "all": 0.7742750560870418, "mean": 0.7754903119760597, "wmean": 0.7742041520422902 }, "spearman": { "all": 0.7904588398710303, "mean": 0.7893808073262332, "wmean": 0.7881208084519268 } } }, "eval_senteval-STS12": 0.6274520092064783, "eval_senteval-STS13": 0.7537361964068123, "eval_senteval-STS14": 0.6776154802540348, "eval_senteval-STS15": 0.7953840242507239, "eval_senteval-STS16": 0.7904588398710303, "eval_senteval-STSBenchmark": 0.7568905857914078, "eval_senteval-SICKRelatedness": 0.6838845953897259, "eval_senteval-avg_sts_7": 0.7264888187386019, "eval_senteval-MR": 80.71, "eval_senteval-CR": 87.05, "eval_senteval-SUBJ": 95.04, "eval_senteval-MPQA": 89.21, "eval_senteval-SST2": 84.06, "eval_senteval-TREC": 80.61, "eval_senteval-MRPC": 70.58, "eval_senteval-avg_transfer": 83.89428571428572 }