| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.7078537910606651, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7005999583866984, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7192800747133679, | |
| 3.077700364928066e-239 | |
| ], | |
| "spearman": [ | |
| 0.7379237505860065, | |
| 4.277474410967731e-258 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.6983884458054606, | |
| 2.940291533445568e-202 | |
| ], | |
| "spearman": [ | |
| 0.7136101793848104, | |
| 4.3889157567246634e-215 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7084868825859301, | |
| "mean": 0.7085074371931644, | |
| "wmean": 0.7083274482665213 | |
| }, | |
| "spearman": { | |
| "all": 0.7134156978222834, | |
| "mean": 0.7173779627858385, | |
| "wmean": 0.7091681993527809 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7474648285643389, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6940199688267912, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7666627901165247, | |
| 6.873435561530908e-98 | |
| ], | |
| "spearman": [ | |
| 0.7253075282545358, | |
| 8.79855875263332e-83 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7471456909275643, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6877954280401497, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7481870904024901, | |
| "mean": 0.753757769869476, | |
| "wmean": 0.7482733900269867 | |
| }, | |
| "spearman": { | |
| "all": 0.6925290217821645, | |
| "mean": 0.7023743083738255, | |
| "wmean": 0.6925064669891857 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 82.2, | |
| "acc": 81.86, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 88.38, | |
| "acc": 87.87, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.78, | |
| "acc": 95.45, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 89.44, | |
| "acc": 89.26, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 84.29, | |
| "acc": 85.83, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 80.47, | |
| "acc": 88.2, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 70.61, | |
| "acc": 69.28, | |
| "f1": 80.28, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.4308787229359875, | |
| 2.951518773589667e-35 | |
| ], | |
| "spearman": [ | |
| 0.43588907674191796, | |
| 3.949785529584259e-36 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.7685772608624017, | |
| 3.0060842440727446e-147 | |
| ], | |
| "spearman": [ | |
| 0.7784022398247612, | |
| 2.1180197452085128e-153 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.48420310815012313, | |
| 2.3512310040226806e-28 | |
| ], | |
| "spearman": [ | |
| 0.5830380352830036, | |
| 3.802237461067692e-43 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7022395609517124, | |
| 1.7609879155318726e-112 | |
| ], | |
| "spearman": [ | |
| 0.6542675353808592, | |
| 7.864813884891985e-93 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.5667702919910472, | |
| 2.7779222845650156e-35 | |
| ], | |
| "spearman": [ | |
| 0.6138135967602001, | |
| 1.1234322018162595e-42 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.5248106860340233, | |
| "mean": 0.5905337889782544, | |
| "wmean": 0.6031731762251964 | |
| }, | |
| "spearman": { | |
| "all": 0.551472253212969, | |
| "mean": 0.6130820967981483, | |
| "wmean": 0.6158124910755702 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.5506969449768325, | |
| 2.2186867879566377e-16 | |
| ], | |
| "spearman": [ | |
| 0.5845168173317136, | |
| 1.0457395906514476e-18 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7701375188545448, | |
| 3.323272248926893e-148 | |
| ], | |
| "spearman": [ | |
| 0.7914045646334811, | |
| 4.792158949648843e-162 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.7396168929190757, | |
| 3.380330666101667e-98 | |
| ], | |
| "spearman": [ | |
| 0.7366350658057337, | |
| 5.06557475163485e-97 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6820859764553678, | |
| "mean": 0.6868171189168177, | |
| "wmean": 0.7310732924460877 | |
| }, | |
| "spearman": { | |
| "all": 0.7229160162899463, | |
| "mean": 0.7041854825903094, | |
| "wmean": 0.7448529159118809 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.4362756518936318, | |
| 2.4689546739502166e-22 | |
| ], | |
| "spearman": [ | |
| 0.4356265722089346, | |
| 2.891472829972546e-22 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7383916191185764, | |
| 6.413175758096686e-53 | |
| ], | |
| "spearman": [ | |
| 0.7137105870085976, | |
| 5.373218095899081e-48 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7468974428675955, | |
| 1.0765636396917894e-134 | |
| ], | |
| "spearman": [ | |
| 0.7245056476016037, | |
| 5.235979722924423e-123 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.796822057327234, | |
| 7.832124393347635e-166 | |
| ], | |
| "spearman": [ | |
| 0.7650434028457997, | |
| 4.13659539181102e-145 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.7857012591480514, | |
| 3.5241880397200474e-158 | |
| ], | |
| "spearman": [ | |
| 0.7967713894609839, | |
| 8.507992228801468e-166 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.6595728752282025, | |
| 7.952319515520837e-95 | |
| ], | |
| "spearman": [ | |
| 0.6114329574279981, | |
| 4.316968711309796e-78 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6649722439404772, | |
| "mean": 0.6939434842638819, | |
| "wmean": 0.7092231346709386 | |
| }, | |
| "spearman": { | |
| "all": 0.6332541978552203, | |
| "mean": 0.6745150927589862, | |
| "wmean": 0.688922715093037 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.6550074272217491, | |
| 2.5452363132722557e-47 | |
| ], | |
| "spearman": [ | |
| 0.6570961934421593, | |
| 1.034283039715521e-47 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.6832984945676881, | |
| 2.8640939149258605e-104 | |
| ], | |
| "spearman": [ | |
| 0.6907992077474371, | |
| 1.9110197999110346e-107 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.6709117751011916, | |
| 2.2248924681984537e-50 | |
| ], | |
| "spearman": [ | |
| 0.6943992418065011, | |
| 2.9220535633271436e-55 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.8086773826726481, | |
| 1.5503098286475848e-174 | |
| ], | |
| "spearman": [ | |
| 0.8205804721828502, | |
| 6.549512205685779e-184 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8326574715302321, | |
| 3.6753187356312923e-194 | |
| ], | |
| "spearman": [ | |
| 0.8494884877285356, | |
| 6.66575077788996e-210 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7334203057310366, | |
| "mean": 0.7301105102187018, | |
| "wmean": 0.7468982374830097 | |
| }, | |
| "spearman": { | |
| "all": 0.7527302594998441, | |
| "mean": 0.7424727205814967, | |
| "wmean": 0.7591539713207883 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.6774976875587965, | |
| 1.7801175736052472e-35 | |
| ], | |
| "spearman": [ | |
| 0.6996537261374689, | |
| 1.1481023213168017e-38 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7723091430702096, | |
| 1.3899358276799273e-50 | |
| ], | |
| "spearman": [ | |
| 0.797440407232274, | |
| 4.088502753480497e-56 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.8114637165686283, | |
| 4.19426284677851e-55 | |
| ], | |
| "spearman": [ | |
| 0.8340872996890834, | |
| 7.881725319141058e-61 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.7526664902410026, | |
| 8.06271051079258e-46 | |
| ], | |
| "spearman": [ | |
| 0.7786484784235972, | |
| 6.807428083289534e-51 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7694288722010691, | |
| 3.6047724274094633e-42 | |
| ], | |
| "spearman": [ | |
| 0.785768908279152, | |
| 4.540569131264119e-45 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7474943176404166, | |
| "mean": 0.7566731819279412, | |
| "wmean": 0.7550483153322336 | |
| }, | |
| "spearman": { | |
| "all": 0.7752096236911921, | |
| "mean": 0.7791197639523151, | |
| "wmean": 0.7776818864535776 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.551472253212969, | |
| "eval_senteval-STS13": 0.7229160162899463, | |
| "eval_senteval-STS14": 0.6332541978552203, | |
| "eval_senteval-STS15": 0.7527302594998441, | |
| "eval_senteval-STS16": 0.7752096236911921, | |
| "eval_senteval-STSBenchmark": 0.7134156978222834, | |
| "eval_senteval-SICKRelatedness": 0.6925290217821645, | |
| "eval_senteval-avg_sts_7": 0.6916467243076598, | |
| "eval_senteval-MR": 82.2, | |
| "eval_senteval-CR": 88.38, | |
| "eval_senteval-SUBJ": 95.78, | |
| "eval_senteval-MPQA": 89.44, | |
| "eval_senteval-SST2": 84.29, | |
| "eval_senteval-TREC": 80.47, | |
| "eval_senteval-MRPC": 70.61, | |
| "eval_senteval-avg_transfer": 84.45285714285716 | |
| } | |