| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.759143306453737, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.732317209733238, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7746081783379711, | |
| 2.2556430824992415e-300 | |
| ], | |
| "spearman": [ | |
| 0.7800424504336374, | |
| 2.5466091811087085e-307 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7580850707783013, | |
| 6.51859239939566e-258 | |
| ], | |
| "spearman": [ | |
| 0.7560754511600052, | |
| 8.850873074119353e-256 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7620304519086235, | |
| "mean": 0.7639455185233365, | |
| "wmean": 0.7616627780381048 | |
| }, | |
| "spearman": { | |
| "all": 0.7488342650348635, | |
| "mean": 0.7561450371089601, | |
| "wmean": 0.7444116088962087 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7660762263543586, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6943387839868075, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7787463535407387, | |
| 6.565381473913046e-103 | |
| ], | |
| "spearman": [ | |
| 0.724076491124677, | |
| 2.2464753888818782e-82 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7585953923013815, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6847050750070173, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7629449380408607, | |
| "mean": 0.7678059907321595, | |
| "wmean": 0.7630014801283258 | |
| }, | |
| "spearman": { | |
| "all": 0.6909722716792152, | |
| "mean": 0.7010401167061673, | |
| "wmean": 0.6910551705512789 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 81.13, | |
| "acc": 80.72, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 87.1, | |
| "acc": 85.35, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.37, | |
| "acc": 94.83, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 88.87, | |
| "acc": 88.84, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 83.72, | |
| "acc": 84.84, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 79.31, | |
| "acc": 85.4, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 71.32, | |
| "acc": 68.93, | |
| "f1": 78.8, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.4802609709856894, | |
| 1.5548549952823705e-44 | |
| ], | |
| "spearman": [ | |
| 0.4768543559246008, | |
| 7.614791350529852e-44 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.8263319209416204, | |
| 1.080221265522035e-188 | |
| ], | |
| "spearman": [ | |
| 0.8252791857619619, | |
| 8.356577398931468e-188 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.5184489923467462, | |
| 6.1406360676578446e-33 | |
| ], | |
| "spearman": [ | |
| 0.6230131131079721, | |
| 1.0590062591812851e-50 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7315874470125636, | |
| 1.414322871092147e-126 | |
| ], | |
| "spearman": [ | |
| 0.6731301571473148, | |
| 4.0960228359103546e-100 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.6259156885670977, | |
| 8.739441951714742e-45 | |
| ], | |
| "spearman": [ | |
| 0.6148988213247114, | |
| 7.33074623929432e-43 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.623383289827507, | |
| "mean": 0.6365090039707434, | |
| "wmean": 0.6487592346944445 | |
| }, | |
| "spearman": { | |
| "all": 0.606079829047924, | |
| "mean": 0.6426351266533122, | |
| "wmean": 0.6476047048746869 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.5661863937046698, | |
| 2.0580012282455086e-17 | |
| ], | |
| "spearman": [ | |
| 0.5912700246061035, | |
| 3.3253838782016723e-19 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7825601400543155, | |
| 4.2299360961974125e-156 | |
| ], | |
| "spearman": [ | |
| 0.7841745697183816, | |
| 3.6476628073513e-157 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.794529783804957, | |
| 3.311330886766186e-123 | |
| ], | |
| "spearman": [ | |
| 0.7834024923042585, | |
| 1.4890472209808275e-117 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7471361547131256, | |
| "mean": 0.7144254391879808, | |
| "wmean": 0.7597736947770001 | |
| }, | |
| "spearman": { | |
| "all": 0.7564536182897523, | |
| "mean": 0.7196156955429145, | |
| "wmean": 0.7595798400813525 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.5270887834584671, | |
| 1.541470939768766e-33 | |
| ], | |
| "spearman": [ | |
| 0.5112851401228272, | |
| 2.4363986591055635e-31 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7735386895034391, | |
| 5.576824423082716e-61 | |
| ], | |
| "spearman": [ | |
| 0.721172124183031, | |
| 1.986170265393856e-49 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7545133819317689, | |
| 5.912159102599232e-139 | |
| ], | |
| "spearman": [ | |
| 0.7126401838119187, | |
| 2.8437071308486684e-117 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8323456329452497, | |
| 6.921690784176308e-194 | |
| ], | |
| "spearman": [ | |
| 0.7937433816131229, | |
| 1.1474485172111374e-163 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8297923551143387, | |
| 1.175057263022769e-191 | |
| ], | |
| "spearman": [ | |
| 0.826236205234858, | |
| 1.3018028294306147e-188 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.7558799639584549, | |
| 9.788201645414497e-140 | |
| ], | |
| "spearman": [ | |
| 0.6845652322701915, | |
| 8.460582099195036e-105 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7159862351857734, | |
| "mean": 0.7455264678186196, | |
| "wmean": 0.7596400159652537 | |
| }, | |
| "spearman": { | |
| "all": 0.6704800621775756, | |
| "mean": 0.7082737112059916, | |
| "wmean": 0.7224849873354 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.7317275686935678, | |
| 4.532297271121567e-64 | |
| ], | |
| "spearman": [ | |
| 0.7407727747177331, | |
| 1.972401483293033e-66 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.7050737483230498, | |
| 9.136794973091405e-114 | |
| ], | |
| "spearman": [ | |
| 0.7064217578173171, | |
| 2.2091874983666628e-114 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.783394736380794, | |
| 4.7974747142716894e-79 | |
| ], | |
| "spearman": [ | |
| 0.7998064345059992, | |
| 1.0770053295383067e-84 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.813583787598092, | |
| 2.5592603175894345e-178 | |
| ], | |
| "spearman": [ | |
| 0.8095733522782433, | |
| 3.220682833365799e-175 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8641888241232765, | |
| 2.5653257492197874e-225 | |
| ], | |
| "spearman": [ | |
| 0.869236692916196, | |
| 4.9395933314044357e-231 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7901758719755425, | |
| "mean": 0.779593733023756, | |
| "wmean": 0.7851018781453998 | |
| }, | |
| "spearman": { | |
| "all": 0.7993930145058169, | |
| "mean": 0.7851622024470977, | |
| "wmean": 0.7888803519059057 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.732325384111771, | |
| 6.031948033432114e-44 | |
| ], | |
| "spearman": [ | |
| 0.7331760524557248, | |
| 4.2918539327373104e-44 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7751129917838497, | |
| 3.6445698860447696e-51 | |
| ], | |
| "spearman": [ | |
| 0.7759120405190852, | |
| 2.4798809912199763e-51 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.8073018229023751, | |
| 3.908432651513822e-54 | |
| ], | |
| "spearman": [ | |
| 0.8201128848961844, | |
| 3.380055386535779e-57 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.8262293698872258, | |
| 2.999128754083028e-62 | |
| ], | |
| "spearman": [ | |
| 0.8467835716176686, | |
| 2.7503849185111653e-68 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7751346076550414, | |
| 3.733475629577834e-43 | |
| ], | |
| "spearman": [ | |
| 0.7850104595896853, | |
| 6.270227463768292e-45 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7817024776005017, | |
| "mean": 0.7832208352680526, | |
| "wmean": 0.7827118895771512 | |
| }, | |
| "spearman": { | |
| "all": 0.7931904794778135, | |
| "mean": 0.7921990018156697, | |
| "wmean": 0.7915153089950118 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.606079829047924, | |
| "eval_senteval-STS13": 0.7564536182897523, | |
| "eval_senteval-STS14": 0.6704800621775756, | |
| "eval_senteval-STS15": 0.7993930145058169, | |
| "eval_senteval-STS16": 0.7931904794778135, | |
| "eval_senteval-STSBenchmark": 0.7488342650348635, | |
| "eval_senteval-SICKRelatedness": 0.6909722716792152, | |
| "eval_senteval-avg_sts_7": 0.7236290771732802, | |
| "eval_senteval-MR": 81.13, | |
| "eval_senteval-CR": 87.1, | |
| "eval_senteval-SUBJ": 95.37, | |
| "eval_senteval-MPQA": 88.87, | |
| "eval_senteval-SST2": 83.72, | |
| "eval_senteval-TREC": 79.31, | |
| "eval_senteval-MRPC": 71.32, | |
| "eval_senteval-avg_transfer": 83.83142857142856 | |
| } | |