| { |
| "method": "Wilcoxon signed-rank test on paired per-sample first-answer-token log-probabilities", |
| "description": "Log P(first correct answer token | input + thoughts) under each model. No answer tokens appended; logits from last position predict first answer token.", |
| "bonferroni_k": 5, |
| "adjusted_alpha": 0.01, |
| "results": { |
| "prosqa_id": { |
| "n": 500, |
| "median_diff": 5.5448455810546875, |
| "mean_diff": 5.900060585021973, |
| "std_diff": 4.304266040095551, |
| "wilcoxon_stat": 2294.0, |
| "p_value": 9.558665789472875e-78, |
| "p_bonferroni": 4.779332894736438e-77, |
| "effect_size_r": 0.8347188840290514, |
| "z_score": 18.664881667917225, |
| "m3_median_logprob": -36.25274085998535, |
| "m5_median_logprob": -30.16720485687256, |
| "m3_mean_logprob": -36.290785385131834, |
| "m5_mean_logprob": -30.39072480010986, |
| "direction": "M5 > M3" |
| }, |
| "7hop": { |
| "n": 1000, |
| "median_diff": 4.609404563903809, |
| "mean_diff": 4.781453802108764, |
| "std_diff": 3.692269596981685, |
| "wilcoxon_stat": 14216.0, |
| "p_value": 3.419355734386437e-147, |
| "p_bonferroni": 1.7096778671932184e-146, |
| "effect_size_r": 0.8170330113820354, |
| "z_score": 25.83685239513508, |
| "m3_median_logprob": -34.421878814697266, |
| "m5_median_logprob": -30.148743629455566, |
| "m3_mean_logprob": -34.93062486076355, |
| "m5_mean_logprob": -30.149171058654787, |
| "direction": "M5 > M3" |
| }, |
| "8hop": { |
| "n": 1000, |
| "median_diff": 4.223997116088867, |
| "mean_diff": 4.518639762878418, |
| "std_diff": 3.4717507040890796, |
| "wilcoxon_stat": 13275.0, |
| "p_value": 2.3666197544095806e-148, |
| "p_bonferroni": 1.1833098772047903e-147, |
| "effect_size_r": 0.8202902881460207, |
| "z_score": 25.93985653057244, |
| "m3_median_logprob": -34.220977783203125, |
| "m5_median_logprob": -29.836036682128906, |
| "m3_mean_logprob": -34.49567288208008, |
| "m5_mean_logprob": -29.97703311920166, |
| "direction": "M5 > M3" |
| }, |
| "dag": { |
| "n": 1000, |
| "median_diff": 5.726862907409668, |
| "mean_diff": 5.837504280090332, |
| "std_diff": 3.5533476769526535, |
| "wilcoxon_stat": 4161.0, |
| "p_value": 7.992937269512403e-160, |
| "p_bonferroni": 3.9964686347562014e-159, |
| "effect_size_r": 0.8518384501300393, |
| "z_score": 26.937497009186796, |
| "m3_median_logprob": -35.94425582885742, |
| "m5_median_logprob": -30.378952980041504, |
| "m3_mean_logprob": -36.28265309143067, |
| "m5_mean_logprob": -30.44514881134033, |
| "direction": "M5 > M3" |
| }, |
| "dense": { |
| "n": 1000, |
| "median_diff": 4.454345703125, |
| "mean_diff": 4.632727506637573, |
| "std_diff": 3.4062677335293263, |
| "wilcoxon_stat": 12447.0, |
| "p_value": 2.237606774559982e-149, |
| "p_bonferroni": 1.118803387279991e-148, |
| "effect_size_r": 0.82315641477788, |
| "z_score": 26.030491412763865, |
| "m3_median_logprob": -34.61273956298828, |
| "m5_median_logprob": -30.307485580444336, |
| "m3_mean_logprob": -35.10507657432556, |
| "m5_mean_logprob": -30.47234906768799, |
| "direction": "M5 > M3" |
| } |
| } |
| } |
