| { |
| "method": "Wilcoxon signed-rank test on paired per-sample first-answer-token log-probabilities", |
| "description": "Log P(first correct answer token | input + thoughts + ### separator). v3: Fixed extraction point (### position) AND tokenization (leading space).", |
| "fixes": [ |
| "Appended ### separator token so logits[-1] is at the answer-predicting position", |
| "Tokenized answer with leading space to match model generation format (### Sally...)" |
| ], |
| "bonferroni_k": 5, |
| "adjusted_alpha": 0.01, |
| "results": { |
| "prosqa_id": { |
| "n": 500, |
| "median_diff": 0.0, |
| "mean_diff": 1.9073484480713887e-09, |
| "std_diff": 3.2808333491232266e-08, |
| "wilcoxon_stat": 59.0, |
| "p_value": 0.22669185385013768, |
| "p_bonferroni": 1.0, |
| "effect_size_r": 0.05406474709320131, |
| "z_score": 1.208924496867323, |
| "m3_median_logprob": 0.0, |
| "m5_median_logprob": 0.0, |
| "m3_mean_logprob": -3.814696910353632e-09, |
| "m5_mean_logprob": -1.9073484622822432e-09, |
| "m3_prob_median": 1.0, |
| "m5_prob_median": 1.0, |
| "direction": "equal" |
| }, |
| "7hop": { |
| "n": 1000, |
| "median_diff": 0.0, |
| "mean_diff": 1.5545431362859574e-05, |
| "std_diff": 0.000491480095991523, |
| "wilcoxon_stat": 817.5, |
| "p_value": 2.825225828337809e-10, |
| "p_bonferroni": 1.4126129141689046e-09, |
| "effect_size_r": 0.19947892842275394, |
| "z_score": 6.308077590256019, |
| "m3_median_logprob": 0.0, |
| "m5_median_logprob": 0.0, |
| "m3_mean_logprob": -1.5582147784044765e-05, |
| "m5_mean_logprob": -3.671642118519003e-08, |
| "m3_prob_median": 1.0, |
| "m5_prob_median": 1.0, |
| "direction": "equal" |
| }, |
| "8hop": { |
| "n": 1000, |
| "median_diff": 0.0, |
| "mean_diff": -2.5510785491178468e-08, |
| "std_diff": 3.456691393519963e-07, |
| "wilcoxon_stat": 556.0, |
| "p_value": 6.337349739272441e-11, |
| "p_bonferroni": 3.1686748696362206e-10, |
| "effect_size_r": 0.20667243975845923, |
| "z_score": 6.535556392206708, |
| "m3_median_logprob": 0.0, |
| "m5_median_logprob": 0.0, |
| "m3_mean_logprob": -1.1801690376955776e-08, |
| "m5_mean_logprob": -3.7312475868134245e-08, |
| "m3_prob_median": 1.0, |
| "m5_prob_median": 1.0, |
| "direction": "equal" |
| }, |
| "dag": { |
| "n": 1000, |
| "median_diff": 0.0, |
| "mean_diff": -3.8504580892606554e-08, |
| "std_diff": 2.2670660050335698e-07, |
| "wilcoxon_stat": 1072.5, |
| "p_value": 7.740160468120871e-16, |
| "p_bonferroni": 3.870080234060436e-15, |
| "effect_size_r": 0.25482388020266233, |
| "z_score": 8.058238636423024, |
| "m3_median_logprob": 0.0, |
| "m5_median_logprob": 0.0, |
| "m3_mean_logprob": -6.675716875292892e-09, |
| "m5_mean_logprob": -4.518029776789945e-08, |
| "m3_prob_median": 1.0, |
| "m5_prob_median": 1.0, |
| "direction": "equal" |
| }, |
| "dense": { |
| "n": 1000, |
| "median_diff": 0.0, |
| "mean_diff": 1.8954253092857698e-06, |
| "std_diff": 6.556349770050734e-05, |
| "wilcoxon_stat": 3283.5, |
| "p_value": 4.2083786008929025e-43, |
| "p_bonferroni": 2.104189300446451e-42, |
| "effect_size_r": 0.4352494406715319, |
| "z_score": 13.763795828363676, |
| "m3_median_logprob": 0.0, |
| "m5_median_logprob": 0.0, |
| "m3_mean_logprob": -2.1150066327138006e-06, |
| "m5_mean_logprob": -2.1958132342803083e-07, |
| "m3_prob_median": 1.0, |
| "m5_prob_median": 1.0, |
| "direction": "equal" |
| } |
| } |
| } |
