| { |
| "normal": { |
| "all": { |
| "targets": 169144, |
| "max_abs_delta": 0.0, |
| "nll": 4.144884785078792, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "length_10-16": { |
| "targets": 56601, |
| "max_abs_delta": 0.0, |
| "nll": 3.639815218349103, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "before_first": { |
| "targets": 83341, |
| "max_abs_delta": 0.0, |
| "nll": 3.6582595163808413, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "after_first": { |
| "targets": 85803, |
| "max_abs_delta": 0.0, |
| "nll": 4.6175470057303505, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "after_second": { |
| "targets": 15328, |
| "max_abs_delta": 0.0, |
| "nll": 4.777278997806058, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "length_17-64": { |
| "targets": 48171, |
| "max_abs_delta": 0.0, |
| "nll": 4.587060177184653, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "length_2-9": { |
| "targets": 25421, |
| "max_abs_delta": 0.0, |
| "nll": 3.312832962033229, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "length_65-128": { |
| "targets": 22018, |
| "max_abs_delta": 0.0, |
| "nll": 4.893068085736666, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "length_129+": { |
| "targets": 16933, |
| "max_abs_delta": 0.0, |
| "nll": 4.851522782663743, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| "C_read_off": { |
| "all": { |
| "targets": 169144, |
| "max_abs_delta": 9.837771174497902, |
| "nll": 4.145392488348739, |
| "delta_nats": 0.0005077032699674228, |
| "ppl_ratio": 1.0005078321730865, |
| "doc_bootstrap_delta_95pct": [ |
| -0.00025010107714061553, |
| 0.0014034107223756765 |
| ] |
| }, |
| "length_10-16": { |
| "targets": 56601, |
| "max_abs_delta": 4.806448265910149, |
| "nll": 3.638001250299956, |
| "delta_nats": -0.0018139680491534834, |
| "ppl_ratio": 0.9981876761965349, |
| "doc_bootstrap_delta_95pct": [ |
| -0.00262191558572389, |
| -0.0008871909215438053 |
| ] |
| }, |
| "before_first": { |
| "targets": 83341, |
| "max_abs_delta": 0.0, |
| "nll": 3.6582595163808413, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "after_first": { |
| "targets": 85803, |
| "max_abs_delta": 9.837771174497902, |
| "nll": 4.618547844417748, |
| "delta_nats": 0.001000838687404517, |
| "ppl_ratio": 1.0010013396935717, |
| "doc_bootstrap_delta_95pct": [ |
| -0.0006520181965873735, |
| 0.0027062239328431454 |
| ] |
| }, |
| "after_second": { |
| "targets": 15328, |
| "max_abs_delta": 0.9968395233154297, |
| "nll": 4.779742083394126, |
| "delta_nats": 0.002463085588067861, |
| "ppl_ratio": 1.0024661214754134, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0007099402930980536, |
| 0.004136419760806341 |
| ] |
| }, |
| "length_17-64": { |
| "targets": 48171, |
| "max_abs_delta": 9.837771174497902, |
| "nll": 4.588660769801097, |
| "delta_nats": 0.00160059261644365, |
| "ppl_ratio": 1.0016018742485047, |
| "doc_bootstrap_delta_95pct": [ |
| -0.0007734870164671116, |
| 0.004409285206532606 |
| ] |
| }, |
| "length_2-9": { |
| "targets": 25421, |
| "max_abs_delta": 0.0, |
| "nll": 3.312832962033229, |
| "delta_nats": 0.0, |
| "ppl_ratio": 1.0, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "length_65-128": { |
| "targets": 22018, |
| "max_abs_delta": 1.317411184310913, |
| "nll": 4.896818332041356, |
| "delta_nats": 0.003750246304688991, |
| "ppl_ratio": 1.0037572872774045, |
| "doc_bootstrap_delta_95pct": [ |
| 0.0024134010107044013, |
| 0.005072407909455955 |
| ] |
| }, |
| "length_129+": { |
| "targets": 16933, |
| "max_abs_delta": 0.9968395233154297, |
| "nll": 4.853227873160534, |
| "delta_nats": 0.0017050904967906946, |
| "ppl_ratio": 1.0017065449901552, |
| "doc_bootstrap_delta_95pct": [ |
| 5.13874762613377e-05, |
| 0.003163192791394106 |
| ] |
| } |
| }, |
| "D_read_off": { |
| "all": { |
| "targets": 169144, |
| "max_abs_delta": 14.808307343628258, |
| "nll": 4.563928485893732, |
| "delta_nats": 0.4190437008148539, |
| "ppl_ratio": 1.5205068007244975, |
| "doc_bootstrap_delta_95pct": [ |
| 0.4084816376528276, |
| 0.42974781810001217 |
| ] |
| }, |
| "length_10-16": { |
| "targets": 56601, |
| "max_abs_delta": 14.808307343628258, |
| "nll": 3.947487452312927, |
| "delta_nats": 0.3076722339636419, |
| "ppl_ratio": 1.3602550704497973, |
| "doc_bootstrap_delta_95pct": [ |
| 0.29860700436242454, |
| 0.3170346994210013 |
| ] |
| }, |
| "before_first": { |
| "targets": 83341, |
| "max_abs_delta": 9.79684591293335, |
| "nll": 3.8673202743180943, |
| "delta_nats": 0.2090607579371381, |
| "ppl_ratio": 1.2325198816208498, |
| "doc_bootstrap_delta_95pct": [ |
| 0.20296126497060282, |
| 0.2153221963653742 |
| ] |
| }, |
| "after_first": { |
| "targets": 85803, |
| "max_abs_delta": 14.808307343628258, |
| "nll": 5.240548475415436, |
| "delta_nats": 0.6230014696850796, |
| "ppl_ratio": 1.864515939768767, |
| "doc_bootstrap_delta_95pct": [ |
| 0.6098227767522754, |
| 0.6367022716297845 |
| ] |
| }, |
| "after_second": { |
| "targets": 15328, |
| "max_abs_delta": 9.862558734603226, |
| "nll": 5.4398346708231395, |
| "delta_nats": 0.6625556730170808, |
| "ppl_ratio": 1.939743354864635, |
| "doc_bootstrap_delta_95pct": [ |
| 0.6337338080399308, |
| 0.6941607536956912 |
| ] |
| }, |
| "length_17-64": { |
| "targets": 48171, |
| "max_abs_delta": 13.812182898342144, |
| "nll": 5.070832321187806, |
| "delta_nats": 0.48377214400309654, |
| "ppl_ratio": 1.6221819796464214, |
| "doc_bootstrap_delta_95pct": [ |
| 0.4675586082944131, |
| 0.49992120057822614 |
| ] |
| }, |
| "length_2-9": { |
| "targets": 25421, |
| "max_abs_delta": 9.79684591293335, |
| "nll": 3.5187845186227595, |
| "delta_nats": 0.20595155658954448, |
| "ppl_ratio": 1.228693680441276, |
| "doc_bootstrap_delta_95pct": [ |
| 0.1965656321666891, |
| 0.2159041335973384 |
| ] |
| }, |
| "length_65-128": { |
| "targets": 22018, |
| "max_abs_delta": 10.74917882680893, |
| "nll": 5.536569118899567, |
| "delta_nats": 0.6435010331629004, |
| "ppl_ratio": 1.9031321581362317, |
| "doc_bootstrap_delta_95pct": [ |
| 0.6137474662493506, |
| 0.6740910758874403 |
| ] |
| }, |
| "length_129+": { |
| "targets": 16933, |
| "max_abs_delta": 9.862558734603226, |
| "nll": 5.486748873671533, |
| "delta_nats": 0.6352260910077897, |
| "ppl_ratio": 1.8874488284296478, |
| "doc_bootstrap_delta_95pct": [ |
| 0.6040921677702079, |
| 0.6676251294704127 |
| ] |
| } |
| }, |
| "interpretation": "within-model read interventions, not retrained ablation, not six-axis ranking; corpus sampled before this model, 100111-word dev subset; document bootstrap ignores cross-document duplicate/dependence" |
| } |
|
|
