| { | |
| "generated": "2026-08-20 11:17:21", | |
| "rule": "macro and corporate are carried side by side as separate columns and are NEVER averaged", | |
| "note": "Ranges over a handful of seeds. No significance test, no p-value.", | |
| "seeds": [ | |
| 42, | |
| 1, | |
| 2 | |
| ], | |
| "tolerance": 0.01, | |
| "precision": { | |
| "fp16": true, | |
| "bf16": false, | |
| "reason": "NVIDIA GeForce GTX 1660 Ti sm_75 no native bf16" | |
| }, | |
| "metrics": { | |
| "macro_f1": { | |
| "name": "macro real held-out F1", | |
| "units": "pts", | |
| "scale": 100.0 | |
| }, | |
| "corp_f1": { | |
| "name": "corporate F1", | |
| "units": "pts", | |
| "scale": 100.0 | |
| }, | |
| "probe_acc": { | |
| "name": "probe accuracy", | |
| "units": "pts", | |
| "scale": 100.0 | |
| }, | |
| "pairs_solved": { | |
| "name": "minimal pairs solved", | |
| "units": "pairs", | |
| "scale": 1.0 | |
| }, | |
| "agree_acc": { | |
| "name": "polarity-agreeing probe acc", | |
| "units": "pts", | |
| "scale": 100.0 | |
| }, | |
| "pb_f1": { | |
| "name": "PhraseBank F1", | |
| "units": "pts", | |
| "scale": 100.0 | |
| } | |
| }, | |
| "configs": [ | |
| { | |
| "name": "u4@20k", | |
| "macro_share": 0.65, | |
| "unfreeze": 4, | |
| "total": 20000, | |
| "epochs": 16, | |
| "criterion": "macro_floored", | |
| "mix_policy": "fixed_total" | |
| }, | |
| { | |
| "name": "u6@32k", | |
| "macro_share": 0.65, | |
| "unfreeze": 6, | |
| "total": 32000, | |
| "epochs": 16, | |
| "criterion": "macro_floored", | |
| "mix_policy": "fixed_total" | |
| } | |
| ], | |
| "per_seed": [ | |
| { | |
| "config": "u4@20k", | |
| "seed": 42, | |
| "status": "LOADED", | |
| "dir": "models/mix65_unfreeze4_macro_floored_s42", | |
| "how": "run.json match in mix65_unfreeze4_macro_floored_s42", | |
| "metrics": { | |
| "macro_f1": 67.59165949395553, | |
| "corp_f1": 81.38670066473745, | |
| "probe_acc": 79.16666666666666, | |
| "pairs_solved": 12.0, | |
| "agree_acc": 57.14285714285714, | |
| "pb_f1": 86.91964125668869 | |
| } | |
| }, | |
| { | |
| "config": "u4@20k", | |
| "seed": 1, | |
| "status": "TRAINED", | |
| "dir": "models/rep_u4_20k_s1", | |
| "how": "trained by this script, 102.94 min", | |
| "metrics": { | |
| "macro_f1": 67.28569186992823, | |
| "corp_f1": 82.06353062408495, | |
| "probe_acc": 70.83333333333334, | |
| "pairs_solved": 7.0, | |
| "agree_acc": 50.0, | |
| "pb_f1": 85.065296617682 | |
| } | |
| }, | |
| { | |
| "config": "u4@20k", | |
| "seed": 2, | |
| "status": "TRAINED", | |
| "dir": "models/rep_u4_20k_s2", | |
| "how": "trained by this script, 136.91 min", | |
| "metrics": { | |
| "macro_f1": 67.41373179920018, | |
| "corp_f1": 82.31776419246067, | |
| "probe_acc": 64.58333333333334, | |
| "pairs_solved": 5.0, | |
| "agree_acc": 57.14285714285714, | |
| "pb_f1": 73.63131856850693 | |
| } | |
| }, | |
| { | |
| "config": "u6@32k", | |
| "seed": 42, | |
| "status": "LOADED", | |
| "dir": "models/mix65_unfreeze6_full32k", | |
| "how": "run.json match in mix65_unfreeze6_full32k", | |
| "metrics": { | |
| "macro_f1": 68.81987346760383, | |
| "corp_f1": 83.97251903033337, | |
| "probe_acc": 77.08333333333334, | |
| "pairs_solved": 8.0, | |
| "agree_acc": 64.28571428571429, | |
| "pb_f1": 73.07008355119325 | |
| } | |
| }, | |
| { | |
| "config": "u6@32k", | |
| "seed": 1, | |
| "status": "TRAINED", | |
| "dir": "models/rep_u6_32k_s1", | |
| "how": "trained by this script, 205.69 min", | |
| "metrics": { | |
| "macro_f1": 69.3386664919747, | |
| "corp_f1": 82.66029840632898, | |
| "probe_acc": 81.25, | |
| "pairs_solved": 11.0, | |
| "agree_acc": 57.14285714285714, | |
| "pb_f1": 65.74260663190499 | |
| } | |
| }, | |
| { | |
| "config": "u6@32k", | |
| "seed": 2, | |
| "status": "TRAINED", | |
| "dir": "models/rep_u6_32k_s2", | |
| "how": "trained by this script, 246.78 min", | |
| "metrics": { | |
| "macro_f1": 69.728009428257, | |
| "corp_f1": 84.15625933222256, | |
| "probe_acc": 81.25, | |
| "pairs_solved": 10.0, | |
| "agree_acc": 64.28571428571429, | |
| "pb_f1": 69.87313438147625 | |
| } | |
| } | |
| ], | |
| "vanilla": { | |
| "macro_f1": 39.03300821786394, | |
| "corp_f1": 70.23790161183014, | |
| "probe_acc": 41.66666666666667, | |
| "pairs_solved": 6.0, | |
| "agree_acc": 71.42857142857143, | |
| "pb_f1": 96.24820403323632 | |
| }, | |
| "aggregate": { | |
| "u4@20k": { | |
| "macro_f1": { | |
| "n": 3, | |
| "mean": 67.43036105436131, | |
| "std": 0.15366016345702269, | |
| "min": 67.28569186992823, | |
| "max": 67.59165949395553, | |
| "range": 0.3059676240273035 | |
| }, | |
| "corp_f1": { | |
| "n": 3, | |
| "mean": 81.9226651604277, | |
| "std": 0.4812505920023736, | |
| "min": 81.38670066473745, | |
| "max": 82.31776419246067, | |
| "range": 0.9310635277232251 | |
| }, | |
| "probe_acc": { | |
| "n": 3, | |
| "mean": 71.52777777777779, | |
| "std": 7.316426217258837, | |
| "min": 64.58333333333334, | |
| "max": 79.16666666666666, | |
| "range": 14.583333333333314 | |
| }, | |
| "pairs_solved": { | |
| "n": 3, | |
| "mean": 8.0, | |
| "std": 3.605551275463989, | |
| "min": 5.0, | |
| "max": 12.0, | |
| "range": 7.0 | |
| }, | |
| "agree_acc": { | |
| "n": 3, | |
| "mean": 54.76190476190476, | |
| "std": 4.12393049421161, | |
| "min": 50.0, | |
| "max": 57.14285714285714, | |
| "range": 7.142857142857139 | |
| }, | |
| "pb_f1": { | |
| "n": 3, | |
| "mean": 81.8720854809592, | |
| "std": 7.196688682239868, | |
| "min": 73.63131856850693, | |
| "max": 86.91964125668869, | |
| "range": 13.288322688181765 | |
| } | |
| }, | |
| "u6@32k": { | |
| "macro_f1": { | |
| "n": 3, | |
| "mean": 69.29551646261184, | |
| "std": 0.4556030888101787, | |
| "min": 68.81987346760383, | |
| "max": 69.728009428257, | |
| "range": 0.9081359606531691 | |
| }, | |
| "corp_f1": { | |
| "n": 3, | |
| "mean": 83.59635892296164, | |
| "std": 0.815841340417078, | |
| "min": 82.66029840632898, | |
| "max": 84.15625933222256, | |
| "range": 1.4959609258935842 | |
| }, | |
| "probe_acc": { | |
| "n": 3, | |
| "mean": 79.86111111111111, | |
| "std": 2.405626121623435, | |
| "min": 77.08333333333334, | |
| "max": 81.25, | |
| "range": 4.166666666666657 | |
| }, | |
| "pairs_solved": { | |
| "n": 3, | |
| "mean": 9.666666666666666, | |
| "std": 1.5275252316519468, | |
| "min": 8.0, | |
| "max": 11.0, | |
| "range": 3.0 | |
| }, | |
| "agree_acc": { | |
| "n": 3, | |
| "mean": 61.904761904761905, | |
| "std": 4.123930494211619, | |
| "min": 57.14285714285714, | |
| "max": 64.28571428571429, | |
| "range": 7.142857142857153 | |
| }, | |
| "pb_f1": { | |
| "n": 3, | |
| "mean": 69.56194152152483, | |
| "std": 3.673637195986982, | |
| "min": 65.74260663190499, | |
| "max": 73.07008355119325, | |
| "range": 7.327476919288259 | |
| } | |
| } | |
| }, | |
| "comparisons": [ | |
| { | |
| "a": "u4@20k", | |
| "b": "u6@32k", | |
| "metric": "macro_f1", | |
| "units": "pts", | |
| "gap": 1.86515540825053, | |
| "spread": 0.9081359606531691, | |
| "half_spread": 0.4540679803265846, | |
| "n_seeds": 3, | |
| "leader": "u6@32k", | |
| "ranges_overlap": false, | |
| "verdict": "RESOLVED", | |
| "a_min": 67.28569186992823, | |
| "a_max": 67.59165949395553, | |
| "b_min": 68.81987346760383, | |
| "b_max": 69.728009428257 | |
| }, | |
| { | |
| "a": "u4@20k", | |
| "b": "u6@32k", | |
| "metric": "corp_f1", | |
| "units": "pts", | |
| "gap": 1.6736937625339436, | |
| "spread": 1.4959609258935842, | |
| "half_spread": 0.7479804629467921, | |
| "n_seeds": 3, | |
| "leader": "u6@32k", | |
| "ranges_overlap": false, | |
| "verdict": "RESOLVED", | |
| "a_min": 81.38670066473745, | |
| "a_max": 82.31776419246067, | |
| "b_min": 82.66029840632898, | |
| "b_max": 84.15625933222256 | |
| }, | |
| { | |
| "a": "u4@20k", | |
| "b": "u6@32k", | |
| "metric": "probe_acc", | |
| "units": "pts", | |
| "gap": 8.333333333333329, | |
| "spread": 14.583333333333314, | |
| "half_spread": 7.291666666666657, | |
| "n_seeds": 3, | |
| "leader": null, | |
| "ranges_overlap": true, | |
| "verdict": "NOT resolved", | |
| "a_min": 64.58333333333334, | |
| "a_max": 79.16666666666666, | |
| "b_min": 77.08333333333334, | |
| "b_max": 81.25 | |
| }, | |
| { | |
| "a": "u4@20k", | |
| "b": "u6@32k", | |
| "metric": "pairs_solved", | |
| "units": "pairs", | |
| "gap": 1.666666666666666, | |
| "spread": 7.0, | |
| "half_spread": 3.5, | |
| "n_seeds": 3, | |
| "leader": null, | |
| "ranges_overlap": true, | |
| "verdict": "NOT resolved", | |
| "a_min": 5.0, | |
| "a_max": 12.0, | |
| "b_min": 8.0, | |
| "b_max": 11.0 | |
| }, | |
| { | |
| "a": "u4@20k", | |
| "b": "u6@32k", | |
| "metric": "agree_acc", | |
| "units": "pts", | |
| "gap": 7.142857142857146, | |
| "spread": 7.142857142857153, | |
| "half_spread": 3.5714285714285765, | |
| "n_seeds": 3, | |
| "leader": null, | |
| "ranges_overlap": true, | |
| "verdict": "NOT resolved", | |
| "a_min": 50.0, | |
| "a_max": 57.14285714285714, | |
| "b_min": 57.14285714285714, | |
| "b_max": 64.28571428571429 | |
| }, | |
| { | |
| "a": "u4@20k", | |
| "b": "u6@32k", | |
| "metric": "pb_f1", | |
| "units": "pts", | |
| "gap": 12.31014395943437, | |
| "spread": 13.288322688181765, | |
| "half_spread": 6.644161344090882, | |
| "n_seeds": 3, | |
| "leader": null, | |
| "ranges_overlap": false, | |
| "verdict": "NOT resolved", | |
| "a_min": 73.63131856850693, | |
| "a_max": 86.91964125668869, | |
| "b_min": 65.74260663190499, | |
| "b_max": 73.07008355119325 | |
| } | |
| ], | |
| "unrankable_metrics": [ | |
| "probe_acc", | |
| "pairs_solved", | |
| "agree_acc", | |
| "pb_f1" | |
| ] | |
| } |