| { |
| "rule": "mean F1 over human200 and ami_dev on a 0.01 grid; moved only when the bootstrap 95% interval of the gain over the previous threshold is above zero; fixed before re-scoring any test set", |
| "heads": { |
| "commitment": { |
| "threshold": 0.54307, |
| "previous": 0.54307, |
| "dev_set": "human200 (Coleman, blind, product definitions) + ami_dev (AMI human annotations)", |
| "gain_mean_f1": 0.017, |
| "gain_ci95": [ |
| -0.0108, |
| 0.0507 |
| ], |
| "f1_human200": 0.8254, |
| "f1_ami_dev": 0.5816, |
| "n_human200": 200, |
| "pos_human200": 33, |
| "n_ami_dev": 5921, |
| "pos_ami_dev": 274, |
| "moved": false |
| }, |
| "open_question": { |
| "threshold": 0.69, |
| "previous": 0.53315, |
| "dev_set": "human200 (Coleman, blind, product definitions) + ami_dev (AMI human annotations)", |
| "gain_mean_f1": 0.0186, |
| "gain_ci95": [ |
| 0.0067, |
| 0.0274 |
| ], |
| "f1_human200": 0.9189, |
| "f1_ami_dev": 0.5628, |
| "n_human200": 200, |
| "pos_human200": 58, |
| "n_ami_dev": 18987, |
| "pos_ami_dev": 777, |
| "moved": true |
| }, |
| "delegate": { |
| "threshold": 0.51, |
| "previous": 0.31651, |
| "dev_set": "human200 (Coleman, blind, product definitions) + ami_dev (AMI human annotations)", |
| "gain_mean_f1": 0.0624, |
| "gain_ci95": [ |
| 0.0096, |
| 0.1131 |
| ], |
| "f1_human200": 0.9, |
| "f1_ami_dev": 0.3333, |
| "n_human200": 200, |
| "pos_human200": 31, |
| "n_ami_dev": 18952, |
| "pos_ami_dev": 22, |
| "moved": true |
| } |
| }, |
| "fitted": "2026-10-07", |
| "previous_rule": "max-F1 on dev; thresholds are fixed before any test run" |
| } |
|
|
