File size: 1,517 Bytes
8a5e2d4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | {
"rule": "mean F1 over human200 and ami_dev on a 0.01 grid; moved only when the bootstrap 95% interval of the gain over the previous threshold is above zero; fixed before re-scoring any test set",
"heads": {
"commitment": {
"threshold": 0.54307,
"previous": 0.54307,
"dev_set": "human200 (Coleman, blind, product definitions) + ami_dev (AMI human annotations)",
"gain_mean_f1": 0.017,
"gain_ci95": [
-0.0108,
0.0507
],
"f1_human200": 0.8254,
"f1_ami_dev": 0.5816,
"n_human200": 200,
"pos_human200": 33,
"n_ami_dev": 5921,
"pos_ami_dev": 274,
"moved": false
},
"open_question": {
"threshold": 0.69,
"previous": 0.53315,
"dev_set": "human200 (Coleman, blind, product definitions) + ami_dev (AMI human annotations)",
"gain_mean_f1": 0.0186,
"gain_ci95": [
0.0067,
0.0274
],
"f1_human200": 0.9189,
"f1_ami_dev": 0.5628,
"n_human200": 200,
"pos_human200": 58,
"n_ami_dev": 18987,
"pos_ami_dev": 777,
"moved": true
},
"delegate": {
"threshold": 0.51,
"previous": 0.31651,
"dev_set": "human200 (Coleman, blind, product definitions) + ami_dev (AMI human annotations)",
"gain_mean_f1": 0.0624,
"gain_ci95": [
0.0096,
0.1131
],
"f1_human200": 0.9,
"f1_ami_dev": 0.3333,
"n_human200": 200,
"pos_human200": 31,
"n_ami_dev": 18952,
"pos_ami_dev": 22,
"moved": true
}
},
"fitted": "2026-10-07",
"previous_rule": "max-F1 on dev; thresholds are fixed before any test run"
}
|