| { | |
| "checkpoint": "models/v002-gru", | |
| "quantized": false, | |
| "platform": "Windows-11-10.0.26200-SP0", | |
| "torch_threads": 2, | |
| "model_load_ms": 8.020800072699785, | |
| "samples": 120, | |
| "success_rate": 0.39166666666666666, | |
| "abstention_rate": 0.4, | |
| "median_observation_ms": 14.639450004324317, | |
| "median_policy_ms": 5.642850010190159, | |
| "median_execution_ms": 15.774849976878613, | |
| "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", | |
| "target_vps_validated": false, | |
| "results": [ | |
| { | |
| "sample_id": 0, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9106427431106567, | |
| "observation_ms": 44.24660000950098, | |
| "policy_ms": 7.870400091633201, | |
| "execution_ms": 3.6689999978989363, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 1, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9873443841934204, | |
| "target_confidence": 0.9999971389770508, | |
| "observation_ms": 14.778300072066486, | |
| "policy_ms": 6.43149996176362, | |
| "execution_ms": 43.75750001054257, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 2, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9888303875923157, | |
| "target_confidence": 1.0, | |
| "observation_ms": 36.495500011369586, | |
| "policy_ms": 5.358200054615736, | |
| "execution_ms": 22.587800049223006, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 3, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9998986721038818, | |
| "observation_ms": 15.025699976831675, | |
| "policy_ms": 4.462999990209937, | |
| "execution_ms": 3.001399920322001, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 4, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9679258465766907, | |
| "observation_ms": 16.171199968084693, | |
| "policy_ms": 5.426099989563227, | |
| "execution_ms": 2.8247000882402062, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 5, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9498040080070496, | |
| "observation_ms": 15.62059996649623, | |
| "policy_ms": 6.210600025951862, | |
| "execution_ms": 18.92640010919422, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 6, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.999823272228241, | |
| "observation_ms": 19.638199941255152, | |
| "policy_ms": 6.917600054293871, | |
| "execution_ms": 19.17179999873042, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 7, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9934561848640442, | |
| "observation_ms": 14.618700020946562, | |
| "policy_ms": 5.980800022371113, | |
| "execution_ms": 17.47959991917014, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 8, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9882636070251465, | |
| "observation_ms": 16.187299974262714, | |
| "policy_ms": 7.8931000316515565, | |
| "execution_ms": 18.755099968984723, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 9, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.6589809656143188, | |
| "target_confidence": 0.999976634979248, | |
| "observation_ms": 15.046400018036366, | |
| "policy_ms": 6.1913999961689115, | |
| "execution_ms": 2.5551000144332647, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 10, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9514384269714355, | |
| "observation_ms": 15.801900066435337, | |
| "policy_ms": 5.364800104871392, | |
| "execution_ms": 19.60490003693849, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 11, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9992782473564148, | |
| "observation_ms": 14.57150001078844, | |
| "policy_ms": 5.289900000207126, | |
| "execution_ms": 2.4638999020680785, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 12, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9710627198219299, | |
| "observation_ms": 14.585500000976026, | |
| "policy_ms": 5.454499972984195, | |
| "execution_ms": 17.143300035968423, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 13, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.6948939561843872, | |
| "observation_ms": 14.131800038740039, | |
| "policy_ms": 5.039899959228933, | |
| "execution_ms": 2.332399948500097, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 14, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.882300096563995, | |
| "policy_ms": 4.618899896740913, | |
| "execution_ms": 19.744499935768545, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 15, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.628600001335144, | |
| "policy_ms": 5.319400108419359, | |
| "execution_ms": 17.34090002719313, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 16, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9597891569137573, | |
| "observation_ms": 15.164899989031255, | |
| "policy_ms": 6.308699958026409, | |
| "execution_ms": 16.590299899689853, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 17, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 20.442000008188188, | |
| "policy_ms": 14.638499938882887, | |
| "execution_ms": 25.483200093731284, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 18, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9969860911369324, | |
| "observation_ms": 22.283100057393312, | |
| "policy_ms": 8.245299919508398, | |
| "execution_ms": 3.1480001052841544, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 19, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.50219995342195, | |
| "policy_ms": 5.641499999910593, | |
| "execution_ms": 20.024199970066547, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 20, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9956907629966736, | |
| "observation_ms": 21.485199918970466, | |
| "policy_ms": 10.731799993664026, | |
| "execution_ms": 3.246000036597252, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 21, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 0.9992986917495728, | |
| "observation_ms": 21.41929999925196, | |
| "policy_ms": 5.92360005248338, | |
| "execution_ms": 3.3164999913424253, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 22, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8012701869010925, | |
| "observation_ms": 19.006700022146106, | |
| "policy_ms": 6.379799917340279, | |
| "execution_ms": 3.13510000705719, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 23, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.0764999454841, | |
| "policy_ms": 5.175700061954558, | |
| "execution_ms": 20.086499978788197, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 24, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.6497063636779785, | |
| "observation_ms": 17.871599993668497, | |
| "policy_ms": 5.860900040715933, | |
| "execution_ms": 2.8192000463604927, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 25, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.6710407137870789, | |
| "observation_ms": 15.921200043521821, | |
| "policy_ms": 4.068300011567771, | |
| "execution_ms": 3.2719000009819865, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 26, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9805163741111755, | |
| "observation_ms": 15.411500004120171, | |
| "policy_ms": 6.655100034549832, | |
| "execution_ms": 16.829799977131188, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 27, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9994151592254639, | |
| "observation_ms": 13.451500097289681, | |
| "policy_ms": 3.9328999118879437, | |
| "execution_ms": 17.0963000273332, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 28, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 13.573499978519976, | |
| "policy_ms": 4.711700021289289, | |
| "execution_ms": 16.50489994790405, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 29, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9940434098243713, | |
| "observation_ms": 14.351799967698753, | |
| "policy_ms": 4.563600057736039, | |
| "execution_ms": 2.69210000988096, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 30, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999980926513672, | |
| "observation_ms": 16.997999977320433, | |
| "policy_ms": 8.311899960972369, | |
| "execution_ms": 18.91580002848059, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 31, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.44539991300553, | |
| "policy_ms": 5.644200020469725, | |
| "execution_ms": 18.05759989656508, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 32, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999985694885254, | |
| "observation_ms": 15.494500054046512, | |
| "policy_ms": 4.54239989630878, | |
| "execution_ms": 16.404900001361966, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 33, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.381199911236763, | |
| "policy_ms": 4.269700031727552, | |
| "execution_ms": 2.302700071595609, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 34, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9988446235656738, | |
| "target_confidence": 0.999969482421875, | |
| "observation_ms": 15.084499958902597, | |
| "policy_ms": 6.35020004119724, | |
| "execution_ms": 11.644899961538613, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 35, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.976399972103536, | |
| "policy_ms": 6.781600066460669, | |
| "execution_ms": 2.4924000026658177, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 36, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.151700026355684, | |
| "policy_ms": 6.173499976284802, | |
| "execution_ms": 18.531200010329485, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 37, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.123800072818995, | |
| "policy_ms": 9.156499989330769, | |
| "execution_ms": 19.456699956208467, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 38, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9331730604171753, | |
| "target_confidence": 0.9992374181747437, | |
| "observation_ms": 16.19349990505725, | |
| "policy_ms": 5.165400099940598, | |
| "execution_ms": 11.042700032703578, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 39, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.94921875, | |
| "observation_ms": 13.69160006288439, | |
| "policy_ms": 5.2671999437734485, | |
| "execution_ms": 20.551699912175536, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 40, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8621384501457214, | |
| "observation_ms": 14.68430005479604, | |
| "policy_ms": 5.860799923539162, | |
| "execution_ms": 2.53950001206249, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 41, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.169800047762692, | |
| "policy_ms": 5.166899994947016, | |
| "execution_ms": 19.90090007893741, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 42, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999990463256836, | |
| "observation_ms": 16.215500072576106, | |
| "policy_ms": 5.8868000051006675, | |
| "execution_ms": 15.761999995447695, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 43, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.84250000026077, | |
| "policy_ms": 4.4160999823361635, | |
| "execution_ms": 15.85169997997582, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 44, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9755481481552124, | |
| "observation_ms": 18.474900047294796, | |
| "policy_ms": 6.045799935236573, | |
| "execution_ms": 2.5333999656140804, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 45, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.774900067597628, | |
| "policy_ms": 4.94549993891269, | |
| "execution_ms": 15.691899927332997, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 46, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999706745147705, | |
| "observation_ms": 13.911699992604554, | |
| "policy_ms": 5.272399983368814, | |
| "execution_ms": 15.609099995344877, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 47, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9941600561141968, | |
| "observation_ms": 13.88399989809841, | |
| "policy_ms": 5.4524000734090805, | |
| "execution_ms": 16.107400064356625, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 48, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999527931213379, | |
| "observation_ms": 12.878799927420914, | |
| "policy_ms": 4.389000008814037, | |
| "execution_ms": 15.67819993942976, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 49, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999653100967407, | |
| "observation_ms": 12.603000039234757, | |
| "policy_ms": 4.5912000350654125, | |
| "execution_ms": 15.382300014607608, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 50, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.26609996613115, | |
| "policy_ms": 4.6440999722108245, | |
| "execution_ms": 2.288100076839328, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 51, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.426000019535422, | |
| "policy_ms": 5.947500001639128, | |
| "execution_ms": 16.165500041097403, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 52, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999997615814209, | |
| "observation_ms": 12.6547000836581, | |
| "policy_ms": 4.678700002841651, | |
| "execution_ms": 17.162400064989924, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 53, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9989532232284546, | |
| "observation_ms": 14.350200071930885, | |
| "policy_ms": 5.14120003208518, | |
| "execution_ms": 2.3252000100910664, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 54, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9810065031051636, | |
| "target_confidence": 0.998680055141449, | |
| "observation_ms": 14.452700037509203, | |
| "policy_ms": 6.582799949683249, | |
| "execution_ms": 18.378900014795363, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 55, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999997615814209, | |
| "observation_ms": 15.689300023950636, | |
| "policy_ms": 5.696299951523542, | |
| "execution_ms": 18.688600044697523, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 56, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999995231628418, | |
| "observation_ms": 15.179599984548986, | |
| "policy_ms": 7.325300015509129, | |
| "execution_ms": 2.467700047418475, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 57, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.71820003259927, | |
| "policy_ms": 6.719099939800799, | |
| "execution_ms": 2.520799986086786, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 58, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.816499944776297, | |
| "policy_ms": 5.723300040699542, | |
| "execution_ms": 16.337000066414475, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 59, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.891999959014356, | |
| "policy_ms": 5.145099945366383, | |
| "execution_ms": 15.593699994497001, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 60, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999799728393555, | |
| "observation_ms": 13.682099990546703, | |
| "policy_ms": 4.811599967069924, | |
| "execution_ms": 16.774000017903745, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 61, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.277099980972707, | |
| "policy_ms": 8.364200009964406, | |
| "execution_ms": 19.13380005862564, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 62, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8035814762115479, | |
| "observation_ms": 15.815299935638905, | |
| "policy_ms": 5.672300001606345, | |
| "execution_ms": 2.436299924738705, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 63, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9838116765022278, | |
| "observation_ms": 14.660199987702072, | |
| "policy_ms": 5.406700074672699, | |
| "execution_ms": 2.404000028036535, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 64, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9968637228012085, | |
| "observation_ms": 14.243200072087348, | |
| "policy_ms": 5.423899972811341, | |
| "execution_ms": 15.442600008100271, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 65, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 13.283299980685115, | |
| "policy_ms": 4.714200040325522, | |
| "execution_ms": 2.179000061005354, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 66, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8451939821243286, | |
| "observation_ms": 15.359399956651032, | |
| "policy_ms": 5.572100053541362, | |
| "execution_ms": 2.367699984461069, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 67, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.154100044630468, | |
| "policy_ms": 5.556099931709468, | |
| "execution_ms": 17.591699957847595, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 68, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.526800041086972, | |
| "policy_ms": 4.182599950581789, | |
| "execution_ms": 2.3926999419927597, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 69, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9994825124740601, | |
| "observation_ms": 14.410000061616302, | |
| "policy_ms": 5.191099946387112, | |
| "execution_ms": 2.831999910995364, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 70, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9999996423721313, | |
| "target_confidence": 0.9999701976776123, | |
| "observation_ms": 14.026100048795342, | |
| "policy_ms": 5.702499998733401, | |
| "execution_ms": 2.484200056642294, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 71, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9580230712890625, | |
| "observation_ms": 12.456300086341798, | |
| "policy_ms": 4.1102000977844, | |
| "execution_ms": 15.295200049877167, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 72, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.999990701675415, | |
| "observation_ms": 14.332700055092573, | |
| "policy_ms": 5.6705999886617064, | |
| "execution_ms": 16.608400037512183, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 73, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.999998927116394, | |
| "observation_ms": 11.805300018750131, | |
| "policy_ms": 5.159699940122664, | |
| "execution_ms": 15.969399944879115, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 74, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9997367262840271, | |
| "target_confidence": 0.5425891280174255, | |
| "observation_ms": 15.75839996803552, | |
| "policy_ms": 5.910899955779314, | |
| "execution_ms": 2.4791999021545053, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 75, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999966621398926, | |
| "observation_ms": 12.371599907055497, | |
| "policy_ms": 4.641800071112812, | |
| "execution_ms": 16.78669999819249, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 76, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.665900052525103, | |
| "policy_ms": 4.241300048306584, | |
| "execution_ms": 15.841100015677512, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 77, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.91079998575151, | |
| "policy_ms": 7.136700092814863, | |
| "execution_ms": 16.29639999009669, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 78, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.315700042061508, | |
| "policy_ms": 5.353999906219542, | |
| "execution_ms": 16.416999977082014, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 79, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.127800034359097, | |
| "policy_ms": 8.579300018027425, | |
| "execution_ms": 16.487499931827188, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 80, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.218700001947582, | |
| "policy_ms": 5.369900027289987, | |
| "execution_ms": 2.751899999566376, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 81, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999608993530273, | |
| "observation_ms": 15.178900095634162, | |
| "policy_ms": 5.437299958430231, | |
| "execution_ms": 2.516100066713989, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 82, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9960330128669739, | |
| "observation_ms": 15.121700009331107, | |
| "policy_ms": 5.732999998144805, | |
| "execution_ms": 2.6709999656304717, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 83, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9999986886978149, | |
| "target_confidence": 0.8480029702186584, | |
| "observation_ms": 13.722400064580142, | |
| "policy_ms": 6.257299915887415, | |
| "execution_ms": 2.3250000085681677, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 84, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9913052916526794, | |
| "observation_ms": 15.474800020456314, | |
| "policy_ms": 5.90430002193898, | |
| "execution_ms": 16.301500028930604, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 85, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9209082722663879, | |
| "observation_ms": 14.292799984104931, | |
| "policy_ms": 6.63459999486804, | |
| "execution_ms": 16.415800084359944, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 86, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 12.9893000703305, | |
| "policy_ms": 4.4371000258252025, | |
| "execution_ms": 2.466200035996735, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 87, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.5589049458503723, | |
| "observation_ms": 15.525400056503713, | |
| "policy_ms": 7.321100099943578, | |
| "execution_ms": 2.8237999649718404, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 88, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9998763799667358, | |
| "observation_ms": 15.69139992352575, | |
| "policy_ms": 5.579799995757639, | |
| "execution_ms": 2.516699954867363, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 89, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9988917708396912, | |
| "observation_ms": 17.7850000327453, | |
| "policy_ms": 6.633299984969199, | |
| "execution_ms": 16.517199925146997, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 90, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999994039535522, | |
| "observation_ms": 15.42800001334399, | |
| "policy_ms": 6.517100031487644, | |
| "execution_ms": 2.6801000349223614, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 91, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999995231628418, | |
| "target_confidence": 0.999991774559021, | |
| "observation_ms": 18.885600031353533, | |
| "policy_ms": 7.807799964211881, | |
| "execution_ms": 21.415700088255107, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 92, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9968438148498535, | |
| "observation_ms": 15.483899973332882, | |
| "policy_ms": 6.061500054784119, | |
| "execution_ms": 16.276399954222143, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 93, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.5529961585998535, | |
| "observation_ms": 12.816700036637485, | |
| "policy_ms": 4.880900029093027, | |
| "execution_ms": 2.2583999671041965, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 94, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9746989607810974, | |
| "observation_ms": 14.847400016151369, | |
| "policy_ms": 8.308400050736964, | |
| "execution_ms": 2.891700016334653, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 95, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.6894709467887878, | |
| "observation_ms": 13.81010003387928, | |
| "policy_ms": 7.412100094370544, | |
| "execution_ms": 2.658099983818829, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 96, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9780493378639221, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.902300016023219, | |
| "policy_ms": 5.481399944983423, | |
| "execution_ms": 16.780099947936833, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 97, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.5665366649627686, | |
| "target_confidence": 0.9999916553497314, | |
| "observation_ms": 13.692499953322113, | |
| "policy_ms": 5.062200012616813, | |
| "execution_ms": 2.353199990466237, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 98, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.573700048960745, | |
| "policy_ms": 6.14269997458905, | |
| "execution_ms": 18.901500036008656, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 99, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999967813491821, | |
| "observation_ms": 19.381299964152277, | |
| "policy_ms": 5.829700035974383, | |
| "execution_ms": 2.478900016285479, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 100, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9519977569580078, | |
| "observation_ms": 14.208900043740869, | |
| "policy_ms": 5.3721999283879995, | |
| "execution_ms": 2.4892999790608883, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 101, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.710799997672439, | |
| "policy_ms": 4.391100024804473, | |
| "execution_ms": 2.3975000949576497, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 102, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999699592590332, | |
| "target_confidence": 0.9877586960792542, | |
| "observation_ms": 16.78659999743104, | |
| "policy_ms": 6.5367999486625195, | |
| "execution_ms": 16.190700000151992, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 103, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9960851669311523, | |
| "target_confidence": 0.9997296929359436, | |
| "observation_ms": 13.554899953305721, | |
| "policy_ms": 3.9613000117242336, | |
| "execution_ms": 17.097899923101068, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 104, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9994691014289856, | |
| "observation_ms": 14.564899960532784, | |
| "policy_ms": 5.859700031578541, | |
| "execution_ms": 15.829900046810508, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 105, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999922513961792, | |
| "observation_ms": 12.78019999153912, | |
| "policy_ms": 3.944299998693168, | |
| "execution_ms": 16.73899998422712, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 106, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999992847442627, | |
| "observation_ms": 15.06660005543381, | |
| "policy_ms": 5.027799983508885, | |
| "execution_ms": 11.834000004455447, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 107, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 16.96310006082058, | |
| "policy_ms": 6.54430000577122, | |
| "execution_ms": 17.723500030115247, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 108, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.147900070995092, | |
| "policy_ms": 6.6255999263376, | |
| "execution_ms": 2.570000011473894, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 109, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9680730104446411, | |
| "target_confidence": 0.9812092185020447, | |
| "observation_ms": 13.616799958981574, | |
| "policy_ms": 5.672800005413592, | |
| "execution_ms": 16.66189997922629, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 110, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.796399998478591, | |
| "policy_ms": 4.303400055505335, | |
| "execution_ms": 16.243300051428378, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 111, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.10790006816387, | |
| "policy_ms": 6.166999926790595, | |
| "execution_ms": 15.69439994636923, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 112, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999850988388062, | |
| "target_confidence": 0.9864395260810852, | |
| "observation_ms": 15.109900035895407, | |
| "policy_ms": 4.6709999442100525, | |
| "execution_ms": 17.726799938827753, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 113, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999536275863647, | |
| "observation_ms": 14.065099996514618, | |
| "policy_ms": 5.889200023375452, | |
| "execution_ms": 2.464100020006299, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 114, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.09929991234094, | |
| "policy_ms": 6.019800086505711, | |
| "execution_ms": 15.787699958309531, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 115, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9998906850814819, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.961399974301457, | |
| "policy_ms": 6.4616999588906765, | |
| "execution_ms": 17.88450009189546, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 116, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.8286662697792053, | |
| "target_confidence": 0.9999352693557739, | |
| "observation_ms": 15.840500011108816, | |
| "policy_ms": 5.301999975927174, | |
| "execution_ms": 2.3964999709278345, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 117, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.605000054463744, | |
| "policy_ms": 4.79289994109422, | |
| "execution_ms": 2.412299974821508, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 118, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.805299997329712, | |
| "policy_ms": 5.196699989028275, | |
| "execution_ms": 16.23519998975098, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 119, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.033100033178926, | |
| "policy_ms": 6.443300051614642, | |
| "execution_ms": 15.90520003810525, | |
| "status": "ok", | |
| "code": "SELECT" | |
| } | |
| ] | |
| } |