Add trained contextual action candidate with browser evaluation and explicit real-web limits
c8e5620 verified | { | |
| "checkpoint": "models/v004-contextual", | |
| "quantized": false, | |
| "platform": "Windows-11-10.0.26200-SP0", | |
| "torch_threads": 2, | |
| "model_load_ms": 94.63169996161014, | |
| "samples": 120, | |
| "success_rate": 0.8166666666666667, | |
| "abstention_rate": 0.10833333333333334, | |
| "median_observation_ms": 19.925549917388707, | |
| "median_policy_ms": 2.0463999826461077, | |
| "median_execution_ms": 23.863700043875724, | |
| "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", | |
| "target_vps_validated": false, | |
| "results": [ | |
| { | |
| "sample_id": 0, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.997082531452179, | |
| "target_confidence": 0.9630829095840454, | |
| "observation_ms": 52.04129999037832, | |
| "policy_ms": 3.6001999396830797, | |
| "execution_ms": 24.167499970644712, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 1, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.381199984811246, | |
| "policy_ms": 1.756099984049797, | |
| "execution_ms": 64.2642000457272, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 2, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 41.61069996189326, | |
| "policy_ms": 2.2362000308930874, | |
| "execution_ms": 25.761299999430776, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 3, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.043700023554265, | |
| "policy_ms": 1.3657000381499529, | |
| "execution_ms": 80.96819999627769, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 4, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 23.297299980185926, | |
| "policy_ms": 1.9205999560654163, | |
| "execution_ms": 62.41819995921105, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 5, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8784183263778687, | |
| "observation_ms": 21.74580004066229, | |
| "policy_ms": 2.3895999183878303, | |
| "execution_ms": 20.609600003808737, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 6, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999873638153076, | |
| "observation_ms": 27.131599956192076, | |
| "policy_ms": 2.516700071282685, | |
| "execution_ms": 28.519999934360385, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 7, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9832378625869751, | |
| "observation_ms": 18.569599953480065, | |
| "policy_ms": 1.7714999848976731, | |
| "execution_ms": 20.5024000024423, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 8, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.716900027357042, | |
| "policy_ms": 2.2554000606760383, | |
| "execution_ms": 20.574199967086315, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 9, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.976699931547046, | |
| "policy_ms": 1.9339000573381782, | |
| "execution_ms": 27.41740003693849, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 10, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.524899962358177, | |
| "policy_ms": 1.7152000218629837, | |
| "execution_ms": 22.37550006248057, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 11, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.5470258593559265, | |
| "observation_ms": 18.770999973639846, | |
| "policy_ms": 1.7818000633269548, | |
| "execution_ms": 3.050200059078634, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 12, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9996076226234436, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 18.240800010971725, | |
| "policy_ms": 1.6741000581532717, | |
| "execution_ms": 14.110499992966652, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 13, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.17189999949187, | |
| "policy_ms": 2.1422000136226416, | |
| "execution_ms": 50.70220003835857, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 14, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.85320001747459, | |
| "policy_ms": 2.1046000765636563, | |
| "execution_ms": 25.676199933514, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 15, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9997895359992981, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.526799950748682, | |
| "policy_ms": 1.6744000604376197, | |
| "execution_ms": 19.506199983879924, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 16, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9971709847450256, | |
| "observation_ms": 19.791599945165217, | |
| "policy_ms": 2.011900069192052, | |
| "execution_ms": 21.349199931137264, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 17, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.411099958233535, | |
| "policy_ms": 1.8891999498009682, | |
| "execution_ms": 21.164899924769998, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 18, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.839700031094253, | |
| "policy_ms": 1.5915000112727284, | |
| "execution_ms": 70.20590000320226, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 19, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 22.31109992135316, | |
| "policy_ms": 1.941800001077354, | |
| "execution_ms": 21.803799900226295, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 20, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.659999943338335, | |
| "policy_ms": 2.439300063997507, | |
| "execution_ms": 51.54939997009933, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 21, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.627000062726438, | |
| "policy_ms": 2.043199958279729, | |
| "execution_ms": 54.44460001308471, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 22, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.43720004055649, | |
| "policy_ms": 2.0836000330746174, | |
| "execution_ms": 52.542699966579676, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 23, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.43700006045401, | |
| "policy_ms": 1.8903999589383602, | |
| "execution_ms": 22.533199982717633, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 24, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.37820005696267, | |
| "policy_ms": 1.7327999230474234, | |
| "execution_ms": 48.637299914844334, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 25, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.0386000322178, | |
| "policy_ms": 1.8211998976767063, | |
| "execution_ms": 32.63259993400425, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 26, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 31.693299999460578, | |
| "policy_ms": 4.196699941530824, | |
| "execution_ms": 34.208000055514276, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 27, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.203899941407144, | |
| "policy_ms": 1.6080000204965472, | |
| "execution_ms": 23.110900074243546, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 28, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.319100072607398, | |
| "policy_ms": 1.617099973373115, | |
| "execution_ms": 27.02000003773719, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 29, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.1315999943763, | |
| "policy_ms": 1.984399976208806, | |
| "execution_ms": 61.0787000041455, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 30, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 24.716699961572886, | |
| "policy_ms": 2.7118000434711576, | |
| "execution_ms": 20.71119996253401, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 31, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.431499997153878, | |
| "policy_ms": 1.9639000529423356, | |
| "execution_ms": 21.217299974523485, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 32, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.999996542930603, | |
| "observation_ms": 17.129699932411313, | |
| "policy_ms": 1.640500035136938, | |
| "execution_ms": 20.674900035373867, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 33, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9980350136756897, | |
| "target_confidence": 0.99998939037323, | |
| "observation_ms": 15.770700061693788, | |
| "policy_ms": 1.789200003258884, | |
| "execution_ms": 3.0932999216020107, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 34, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.88559991493821, | |
| "policy_ms": 2.4881999706849456, | |
| "execution_ms": 21.22969995252788, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 35, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.057199962437153, | |
| "policy_ms": 2.530900062993169, | |
| "execution_ms": 35.879299975931644, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 36, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9625697731971741, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.18990002013743, | |
| "policy_ms": 2.424499951303005, | |
| "execution_ms": 25.121700018644333, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 37, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 23.415900068357587, | |
| "policy_ms": 2.4779000086709857, | |
| "execution_ms": 23.03289994597435, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 38, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.38509999308735, | |
| "policy_ms": 1.792999915778637, | |
| "execution_ms": 21.66259998921305, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 39, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9615044593811035, | |
| "observation_ms": 17.784499912522733, | |
| "policy_ms": 1.5483000315725803, | |
| "execution_ms": 13.7020000256598, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 40, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9991171956062317, | |
| "target_confidence": 0.8294482827186584, | |
| "observation_ms": 18.787800101563334, | |
| "policy_ms": 2.069000038318336, | |
| "execution_ms": 3.284999984316528, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 41, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.18359992466867, | |
| "policy_ms": 1.8230000277981162, | |
| "execution_ms": 23.239500005729496, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 42, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 21.46099996753037, | |
| "policy_ms": 2.3865000111982226, | |
| "execution_ms": 26.55770001001656, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 43, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.84760006982833, | |
| "policy_ms": 1.6396999126300216, | |
| "execution_ms": 27.551600011065602, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 44, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 59.78959996718913, | |
| "policy_ms": 2.021700027398765, | |
| "execution_ms": 45.08690000511706, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 45, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.197900033555925, | |
| "policy_ms": 1.7804999370127916, | |
| "execution_ms": 23.228899925015867, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 46, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.208099973388016, | |
| "policy_ms": 1.6938999760895967, | |
| "execution_ms": 22.70880003925413, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 47, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9346925020217896, | |
| "target_confidence": 0.9996263980865479, | |
| "observation_ms": 19.146199920214713, | |
| "policy_ms": 1.9916000310331583, | |
| "execution_ms": 14.529700041748583, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 48, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9997907280921936, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.851600005291402, | |
| "policy_ms": 1.7500000540167093, | |
| "execution_ms": 22.093100007623434, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 49, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.993399942293763, | |
| "policy_ms": 1.2751000467687845, | |
| "execution_ms": 22.90139999240637, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 50, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.416999934241176, | |
| "policy_ms": 1.7480000387877226, | |
| "execution_ms": 55.009900010190904, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 51, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.035200006328523, | |
| "policy_ms": 1.66229996830225, | |
| "execution_ms": 22.664399934001267, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 52, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.522900042124093, | |
| "policy_ms": 1.9642000552266836, | |
| "execution_ms": 22.66699995379895, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 53, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.8216633796691895, | |
| "target_confidence": 0.999998927116394, | |
| "observation_ms": 18.218699959106743, | |
| "policy_ms": 1.763500040397048, | |
| "execution_ms": 2.8200000524520874, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 54, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.149499992839992, | |
| "policy_ms": 2.4216000456362963, | |
| "execution_ms": 23.797200061380863, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 55, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9997102618217468, | |
| "observation_ms": 19.446899998001754, | |
| "policy_ms": 2.3861000081524253, | |
| "execution_ms": 22.44800003245473, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 56, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9994478821754456, | |
| "observation_ms": 19.34529992286116, | |
| "policy_ms": 1.6651999903842807, | |
| "execution_ms": 2.9511998873203993, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 57, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.44330001808703, | |
| "policy_ms": 2.0528999157249928, | |
| "execution_ms": 54.767899913713336, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 58, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.317499991506338, | |
| "policy_ms": 2.5025999639183283, | |
| "execution_ms": 22.964300005696714, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 59, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.604999990202487, | |
| "policy_ms": 1.416000071913004, | |
| "execution_ms": 21.546700038015842, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 60, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.402900081127882, | |
| "policy_ms": 2.1657999604940414, | |
| "execution_ms": 23.002700065262616, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 61, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 19.693100010044873, | |
| "policy_ms": 2.4681999348104, | |
| "execution_ms": 23.930200026370585, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 62, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.8983810544013977, | |
| "target_confidence": 1.0, | |
| "observation_ms": 23.263600072823465, | |
| "policy_ms": 2.2478000028058887, | |
| "execution_ms": 15.640300000086427, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 63, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9983367919921875, | |
| "target_confidence": 0.85873943567276, | |
| "observation_ms": 21.32920001167804, | |
| "policy_ms": 2.138499985449016, | |
| "execution_ms": 3.9785000262781978, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 64, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9829460382461548, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.072199986316264, | |
| "policy_ms": 1.874100067652762, | |
| "execution_ms": 22.601000033318996, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 65, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.5237547159194946, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.744299934245646, | |
| "policy_ms": 1.6528000123798847, | |
| "execution_ms": 3.1644999980926514, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 66, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 23.615000071004033, | |
| "policy_ms": 2.410199958831072, | |
| "execution_ms": 42.36810002475977, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 67, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.37950001936406, | |
| "policy_ms": 1.804900006391108, | |
| "execution_ms": 23.62260001245886, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 68, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.995500053279102, | |
| "policy_ms": 1.645199954509735, | |
| "execution_ms": 32.227500109001994, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 69, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9998936653137207, | |
| "target_confidence": 0.5836381912231445, | |
| "observation_ms": 18.146500107832253, | |
| "policy_ms": 1.6587999416515231, | |
| "execution_ms": 3.0242999782785773, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 70, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.674799987114966, | |
| "policy_ms": 2.2829000372439623, | |
| "execution_ms": 58.3354999544099, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 71, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.07989992853254, | |
| "policy_ms": 1.7679999582469463, | |
| "execution_ms": 22.723899921402335, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 72, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "O", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.6719118356704712, | |
| "observation_ms": 20.61340003274381, | |
| "policy_ms": 2.3207999765872955, | |
| "execution_ms": 3.163499990478158, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 73, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9052738547325134, | |
| "observation_ms": 16.669899923726916, | |
| "policy_ms": 1.638100016862154, | |
| "execution_ms": 22.68329996149987, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 74, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 21.329900017008185, | |
| "policy_ms": 1.9255999941378832, | |
| "execution_ms": 23.794899927452207, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 75, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 43.32369996700436, | |
| "policy_ms": 1.9292000215500593, | |
| "execution_ms": 25.223999982699752, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 76, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.537899966351688, | |
| "policy_ms": 1.6442000633105636, | |
| "execution_ms": 22.20050001051277, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 77, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.999998927116394, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.53789999242872, | |
| "policy_ms": 2.0939999958500266, | |
| "execution_ms": 21.898900042288005, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 78, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.23720005247742, | |
| "policy_ms": 1.4942999696359038, | |
| "execution_ms": 20.16330009792, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 79, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.72490006685257, | |
| "policy_ms": 2.668099943548441, | |
| "execution_ms": 20.48149995971471, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 80, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.801099901087582, | |
| "policy_ms": 2.180800074711442, | |
| "execution_ms": 39.21910002827644, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 81, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999997615814209, | |
| "target_confidence": 0.9999921321868896, | |
| "observation_ms": 17.265999922528863, | |
| "policy_ms": 1.5931000234559178, | |
| "execution_ms": 39.03109999373555, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 82, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.587499951943755, | |
| "policy_ms": 1.7686000792309642, | |
| "execution_ms": 54.260200005955994, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 83, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9128974080085754, | |
| "target_confidence": 1.0, | |
| "observation_ms": 21.60480001475662, | |
| "policy_ms": 1.6494999872520566, | |
| "execution_ms": 26.27339994069189, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 84, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 22.096799919381738, | |
| "policy_ms": 2.284600050188601, | |
| "execution_ms": 23.54269998613745, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 85, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.568500040099025, | |
| "policy_ms": 2.6114999782294035, | |
| "execution_ms": 22.038600058294833, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 86, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.049999933689833, | |
| "policy_ms": 1.8260000506415963, | |
| "execution_ms": 33.05269998963922, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 87, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 21.442299941554666, | |
| "policy_ms": 1.7699000891298056, | |
| "execution_ms": 46.82319995481521, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 88, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 25.167400017380714, | |
| "policy_ms": 2.3444999242201447, | |
| "execution_ms": 60.75640011113137, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 89, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.64460003748536, | |
| "policy_ms": 2.152100089006126, | |
| "execution_ms": 22.202300024218857, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 90, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9992997646331787, | |
| "target_confidence": 0.9984721541404724, | |
| "observation_ms": 23.232600069604814, | |
| "policy_ms": 2.1249999990686774, | |
| "execution_ms": 53.32009994890541, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 91, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 24.84650001861155, | |
| "policy_ms": 2.548099961131811, | |
| "execution_ms": 27.10760000627488, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 92, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999769926071167, | |
| "target_confidence": 0.9999783039093018, | |
| "observation_ms": 21.894300007261336, | |
| "policy_ms": 2.4238000623881817, | |
| "execution_ms": 24.46879993658513, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 93, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.086599932052195, | |
| "policy_ms": 1.950700068846345, | |
| "execution_ms": 26.81089995894581, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 94, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 27.7623999863863, | |
| "policy_ms": 1.9869999960064888, | |
| "execution_ms": 55.17740000505, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 95, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 24.448600015603006, | |
| "policy_ms": 1.9808000652119517, | |
| "execution_ms": 56.22260004747659, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 96, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 27.35049999319017, | |
| "policy_ms": 2.7284000534564257, | |
| "execution_ms": 56.82669999077916, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 97, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 26.53720008675009, | |
| "policy_ms": 2.753300010226667, | |
| "execution_ms": 28.886200045235455, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 98, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.75300000049174, | |
| "policy_ms": 1.8582999473437667, | |
| "execution_ms": 27.006000047549605, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 99, | |
| "template": "table", | |
| "success": true, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.5633274912834167, | |
| "target_confidence": 1.0, | |
| "observation_ms": 26.3063000747934, | |
| "policy_ms": 2.529999939724803, | |
| "execution_ms": 4.016100079752505, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 100, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9986655712127686, | |
| "target_confidence": 0.9999961853027344, | |
| "observation_ms": 24.024399928748608, | |
| "policy_ms": 2.307699993252754, | |
| "execution_ms": 3.3103000605478883, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 101, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9980510473251343, | |
| "target_confidence": 1.0, | |
| "observation_ms": 20.079100038856268, | |
| "policy_ms": 1.4865999110043049, | |
| "execution_ms": 37.717900006100535, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 102, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999996423721313, | |
| "target_confidence": 1.0, | |
| "observation_ms": 23.466199985705316, | |
| "policy_ms": 2.661900012753904, | |
| "execution_ms": 25.023799971677363, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 103, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.584399997256696, | |
| "policy_ms": 2.478900016285479, | |
| "execution_ms": 24.177800049073994, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 104, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9814552664756775, | |
| "target_confidence": 0.9991796612739563, | |
| "observation_ms": 21.240900037810206, | |
| "policy_ms": 2.6496999198570848, | |
| "execution_ms": 15.518000000156462, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 105, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.418700085021555, | |
| "policy_ms": 2.1063999738544226, | |
| "execution_ms": 24.387600016780198, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 106, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.328000023961067, | |
| "policy_ms": 2.288699964992702, | |
| "execution_ms": 24.971900042146444, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 107, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9998756647109985, | |
| "observation_ms": 24.564399966038764, | |
| "policy_ms": 2.544500050134957, | |
| "execution_ms": 22.7174999890849, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 108, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9161800742149353, | |
| "target_confidence": 0.9998284578323364, | |
| "observation_ms": 22.466100053861737, | |
| "policy_ms": 2.624599961563945, | |
| "execution_ms": 3.638099995441735, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 109, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 31.51220001745969, | |
| "policy_ms": 2.2133999736979604, | |
| "execution_ms": 27.044099988415837, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 110, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.84640008211136, | |
| "policy_ms": 2.81360000371933, | |
| "execution_ms": 23.55599997099489, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 111, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 23.418699973262846, | |
| "policy_ms": 2.1739000221714377, | |
| "execution_ms": 25.060700019821525, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 112, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 23.06119992863387, | |
| "policy_ms": 2.1838999819010496, | |
| "execution_ms": 27.73279999382794, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 113, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 26.08530002180487, | |
| "policy_ms": 2.0496000070124865, | |
| "execution_ms": 52.54499998409301, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 114, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 21.78979991003871, | |
| "policy_ms": 2.9105000430718064, | |
| "execution_ms": 27.780500007793307, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 115, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999645948410034, | |
| "target_confidence": 1.0, | |
| "observation_ms": 21.985100000165403, | |
| "policy_ms": 2.178400056436658, | |
| "execution_ms": 25.093400035984814, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 116, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 21.745900041423738, | |
| "policy_ms": 2.273899968713522, | |
| "execution_ms": 25.786099955439568, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 117, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 22.26999995764345, | |
| "policy_ms": 2.4512000381946564, | |
| "execution_ms": 59.026199975050986, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 118, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 21.8062000349164, | |
| "policy_ms": 2.3586000315845013, | |
| "execution_ms": 23.766899947077036, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 119, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999450445175171, | |
| "observation_ms": 25.91029996983707, | |
| "policy_ms": 2.724300022237003, | |
| "execution_ms": 26.5594000229612, | |
| "status": "ok", | |
| "code": "SELECT" | |
| } | |
| ] | |
| } |