{ "checkpoint": "models/v004-contextual", "quantized": false, "platform": "Windows-11-10.0.26200-SP0", "torch_threads": 2, "model_load_ms": 94.63169996161014, "samples": 120, "success_rate": 0.8166666666666667, "abstention_rate": 0.10833333333333334, "median_observation_ms": 19.925549917388707, "median_policy_ms": 2.0463999826461077, "median_execution_ms": 23.863700043875724, "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", "target_vps_validated": false, "results": [ { "sample_id": 0, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.997082531452179, "target_confidence": 0.9630829095840454, "observation_ms": 52.04129999037832, "policy_ms": 3.6001999396830797, "execution_ms": 24.167499970644712, "status": "ok", "code": "ASK_USER" }, { "sample_id": 1, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.381199984811246, "policy_ms": 1.756099984049797, "execution_ms": 64.2642000457272, "status": "ok", "code": "TYPE" }, { "sample_id": 2, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 41.61069996189326, "policy_ms": 2.2362000308930874, "execution_ms": 25.761299999430776, "status": "ok", "code": "TYPE" }, { "sample_id": 3, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.043700023554265, "policy_ms": 1.3657000381499529, "execution_ms": 80.96819999627769, "status": "ok", "code": "CLICK" }, { "sample_id": 4, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 23.297299980185926, "policy_ms": 1.9205999560654163, "execution_ms": 62.41819995921105, "status": "ok", "code": "CLICK" }, { "sample_id": 5, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 0.8784183263778687, "observation_ms": 21.74580004066229, "policy_ms": 2.3895999183878303, "execution_ms": 20.609600003808737, "status": "ok", "code": "TYPE" }, { "sample_id": 6, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999873638153076, "observation_ms": 27.131599956192076, "policy_ms": 2.516700071282685, "execution_ms": 28.519999934360385, "status": "ok", "code": "SELECT" }, { "sample_id": 7, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9832378625869751, "observation_ms": 18.569599953480065, "policy_ms": 1.7714999848976731, "execution_ms": 20.5024000024423, "status": "ok", "code": "SELECT" }, { "sample_id": 8, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.716900027357042, "policy_ms": 2.2554000606760383, "execution_ms": 20.574199967086315, "status": "ok", "code": "SELECT" }, { "sample_id": 9, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.976699931547046, "policy_ms": 1.9339000573381782, "execution_ms": 27.41740003693849, "status": "ok", "code": "TYPE" }, { "sample_id": 10, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.524899962358177, "policy_ms": 1.7152000218629837, "execution_ms": 22.37550006248057, "status": "ok", "code": "SELECT" }, { "sample_id": 11, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.5470258593559265, "observation_ms": 18.770999973639846, "policy_ms": 1.7818000633269548, "execution_ms": 3.050200059078634, "status": "ok", "code": "ASK_USER" }, { "sample_id": 12, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9996076226234436, "target_confidence": 0.9999998807907104, "observation_ms": 18.240800010971725, "policy_ms": 1.6741000581532717, "execution_ms": 14.110499992966652, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 13, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.17189999949187, "policy_ms": 2.1422000136226416, "execution_ms": 50.70220003835857, "status": "ok", "code": "CLICK" }, { "sample_id": 14, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.85320001747459, "policy_ms": 2.1046000765636563, "execution_ms": 25.676199933514, "status": "ok", "code": "SELECT" }, { "sample_id": 15, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9997895359992981, "target_confidence": 1.0, "observation_ms": 15.526799950748682, "policy_ms": 1.6744000604376197, "execution_ms": 19.506199983879924, "status": "ok", "code": "TYPE" }, { "sample_id": 16, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9971709847450256, "observation_ms": 19.791599945165217, "policy_ms": 2.011900069192052, "execution_ms": 21.349199931137264, "status": "ok", "code": "SELECT" }, { "sample_id": 17, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.411099958233535, "policy_ms": 1.8891999498009682, "execution_ms": 21.164899924769998, "status": "ok", "code": "SELECT" }, { "sample_id": 18, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.839700031094253, "policy_ms": 1.5915000112727284, "execution_ms": 70.20590000320226, "status": "ok", "code": "CLICK" }, { "sample_id": 19, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 22.31109992135316, "policy_ms": 1.941800001077354, "execution_ms": 21.803799900226295, "status": "ok", "code": "SELECT" }, { "sample_id": 20, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.659999943338335, "policy_ms": 2.439300063997507, "execution_ms": 51.54939997009933, "status": "ok", "code": "CLICK" }, { "sample_id": 21, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.627000062726438, "policy_ms": 2.043199958279729, "execution_ms": 54.44460001308471, "status": "ok", "code": "CLICK" }, { "sample_id": 22, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.43720004055649, "policy_ms": 2.0836000330746174, "execution_ms": 52.542699966579676, "status": "ok", "code": "CLICK" }, { "sample_id": 23, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.43700006045401, "policy_ms": 1.8903999589383602, "execution_ms": 22.533199982717633, "status": "ok", "code": "SELECT" }, { "sample_id": 24, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.37820005696267, "policy_ms": 1.7327999230474234, "execution_ms": 48.637299914844334, "status": "ok", "code": "CLICK" }, { "sample_id": 25, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.0386000322178, "policy_ms": 1.8211998976767063, "execution_ms": 32.63259993400425, "status": "ok", "code": "CLICK" }, { "sample_id": 26, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 31.693299999460578, "policy_ms": 4.196699941530824, "execution_ms": 34.208000055514276, "status": "ok", "code": "TYPE" }, { "sample_id": 27, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.203899941407144, "policy_ms": 1.6080000204965472, "execution_ms": 23.110900074243546, "status": "ok", "code": "SELECT" }, { "sample_id": 28, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.319100072607398, "policy_ms": 1.617099973373115, "execution_ms": 27.02000003773719, "status": "ok", "code": "TYPE" }, { "sample_id": 29, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.1315999943763, "policy_ms": 1.984399976208806, "execution_ms": 61.0787000041455, "status": "ok", "code": "CLICK" }, { "sample_id": 30, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999998807907104, "target_confidence": 0.9999998807907104, "observation_ms": 24.716699961572886, "policy_ms": 2.7118000434711576, "execution_ms": 20.71119996253401, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 31, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.431499997153878, "policy_ms": 1.9639000529423356, "execution_ms": 21.217299974523485, "status": "ok", "code": "SELECT" }, { "sample_id": 32, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.999996542930603, "observation_ms": 17.129699932411313, "policy_ms": 1.640500035136938, "execution_ms": 20.674900035373867, "status": "ok", "code": "SELECT" }, { "sample_id": 33, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9980350136756897, "target_confidence": 0.99998939037323, "observation_ms": 15.770700061693788, "policy_ms": 1.789200003258884, "execution_ms": 3.0932999216020107, "status": "ok", "code": "ASK_USER" }, { "sample_id": 34, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.88559991493821, "policy_ms": 2.4881999706849456, "execution_ms": 21.22969995252788, "status": "ok", "code": "TYPE" }, { "sample_id": 35, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.057199962437153, "policy_ms": 2.530900062993169, "execution_ms": 35.879299975931644, "status": "ok", "code": "CLICK" }, { "sample_id": 36, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9625697731971741, "target_confidence": 1.0, "observation_ms": 19.18990002013743, "policy_ms": 2.424499951303005, "execution_ms": 25.121700018644333, "status": "ok", "code": "TYPE" }, { "sample_id": 37, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 23.415900068357587, "policy_ms": 2.4779000086709857, "execution_ms": 23.03289994597435, "status": "ok", "code": "SELECT" }, { "sample_id": 38, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.38509999308735, "policy_ms": 1.792999915778637, "execution_ms": 21.66259998921305, "status": "ok", "code": "TYPE" }, { "sample_id": 39, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9615044593811035, "observation_ms": 17.784499912522733, "policy_ms": 1.5483000315725803, "execution_ms": 13.7020000256598, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 40, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9991171956062317, "target_confidence": 0.8294482827186584, "observation_ms": 18.787800101563334, "policy_ms": 2.069000038318336, "execution_ms": 3.284999984316528, "status": "ok", "code": "ASK_USER" }, { "sample_id": 41, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.18359992466867, "policy_ms": 1.8230000277981162, "execution_ms": 23.239500005729496, "status": "ok", "code": "SELECT" }, { "sample_id": 42, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 21.46099996753037, "policy_ms": 2.3865000111982226, "execution_ms": 26.55770001001656, "status": "ok", "code": "SELECT" }, { "sample_id": 43, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.84760006982833, "policy_ms": 1.6396999126300216, "execution_ms": 27.551600011065602, "status": "ok", "code": "SELECT" }, { "sample_id": 44, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 59.78959996718913, "policy_ms": 2.021700027398765, "execution_ms": 45.08690000511706, "status": "ok", "code": "CLICK" }, { "sample_id": 45, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.197900033555925, "policy_ms": 1.7804999370127916, "execution_ms": 23.228899925015867, "status": "ok", "code": "SELECT" }, { "sample_id": 46, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.208099973388016, "policy_ms": 1.6938999760895967, "execution_ms": 22.70880003925413, "status": "ok", "code": "SELECT" }, { "sample_id": 47, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9346925020217896, "target_confidence": 0.9996263980865479, "observation_ms": 19.146199920214713, "policy_ms": 1.9916000310331583, "execution_ms": 14.529700041748583, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 48, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9997907280921936, "target_confidence": 1.0, "observation_ms": 18.851600005291402, "policy_ms": 1.7500000540167093, "execution_ms": 22.093100007623434, "status": "ok", "code": "TYPE" }, { "sample_id": 49, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999998807907104, "target_confidence": 1.0, "observation_ms": 16.993399942293763, "policy_ms": 1.2751000467687845, "execution_ms": 22.90139999240637, "status": "ok", "code": "TYPE" }, { "sample_id": 50, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.416999934241176, "policy_ms": 1.7480000387877226, "execution_ms": 55.009900010190904, "status": "ok", "code": "CLICK" }, { "sample_id": 51, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.035200006328523, "policy_ms": 1.66229996830225, "execution_ms": 22.664399934001267, "status": "ok", "code": "SELECT" }, { "sample_id": 52, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.522900042124093, "policy_ms": 1.9642000552266836, "execution_ms": 22.66699995379895, "status": "ok", "code": "SELECT" }, { "sample_id": 53, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.8216633796691895, "target_confidence": 0.999998927116394, "observation_ms": 18.218699959106743, "policy_ms": 1.763500040397048, "execution_ms": 2.8200000524520874, "status": "ok", "code": "ASK_USER" }, { "sample_id": 54, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.149499992839992, "policy_ms": 2.4216000456362963, "execution_ms": 23.797200061380863, "status": "ok", "code": "TYPE" }, { "sample_id": 55, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9997102618217468, "observation_ms": 19.446899998001754, "policy_ms": 2.3861000081524253, "execution_ms": 22.44800003245473, "status": "ok", "code": "SELECT" }, { "sample_id": 56, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9994478821754456, "observation_ms": 19.34529992286116, "policy_ms": 1.6651999903842807, "execution_ms": 2.9511998873203993, "status": "ok", "code": "ASK_USER" }, { "sample_id": 57, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.44330001808703, "policy_ms": 2.0528999157249928, "execution_ms": 54.767899913713336, "status": "ok", "code": "CLICK" }, { "sample_id": 58, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.317499991506338, "policy_ms": 2.5025999639183283, "execution_ms": 22.964300005696714, "status": "ok", "code": "SELECT" }, { "sample_id": 59, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.604999990202487, "policy_ms": 1.416000071913004, "execution_ms": 21.546700038015842, "status": "ok", "code": "SELECT" }, { "sample_id": 60, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.402900081127882, "policy_ms": 2.1657999604940414, "execution_ms": 23.002700065262616, "status": "ok", "code": "SELECT" }, { "sample_id": 61, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 19.693100010044873, "policy_ms": 2.4681999348104, "execution_ms": 23.930200026370585, "status": "ok", "code": "SELECT" }, { "sample_id": 62, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.8983810544013977, "target_confidence": 1.0, "observation_ms": 23.263600072823465, "policy_ms": 2.2478000028058887, "execution_ms": 15.640300000086427, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 63, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9983367919921875, "target_confidence": 0.85873943567276, "observation_ms": 21.32920001167804, "policy_ms": 2.138499985449016, "execution_ms": 3.9785000262781978, "status": "ok", "code": "ASK_USER" }, { "sample_id": 64, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9829460382461548, "target_confidence": 1.0, "observation_ms": 20.072199986316264, "policy_ms": 1.874100067652762, "execution_ms": 22.601000033318996, "status": "ok", "code": "TYPE" }, { "sample_id": 65, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.5237547159194946, "target_confidence": 1.0, "observation_ms": 19.744299934245646, "policy_ms": 1.6528000123798847, "execution_ms": 3.1644999980926514, "status": "ok", "code": "ASK_USER" }, { "sample_id": 66, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 23.615000071004033, "policy_ms": 2.410199958831072, "execution_ms": 42.36810002475977, "status": "ok", "code": "CLICK" }, { "sample_id": 67, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.37950001936406, "policy_ms": 1.804900006391108, "execution_ms": 23.62260001245886, "status": "ok", "code": "TYPE" }, { "sample_id": 68, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.995500053279102, "policy_ms": 1.645199954509735, "execution_ms": 32.227500109001994, "status": "ok", "code": "CLICK" }, { "sample_id": 69, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9998936653137207, "target_confidence": 0.5836381912231445, "observation_ms": 18.146500107832253, "policy_ms": 1.6587999416515231, "execution_ms": 3.0242999782785773, "status": "ok", "code": "ASK_USER" }, { "sample_id": 70, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.674799987114966, "policy_ms": 2.2829000372439623, "execution_ms": 58.3354999544099, "status": "ok", "code": "CLICK" }, { "sample_id": 71, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.07989992853254, "policy_ms": 1.7679999582469463, "execution_ms": 22.723899921402335, "status": "ok", "code": "SELECT" }, { "sample_id": 72, "template": "table", "success": false, "abstained": true, "expected_action": "O", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.6719118356704712, "observation_ms": 20.61340003274381, "policy_ms": 2.3207999765872955, "execution_ms": 3.163499990478158, "status": "ok", "code": "ASK_USER" }, { "sample_id": 73, "template": "nested", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9052738547325134, "observation_ms": 16.669899923726916, "policy_ms": 1.638100016862154, "execution_ms": 22.68329996149987, "status": "ok", "code": "SELECT" }, { "sample_id": 74, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 21.329900017008185, "policy_ms": 1.9255999941378832, "execution_ms": 23.794899927452207, "status": "ok", "code": "TYPE" }, { "sample_id": 75, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 43.32369996700436, "policy_ms": 1.9292000215500593, "execution_ms": 25.223999982699752, "status": "ok", "code": "SELECT" }, { "sample_id": 76, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.537899966351688, "policy_ms": 1.6442000633105636, "execution_ms": 22.20050001051277, "status": "ok", "code": "SELECT" }, { "sample_id": 77, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.999998927116394, "target_confidence": 1.0, "observation_ms": 19.53789999242872, "policy_ms": 2.0939999958500266, "execution_ms": 21.898900042288005, "status": "ok", "code": "TYPE" }, { "sample_id": 78, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.23720005247742, "policy_ms": 1.4942999696359038, "execution_ms": 20.16330009792, "status": "ok", "code": "SELECT" }, { "sample_id": 79, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.72490006685257, "policy_ms": 2.668099943548441, "execution_ms": 20.48149995971471, "status": "ok", "code": "SELECT" }, { "sample_id": 80, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.801099901087582, "policy_ms": 2.180800074711442, "execution_ms": 39.21910002827644, "status": "ok", "code": "CLICK" }, { "sample_id": 81, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999997615814209, "target_confidence": 0.9999921321868896, "observation_ms": 17.265999922528863, "policy_ms": 1.5931000234559178, "execution_ms": 39.03109999373555, "status": "ok", "code": "CLICK" }, { "sample_id": 82, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.587499951943755, "policy_ms": 1.7686000792309642, "execution_ms": 54.260200005955994, "status": "ok", "code": "CLICK" }, { "sample_id": 83, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9128974080085754, "target_confidence": 1.0, "observation_ms": 21.60480001475662, "policy_ms": 1.6494999872520566, "execution_ms": 26.27339994069189, "status": "ok", "code": "TYPE" }, { "sample_id": 84, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 22.096799919381738, "policy_ms": 2.284600050188601, "execution_ms": 23.54269998613745, "status": "ok", "code": "TYPE" }, { "sample_id": 85, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.568500040099025, "policy_ms": 2.6114999782294035, "execution_ms": 22.038600058294833, "status": "ok", "code": "SELECT" }, { "sample_id": 86, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.049999933689833, "policy_ms": 1.8260000506415963, "execution_ms": 33.05269998963922, "status": "ok", "code": "CLICK" }, { "sample_id": 87, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 21.442299941554666, "policy_ms": 1.7699000891298056, "execution_ms": 46.82319995481521, "status": "ok", "code": "CLICK" }, { "sample_id": 88, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 25.167400017380714, "policy_ms": 2.3444999242201447, "execution_ms": 60.75640011113137, "status": "ok", "code": "CLICK" }, { "sample_id": 89, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 20.64460003748536, "policy_ms": 2.152100089006126, "execution_ms": 22.202300024218857, "status": "ok", "code": "SELECT" }, { "sample_id": 90, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9992997646331787, "target_confidence": 0.9984721541404724, "observation_ms": 23.232600069604814, "policy_ms": 2.1249999990686774, "execution_ms": 53.32009994890541, "status": "ok", "code": "CLICK" }, { "sample_id": 91, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 24.84650001861155, "policy_ms": 2.548099961131811, "execution_ms": 27.10760000627488, "status": "ok", "code": "TYPE" }, { "sample_id": 92, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999769926071167, "target_confidence": 0.9999783039093018, "observation_ms": 21.894300007261336, "policy_ms": 2.4238000623881817, "execution_ms": 24.46879993658513, "status": "ok", "code": "TYPE" }, { "sample_id": 93, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.086599932052195, "policy_ms": 1.950700068846345, "execution_ms": 26.81089995894581, "status": "ok", "code": "TYPE" }, { "sample_id": 94, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 27.7623999863863, "policy_ms": 1.9869999960064888, "execution_ms": 55.17740000505, "status": "ok", "code": "CLICK" }, { "sample_id": 95, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 24.448600015603006, "policy_ms": 1.9808000652119517, "execution_ms": 56.22260004747659, "status": "ok", "code": "CLICK" }, { "sample_id": 96, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 27.35049999319017, "policy_ms": 2.7284000534564257, "execution_ms": 56.82669999077916, "status": "ok", "code": "TYPE" }, { "sample_id": 97, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 26.53720008675009, "policy_ms": 2.753300010226667, "execution_ms": 28.886200045235455, "status": "ok", "code": "TYPE" }, { "sample_id": 98, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.75300000049174, "policy_ms": 1.8582999473437667, "execution_ms": 27.006000047549605, "status": "ok", "code": "SELECT" }, { "sample_id": 99, "template": "table", "success": true, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.5633274912834167, "target_confidence": 1.0, "observation_ms": 26.3063000747934, "policy_ms": 2.529999939724803, "execution_ms": 4.016100079752505, "status": "ok", "code": "ASK_USER" }, { "sample_id": 100, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9986655712127686, "target_confidence": 0.9999961853027344, "observation_ms": 24.024399928748608, "policy_ms": 2.307699993252754, "execution_ms": 3.3103000605478883, "status": "ok", "code": "ASK_USER" }, { "sample_id": 101, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9980510473251343, "target_confidence": 1.0, "observation_ms": 20.079100038856268, "policy_ms": 1.4865999110043049, "execution_ms": 37.717900006100535, "status": "ok", "code": "CLICK" }, { "sample_id": 102, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999996423721313, "target_confidence": 1.0, "observation_ms": 23.466199985705316, "policy_ms": 2.661900012753904, "execution_ms": 25.023799971677363, "status": "ok", "code": "TYPE" }, { "sample_id": 103, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.584399997256696, "policy_ms": 2.478900016285479, "execution_ms": 24.177800049073994, "status": "ok", "code": "TYPE" }, { "sample_id": 104, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9814552664756775, "target_confidence": 0.9991796612739563, "observation_ms": 21.240900037810206, "policy_ms": 2.6496999198570848, "execution_ms": 15.518000000156462, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 105, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.418700085021555, "policy_ms": 2.1063999738544226, "execution_ms": 24.387600016780198, "status": "ok", "code": "TYPE" }, { "sample_id": 106, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.328000023961067, "policy_ms": 2.288699964992702, "execution_ms": 24.971900042146444, "status": "ok", "code": "TYPE" }, { "sample_id": 107, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9998756647109985, "observation_ms": 24.564399966038764, "policy_ms": 2.544500050134957, "execution_ms": 22.7174999890849, "status": "ok", "code": "SELECT" }, { "sample_id": 108, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9161800742149353, "target_confidence": 0.9998284578323364, "observation_ms": 22.466100053861737, "policy_ms": 2.624599961563945, "execution_ms": 3.638099995441735, "status": "ok", "code": "ASK_USER" }, { "sample_id": 109, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 31.51220001745969, "policy_ms": 2.2133999736979604, "execution_ms": 27.044099988415837, "status": "ok", "code": "TYPE" }, { "sample_id": 110, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.84640008211136, "policy_ms": 2.81360000371933, "execution_ms": 23.55599997099489, "status": "ok", "code": "SELECT" }, { "sample_id": 111, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 23.418699973262846, "policy_ms": 2.1739000221714377, "execution_ms": 25.060700019821525, "status": "ok", "code": "SELECT" }, { "sample_id": 112, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 23.06119992863387, "policy_ms": 2.1838999819010496, "execution_ms": 27.73279999382794, "status": "ok", "code": "TYPE" }, { "sample_id": 113, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 26.08530002180487, "policy_ms": 2.0496000070124865, "execution_ms": 52.54499998409301, "status": "ok", "code": "CLICK" }, { "sample_id": 114, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 21.78979991003871, "policy_ms": 2.9105000430718064, "execution_ms": 27.780500007793307, "status": "ok", "code": "SELECT" }, { "sample_id": 115, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999645948410034, "target_confidence": 1.0, "observation_ms": 21.985100000165403, "policy_ms": 2.178400056436658, "execution_ms": 25.093400035984814, "status": "ok", "code": "TYPE" }, { "sample_id": 116, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 21.745900041423738, "policy_ms": 2.273899968713522, "execution_ms": 25.786099955439568, "status": "ok", "code": "TYPE" }, { "sample_id": 117, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 22.26999995764345, "policy_ms": 2.4512000381946564, "execution_ms": 59.026199975050986, "status": "ok", "code": "CLICK" }, { "sample_id": 118, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 21.8062000349164, "policy_ms": 2.3586000315845013, "execution_ms": 23.766899947077036, "status": "ok", "code": "SELECT" }, { "sample_id": 119, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999450445175171, "observation_ms": 25.91029996983707, "policy_ms": 2.724300022237003, "execution_ms": 26.5594000229612, "status": "ok", "code": "SELECT" } ] }