{ "checkpoint": "models/v000-mean", "quantized": false, "platform": "Windows-11-10.0.26200-SP0", "torch_threads": 2, "model_load_ms": 9.840800077654421, "samples": 120, "success_rate": 0.6166666666666667, "abstention_rate": 0.15833333333333333, "median_observation_ms": 15.722550044301897, "median_policy_ms": 2.642350038513541, "median_execution_ms": 17.39390002330765, "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", "target_vps_validated": false, "results": [ { "sample_id": 0, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.6028217077255249, "observation_ms": 41.0907999612391, "policy_ms": 4.931100062094629, "execution_ms": 5.35310001578182, "status": "ok", "code": "ASK_USER" }, { "sample_id": 1, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999998807907104, "target_confidence": 1.0, "observation_ms": 16.348800039850175, "policy_ms": 2.6072999462485313, "execution_ms": 46.392700052820146, "status": "ok", "code": "TYPE" }, { "sample_id": 2, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 29.954700032249093, "policy_ms": 2.511699916794896, "execution_ms": 31.195999938063323, "status": "ok", "code": "TYPE" }, { "sample_id": 3, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999649524688721, "target_confidence": 1.0, "observation_ms": 15.29299991670996, "policy_ms": 2.1188000682741404, "execution_ms": 32.14030002709478, "status": "ok", "code": "CLICK" }, { "sample_id": 4, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9261236786842346, "target_confidence": 1.0, "observation_ms": 17.166699981316924, "policy_ms": 3.0170000391080976, "execution_ms": 2.691800007596612, "status": "ok", "code": "ASK_USER" }, { "sample_id": 5, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9993553757667542, "observation_ms": 18.30679993145168, "policy_ms": 2.898600068874657, "execution_ms": 13.020699960179627, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 6, "template": "nested", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9194011688232422, "observation_ms": 20.474000019021332, "policy_ms": 3.6683999933302402, "execution_ms": 21.632199990563095, "status": "ok", "code": "SELECT" }, { "sample_id": 7, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9984651803970337, "observation_ms": 18.28419999219477, "policy_ms": 3.329100087285042, "execution_ms": 23.86119996663183, "status": "ok", "code": "SELECT" }, { "sample_id": 8, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999978542327881, "observation_ms": 18.28289998229593, "policy_ms": 3.1875999411568046, "execution_ms": 18.8400000333786, "status": "ok", "code": "SELECT" }, { "sample_id": 9, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.720268964767456, "target_confidence": 1.0, "observation_ms": 15.088799991644919, "policy_ms": 1.9933999283239245, "execution_ms": 2.431600005365908, "status": "ok", "code": "ASK_USER" }, { "sample_id": 10, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.001499887555838, "policy_ms": 2.0283000776544213, "execution_ms": 18.627900048159063, "status": "ok", "code": "SELECT" }, { "sample_id": 11, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.6273818016052246, "observation_ms": 15.724600059911609, "policy_ms": 2.161799930036068, "execution_ms": 2.5417000288143754, "status": "ok", "code": "ASK_USER" }, { "sample_id": 12, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.226400017738342, "policy_ms": 2.511899918317795, "execution_ms": 13.394499896094203, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 13, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999972581863403, "target_confidence": 1.0, "observation_ms": 14.533399953506887, "policy_ms": 2.852100064046681, "execution_ms": 47.78600006829947, "status": "ok", "code": "CLICK" }, { "sample_id": 14, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.751499915495515, "policy_ms": 2.3541999980807304, "execution_ms": 17.82499998807907, "status": "ok", "code": "SELECT" }, { "sample_id": 15, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.72710003145039, "policy_ms": 1.6965999966487288, "execution_ms": 12.583900010213256, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 16, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.40279996395111, "policy_ms": 3.0305000254884362, "execution_ms": 18.68630002718419, "status": "ok", "code": "SELECT" }, { "sample_id": 17, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.897099907509983, "policy_ms": 2.886700094677508, "execution_ms": 19.381599966436625, "status": "ok", "code": "SELECT" }, { "sample_id": 18, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9988070726394653, "target_confidence": 1.0, "observation_ms": 15.318899997510016, "policy_ms": 2.181699965149164, "execution_ms": 55.76350004412234, "status": "ok", "code": "CLICK" }, { "sample_id": 19, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.741200048476458, "policy_ms": 2.4813999189063907, "execution_ms": 21.54819993302226, "status": "ok", "code": "SELECT" }, { "sample_id": 20, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9989357590675354, "target_confidence": 1.0, "observation_ms": 19.367799977771938, "policy_ms": 3.077700035646558, "execution_ms": 28.341200086288154, "status": "ok", "code": "CLICK" }, { "sample_id": 21, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.52290001604706, "policy_ms": 2.6355000445619226, "execution_ms": 54.26959996111691, "status": "ok", "code": "CLICK" }, { "sample_id": 22, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.09010004904121, "policy_ms": 2.549300086684525, "execution_ms": 55.372499977238476, "status": "ok", "code": "CLICK" }, { "sample_id": 23, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.668199979700148, "policy_ms": 3.1169999856501818, "execution_ms": 19.12760001141578, "status": "ok", "code": "SELECT" }, { "sample_id": 24, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999839067459106, "target_confidence": 1.0, "observation_ms": 18.24030000716448, "policy_ms": 2.835199935361743, "execution_ms": 53.59849997330457, "status": "ok", "code": "CLICK" }, { "sample_id": 25, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.502800000831485, "policy_ms": 2.149400068446994, "execution_ms": 57.814099942333996, "status": "ok", "code": "CLICK" }, { "sample_id": 26, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.772700055502355, "policy_ms": 3.4591000294312835, "execution_ms": 11.525699985213578, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 27, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.285500071011484, "policy_ms": 1.628599944524467, "execution_ms": 17.25599996279925, "status": "ok", "code": "SELECT" }, { "sample_id": 28, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.44849995803088, "policy_ms": 1.9515000749379396, "execution_ms": 11.138500063680112, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 29, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.951699900440872, "policy_ms": 2.086500055156648, "execution_ms": 36.58199997153133, "status": "ok", "code": "CLICK" }, { "sample_id": 30, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.507000032812357, "policy_ms": 2.9299999587237835, "execution_ms": 10.971100069582462, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 31, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.220500041730702, "policy_ms": 2.926700050011277, "execution_ms": 20.08299995213747, "status": "ok", "code": "SELECT" }, { "sample_id": 32, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.720500028692186, "policy_ms": 2.1554999984800816, "execution_ms": 18.35559995379299, "status": "ok", "code": "SELECT" }, { "sample_id": 33, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999918937683105, "observation_ms": 14.345600036904216, "policy_ms": 1.8833000212907791, "execution_ms": 2.653100062161684, "status": "ok", "code": "ASK_USER" }, { "sample_id": 34, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9991865754127502, "target_confidence": 1.0, "observation_ms": 17.341499915346503, "policy_ms": 3.9911000058054924, "execution_ms": 18.907299963757396, "status": "ok", "code": "TYPE" }, { "sample_id": 35, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999992847442627, "target_confidence": 1.0, "observation_ms": 14.940900029614568, "policy_ms": 1.7117999959737062, "execution_ms": 55.279399966821074, "status": "ok", "code": "CLICK" }, { "sample_id": 36, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.2782999901101, "policy_ms": 2.2227000445127487, "execution_ms": 11.639100033789873, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 37, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.447200069203973, "policy_ms": 2.64920003246516, "execution_ms": 22.73369999602437, "status": "ok", "code": "SELECT" }, { "sample_id": 38, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9380846619606018, "target_confidence": 1.0, "observation_ms": 18.356399959884584, "policy_ms": 2.828199998475611, "execution_ms": 11.570300091989338, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 39, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9524294137954712, "observation_ms": 13.235999969765544, "policy_ms": 3.237399971112609, "execution_ms": 12.619900051504374, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 40, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9989209175109863, "observation_ms": 14.78179998230189, "policy_ms": 2.5893000420182943, "execution_ms": 2.7925000758841634, "status": "ok", "code": "ASK_USER" }, { "sample_id": 41, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.161800034344196, "policy_ms": 2.2444999776780605, "execution_ms": 20.212199888192117, "status": "ok", "code": "SELECT" }, { "sample_id": 42, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.984900062903762, "policy_ms": 3.330400097183883, "execution_ms": 18.50799995008856, "status": "ok", "code": "SELECT" }, { "sample_id": 43, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.629499986767769, "policy_ms": 1.6976000042632222, "execution_ms": 18.62929994240403, "status": "ok", "code": "SELECT" }, { "sample_id": 44, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.890900071710348, "policy_ms": 3.423499991185963, "execution_ms": 44.28839997854084, "status": "ok", "code": "CLICK" }, { "sample_id": 45, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.07880005799234, "policy_ms": 1.881699892692268, "execution_ms": 21.7675999738276, "status": "ok", "code": "SELECT" }, { "sample_id": 46, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.39379998575896, "policy_ms": 2.288699964992702, "execution_ms": 17.625999986194074, "status": "ok", "code": "SELECT" }, { "sample_id": 47, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999927282333374, "observation_ms": 14.32419999036938, "policy_ms": 2.470299950800836, "execution_ms": 12.402099906466901, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 48, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.879200004041195, "policy_ms": 2.7562000323086977, "execution_ms": 12.391900061629713, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 49, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.33019996713847, "policy_ms": 1.9475000444799662, "execution_ms": 11.631699977442622, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 50, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.6744232177734375, "target_confidence": 1.0, "observation_ms": 16.53129991609603, "policy_ms": 2.243799972347915, "execution_ms": 2.6498999213799834, "status": "ok", "code": "ASK_USER" }, { "sample_id": 51, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.148900052532554, "policy_ms": 2.399299992248416, "execution_ms": 16.850900021381676, "status": "ok", "code": "SELECT" }, { "sample_id": 52, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.341399936005473, "policy_ms": 2.228700090199709, "execution_ms": 20.004799938760698, "status": "ok", "code": "SELECT" }, { "sample_id": 53, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 16.835800022818148, "policy_ms": 2.710500033572316, "execution_ms": 2.8714999789372087, "status": "ok", "code": "ASK_USER" }, { "sample_id": 54, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.999863862991333, "target_confidence": 1.0, "observation_ms": 16.745399916544557, "policy_ms": 2.9193999944254756, "execution_ms": 18.452899996191263, "status": "ok", "code": "TYPE" }, { "sample_id": 55, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.656399937346578, "policy_ms": 2.845000009983778, "execution_ms": 18.603299977257848, "status": "ok", "code": "SELECT" }, { "sample_id": 56, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999996423721313, "observation_ms": 15.194299980066717, "policy_ms": 3.7309000035747886, "execution_ms": 2.539800014346838, "status": "ok", "code": "ASK_USER" }, { "sample_id": 57, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.091400011442602, "policy_ms": 2.4770000018179417, "execution_ms": 46.00049997679889, "status": "ok", "code": "CLICK" }, { "sample_id": 58, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.82049995381385, "policy_ms": 3.619199967943132, "execution_ms": 17.847699928097427, "status": "ok", "code": "SELECT" }, { "sample_id": 59, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.140800038352609, "policy_ms": 3.8938000798225403, "execution_ms": 18.620199989527464, "status": "ok", "code": "SELECT" }, { "sample_id": 60, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.123799956403673, "policy_ms": 2.451400039717555, "execution_ms": 18.99300003424287, "status": "ok", "code": "SELECT" }, { "sample_id": 61, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.06960003543645, "policy_ms": 4.310499993152916, "execution_ms": 18.57810001820326, "status": "ok", "code": "SELECT" }, { "sample_id": 62, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9984056353569031, "target_confidence": 1.0, "observation_ms": 16.65180001873523, "policy_ms": 2.936400007456541, "execution_ms": 11.463699978776276, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 63, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9954480528831482, "observation_ms": 16.483900020830333, "policy_ms": 4.453599918633699, "execution_ms": 3.3304999815300107, "status": "ok", "code": "ASK_USER" }, { "sample_id": 64, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.926200012676418, "policy_ms": 2.8759000124409795, "execution_ms": 11.980699957348406, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 65, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.666200080886483, "policy_ms": 2.0320999901741743, "execution_ms": 2.2483000066131353, "status": "ok", "code": "ASK_USER" }, { "sample_id": 66, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9823814034461975, "target_confidence": 1.0, "observation_ms": 17.019399907439947, "policy_ms": 3.4302000422030687, "execution_ms": 48.08139998931438, "status": "ok", "code": "CLICK" }, { "sample_id": 67, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9998229146003723, "target_confidence": 1.0, "observation_ms": 16.775500029325485, "policy_ms": 1.930900034494698, "execution_ms": 18.81799998227507, "status": "ok", "code": "TYPE" }, { "sample_id": 68, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.999362051486969, "target_confidence": 1.0, "observation_ms": 14.861600007861853, "policy_ms": 1.7525999573990703, "execution_ms": 2.316199941560626, "status": "ok", "code": "ASK_USER" }, { "sample_id": 69, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.999445378780365, "observation_ms": 20.707699935883284, "policy_ms": 2.239100052975118, "execution_ms": 3.15140001475811, "status": "ok", "code": "ASK_USER" }, { "sample_id": 70, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.91260007303208, "policy_ms": 2.6763000059872866, "execution_ms": 26.25090000219643, "status": "ok", "code": "CLICK" }, { "sample_id": 71, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.43979998677969, "policy_ms": 2.317399950698018, "execution_ms": 16.30629994906485, "status": "ok", "code": "SELECT" }, { "sample_id": 72, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.8963271975517273, "observation_ms": 16.73909998498857, "policy_ms": 2.9506999999284744, "execution_ms": 11.113999993540347, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 73, "template": "nested", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999988079071045, "observation_ms": 13.182500028051436, "policy_ms": 1.789500005543232, "execution_ms": 16.17940003052354, "status": "ok", "code": "SELECT" }, { "sample_id": 74, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.6012862920761108, "target_confidence": 1.0, "observation_ms": 14.6752999862656, "policy_ms": 3.1855000415816903, "execution_ms": 2.329000039026141, "status": "ok", "code": "ASK_USER" }, { "sample_id": 75, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.364199967123568, "policy_ms": 1.8962000031024218, "execution_ms": 18.064500065520406, "status": "ok", "code": "SELECT" }, { "sample_id": 76, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.066100073046982, "policy_ms": 1.602599979378283, "execution_ms": 17.419800045900047, "status": "ok", "code": "SELECT" }, { "sample_id": 77, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.11209991481155, "policy_ms": 3.523800056427717, "execution_ms": 12.085000053048134, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 78, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.945799998007715, "policy_ms": 2.0475999917834997, "execution_ms": 18.963399925269186, "status": "ok", "code": "SELECT" }, { "sample_id": 79, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.105900031514466, "policy_ms": 4.0596999460831285, "execution_ms": 17.803399940021336, "status": "ok", "code": "SELECT" }, { "sample_id": 80, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.763400006107986, "policy_ms": 2.6825000531971455, "execution_ms": 55.71610003244132, "status": "ok", "code": "CLICK" }, { "sample_id": 81, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999572038650513, "target_confidence": 1.0, "observation_ms": 15.868999995291233, "policy_ms": 2.6097999652847648, "execution_ms": 54.374199942685664, "status": "ok", "code": "CLICK" }, { "sample_id": 82, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.8975507616996765, "target_confidence": 1.0, "observation_ms": 15.959199983626604, "policy_ms": 3.017399925738573, "execution_ms": 56.86929996591061, "status": "ok", "code": "CLICK" }, { "sample_id": 83, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9982919096946716, "target_confidence": 1.0, "observation_ms": 19.22529994044453, "policy_ms": 2.697200048714876, "execution_ms": 11.397000052966177, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 84, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.615900025703013, "policy_ms": 3.63880000077188, "execution_ms": 13.063999940641224, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 85, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.511700021103024, "policy_ms": 2.7857000241056085, "execution_ms": 17.01329997740686, "status": "ok", "code": "SELECT" }, { "sample_id": 86, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999996423721313, "target_confidence": 1.0, "observation_ms": 14.36120003927499, "policy_ms": 1.8855000380426645, "execution_ms": 24.41690000705421, "status": "ok", "code": "CLICK" }, { "sample_id": 87, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9854127764701843, "target_confidence": 1.0, "observation_ms": 19.69929994083941, "policy_ms": 2.6805000379681587, "execution_ms": 50.03080004826188, "status": "ok", "code": "CLICK" }, { "sample_id": 88, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9993982315063477, "target_confidence": 1.0, "observation_ms": 19.228799967095256, "policy_ms": 3.0580000020563602, "execution_ms": 49.42660010419786, "status": "ok", "code": "CLICK" }, { "sample_id": 89, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 22.761099971830845, "policy_ms": 5.15910005196929, "execution_ms": 22.098699933849275, "status": "ok", "code": "SELECT" }, { "sample_id": 90, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9993982315063477, "target_confidence": 0.9999520778656006, "observation_ms": 15.120600000955164, "policy_ms": 3.2110000029206276, "execution_ms": 27.894600061699748, "status": "ok", "code": "CLICK" }, { "sample_id": 91, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9313865303993225, "target_confidence": 1.0, "observation_ms": 13.383200042881072, "policy_ms": 2.8343999292701483, "execution_ms": 16.3021000334993, "status": "ok", "code": "TYPE" }, { "sample_id": 92, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9995154142379761, "observation_ms": 15.163899981416762, "policy_ms": 2.8198999352753162, "execution_ms": 10.540399933233857, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 93, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.40700006019324, "policy_ms": 1.7691999673843384, "execution_ms": 10.332000092603266, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 94, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9896392822265625, "target_confidence": 1.0, "observation_ms": 14.853800064884126, "policy_ms": 2.810999983921647, "execution_ms": 29.672999982722104, "status": "ok", "code": "CLICK" }, { "sample_id": 95, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9500922560691833, "target_confidence": 1.0, "observation_ms": 13.254300109110773, "policy_ms": 2.440100070089102, "execution_ms": 23.67790008429438, "status": "ok", "code": "CLICK" }, { "sample_id": 96, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9655914902687073, "target_confidence": 1.0, "observation_ms": 13.774699997156858, "policy_ms": 2.7686000103130937, "execution_ms": 9.721900103613734, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 97, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9535804390907288, "target_confidence": 1.0, "observation_ms": 11.383499950170517, "policy_ms": 2.25639995187521, "execution_ms": 16.029999940656126, "status": "ok", "code": "TYPE" }, { "sample_id": 98, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.237299979664385, "policy_ms": 1.9378999713808298, "execution_ms": 17.197499983012676, "status": "ok", "code": "SELECT" }, { "sample_id": 99, "template": "table", "success": true, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.470300081185997, "policy_ms": 3.1556999310851097, "execution_ms": 2.621099934913218, "status": "ok", "code": "ASK_USER" }, { "sample_id": 100, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 13.75969999935478, "policy_ms": 2.693300019018352, "execution_ms": 2.2585000842809677, "status": "ok", "code": "ASK_USER" }, { "sample_id": 101, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.587200035341084, "policy_ms": 2.363799954764545, "execution_ms": 2.671200083568692, "status": "ok", "code": "ASK_USER" }, { "sample_id": 102, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9827704429626465, "target_confidence": 1.0, "observation_ms": 17.5406999187544, "policy_ms": 3.0599000165238976, "execution_ms": 17.11390004493296, "status": "ok", "code": "TYPE" }, { "sample_id": 103, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999997615814209, "target_confidence": 1.0, "observation_ms": 12.314500054344535, "policy_ms": 1.8863000441342592, "execution_ms": 16.594600048847497, "status": "ok", "code": "TYPE" }, { "sample_id": 104, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9209970831871033, "observation_ms": 14.165100059472024, "policy_ms": 2.4361000396311283, "execution_ms": 15.23400004953146, "status": "ok", "code": "SELECT" }, { "sample_id": 105, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.003300013020635, "policy_ms": 1.6154000768437982, "execution_ms": 18.52439995855093, "status": "ok", "code": "TYPE" }, { "sample_id": 106, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9997993111610413, "target_confidence": 1.0, "observation_ms": 13.28109996393323, "policy_ms": 1.9484000513330102, "execution_ms": 10.351400007493794, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 107, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9982926249504089, "observation_ms": 14.476899988949299, "policy_ms": 3.381800022907555, "execution_ms": 15.20399993751198, "status": "ok", "code": "SELECT" }, { "sample_id": 108, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.83090000692755, "policy_ms": 3.055799985304475, "execution_ms": 2.2097000619396567, "status": "ok", "code": "ASK_USER" }, { "sample_id": 109, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9956113696098328, "target_confidence": 1.0, "observation_ms": 15.895100077614188, "policy_ms": 3.56480001937598, "execution_ms": 12.63380004093051, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 110, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.509899959899485, "policy_ms": 1.84599997010082, "execution_ms": 15.045200008898973, "status": "ok", "code": "SELECT" }, { "sample_id": 111, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.48460002616048, "policy_ms": 2.6233999524265528, "execution_ms": 16.111999982967973, "status": "ok", "code": "SELECT" }, { "sample_id": 112, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.8389744758605957, "target_confidence": 1.0, "observation_ms": 17.920400016009808, "policy_ms": 2.507500001229346, "execution_ms": 2.3775999434292316, "status": "ok", "code": "ASK_USER" }, { "sample_id": 113, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.258200022391975, "policy_ms": 2.506399992853403, "execution_ms": 22.85220008343458, "status": "ok", "code": "CLICK" }, { "sample_id": 114, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.853500062599778, "policy_ms": 3.9912000065669417, "execution_ms": 19.208399928174913, "status": "ok", "code": "SELECT" }, { "sample_id": 115, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9923478364944458, "target_confidence": 1.0, "observation_ms": 13.800600077956915, "policy_ms": 2.797399996779859, "execution_ms": 16.694299993105233, "status": "ok", "code": "TYPE" }, { "sample_id": 116, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9923343062400818, "target_confidence": 1.0, "observation_ms": 14.609100064262748, "policy_ms": 2.129599917680025, "execution_ms": 17.368000000715256, "status": "ok", "code": "TYPE" }, { "sample_id": 117, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999951124191284, "target_confidence": 1.0, "observation_ms": 15.091100009158254, "policy_ms": 1.891499967314303, "execution_ms": 21.638500038534403, "status": "ok", "code": "CLICK" }, { "sample_id": 118, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.210700009949505, "policy_ms": 2.4665999226272106, "execution_ms": 14.696700032800436, "status": "ok", "code": "SELECT" }, { "sample_id": 119, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999986886978149, "observation_ms": 13.8888000510633, "policy_ms": 3.2760000322014093, "execution_ms": 15.402200049720705, "status": "ok", "code": "SELECT" } ] }