| { | |
| "checkpoint": "models/v000-mean", | |
| "quantized": false, | |
| "platform": "Windows-11-10.0.26200-SP0", | |
| "torch_threads": 2, | |
| "model_load_ms": 9.840800077654421, | |
| "samples": 120, | |
| "success_rate": 0.6166666666666667, | |
| "abstention_rate": 0.15833333333333333, | |
| "median_observation_ms": 15.722550044301897, | |
| "median_policy_ms": 2.642350038513541, | |
| "median_execution_ms": 17.39390002330765, | |
| "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", | |
| "target_vps_validated": false, | |
| "results": [ | |
| { | |
| "sample_id": 0, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.6028217077255249, | |
| "observation_ms": 41.0907999612391, | |
| "policy_ms": 4.931100062094629, | |
| "execution_ms": 5.35310001578182, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 1, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.348800039850175, | |
| "policy_ms": 2.6072999462485313, | |
| "execution_ms": 46.392700052820146, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 2, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 29.954700032249093, | |
| "policy_ms": 2.511699916794896, | |
| "execution_ms": 31.195999938063323, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 3, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999649524688721, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.29299991670996, | |
| "policy_ms": 2.1188000682741404, | |
| "execution_ms": 32.14030002709478, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 4, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9261236786842346, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.166699981316924, | |
| "policy_ms": 3.0170000391080976, | |
| "execution_ms": 2.691800007596612, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 5, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9993553757667542, | |
| "observation_ms": 18.30679993145168, | |
| "policy_ms": 2.898600068874657, | |
| "execution_ms": 13.020699960179627, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 6, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9194011688232422, | |
| "observation_ms": 20.474000019021332, | |
| "policy_ms": 3.6683999933302402, | |
| "execution_ms": 21.632199990563095, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 7, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9984651803970337, | |
| "observation_ms": 18.28419999219477, | |
| "policy_ms": 3.329100087285042, | |
| "execution_ms": 23.86119996663183, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 8, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999978542327881, | |
| "observation_ms": 18.28289998229593, | |
| "policy_ms": 3.1875999411568046, | |
| "execution_ms": 18.8400000333786, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 9, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.720268964767456, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.088799991644919, | |
| "policy_ms": 1.9933999283239245, | |
| "execution_ms": 2.431600005365908, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 10, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.001499887555838, | |
| "policy_ms": 2.0283000776544213, | |
| "execution_ms": 18.627900048159063, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 11, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.6273818016052246, | |
| "observation_ms": 15.724600059911609, | |
| "policy_ms": 2.161799930036068, | |
| "execution_ms": 2.5417000288143754, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 12, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.226400017738342, | |
| "policy_ms": 2.511899918317795, | |
| "execution_ms": 13.394499896094203, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 13, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999972581863403, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.533399953506887, | |
| "policy_ms": 2.852100064046681, | |
| "execution_ms": 47.78600006829947, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 14, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.751499915495515, | |
| "policy_ms": 2.3541999980807304, | |
| "execution_ms": 17.82499998807907, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 15, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.72710003145039, | |
| "policy_ms": 1.6965999966487288, | |
| "execution_ms": 12.583900010213256, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 16, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.40279996395111, | |
| "policy_ms": 3.0305000254884362, | |
| "execution_ms": 18.68630002718419, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 17, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.897099907509983, | |
| "policy_ms": 2.886700094677508, | |
| "execution_ms": 19.381599966436625, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 18, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9988070726394653, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.318899997510016, | |
| "policy_ms": 2.181699965149164, | |
| "execution_ms": 55.76350004412234, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 19, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.741200048476458, | |
| "policy_ms": 2.4813999189063907, | |
| "execution_ms": 21.54819993302226, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 20, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9989357590675354, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.367799977771938, | |
| "policy_ms": 3.077700035646558, | |
| "execution_ms": 28.341200086288154, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 21, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.52290001604706, | |
| "policy_ms": 2.6355000445619226, | |
| "execution_ms": 54.26959996111691, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 22, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.09010004904121, | |
| "policy_ms": 2.549300086684525, | |
| "execution_ms": 55.372499977238476, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 23, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.668199979700148, | |
| "policy_ms": 3.1169999856501818, | |
| "execution_ms": 19.12760001141578, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 24, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999839067459106, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.24030000716448, | |
| "policy_ms": 2.835199935361743, | |
| "execution_ms": 53.59849997330457, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 25, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.502800000831485, | |
| "policy_ms": 2.149400068446994, | |
| "execution_ms": 57.814099942333996, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 26, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.772700055502355, | |
| "policy_ms": 3.4591000294312835, | |
| "execution_ms": 11.525699985213578, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 27, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.285500071011484, | |
| "policy_ms": 1.628599944524467, | |
| "execution_ms": 17.25599996279925, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 28, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.44849995803088, | |
| "policy_ms": 1.9515000749379396, | |
| "execution_ms": 11.138500063680112, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 29, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.951699900440872, | |
| "policy_ms": 2.086500055156648, | |
| "execution_ms": 36.58199997153133, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 30, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.507000032812357, | |
| "policy_ms": 2.9299999587237835, | |
| "execution_ms": 10.971100069582462, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 31, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.220500041730702, | |
| "policy_ms": 2.926700050011277, | |
| "execution_ms": 20.08299995213747, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 32, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.720500028692186, | |
| "policy_ms": 2.1554999984800816, | |
| "execution_ms": 18.35559995379299, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 33, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999918937683105, | |
| "observation_ms": 14.345600036904216, | |
| "policy_ms": 1.8833000212907791, | |
| "execution_ms": 2.653100062161684, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 34, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9991865754127502, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.341499915346503, | |
| "policy_ms": 3.9911000058054924, | |
| "execution_ms": 18.907299963757396, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 35, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999992847442627, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.940900029614568, | |
| "policy_ms": 1.7117999959737062, | |
| "execution_ms": 55.279399966821074, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 36, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.2782999901101, | |
| "policy_ms": 2.2227000445127487, | |
| "execution_ms": 11.639100033789873, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 37, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.447200069203973, | |
| "policy_ms": 2.64920003246516, | |
| "execution_ms": 22.73369999602437, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 38, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9380846619606018, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.356399959884584, | |
| "policy_ms": 2.828199998475611, | |
| "execution_ms": 11.570300091989338, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 39, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9524294137954712, | |
| "observation_ms": 13.235999969765544, | |
| "policy_ms": 3.237399971112609, | |
| "execution_ms": 12.619900051504374, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 40, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9989209175109863, | |
| "observation_ms": 14.78179998230189, | |
| "policy_ms": 2.5893000420182943, | |
| "execution_ms": 2.7925000758841634, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 41, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.161800034344196, | |
| "policy_ms": 2.2444999776780605, | |
| "execution_ms": 20.212199888192117, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 42, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.984900062903762, | |
| "policy_ms": 3.330400097183883, | |
| "execution_ms": 18.50799995008856, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 43, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.629499986767769, | |
| "policy_ms": 1.6976000042632222, | |
| "execution_ms": 18.62929994240403, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 44, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.890900071710348, | |
| "policy_ms": 3.423499991185963, | |
| "execution_ms": 44.28839997854084, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 45, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.07880005799234, | |
| "policy_ms": 1.881699892692268, | |
| "execution_ms": 21.7675999738276, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 46, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.39379998575896, | |
| "policy_ms": 2.288699964992702, | |
| "execution_ms": 17.625999986194074, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 47, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999927282333374, | |
| "observation_ms": 14.32419999036938, | |
| "policy_ms": 2.470299950800836, | |
| "execution_ms": 12.402099906466901, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 48, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.879200004041195, | |
| "policy_ms": 2.7562000323086977, | |
| "execution_ms": 12.391900061629713, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 49, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.33019996713847, | |
| "policy_ms": 1.9475000444799662, | |
| "execution_ms": 11.631699977442622, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 50, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.6744232177734375, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.53129991609603, | |
| "policy_ms": 2.243799972347915, | |
| "execution_ms": 2.6498999213799834, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 51, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.148900052532554, | |
| "policy_ms": 2.399299992248416, | |
| "execution_ms": 16.850900021381676, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 52, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.341399936005473, | |
| "policy_ms": 2.228700090199709, | |
| "execution_ms": 20.004799938760698, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 53, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 16.835800022818148, | |
| "policy_ms": 2.710500033572316, | |
| "execution_ms": 2.8714999789372087, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 54, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.999863862991333, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.745399916544557, | |
| "policy_ms": 2.9193999944254756, | |
| "execution_ms": 18.452899996191263, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 55, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.656399937346578, | |
| "policy_ms": 2.845000009983778, | |
| "execution_ms": 18.603299977257848, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 56, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999996423721313, | |
| "observation_ms": 15.194299980066717, | |
| "policy_ms": 3.7309000035747886, | |
| "execution_ms": 2.539800014346838, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 57, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.091400011442602, | |
| "policy_ms": 2.4770000018179417, | |
| "execution_ms": 46.00049997679889, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 58, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.82049995381385, | |
| "policy_ms": 3.619199967943132, | |
| "execution_ms": 17.847699928097427, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 59, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.140800038352609, | |
| "policy_ms": 3.8938000798225403, | |
| "execution_ms": 18.620199989527464, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 60, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.123799956403673, | |
| "policy_ms": 2.451400039717555, | |
| "execution_ms": 18.99300003424287, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 61, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.06960003543645, | |
| "policy_ms": 4.310499993152916, | |
| "execution_ms": 18.57810001820326, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 62, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9984056353569031, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.65180001873523, | |
| "policy_ms": 2.936400007456541, | |
| "execution_ms": 11.463699978776276, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 63, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9954480528831482, | |
| "observation_ms": 16.483900020830333, | |
| "policy_ms": 4.453599918633699, | |
| "execution_ms": 3.3304999815300107, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 64, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.926200012676418, | |
| "policy_ms": 2.8759000124409795, | |
| "execution_ms": 11.980699957348406, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 65, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.666200080886483, | |
| "policy_ms": 2.0320999901741743, | |
| "execution_ms": 2.2483000066131353, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 66, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9823814034461975, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.019399907439947, | |
| "policy_ms": 3.4302000422030687, | |
| "execution_ms": 48.08139998931438, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 67, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9998229146003723, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.775500029325485, | |
| "policy_ms": 1.930900034494698, | |
| "execution_ms": 18.81799998227507, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 68, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.999362051486969, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.861600007861853, | |
| "policy_ms": 1.7525999573990703, | |
| "execution_ms": 2.316199941560626, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 69, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.999445378780365, | |
| "observation_ms": 20.707699935883284, | |
| "policy_ms": 2.239100052975118, | |
| "execution_ms": 3.15140001475811, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 70, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.91260007303208, | |
| "policy_ms": 2.6763000059872866, | |
| "execution_ms": 26.25090000219643, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 71, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.43979998677969, | |
| "policy_ms": 2.317399950698018, | |
| "execution_ms": 16.30629994906485, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 72, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8963271975517273, | |
| "observation_ms": 16.73909998498857, | |
| "policy_ms": 2.9506999999284744, | |
| "execution_ms": 11.113999993540347, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 73, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999988079071045, | |
| "observation_ms": 13.182500028051436, | |
| "policy_ms": 1.789500005543232, | |
| "execution_ms": 16.17940003052354, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 74, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.6012862920761108, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.6752999862656, | |
| "policy_ms": 3.1855000415816903, | |
| "execution_ms": 2.329000039026141, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 75, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.364199967123568, | |
| "policy_ms": 1.8962000031024218, | |
| "execution_ms": 18.064500065520406, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 76, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.066100073046982, | |
| "policy_ms": 1.602599979378283, | |
| "execution_ms": 17.419800045900047, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 77, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.11209991481155, | |
| "policy_ms": 3.523800056427717, | |
| "execution_ms": 12.085000053048134, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 78, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.945799998007715, | |
| "policy_ms": 2.0475999917834997, | |
| "execution_ms": 18.963399925269186, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 79, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.105900031514466, | |
| "policy_ms": 4.0596999460831285, | |
| "execution_ms": 17.803399940021336, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 80, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.763400006107986, | |
| "policy_ms": 2.6825000531971455, | |
| "execution_ms": 55.71610003244132, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 81, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999572038650513, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.868999995291233, | |
| "policy_ms": 2.6097999652847648, | |
| "execution_ms": 54.374199942685664, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 82, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.8975507616996765, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.959199983626604, | |
| "policy_ms": 3.017399925738573, | |
| "execution_ms": 56.86929996591061, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 83, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9982919096946716, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.22529994044453, | |
| "policy_ms": 2.697200048714876, | |
| "execution_ms": 11.397000052966177, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 84, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.615900025703013, | |
| "policy_ms": 3.63880000077188, | |
| "execution_ms": 13.063999940641224, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 85, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.511700021103024, | |
| "policy_ms": 2.7857000241056085, | |
| "execution_ms": 17.01329997740686, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 86, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999996423721313, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.36120003927499, | |
| "policy_ms": 1.8855000380426645, | |
| "execution_ms": 24.41690000705421, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 87, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9854127764701843, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.69929994083941, | |
| "policy_ms": 2.6805000379681587, | |
| "execution_ms": 50.03080004826188, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 88, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9993982315063477, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.228799967095256, | |
| "policy_ms": 3.0580000020563602, | |
| "execution_ms": 49.42660010419786, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 89, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 22.761099971830845, | |
| "policy_ms": 5.15910005196929, | |
| "execution_ms": 22.098699933849275, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 90, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9993982315063477, | |
| "target_confidence": 0.9999520778656006, | |
| "observation_ms": 15.120600000955164, | |
| "policy_ms": 3.2110000029206276, | |
| "execution_ms": 27.894600061699748, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 91, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9313865303993225, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.383200042881072, | |
| "policy_ms": 2.8343999292701483, | |
| "execution_ms": 16.3021000334993, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 92, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9995154142379761, | |
| "observation_ms": 15.163899981416762, | |
| "policy_ms": 2.8198999352753162, | |
| "execution_ms": 10.540399933233857, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 93, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.40700006019324, | |
| "policy_ms": 1.7691999673843384, | |
| "execution_ms": 10.332000092603266, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 94, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9896392822265625, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.853800064884126, | |
| "policy_ms": 2.810999983921647, | |
| "execution_ms": 29.672999982722104, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 95, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9500922560691833, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.254300109110773, | |
| "policy_ms": 2.440100070089102, | |
| "execution_ms": 23.67790008429438, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 96, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9655914902687073, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.774699997156858, | |
| "policy_ms": 2.7686000103130937, | |
| "execution_ms": 9.721900103613734, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 97, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9535804390907288, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.383499950170517, | |
| "policy_ms": 2.25639995187521, | |
| "execution_ms": 16.029999940656126, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 98, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.237299979664385, | |
| "policy_ms": 1.9378999713808298, | |
| "execution_ms": 17.197499983012676, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 99, | |
| "template": "table", | |
| "success": true, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.470300081185997, | |
| "policy_ms": 3.1556999310851097, | |
| "execution_ms": 2.621099934913218, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 100, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 13.75969999935478, | |
| "policy_ms": 2.693300019018352, | |
| "execution_ms": 2.2585000842809677, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 101, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.587200035341084, | |
| "policy_ms": 2.363799954764545, | |
| "execution_ms": 2.671200083568692, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 102, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9827704429626465, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.5406999187544, | |
| "policy_ms": 3.0599000165238976, | |
| "execution_ms": 17.11390004493296, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 103, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999997615814209, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.314500054344535, | |
| "policy_ms": 1.8863000441342592, | |
| "execution_ms": 16.594600048847497, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 104, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9209970831871033, | |
| "observation_ms": 14.165100059472024, | |
| "policy_ms": 2.4361000396311283, | |
| "execution_ms": 15.23400004953146, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 105, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.003300013020635, | |
| "policy_ms": 1.6154000768437982, | |
| "execution_ms": 18.52439995855093, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 106, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9997993111610413, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.28109996393323, | |
| "policy_ms": 1.9484000513330102, | |
| "execution_ms": 10.351400007493794, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 107, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9982926249504089, | |
| "observation_ms": 14.476899988949299, | |
| "policy_ms": 3.381800022907555, | |
| "execution_ms": 15.20399993751198, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 108, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.83090000692755, | |
| "policy_ms": 3.055799985304475, | |
| "execution_ms": 2.2097000619396567, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 109, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9956113696098328, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.895100077614188, | |
| "policy_ms": 3.56480001937598, | |
| "execution_ms": 12.63380004093051, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 110, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.509899959899485, | |
| "policy_ms": 1.84599997010082, | |
| "execution_ms": 15.045200008898973, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 111, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.48460002616048, | |
| "policy_ms": 2.6233999524265528, | |
| "execution_ms": 16.111999982967973, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 112, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.8389744758605957, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.920400016009808, | |
| "policy_ms": 2.507500001229346, | |
| "execution_ms": 2.3775999434292316, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 113, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.258200022391975, | |
| "policy_ms": 2.506399992853403, | |
| "execution_ms": 22.85220008343458, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 114, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.853500062599778, | |
| "policy_ms": 3.9912000065669417, | |
| "execution_ms": 19.208399928174913, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 115, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9923478364944458, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.800600077956915, | |
| "policy_ms": 2.797399996779859, | |
| "execution_ms": 16.694299993105233, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 116, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9923343062400818, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.609100064262748, | |
| "policy_ms": 2.129599917680025, | |
| "execution_ms": 17.368000000715256, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 117, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999951124191284, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.091100009158254, | |
| "policy_ms": 1.891499967314303, | |
| "execution_ms": 21.638500038534403, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 118, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.210700009949505, | |
| "policy_ms": 2.4665999226272106, | |
| "execution_ms": 14.696700032800436, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 119, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999986886978149, | |
| "observation_ms": 13.8888000510633, | |
| "policy_ms": 3.2760000322014093, | |
| "execution_ms": 15.402200049720705, | |
| "status": "ok", | |
| "code": "SELECT" | |
| } | |
| ] | |
| } |