| { | |
| "checkpoint": "models/v003-transformer", | |
| "quantized": false, | |
| "platform": "Windows-11-10.0.26200-SP0", | |
| "torch_threads": 2, | |
| "model_load_ms": 11.723599978722632, | |
| "samples": 120, | |
| "success_rate": 0.49166666666666664, | |
| "abstention_rate": 0.15, | |
| "median_observation_ms": 14.34735005022958, | |
| "median_policy_ms": 4.208899976219982, | |
| "median_execution_ms": 16.347200027666986, | |
| "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", | |
| "target_vps_validated": false, | |
| "results": [ | |
| { | |
| "sample_id": 0, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 43.33179991226643, | |
| "policy_ms": 6.381200044415891, | |
| "execution_ms": 3.6983999889343977, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 1, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999996423721313, | |
| "observation_ms": 15.646400046534836, | |
| "policy_ms": 3.6445000441744924, | |
| "execution_ms": 43.543800013139844, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 2, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999963045120239, | |
| "target_confidence": 0.9999997615814209, | |
| "observation_ms": 29.71329994034022, | |
| "policy_ms": 4.3830000795423985, | |
| "execution_ms": 18.738399958238006, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 3, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.353300095535815, | |
| "policy_ms": 3.6409999011084437, | |
| "execution_ms": 58.37500002235174, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 4, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.350400052033365, | |
| "policy_ms": 4.532700055278838, | |
| "execution_ms": 32.714600092731416, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 5, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.697097897529602, | |
| "observation_ms": 16.117900027893484, | |
| "policy_ms": 4.587300005368888, | |
| "execution_ms": 2.7254000306129456, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 6, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.944299986585975, | |
| "policy_ms": 4.615599988028407, | |
| "execution_ms": 16.94929995574057, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 7, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999443292617798, | |
| "observation_ms": 15.260900021530688, | |
| "policy_ms": 5.21530001424253, | |
| "execution_ms": 16.81660010945052, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 8, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.323400010354817, | |
| "policy_ms": 4.924600012600422, | |
| "execution_ms": 17.00610003899783, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 9, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.643000020645559, | |
| "policy_ms": 3.2848999835550785, | |
| "execution_ms": 11.316700023598969, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 10, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.301400002092123, | |
| "policy_ms": 3.5921999951824546, | |
| "execution_ms": 16.197599936276674, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 11, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9445252418518066, | |
| "observation_ms": 13.399999937973917, | |
| "policy_ms": 3.580200020223856, | |
| "execution_ms": 2.6394000742584467, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 12, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9546310901641846, | |
| "observation_ms": 15.233399928547442, | |
| "policy_ms": 3.8636999670416117, | |
| "execution_ms": 16.240400029346347, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 13, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.776699943467975, | |
| "policy_ms": 4.08179999794811, | |
| "execution_ms": 26.79060003720224, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 14, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9981576800346375, | |
| "observation_ms": 13.14350008033216, | |
| "policy_ms": 5.236499942839146, | |
| "execution_ms": 15.965000027790666, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 15, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.12550001218915, | |
| "policy_ms": 3.0944999307394028, | |
| "execution_ms": 16.255500027909875, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 16, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9995447993278503, | |
| "observation_ms": 14.992900076322258, | |
| "policy_ms": 5.350299994461238, | |
| "execution_ms": 16.49599999655038, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 17, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.71680000051856, | |
| "policy_ms": 4.569399985484779, | |
| "execution_ms": 17.183199990540743, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 18, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.53019997663796, | |
| "policy_ms": 4.76769998203963, | |
| "execution_ms": 30.86950001306832, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 19, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.703300014138222, | |
| "policy_ms": 5.213299999013543, | |
| "execution_ms": 16.328300000168383, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 20, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9988501071929932, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.607999986968935, | |
| "policy_ms": 5.30409999191761, | |
| "execution_ms": 54.43910008762032, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 21, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.421499985270202, | |
| "policy_ms": 4.855699953623116, | |
| "execution_ms": 57.28349997662008, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 22, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.5187785625457764, | |
| "observation_ms": 15.052200062200427, | |
| "policy_ms": 3.6743999226018786, | |
| "execution_ms": 2.2890999680384994, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 23, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.341400004923344, | |
| "policy_ms": 4.631799994967878, | |
| "execution_ms": 19.049499998800457, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 24, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.394600039348006, | |
| "policy_ms": 4.865199909545481, | |
| "execution_ms": 55.08479999843985, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 25, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.264099903404713, | |
| "policy_ms": 3.420600085519254, | |
| "execution_ms": 24.465400027111173, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 26, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999994039535522, | |
| "observation_ms": 14.181600068695843, | |
| "policy_ms": 4.615299985744059, | |
| "execution_ms": 16.70240005478263, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 27, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999845027923584, | |
| "observation_ms": 13.807500014081597, | |
| "policy_ms": 3.414599923416972, | |
| "execution_ms": 17.513599945232272, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 28, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.458800036460161, | |
| "policy_ms": 4.299599910154939, | |
| "execution_ms": 15.959499985910952, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 29, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.318099896423519, | |
| "policy_ms": 3.613900044001639, | |
| "execution_ms": 30.32110002823174, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 30, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9935959577560425, | |
| "observation_ms": 15.342000056989491, | |
| "policy_ms": 6.037199986167252, | |
| "execution_ms": 18.762000021524727, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 31, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.748099911026657, | |
| "policy_ms": 4.332500044256449, | |
| "execution_ms": 15.600200043991208, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 32, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.30940006300807, | |
| "policy_ms": 3.552400041371584, | |
| "execution_ms": 15.391999972052872, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 33, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999959468841553, | |
| "observation_ms": 13.883200008422136, | |
| "policy_ms": 3.6560000153258443, | |
| "execution_ms": 2.3434999166056514, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 34, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9908668994903564, | |
| "observation_ms": 19.056000048294663, | |
| "policy_ms": 5.479299928992987, | |
| "execution_ms": 16.300899907946587, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 35, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.999966025352478, | |
| "observation_ms": 12.62619998306036, | |
| "policy_ms": 2.8918000170961022, | |
| "execution_ms": 28.523200075142086, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 36, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.42769991606474, | |
| "policy_ms": 3.3411000622436404, | |
| "execution_ms": 14.98800003901124, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 37, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.420899911783636, | |
| "policy_ms": 4.462899989448488, | |
| "execution_ms": 15.945799998007715, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 38, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999078512191772, | |
| "observation_ms": 14.410600066184998, | |
| "policy_ms": 4.018999985419214, | |
| "execution_ms": 15.776899992488325, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 39, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.7229601740837097, | |
| "observation_ms": 13.835499994456768, | |
| "policy_ms": 3.4531999845057726, | |
| "execution_ms": 2.393100061453879, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 40, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.8199255466461182, | |
| "target_confidence": 0.9622209072113037, | |
| "observation_ms": 13.003399944864213, | |
| "policy_ms": 4.216299974359572, | |
| "execution_ms": 2.3136999225243926, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 41, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.038600027561188, | |
| "policy_ms": 3.758199978619814, | |
| "execution_ms": 19.614299992099404, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 42, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9906959533691406, | |
| "observation_ms": 15.380099997855723, | |
| "policy_ms": 5.174900055862963, | |
| "execution_ms": 16.194800031371415, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 43, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.09920000191778, | |
| "policy_ms": 2.9875999316573143, | |
| "execution_ms": 16.857200069352984, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 44, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 19.488100078888237, | |
| "policy_ms": 4.6657000202685595, | |
| "execution_ms": 45.196899911388755, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 45, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.174100011587143, | |
| "policy_ms": 3.612800035625696, | |
| "execution_ms": 16.329400008544326, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 46, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.86790000833571, | |
| "policy_ms": 3.732500015757978, | |
| "execution_ms": 15.6872000079602, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 47, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9994713664054871, | |
| "observation_ms": 14.796999981626868, | |
| "policy_ms": 4.007799900136888, | |
| "execution_ms": 17.369800014421344, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 48, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999954700469971, | |
| "observation_ms": 17.278900020755827, | |
| "policy_ms": 3.432299941778183, | |
| "execution_ms": 16.455199918709695, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 49, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.219800031743944, | |
| "policy_ms": 3.06679995264858, | |
| "execution_ms": 15.623299987055361, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 50, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.64410000294447, | |
| "policy_ms": 2.9203000012785196, | |
| "execution_ms": 28.96669995971024, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 51, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.099399977363646, | |
| "policy_ms": 4.649800015613437, | |
| "execution_ms": 15.478800050914288, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 52, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999995231628418, | |
| "observation_ms": 14.24969988875091, | |
| "policy_ms": 3.487100009806454, | |
| "execution_ms": 16.336900065653026, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 53, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9994637370109558, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.493999955244362, | |
| "policy_ms": 4.184499965049326, | |
| "execution_ms": 2.6196999242529273, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 54, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.671700006350875, | |
| "policy_ms": 5.404000054113567, | |
| "execution_ms": 16.411299933679402, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 55, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.105199931189418, | |
| "policy_ms": 4.387599998153746, | |
| "execution_ms": 15.674199908971786, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 56, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9999959468841553, | |
| "target_confidence": 0.9999991655349731, | |
| "observation_ms": 13.736499939113855, | |
| "policy_ms": 3.7805000320076942, | |
| "execution_ms": 2.331000054255128, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 57, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.99505215883255, | |
| "observation_ms": 13.307199929840863, | |
| "policy_ms": 4.483100026845932, | |
| "execution_ms": 24.10740009509027, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 58, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.891300001181662, | |
| "policy_ms": 4.842999973334372, | |
| "execution_ms": 16.17439999245107, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 59, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.18480004556477, | |
| "policy_ms": 3.7406999617815018, | |
| "execution_ms": 16.027100034989417, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 60, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9976951479911804, | |
| "observation_ms": 13.399000046774745, | |
| "policy_ms": 4.2375000193715096, | |
| "execution_ms": 15.702300006523728, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 61, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.998699982650578, | |
| "policy_ms": 5.065600038506091, | |
| "execution_ms": 16.46650000475347, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 62, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8148142695426941, | |
| "observation_ms": 14.55389999318868, | |
| "policy_ms": 4.31150000076741, | |
| "execution_ms": 2.500499947927892, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 63, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.428599948994815, | |
| "policy_ms": 7.091199979186058, | |
| "execution_ms": 3.336200024932623, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 64, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.28289998229593, | |
| "policy_ms": 4.436500021256506, | |
| "execution_ms": 16.73330005723983, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 65, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9999277591705322, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.230200089514256, | |
| "policy_ms": 3.7158000050112605, | |
| "execution_ms": 2.4012000067159534, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 66, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.288099948316813, | |
| "policy_ms": 4.775100038386881, | |
| "execution_ms": 55.68130000028759, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 67, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9995371103286743, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.914499992504716, | |
| "policy_ms": 3.272100002504885, | |
| "execution_ms": 18.482700106687844, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 68, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.357700081542134, | |
| "policy_ms": 2.761899959295988, | |
| "execution_ms": 25.26570006739348, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 69, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.999994158744812, | |
| "target_confidence": 0.9998956918716431, | |
| "observation_ms": 12.45879998896271, | |
| "policy_ms": 3.260800032876432, | |
| "execution_ms": 2.2203000262379646, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 70, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.865799992345273, | |
| "policy_ms": 4.4874000595882535, | |
| "execution_ms": 45.95060006249696, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 71, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.713100062683225, | |
| "policy_ms": 3.830699948593974, | |
| "execution_ms": 17.074699979275465, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 72, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999479055404663, | |
| "observation_ms": 17.204499919898808, | |
| "policy_ms": 6.514200009405613, | |
| "execution_ms": 17.975800088606775, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 73, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.782400008291006, | |
| "policy_ms": 2.8341999277472496, | |
| "execution_ms": 14.98929993249476, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 74, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999997615814209, | |
| "observation_ms": 14.468500041402876, | |
| "policy_ms": 5.100599955767393, | |
| "execution_ms": 15.640300000086427, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 75, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.580499984323978, | |
| "policy_ms": 2.9826000100001693, | |
| "execution_ms": 15.16129996161908, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 76, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.51810002233833, | |
| "policy_ms": 2.7628999669104815, | |
| "execution_ms": 16.3880999898538, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 77, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.518700026907027, | |
| "policy_ms": 5.012999987229705, | |
| "execution_ms": 16.357499989680946, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 78, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.556200010702014, | |
| "policy_ms": 4.080199985764921, | |
| "execution_ms": 16.73739997204393, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 79, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.642500085756183, | |
| "policy_ms": 5.334799992851913, | |
| "execution_ms": 15.943299978971481, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 80, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.948100036941469, | |
| "policy_ms": 4.190700012259185, | |
| "execution_ms": 55.901499930769205, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 81, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.45890003722161, | |
| "policy_ms": 3.8262000307440758, | |
| "execution_ms": 23.478199960663915, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 82, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.99989914894104, | |
| "observation_ms": 15.5745999654755, | |
| "policy_ms": 6.229200051166117, | |
| "execution_ms": 51.97759997099638, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 83, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9656752943992615, | |
| "observation_ms": 14.372300007380545, | |
| "policy_ms": 5.383600015193224, | |
| "execution_ms": 16.966499970294535, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 84, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8647266626358032, | |
| "observation_ms": 16.160600003786385, | |
| "policy_ms": 5.140200024470687, | |
| "execution_ms": 15.700799995101988, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 85, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9447211623191833, | |
| "observation_ms": 13.18640005774796, | |
| "policy_ms": 4.051100113429129, | |
| "execution_ms": 15.234699938446283, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 86, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.797399937175214, | |
| "policy_ms": 3.194599994458258, | |
| "execution_ms": 29.889099998399615, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 87, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.166900004260242, | |
| "policy_ms": 4.183799959719181, | |
| "execution_ms": 33.83550001308322, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 88, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 17.07990001887083, | |
| "policy_ms": 5.7897999649867415, | |
| "execution_ms": 50.41929997969419, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 89, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.78339999448508, | |
| "policy_ms": 5.097899935208261, | |
| "execution_ms": 15.670100110583007, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 90, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.454099979251623, | |
| "policy_ms": 5.093200015835464, | |
| "execution_ms": 23.459500051103532, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 91, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999957084655762, | |
| "target_confidence": 0.9984277486801147, | |
| "observation_ms": 13.411599909886718, | |
| "policy_ms": 3.9958000415936112, | |
| "execution_ms": 15.878699952736497, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 92, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999996423721313, | |
| "observation_ms": 14.981499989517033, | |
| "policy_ms": 4.761700052767992, | |
| "execution_ms": 18.03700008895248, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 93, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 14.161000028252602, | |
| "policy_ms": 3.060900024138391, | |
| "execution_ms": 14.705299981869757, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 94, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.453200088813901, | |
| "policy_ms": 4.6984999207779765, | |
| "execution_ms": 24.97989998664707, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 95, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.531000004149973, | |
| "policy_ms": 4.265900002792478, | |
| "execution_ms": 21.575899911113083, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 96, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8603801131248474, | |
| "observation_ms": 14.894000021740794, | |
| "policy_ms": 4.233099985867739, | |
| "execution_ms": 17.60569994803518, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 97, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999197721481323, | |
| "observation_ms": 18.818299984559417, | |
| "policy_ms": 3.515999997034669, | |
| "execution_ms": 18.676399951800704, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 98, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.14169995021075, | |
| "policy_ms": 3.1015999848023057, | |
| "execution_ms": 17.227199976332486, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 99, | |
| "template": "table", | |
| "success": true, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8986082077026367, | |
| "observation_ms": 15.760399983264506, | |
| "policy_ms": 4.673700081184506, | |
| "execution_ms": 2.6542999548837543, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 100, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999239444732666, | |
| "observation_ms": 14.298899914138019, | |
| "policy_ms": 4.248299985192716, | |
| "execution_ms": 2.7375000063329935, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 101, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.684700031764805, | |
| "policy_ms": 3.019999945536256, | |
| "execution_ms": 2.2021998884156346, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 102, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9922630786895752, | |
| "observation_ms": 17.184600001201034, | |
| "policy_ms": 4.535899963229895, | |
| "execution_ms": 15.5123999575153, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 103, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999997615814209, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.034500065259635, | |
| "policy_ms": 2.9252999229356647, | |
| "execution_ms": 15.785699943080544, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 104, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.99896240234375, | |
| "observation_ms": 13.376299990341067, | |
| "policy_ms": 4.238800029270351, | |
| "execution_ms": 15.408799983561039, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 105, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9708772897720337, | |
| "observation_ms": 11.8101000552997, | |
| "policy_ms": 3.440900007262826, | |
| "execution_ms": 11.12519996240735, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 106, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.58169994596392, | |
| "policy_ms": 3.676300053484738, | |
| "execution_ms": 16.66079997085035, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 107, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.241799992509186, | |
| "policy_ms": 7.07180006429553, | |
| "execution_ms": 16.849200008437037, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 108, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.715700061060488, | |
| "policy_ms": 4.35160007327795, | |
| "execution_ms": 2.422400051727891, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 109, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.0886999219656, | |
| "policy_ms": 6.280199973843992, | |
| "execution_ms": 17.354499897919595, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 110, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.413999997079372, | |
| "policy_ms": 3.1624999828636646, | |
| "execution_ms": 14.989799936302006, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 111, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.437800040468574, | |
| "policy_ms": 4.201499978080392, | |
| "execution_ms": 15.381600009277463, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 112, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999994039535522, | |
| "target_confidence": 0.9972097277641296, | |
| "observation_ms": 13.343900092877448, | |
| "policy_ms": 4.147200030274689, | |
| "execution_ms": 10.750599903985858, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 113, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.769999913871288, | |
| "policy_ms": 4.013400059193373, | |
| "execution_ms": 37.7233000472188, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 114, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.909100020304322, | |
| "policy_ms": 5.094700027257204, | |
| "execution_ms": 16.464499989524484, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 115, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.641499914228916, | |
| "policy_ms": 3.7711000768467784, | |
| "execution_ms": 15.856500016525388, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 116, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.7579777240753174, | |
| "observation_ms": 16.42950007226318, | |
| "policy_ms": 3.464199951849878, | |
| "execution_ms": 2.3365000961348414, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 117, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.082099914550781, | |
| "policy_ms": 3.243400016799569, | |
| "execution_ms": 43.12389995902777, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 118, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999833106994629, | |
| "observation_ms": 12.96540000475943, | |
| "policy_ms": 3.94099997356534, | |
| "execution_ms": 16.32010005414486, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 119, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9794986844062805, | |
| "observation_ms": 16.84259995818138, | |
| "policy_ms": 4.989200038835406, | |
| "execution_ms": 15.340300044044852, | |
| "status": "ok", | |
| "code": "SELECT" | |
| } | |
| ] | |
| } |