{ "checkpoint": "models/v003-transformer", "quantized": false, "platform": "Windows-11-10.0.26200-SP0", "torch_threads": 2, "model_load_ms": 11.723599978722632, "samples": 120, "success_rate": 0.49166666666666664, "abstention_rate": 0.15, "median_observation_ms": 14.34735005022958, "median_policy_ms": 4.208899976219982, "median_execution_ms": 16.347200027666986, "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", "target_vps_validated": false, "results": [ { "sample_id": 0, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 43.33179991226643, "policy_ms": 6.381200044415891, "execution_ms": 3.6983999889343977, "status": "ok", "code": "ASK_USER" }, { "sample_id": 1, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999996423721313, "observation_ms": 15.646400046534836, "policy_ms": 3.6445000441744924, "execution_ms": 43.543800013139844, "status": "ok", "code": "SELECT" }, { "sample_id": 2, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999963045120239, "target_confidence": 0.9999997615814209, "observation_ms": 29.71329994034022, "policy_ms": 4.3830000795423985, "execution_ms": 18.738399958238006, "status": "ok", "code": "SELECT" }, { "sample_id": 3, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.353300095535815, "policy_ms": 3.6409999011084437, "execution_ms": 58.37500002235174, "status": "ok", "code": "CLICK" }, { "sample_id": 4, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.350400052033365, "policy_ms": 4.532700055278838, "execution_ms": 32.714600092731416, "status": "ok", "code": "CLICK" }, { "sample_id": 5, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.697097897529602, "observation_ms": 16.117900027893484, "policy_ms": 4.587300005368888, "execution_ms": 2.7254000306129456, "status": "ok", "code": "ASK_USER" }, { "sample_id": 6, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.944299986585975, "policy_ms": 4.615599988028407, "execution_ms": 16.94929995574057, "status": "ok", "code": "SELECT" }, { "sample_id": 7, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999443292617798, "observation_ms": 15.260900021530688, "policy_ms": 5.21530001424253, "execution_ms": 16.81660010945052, "status": "ok", "code": "SELECT" }, { "sample_id": 8, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.323400010354817, "policy_ms": 4.924600012600422, "execution_ms": 17.00610003899783, "status": "ok", "code": "SELECT" }, { "sample_id": 9, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999998807907104, "target_confidence": 1.0, "observation_ms": 15.643000020645559, "policy_ms": 3.2848999835550785, "execution_ms": 11.316700023598969, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 10, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.301400002092123, "policy_ms": 3.5921999951824546, "execution_ms": 16.197599936276674, "status": "ok", "code": "SELECT" }, { "sample_id": 11, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9445252418518066, "observation_ms": 13.399999937973917, "policy_ms": 3.580200020223856, "execution_ms": 2.6394000742584467, "status": "ok", "code": "ASK_USER" }, { "sample_id": 12, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9546310901641846, "observation_ms": 15.233399928547442, "policy_ms": 3.8636999670416117, "execution_ms": 16.240400029346347, "status": "ok", "code": "SELECT" }, { "sample_id": 13, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.776699943467975, "policy_ms": 4.08179999794811, "execution_ms": 26.79060003720224, "status": "ok", "code": "CLICK" }, { "sample_id": 14, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9981576800346375, "observation_ms": 13.14350008033216, "policy_ms": 5.236499942839146, "execution_ms": 15.965000027790666, "status": "ok", "code": "SELECT" }, { "sample_id": 15, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.12550001218915, "policy_ms": 3.0944999307394028, "execution_ms": 16.255500027909875, "status": "ok", "code": "SELECT" }, { "sample_id": 16, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9995447993278503, "observation_ms": 14.992900076322258, "policy_ms": 5.350299994461238, "execution_ms": 16.49599999655038, "status": "ok", "code": "SELECT" }, { "sample_id": 17, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.71680000051856, "policy_ms": 4.569399985484779, "execution_ms": 17.183199990540743, "status": "ok", "code": "SELECT" }, { "sample_id": 18, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.53019997663796, "policy_ms": 4.76769998203963, "execution_ms": 30.86950001306832, "status": "ok", "code": "CLICK" }, { "sample_id": 19, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.703300014138222, "policy_ms": 5.213299999013543, "execution_ms": 16.328300000168383, "status": "ok", "code": "SELECT" }, { "sample_id": 20, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9988501071929932, "target_confidence": 1.0, "observation_ms": 15.607999986968935, "policy_ms": 5.30409999191761, "execution_ms": 54.43910008762032, "status": "ok", "code": "CLICK" }, { "sample_id": 21, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.421499985270202, "policy_ms": 4.855699953623116, "execution_ms": 57.28349997662008, "status": "ok", "code": "CLICK" }, { "sample_id": 22, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.5187785625457764, "observation_ms": 15.052200062200427, "policy_ms": 3.6743999226018786, "execution_ms": 2.2890999680384994, "status": "ok", "code": "ASK_USER" }, { "sample_id": 23, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.341400004923344, "policy_ms": 4.631799994967878, "execution_ms": 19.049499998800457, "status": "ok", "code": "SELECT" }, { "sample_id": 24, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.394600039348006, "policy_ms": 4.865199909545481, "execution_ms": 55.08479999843985, "status": "ok", "code": "CLICK" }, { "sample_id": 25, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.264099903404713, "policy_ms": 3.420600085519254, "execution_ms": 24.465400027111173, "status": "ok", "code": "CLICK" }, { "sample_id": 26, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999994039535522, "observation_ms": 14.181600068695843, "policy_ms": 4.615299985744059, "execution_ms": 16.70240005478263, "status": "ok", "code": "SELECT" }, { "sample_id": 27, "template": "nested", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999845027923584, "observation_ms": 13.807500014081597, "policy_ms": 3.414599923416972, "execution_ms": 17.513599945232272, "status": "ok", "code": "SELECT" }, { "sample_id": 28, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.458800036460161, "policy_ms": 4.299599910154939, "execution_ms": 15.959499985910952, "status": "ok", "code": "SELECT" }, { "sample_id": 29, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.318099896423519, "policy_ms": 3.613900044001639, "execution_ms": 30.32110002823174, "status": "ok", "code": "CLICK" }, { "sample_id": 30, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9935959577560425, "observation_ms": 15.342000056989491, "policy_ms": 6.037199986167252, "execution_ms": 18.762000021524727, "status": "ok", "code": "SELECT" }, { "sample_id": 31, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.748099911026657, "policy_ms": 4.332500044256449, "execution_ms": 15.600200043991208, "status": "ok", "code": "SELECT" }, { "sample_id": 32, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.30940006300807, "policy_ms": 3.552400041371584, "execution_ms": 15.391999972052872, "status": "ok", "code": "SELECT" }, { "sample_id": 33, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999959468841553, "observation_ms": 13.883200008422136, "policy_ms": 3.6560000153258443, "execution_ms": 2.3434999166056514, "status": "ok", "code": "ASK_USER" }, { "sample_id": 34, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9908668994903564, "observation_ms": 19.056000048294663, "policy_ms": 5.479299928992987, "execution_ms": 16.300899907946587, "status": "ok", "code": "SELECT" }, { "sample_id": 35, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 0.999966025352478, "observation_ms": 12.62619998306036, "policy_ms": 2.8918000170961022, "execution_ms": 28.523200075142086, "status": "ok", "code": "CLICK" }, { "sample_id": 36, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.42769991606474, "policy_ms": 3.3411000622436404, "execution_ms": 14.98800003901124, "status": "ok", "code": "SELECT" }, { "sample_id": 37, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.420899911783636, "policy_ms": 4.462899989448488, "execution_ms": 15.945799998007715, "status": "ok", "code": "SELECT" }, { "sample_id": 38, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999078512191772, "observation_ms": 14.410600066184998, "policy_ms": 4.018999985419214, "execution_ms": 15.776899992488325, "status": "ok", "code": "SELECT" }, { "sample_id": 39, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.7229601740837097, "observation_ms": 13.835499994456768, "policy_ms": 3.4531999845057726, "execution_ms": 2.393100061453879, "status": "ok", "code": "ASK_USER" }, { "sample_id": 40, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.8199255466461182, "target_confidence": 0.9622209072113037, "observation_ms": 13.003399944864213, "policy_ms": 4.216299974359572, "execution_ms": 2.3136999225243926, "status": "ok", "code": "ASK_USER" }, { "sample_id": 41, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.038600027561188, "policy_ms": 3.758199978619814, "execution_ms": 19.614299992099404, "status": "ok", "code": "SELECT" }, { "sample_id": 42, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9906959533691406, "observation_ms": 15.380099997855723, "policy_ms": 5.174900055862963, "execution_ms": 16.194800031371415, "status": "ok", "code": "SELECT" }, { "sample_id": 43, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.09920000191778, "policy_ms": 2.9875999316573143, "execution_ms": 16.857200069352984, "status": "ok", "code": "SELECT" }, { "sample_id": 44, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.488100078888237, "policy_ms": 4.6657000202685595, "execution_ms": 45.196899911388755, "status": "ok", "code": "CLICK" }, { "sample_id": 45, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.174100011587143, "policy_ms": 3.612800035625696, "execution_ms": 16.329400008544326, "status": "ok", "code": "SELECT" }, { "sample_id": 46, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.86790000833571, "policy_ms": 3.732500015757978, "execution_ms": 15.6872000079602, "status": "ok", "code": "SELECT" }, { "sample_id": 47, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9994713664054871, "observation_ms": 14.796999981626868, "policy_ms": 4.007799900136888, "execution_ms": 17.369800014421344, "status": "ok", "code": "SELECT" }, { "sample_id": 48, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999954700469971, "observation_ms": 17.278900020755827, "policy_ms": 3.432299941778183, "execution_ms": 16.455199918709695, "status": "ok", "code": "SELECT" }, { "sample_id": 49, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.219800031743944, "policy_ms": 3.06679995264858, "execution_ms": 15.623299987055361, "status": "ok", "code": "SELECT" }, { "sample_id": 50, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.64410000294447, "policy_ms": 2.9203000012785196, "execution_ms": 28.96669995971024, "status": "ok", "code": "CLICK" }, { "sample_id": 51, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.099399977363646, "policy_ms": 4.649800015613437, "execution_ms": 15.478800050914288, "status": "ok", "code": "SELECT" }, { "sample_id": 52, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999995231628418, "observation_ms": 14.24969988875091, "policy_ms": 3.487100009806454, "execution_ms": 16.336900065653026, "status": "ok", "code": "SELECT" }, { "sample_id": 53, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9994637370109558, "target_confidence": 1.0, "observation_ms": 13.493999955244362, "policy_ms": 4.184499965049326, "execution_ms": 2.6196999242529273, "status": "ok", "code": "ASK_USER" }, { "sample_id": 54, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.671700006350875, "policy_ms": 5.404000054113567, "execution_ms": 16.411299933679402, "status": "ok", "code": "SELECT" }, { "sample_id": 55, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.105199931189418, "policy_ms": 4.387599998153746, "execution_ms": 15.674199908971786, "status": "ok", "code": "SELECT" }, { "sample_id": 56, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9999959468841553, "target_confidence": 0.9999991655349731, "observation_ms": 13.736499939113855, "policy_ms": 3.7805000320076942, "execution_ms": 2.331000054255128, "status": "ok", "code": "ASK_USER" }, { "sample_id": 57, "template": "table", "success": false, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 0.99505215883255, "observation_ms": 13.307199929840863, "policy_ms": 4.483100026845932, "execution_ms": 24.10740009509027, "status": "ok", "code": "CLICK" }, { "sample_id": 58, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.891300001181662, "policy_ms": 4.842999973334372, "execution_ms": 16.17439999245107, "status": "ok", "code": "SELECT" }, { "sample_id": 59, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.18480004556477, "policy_ms": 3.7406999617815018, "execution_ms": 16.027100034989417, "status": "ok", "code": "SELECT" }, { "sample_id": 60, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9976951479911804, "observation_ms": 13.399000046774745, "policy_ms": 4.2375000193715096, "execution_ms": 15.702300006523728, "status": "ok", "code": "SELECT" }, { "sample_id": 61, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.998699982650578, "policy_ms": 5.065600038506091, "execution_ms": 16.46650000475347, "status": "ok", "code": "SELECT" }, { "sample_id": 62, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.8148142695426941, "observation_ms": 14.55389999318868, "policy_ms": 4.31150000076741, "execution_ms": 2.500499947927892, "status": "ok", "code": "ASK_USER" }, { "sample_id": 63, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.428599948994815, "policy_ms": 7.091199979186058, "execution_ms": 3.336200024932623, "status": "ok", "code": "ASK_USER" }, { "sample_id": 64, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.28289998229593, "policy_ms": 4.436500021256506, "execution_ms": 16.73330005723983, "status": "ok", "code": "SELECT" }, { "sample_id": 65, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9999277591705322, "target_confidence": 1.0, "observation_ms": 14.230200089514256, "policy_ms": 3.7158000050112605, "execution_ms": 2.4012000067159534, "status": "ok", "code": "ASK_USER" }, { "sample_id": 66, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.288099948316813, "policy_ms": 4.775100038386881, "execution_ms": 55.68130000028759, "status": "ok", "code": "CLICK" }, { "sample_id": 67, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9995371103286743, "target_confidence": 1.0, "observation_ms": 15.914499992504716, "policy_ms": 3.272100002504885, "execution_ms": 18.482700106687844, "status": "ok", "code": "SELECT" }, { "sample_id": 68, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.357700081542134, "policy_ms": 2.761899959295988, "execution_ms": 25.26570006739348, "status": "ok", "code": "CLICK" }, { "sample_id": 69, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.999994158744812, "target_confidence": 0.9998956918716431, "observation_ms": 12.45879998896271, "policy_ms": 3.260800032876432, "execution_ms": 2.2203000262379646, "status": "ok", "code": "ASK_USER" }, { "sample_id": 70, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.865799992345273, "policy_ms": 4.4874000595882535, "execution_ms": 45.95060006249696, "status": "ok", "code": "CLICK" }, { "sample_id": 71, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.713100062683225, "policy_ms": 3.830699948593974, "execution_ms": 17.074699979275465, "status": "ok", "code": "SELECT" }, { "sample_id": 72, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999479055404663, "observation_ms": 17.204499919898808, "policy_ms": 6.514200009405613, "execution_ms": 17.975800088606775, "status": "ok", "code": "SELECT" }, { "sample_id": 73, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.782400008291006, "policy_ms": 2.8341999277472496, "execution_ms": 14.98929993249476, "status": "ok", "code": "SELECT" }, { "sample_id": 74, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999997615814209, "observation_ms": 14.468500041402876, "policy_ms": 5.100599955767393, "execution_ms": 15.640300000086427, "status": "ok", "code": "SELECT" }, { "sample_id": 75, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.580499984323978, "policy_ms": 2.9826000100001693, "execution_ms": 15.16129996161908, "status": "ok", "code": "SELECT" }, { "sample_id": 76, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.51810002233833, "policy_ms": 2.7628999669104815, "execution_ms": 16.3880999898538, "status": "ok", "code": "SELECT" }, { "sample_id": 77, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.518700026907027, "policy_ms": 5.012999987229705, "execution_ms": 16.357499989680946, "status": "ok", "code": "SELECT" }, { "sample_id": 78, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.556200010702014, "policy_ms": 4.080199985764921, "execution_ms": 16.73739997204393, "status": "ok", "code": "SELECT" }, { "sample_id": 79, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.642500085756183, "policy_ms": 5.334799992851913, "execution_ms": 15.943299978971481, "status": "ok", "code": "SELECT" }, { "sample_id": 80, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.948100036941469, "policy_ms": 4.190700012259185, "execution_ms": 55.901499930769205, "status": "ok", "code": "CLICK" }, { "sample_id": 81, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.45890003722161, "policy_ms": 3.8262000307440758, "execution_ms": 23.478199960663915, "status": "ok", "code": "CLICK" }, { "sample_id": 82, "template": "nested", "success": false, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 0.99989914894104, "observation_ms": 15.5745999654755, "policy_ms": 6.229200051166117, "execution_ms": 51.97759997099638, "status": "ok", "code": "CLICK" }, { "sample_id": 83, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9656752943992615, "observation_ms": 14.372300007380545, "policy_ms": 5.383600015193224, "execution_ms": 16.966499970294535, "status": "ok", "code": "SELECT" }, { "sample_id": 84, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.8647266626358032, "observation_ms": 16.160600003786385, "policy_ms": 5.140200024470687, "execution_ms": 15.700799995101988, "status": "ok", "code": "SELECT" }, { "sample_id": 85, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9447211623191833, "observation_ms": 13.18640005774796, "policy_ms": 4.051100113429129, "execution_ms": 15.234699938446283, "status": "ok", "code": "SELECT" }, { "sample_id": 86, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.797399937175214, "policy_ms": 3.194599994458258, "execution_ms": 29.889099998399615, "status": "ok", "code": "CLICK" }, { "sample_id": 87, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999998807907104, "target_confidence": 1.0, "observation_ms": 15.166900004260242, "policy_ms": 4.183799959719181, "execution_ms": 33.83550001308322, "status": "ok", "code": "CLICK" }, { "sample_id": 88, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.07990001887083, "policy_ms": 5.7897999649867415, "execution_ms": 50.41929997969419, "status": "ok", "code": "CLICK" }, { "sample_id": 89, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.78339999448508, "policy_ms": 5.097899935208261, "execution_ms": 15.670100110583007, "status": "ok", "code": "SELECT" }, { "sample_id": 90, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.454099979251623, "policy_ms": 5.093200015835464, "execution_ms": 23.459500051103532, "status": "ok", "code": "CLICK" }, { "sample_id": 91, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999957084655762, "target_confidence": 0.9984277486801147, "observation_ms": 13.411599909886718, "policy_ms": 3.9958000415936112, "execution_ms": 15.878699952736497, "status": "ok", "code": "SELECT" }, { "sample_id": 92, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999996423721313, "observation_ms": 14.981499989517033, "policy_ms": 4.761700052767992, "execution_ms": 18.03700008895248, "status": "ok", "code": "SELECT" }, { "sample_id": 93, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 14.161000028252602, "policy_ms": 3.060900024138391, "execution_ms": 14.705299981869757, "status": "ok", "code": "SELECT" }, { "sample_id": 94, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.453200088813901, "policy_ms": 4.6984999207779765, "execution_ms": 24.97989998664707, "status": "ok", "code": "CLICK" }, { "sample_id": 95, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.531000004149973, "policy_ms": 4.265900002792478, "execution_ms": 21.575899911113083, "status": "ok", "code": "CLICK" }, { "sample_id": 96, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.8603801131248474, "observation_ms": 14.894000021740794, "policy_ms": 4.233099985867739, "execution_ms": 17.60569994803518, "status": "ok", "code": "SELECT" }, { "sample_id": 97, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999197721481323, "observation_ms": 18.818299984559417, "policy_ms": 3.515999997034669, "execution_ms": 18.676399951800704, "status": "ok", "code": "SELECT" }, { "sample_id": 98, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.14169995021075, "policy_ms": 3.1015999848023057, "execution_ms": 17.227199976332486, "status": "ok", "code": "SELECT" }, { "sample_id": 99, "template": "table", "success": true, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.8986082077026367, "observation_ms": 15.760399983264506, "policy_ms": 4.673700081184506, "execution_ms": 2.6542999548837543, "status": "ok", "code": "ASK_USER" }, { "sample_id": 100, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999239444732666, "observation_ms": 14.298899914138019, "policy_ms": 4.248299985192716, "execution_ms": 2.7375000063329935, "status": "ok", "code": "ASK_USER" }, { "sample_id": 101, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 11.684700031764805, "policy_ms": 3.019999945536256, "execution_ms": 2.2021998884156346, "status": "ok", "code": "ASK_USER" }, { "sample_id": 102, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9922630786895752, "observation_ms": 17.184600001201034, "policy_ms": 4.535899963229895, "execution_ms": 15.5123999575153, "status": "ok", "code": "SELECT" }, { "sample_id": 103, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999997615814209, "target_confidence": 1.0, "observation_ms": 13.034500065259635, "policy_ms": 2.9252999229356647, "execution_ms": 15.785699943080544, "status": "ok", "code": "SELECT" }, { "sample_id": 104, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.99896240234375, "observation_ms": 13.376299990341067, "policy_ms": 4.238800029270351, "execution_ms": 15.408799983561039, "status": "ok", "code": "SELECT" }, { "sample_id": 105, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9708772897720337, "observation_ms": 11.8101000552997, "policy_ms": 3.440900007262826, "execution_ms": 11.12519996240735, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 106, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 11.58169994596392, "policy_ms": 3.676300053484738, "execution_ms": 16.66079997085035, "status": "ok", "code": "SELECT" }, { "sample_id": 107, "template": "nested", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.241799992509186, "policy_ms": 7.07180006429553, "execution_ms": 16.849200008437037, "status": "ok", "code": "SELECT" }, { "sample_id": 108, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.715700061060488, "policy_ms": 4.35160007327795, "execution_ms": 2.422400051727891, "status": "ok", "code": "ASK_USER" }, { "sample_id": 109, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.0886999219656, "policy_ms": 6.280199973843992, "execution_ms": 17.354499897919595, "status": "ok", "code": "SELECT" }, { "sample_id": 110, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.413999997079372, "policy_ms": 3.1624999828636646, "execution_ms": 14.989799936302006, "status": "ok", "code": "SELECT" }, { "sample_id": 111, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.437800040468574, "policy_ms": 4.201499978080392, "execution_ms": 15.381600009277463, "status": "ok", "code": "SELECT" }, { "sample_id": 112, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999994039535522, "target_confidence": 0.9972097277641296, "observation_ms": 13.343900092877448, "policy_ms": 4.147200030274689, "execution_ms": 10.750599903985858, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 113, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999998807907104, "target_confidence": 1.0, "observation_ms": 12.769999913871288, "policy_ms": 4.013400059193373, "execution_ms": 37.7233000472188, "status": "ok", "code": "CLICK" }, { "sample_id": 114, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.909100020304322, "policy_ms": 5.094700027257204, "execution_ms": 16.464499989524484, "status": "ok", "code": "SELECT" }, { "sample_id": 115, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.641499914228916, "policy_ms": 3.7711000768467784, "execution_ms": 15.856500016525388, "status": "ok", "code": "SELECT" }, { "sample_id": 116, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.7579777240753174, "observation_ms": 16.42950007226318, "policy_ms": 3.464199951849878, "execution_ms": 2.3365000961348414, "status": "ok", "code": "ASK_USER" }, { "sample_id": 117, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.082099914550781, "policy_ms": 3.243400016799569, "execution_ms": 43.12389995902777, "status": "ok", "code": "CLICK" }, { "sample_id": 118, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999833106994629, "observation_ms": 12.96540000475943, "policy_ms": 3.94099997356534, "execution_ms": 16.32010005414486, "status": "ok", "code": "SELECT" }, { "sample_id": 119, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9794986844062805, "observation_ms": 16.84259995818138, "policy_ms": 4.989200038835406, "execution_ms": 15.340300044044852, "status": "ok", "code": "SELECT" } ] }