{ "checkpoint": "models/v001-no-lexical", "quantized": false, "platform": "Windows-11-10.0.26200-SP0", "torch_threads": 2, "model_load_ms": 7.755500031635165, "samples": 120, "success_rate": 0.5833333333333334, "abstention_rate": 0.175, "median_observation_ms": 14.387400005944073, "median_policy_ms": 2.590400050394237, "median_execution_ms": 15.836199978366494, "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", "target_vps_validated": false, "results": [ { "sample_id": 0, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999039173126221, "observation_ms": 48.155799973756075, "policy_ms": 4.100500023923814, "execution_ms": 3.78839997574687, "status": "ok", "code": "ASK_USER" }, { "sample_id": 1, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999996423721313, "target_confidence": 1.0, "observation_ms": 14.890599995851517, "policy_ms": 2.4492000229656696, "execution_ms": 39.766900008544326, "status": "ok", "code": "TYPE" }, { "sample_id": 2, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999998807907104, "target_confidence": 1.0, "observation_ms": 37.80180006287992, "policy_ms": 2.3002999369055033, "execution_ms": 18.543299986049533, "status": "ok", "code": "TYPE" }, { "sample_id": 3, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999998807907104, "target_confidence": 1.0, "observation_ms": 14.828799990937114, "policy_ms": 2.4418000830337405, "execution_ms": 31.127299997024238, "status": "ok", "code": "CLICK" }, { "sample_id": 4, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.8566849827766418, "target_confidence": 1.0, "observation_ms": 15.973599976859987, "policy_ms": 2.631800016388297, "execution_ms": 2.5369999930262566, "status": "ok", "code": "ASK_USER" }, { "sample_id": 5, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.66629996523261, "policy_ms": 2.7742000529542565, "execution_ms": 10.593199986033142, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 6, "template": "nested", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.330099992454052, "policy_ms": 2.7721000369638205, "execution_ms": 19.744400051422417, "status": "ok", "code": "SELECT" }, { "sample_id": 7, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.999404788017273, "observation_ms": 15.696700080297887, "policy_ms": 2.842400106601417, "execution_ms": 17.253799946047366, "status": "ok", "code": "SELECT" }, { "sample_id": 8, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9946579337120056, "observation_ms": 15.824000001884997, "policy_ms": 2.9687000205740333, "execution_ms": 16.37490000575781, "status": "ok", "code": "SELECT" }, { "sample_id": 9, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.5657387971878052, "target_confidence": 1.0, "observation_ms": 13.3670000359416, "policy_ms": 1.8517000135034323, "execution_ms": 2.1585000213235617, "status": "ok", "code": "ASK_USER" }, { "sample_id": 10, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.34619999397546, "policy_ms": 2.0665000192821026, "execution_ms": 16.778600052930415, "status": "ok", "code": "SELECT" }, { "sample_id": 11, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999995231628418, "observation_ms": 18.10390001628548, "policy_ms": 2.118099946528673, "execution_ms": 2.3756999289616942, "status": "ok", "code": "ASK_USER" }, { "sample_id": 12, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999984502792358, "observation_ms": 14.613799983635545, "policy_ms": 2.5136000476777554, "execution_ms": 10.11360005941242, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 13, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.006900019012392, "policy_ms": 2.845599898137152, "execution_ms": 57.12309991940856, "status": "ok", "code": "CLICK" }, { "sample_id": 14, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.414299977943301, "policy_ms": 2.4254999589174986, "execution_ms": 17.154800007119775, "status": "ok", "code": "SELECT" }, { "sample_id": 15, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.039400012232363, "policy_ms": 1.5656000468879938, "execution_ms": 13.976700021885335, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 16, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.034599954262376, "policy_ms": 3.0512999510392547, "execution_ms": 18.35849997587502, "status": "ok", "code": "SELECT" }, { "sample_id": 17, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.637500026263297, "policy_ms": 3.009999985806644, "execution_ms": 15.937500051222742, "status": "ok", "code": "SELECT" }, { "sample_id": 18, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999964237213135, "target_confidence": 1.0, "observation_ms": 13.905499945394695, "policy_ms": 2.189300023019314, "execution_ms": 28.85450003668666, "status": "ok", "code": "CLICK" }, { "sample_id": 19, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.958699932321906, "policy_ms": 2.4140001041814685, "execution_ms": 16.44000003580004, "status": "ok", "code": "SELECT" }, { "sample_id": 20, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999957084655762, "target_confidence": 1.0, "observation_ms": 15.601499937474728, "policy_ms": 2.930799964815378, "execution_ms": 26.539099984802306, "status": "ok", "code": "CLICK" }, { "sample_id": 21, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.49610001873225, "policy_ms": 2.6504999259486794, "execution_ms": 59.45119995158166, "status": "ok", "code": "CLICK" }, { "sample_id": 22, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.534300007857382, "policy_ms": 2.8038000455126166, "execution_ms": 54.541900055482984, "status": "ok", "code": "CLICK" }, { "sample_id": 23, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.405699981376529, "policy_ms": 2.187600010074675, "execution_ms": 15.721399919129908, "status": "ok", "code": "SELECT" }, { "sample_id": 24, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999997615814209, "target_confidence": 1.0, "observation_ms": 14.66660003643483, "policy_ms": 3.554799943231046, "execution_ms": 48.90890000388026, "status": "ok", "code": "CLICK" }, { "sample_id": 25, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.233899953775108, "policy_ms": 1.8354000058025122, "execution_ms": 25.838000001385808, "status": "ok", "code": "CLICK" }, { "sample_id": 26, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 18.253899994306266, "policy_ms": 3.1340999994426966, "execution_ms": 13.132599997334182, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 27, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.520599972456694, "policy_ms": 1.7303000204265118, "execution_ms": 16.68460003565997, "status": "ok", "code": "SELECT" }, { "sample_id": 28, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.085299986414611, "policy_ms": 1.9613999174907804, "execution_ms": 9.540400002151728, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 29, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.150600017979741, "policy_ms": 2.2929999977350235, "execution_ms": 24.475399986840785, "status": "ok", "code": "CLICK" }, { "sample_id": 30, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.633000013418496, "policy_ms": 3.142800065688789, "execution_ms": 10.01420000102371, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 31, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.44879998639226, "policy_ms": 2.66519992146641, "execution_ms": 16.52860001195222, "status": "ok", "code": "SELECT" }, { "sample_id": 32, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.910100049339235, "policy_ms": 2.1643999498337507, "execution_ms": 15.578899998217821, "status": "ok", "code": "SELECT" }, { "sample_id": 33, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9468345046043396, "observation_ms": 13.298999983817339, "policy_ms": 2.1152000408619642, "execution_ms": 2.374800038523972, "status": "ok", "code": "ASK_USER" }, { "sample_id": 34, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.98715740442276, "target_confidence": 1.0, "observation_ms": 14.753099996596575, "policy_ms": 3.1820000149309635, "execution_ms": 18.66950001567602, "status": "ok", "code": "TYPE" }, { "sample_id": 35, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.49889999255538, "policy_ms": 1.9891000119969249, "execution_ms": 30.96399991773069, "status": "ok", "code": "CLICK" }, { "sample_id": 36, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.855400029569864, "policy_ms": 1.9989999709650874, "execution_ms": 10.228500002995133, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 37, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.236199902370572, "policy_ms": 2.596600097604096, "execution_ms": 15.496000065468252, "status": "ok", "code": "SELECT" }, { "sample_id": 38, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9890561103820801, "target_confidence": 1.0, "observation_ms": 16.398599953390658, "policy_ms": 2.4811000330373645, "execution_ms": 10.626500006765127, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 39, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9998949766159058, "observation_ms": 12.702600099146366, "policy_ms": 2.3571999045088887, "execution_ms": 10.468199965544045, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 40, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9305520057678223, "observation_ms": 14.912900049239397, "policy_ms": 2.680000034160912, "execution_ms": 2.3592000361531973, "status": "ok", "code": "ASK_USER" }, { "sample_id": 41, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.922900030389428, "policy_ms": 2.2016000002622604, "execution_ms": 16.15399995353073, "status": "ok", "code": "SELECT" }, { "sample_id": 42, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.27980000153184, "policy_ms": 2.9817000031471252, "execution_ms": 20.689299912191927, "status": "ok", "code": "SELECT" }, { "sample_id": 43, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.495000010356307, "policy_ms": 1.7315000295639038, "execution_ms": 16.5631000418216, "status": "ok", "code": "SELECT" }, { "sample_id": 44, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.022499999962747, "policy_ms": 2.5951999705284834, "execution_ms": 28.8407999323681, "status": "ok", "code": "CLICK" }, { "sample_id": 45, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.994099990464747, "policy_ms": 1.922299969010055, "execution_ms": 15.168599900789559, "status": "ok", "code": "SELECT" }, { "sample_id": 46, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.887299992144108, "policy_ms": 2.1222999785095453, "execution_ms": 15.42199996765703, "status": "ok", "code": "SELECT" }, { "sample_id": 47, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.8051379323005676, "observation_ms": 15.364399994723499, "policy_ms": 2.498099929653108, "execution_ms": 2.7083000168204308, "status": "ok", "code": "ASK_USER" }, { "sample_id": 48, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.14170001912862, "policy_ms": 1.8744999542832375, "execution_ms": 9.740900015458465, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 49, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.013899977318943, "policy_ms": 2.550999983213842, "execution_ms": 11.068299994803965, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 50, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9796527624130249, "target_confidence": 1.0, "observation_ms": 12.284099939279258, "policy_ms": 1.7565001035109162, "execution_ms": 22.017300012521446, "status": "ok", "code": "CLICK" }, { "sample_id": 51, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.864399981684983, "policy_ms": 2.271600067615509, "execution_ms": 15.552200027741492, "status": "ok", "code": "SELECT" }, { "sample_id": 52, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.490600045770407, "policy_ms": 2.149399952031672, "execution_ms": 14.547700062394142, "status": "ok", "code": "SELECT" }, { "sample_id": 53, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9997931122779846, "observation_ms": 15.169100021012127, "policy_ms": 3.2730000093579292, "execution_ms": 2.885499969124794, "status": "ok", "code": "ASK_USER" }, { "sample_id": 54, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9972251653671265, "target_confidence": 1.0, "observation_ms": 14.367499970830977, "policy_ms": 3.09520005248487, "execution_ms": 16.815399983897805, "status": "ok", "code": "TYPE" }, { "sample_id": 55, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999972581863403, "observation_ms": 15.827800030820072, "policy_ms": 3.6145999329164624, "execution_ms": 17.35270000062883, "status": "ok", "code": "SELECT" }, { "sample_id": 56, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.40730004105717, "policy_ms": 2.7208999963477254, "execution_ms": 2.3763999342918396, "status": "ok", "code": "ASK_USER" }, { "sample_id": 57, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.356899959035218, "policy_ms": 2.341100014746189, "execution_ms": 48.094899975694716, "status": "ok", "code": "CLICK" }, { "sample_id": 58, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.218099997378886, "policy_ms": 3.1998000340536237, "execution_ms": 15.435200068168342, "status": "ok", "code": "SELECT" }, { "sample_id": 59, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.307899935171008, "policy_ms": 2.1096999989822507, "execution_ms": 15.629699919372797, "status": "ok", "code": "SELECT" }, { "sample_id": 60, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.890400015749037, "policy_ms": 2.027299953624606, "execution_ms": 15.748900012113154, "status": "ok", "code": "SELECT" }, { "sample_id": 61, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.705399935133755, "policy_ms": 3.4446000354364514, "execution_ms": 16.00020006299019, "status": "ok", "code": "SELECT" }, { "sample_id": 62, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.999985933303833, "target_confidence": 1.0, "observation_ms": 15.249400050379336, "policy_ms": 2.678500022739172, "execution_ms": 10.372799937613308, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 63, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9900351166725159, "observation_ms": 15.276299905963242, "policy_ms": 2.547200070694089, "execution_ms": 2.489199978299439, "status": "ok", "code": "ASK_USER" }, { "sample_id": 64, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999909400939941, "observation_ms": 14.591599931009114, "policy_ms": 2.7464000741019845, "execution_ms": 10.19070006441325, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 65, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.814099947921932, "policy_ms": 1.896600006148219, "execution_ms": 2.2869999520480633, "status": "ok", "code": "ASK_USER" }, { "sample_id": 66, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.999858021736145, "target_confidence": 1.0, "observation_ms": 15.144100063480437, "policy_ms": 2.86400003824383, "execution_ms": 34.816599916666746, "status": "ok", "code": "CLICK" }, { "sample_id": 67, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9981772899627686, "target_confidence": 1.0, "observation_ms": 12.527500046417117, "policy_ms": 1.9356999546289444, "execution_ms": 16.11970004159957, "status": "ok", "code": "TYPE" }, { "sample_id": 68, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9811867475509644, "target_confidence": 1.0, "observation_ms": 11.66930003091693, "policy_ms": 1.8606000812724233, "execution_ms": 2.3653999669477344, "status": "ok", "code": "ASK_USER" }, { "sample_id": 69, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.11489993147552, "policy_ms": 1.9087999826297164, "execution_ms": 2.070999937132001, "status": "ok", "code": "ASK_USER" }, { "sample_id": 70, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.464600080624223, "policy_ms": 2.668699948117137, "execution_ms": 23.93499994650483, "status": "ok", "code": "CLICK" }, { "sample_id": 71, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.999996542930603, "observation_ms": 12.95220002066344, "policy_ms": 2.4679999332875013, "execution_ms": 16.157799982465804, "status": "ok", "code": "SELECT" }, { "sample_id": 72, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9893146753311157, "observation_ms": 15.717100002802908, "policy_ms": 3.7606999976560473, "execution_ms": 16.984299989417195, "status": "ok", "code": "SELECT" }, { "sample_id": 73, "template": "nested", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.583000098355114, "policy_ms": 1.9235999789088964, "execution_ms": 15.615000040270388, "status": "ok", "code": "SELECT" }, { "sample_id": 74, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.8770361542701721, "target_confidence": 1.0, "observation_ms": 15.939899953082204, "policy_ms": 2.9320999747142196, "execution_ms": 10.032300022430718, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 75, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.446000007912517, "policy_ms": 1.9451999105513096, "execution_ms": 15.01169998664409, "status": "ok", "code": "SELECT" }, { "sample_id": 76, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.012399965897202, "policy_ms": 1.7288000090047717, "execution_ms": 16.6993000311777, "status": "ok", "code": "SELECT" }, { "sample_id": 77, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 12.958699953742325, "policy_ms": 2.9100000392645597, "execution_ms": 9.939000010490417, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 78, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.985300039872527, "policy_ms": 1.9712999928742647, "execution_ms": 14.669200056232512, "status": "ok", "code": "SELECT" }, { "sample_id": 79, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.078799962997437, "policy_ms": 3.023100085556507, "execution_ms": 15.87969996035099, "status": "ok", "code": "SELECT" }, { "sample_id": 80, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.870000024326146, "policy_ms": 2.5908000534400344, "execution_ms": 54.726800066418946, "status": "ok", "code": "CLICK" }, { "sample_id": 81, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999955892562866, "target_confidence": 1.0, "observation_ms": 14.03229997958988, "policy_ms": 2.6039000367745757, "execution_ms": 21.69069997034967, "status": "ok", "code": "CLICK" }, { "sample_id": 82, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9874667525291443, "target_confidence": 1.0, "observation_ms": 13.66629998665303, "policy_ms": 3.2644999446347356, "execution_ms": 22.565199993550777, "status": "ok", "code": "CLICK" }, { "sample_id": 83, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999885559082031, "target_confidence": 1.0, "observation_ms": 13.963400037027895, "policy_ms": 2.765199984423816, "execution_ms": 9.677199996076524, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 84, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.606799930334091, "policy_ms": 2.8017000295221806, "execution_ms": 10.312799946404994, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 85, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.567699981853366, "policy_ms": 3.14990000333637, "execution_ms": 16.22500002849847, "status": "ok", "code": "SELECT" }, { "sample_id": 86, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.63089999742806, "policy_ms": 1.9193000625818968, "execution_ms": 26.472300058230758, "status": "ok", "code": "CLICK" }, { "sample_id": 87, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9978881478309631, "target_confidence": 1.0, "observation_ms": 14.111700002104044, "policy_ms": 3.0320000369101763, "execution_ms": 22.802699939347804, "status": "ok", "code": "CLICK" }, { "sample_id": 88, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999955892562866, "target_confidence": 1.0, "observation_ms": 15.391700086183846, "policy_ms": 3.0305999098345637, "execution_ms": 39.677699911408126, "status": "ok", "code": "CLICK" }, { "sample_id": 89, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.580999945290387, "policy_ms": 3.050200059078634, "execution_ms": 16.045400057919323, "status": "ok", "code": "SELECT" }, { "sample_id": 90, "template": "table", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999955892562866, "target_confidence": 0.9998617172241211, "observation_ms": 16.759400023147464, "policy_ms": 3.5053000319749117, "execution_ms": 23.27140001580119, "status": "ok", "code": "CLICK" }, { "sample_id": 91, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.7773793935775757, "target_confidence": 1.0, "observation_ms": 14.848099905066192, "policy_ms": 2.7843000134453177, "execution_ms": 2.3981999838724732, "status": "ok", "code": "ASK_USER" }, { "sample_id": 92, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 13.715100008994341, "policy_ms": 3.015700029209256, "execution_ms": 9.715899941511452, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 93, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.955100042745471, "policy_ms": 2.0581999560818076, "execution_ms": 12.647299910895526, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 94, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.999945878982544, "target_confidence": 1.0, "observation_ms": 14.752999995835125, "policy_ms": 2.7809999883174896, "execution_ms": 54.606499965302646, "status": "ok", "code": "CLICK" }, { "sample_id": 95, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9988864064216614, "target_confidence": 1.0, "observation_ms": 13.102500000968575, "policy_ms": 2.5030000833794475, "execution_ms": 23.343699984252453, "status": "ok", "code": "CLICK" }, { "sample_id": 96, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9808314442634583, "target_confidence": 1.0, "observation_ms": 13.83519999217242, "policy_ms": 2.760999952442944, "execution_ms": 9.711199905723333, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 97, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9050796627998352, "target_confidence": 1.0, "observation_ms": 11.939999996684492, "policy_ms": 1.9660999532788992, "execution_ms": 16.0264999140054, "status": "ok", "code": "TYPE" }, { "sample_id": 98, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 11.875799973495305, "policy_ms": 2.076099975965917, "execution_ms": 15.792699996381998, "status": "ok", "code": "SELECT" }, { "sample_id": 99, "template": "table", "success": true, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.462799929082394, "policy_ms": 3.6057999823242426, "execution_ms": 2.986599924042821, "status": "ok", "code": "ASK_USER" }, { "sample_id": 100, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999902248382568, "observation_ms": 15.135299996472895, "policy_ms": 2.6094999630004168, "execution_ms": 2.2055000299587846, "status": "ok", "code": "ASK_USER" }, { "sample_id": 101, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 11.98309997562319, "policy_ms": 1.9260999979451299, "execution_ms": 2.0203000167384744, "status": "ok", "code": "ASK_USER" }, { "sample_id": 102, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.8218594193458557, "target_confidence": 1.0, "observation_ms": 15.545599977485836, "policy_ms": 3.12780006788671, "execution_ms": 2.2397999418899417, "status": "ok", "code": "ASK_USER" }, { "sample_id": 103, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999995231628418, "target_confidence": 1.0, "observation_ms": 11.621500016190112, "policy_ms": 1.8804999999701977, "execution_ms": 18.729500006884336, "status": "ok", "code": "TYPE" }, { "sample_id": 104, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.6922178864479065, "observation_ms": 16.361800022423267, "policy_ms": 2.5900000473484397, "execution_ms": 2.6360000483691692, "status": "ok", "code": "ASK_USER" }, { "sample_id": 105, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.549699982628226, "policy_ms": 1.6466000815853477, "execution_ms": 17.236400046385825, "status": "ok", "code": "TYPE" }, { "sample_id": 106, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999979734420776, "target_confidence": 1.0, "observation_ms": 13.705999939702451, "policy_ms": 1.84449995867908, "execution_ms": 12.095400015823543, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 107, "template": "nested", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9943962097167969, "observation_ms": 15.221600071527064, "policy_ms": 3.771199961192906, "execution_ms": 15.91529999859631, "status": "ok", "code": "SELECT" }, { "sample_id": 108, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.304100070148706, "policy_ms": 2.8521999483928084, "execution_ms": 2.2107999539002776, "status": "ok", "code": "ASK_USER" }, { "sample_id": 109, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9992850422859192, "target_confidence": 1.0, "observation_ms": 14.743900042958558, "policy_ms": 2.9267999343574047, "execution_ms": 10.188999935053289, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 110, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.455100077204406, "policy_ms": 1.881500007584691, "execution_ms": 16.300100018270314, "status": "ok", "code": "SELECT" }, { "sample_id": 111, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.463100000284612, "policy_ms": 2.7065000031143427, "execution_ms": 15.26350004132837, "status": "ok", "code": "SELECT" }, { "sample_id": 112, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.7560167908668518, "target_confidence": 1.0, "observation_ms": 13.202299945987761, "policy_ms": 2.3172999499365687, "execution_ms": 2.1214999724179506, "status": "ok", "code": "ASK_USER" }, { "sample_id": 113, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.698400067165494, "policy_ms": 2.3204999743029475, "execution_ms": 45.52929999772459, "status": "ok", "code": "CLICK" }, { "sample_id": 114, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.901500009931624, "policy_ms": 3.0267999973148108, "execution_ms": 17.193299951031804, "status": "ok", "code": "SELECT" }, { "sample_id": 115, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9989173412322998, "target_confidence": 1.0, "observation_ms": 16.38369995635003, "policy_ms": 2.64620000962168, "execution_ms": 22.608300088904798, "status": "ok", "code": "TYPE" }, { "sample_id": 116, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9819120168685913, "target_confidence": 1.0, "observation_ms": 16.344000003300607, "policy_ms": 2.8233000775799155, "execution_ms": 22.272499976679683, "status": "ok", "code": "TYPE" }, { "sample_id": 117, "template": "nested", "success": true, "abstained": false, "expected_action": "C", "predicted_action": "C", "action_confidence": 0.9999998807907104, "target_confidence": 1.0, "observation_ms": 13.907600077800453, "policy_ms": 1.9036999437958002, "execution_ms": 25.179299991577864, "status": "ok", "code": "CLICK" }, { "sample_id": 118, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.256300007924438, "policy_ms": 2.396099967882037, "execution_ms": 15.387500054202974, "status": "ok", "code": "SELECT" }, { "sample_id": 119, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9981478452682495, "observation_ms": 14.962199958972633, "policy_ms": 3.0468000331893563, "execution_ms": 15.09010000154376, "status": "ok", "code": "SELECT" } ] }