| { | |
| "checkpoint": "models/v001-no-lexical", | |
| "quantized": false, | |
| "platform": "Windows-11-10.0.26200-SP0", | |
| "torch_threads": 2, | |
| "model_load_ms": 7.755500031635165, | |
| "samples": 120, | |
| "success_rate": 0.5833333333333334, | |
| "abstention_rate": 0.175, | |
| "median_observation_ms": 14.387400005944073, | |
| "median_policy_ms": 2.590400050394237, | |
| "median_execution_ms": 15.836199978366494, | |
| "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", | |
| "target_vps_validated": false, | |
| "results": [ | |
| { | |
| "sample_id": 0, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999039173126221, | |
| "observation_ms": 48.155799973756075, | |
| "policy_ms": 4.100500023923814, | |
| "execution_ms": 3.78839997574687, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 1, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999996423721313, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.890599995851517, | |
| "policy_ms": 2.4492000229656696, | |
| "execution_ms": 39.766900008544326, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 2, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 1.0, | |
| "observation_ms": 37.80180006287992, | |
| "policy_ms": 2.3002999369055033, | |
| "execution_ms": 18.543299986049533, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 3, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.828799990937114, | |
| "policy_ms": 2.4418000830337405, | |
| "execution_ms": 31.127299997024238, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 4, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.8566849827766418, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.973599976859987, | |
| "policy_ms": 2.631800016388297, | |
| "execution_ms": 2.5369999930262566, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 5, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.66629996523261, | |
| "policy_ms": 2.7742000529542565, | |
| "execution_ms": 10.593199986033142, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 6, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.330099992454052, | |
| "policy_ms": 2.7721000369638205, | |
| "execution_ms": 19.744400051422417, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 7, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.999404788017273, | |
| "observation_ms": 15.696700080297887, | |
| "policy_ms": 2.842400106601417, | |
| "execution_ms": 17.253799946047366, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 8, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9946579337120056, | |
| "observation_ms": 15.824000001884997, | |
| "policy_ms": 2.9687000205740333, | |
| "execution_ms": 16.37490000575781, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 9, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.5657387971878052, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.3670000359416, | |
| "policy_ms": 1.8517000135034323, | |
| "execution_ms": 2.1585000213235617, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 10, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.34619999397546, | |
| "policy_ms": 2.0665000192821026, | |
| "execution_ms": 16.778600052930415, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 11, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999995231628418, | |
| "observation_ms": 18.10390001628548, | |
| "policy_ms": 2.118099946528673, | |
| "execution_ms": 2.3756999289616942, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 12, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999984502792358, | |
| "observation_ms": 14.613799983635545, | |
| "policy_ms": 2.5136000476777554, | |
| "execution_ms": 10.11360005941242, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 13, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.006900019012392, | |
| "policy_ms": 2.845599898137152, | |
| "execution_ms": 57.12309991940856, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 14, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.414299977943301, | |
| "policy_ms": 2.4254999589174986, | |
| "execution_ms": 17.154800007119775, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 15, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.039400012232363, | |
| "policy_ms": 1.5656000468879938, | |
| "execution_ms": 13.976700021885335, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 16, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.034599954262376, | |
| "policy_ms": 3.0512999510392547, | |
| "execution_ms": 18.35849997587502, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 17, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.637500026263297, | |
| "policy_ms": 3.009999985806644, | |
| "execution_ms": 15.937500051222742, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 18, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999964237213135, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.905499945394695, | |
| "policy_ms": 2.189300023019314, | |
| "execution_ms": 28.85450003668666, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 19, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.958699932321906, | |
| "policy_ms": 2.4140001041814685, | |
| "execution_ms": 16.44000003580004, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 20, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999957084655762, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.601499937474728, | |
| "policy_ms": 2.930799964815378, | |
| "execution_ms": 26.539099984802306, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 21, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.49610001873225, | |
| "policy_ms": 2.6504999259486794, | |
| "execution_ms": 59.45119995158166, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 22, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.534300007857382, | |
| "policy_ms": 2.8038000455126166, | |
| "execution_ms": 54.541900055482984, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 23, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.405699981376529, | |
| "policy_ms": 2.187600010074675, | |
| "execution_ms": 15.721399919129908, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 24, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999997615814209, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.66660003643483, | |
| "policy_ms": 3.554799943231046, | |
| "execution_ms": 48.90890000388026, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 25, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.233899953775108, | |
| "policy_ms": 1.8354000058025122, | |
| "execution_ms": 25.838000001385808, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 26, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 18.253899994306266, | |
| "policy_ms": 3.1340999994426966, | |
| "execution_ms": 13.132599997334182, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 27, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.520599972456694, | |
| "policy_ms": 1.7303000204265118, | |
| "execution_ms": 16.68460003565997, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 28, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.085299986414611, | |
| "policy_ms": 1.9613999174907804, | |
| "execution_ms": 9.540400002151728, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 29, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.150600017979741, | |
| "policy_ms": 2.2929999977350235, | |
| "execution_ms": 24.475399986840785, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 30, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.633000013418496, | |
| "policy_ms": 3.142800065688789, | |
| "execution_ms": 10.01420000102371, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 31, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.44879998639226, | |
| "policy_ms": 2.66519992146641, | |
| "execution_ms": 16.52860001195222, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 32, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.910100049339235, | |
| "policy_ms": 2.1643999498337507, | |
| "execution_ms": 15.578899998217821, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 33, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9468345046043396, | |
| "observation_ms": 13.298999983817339, | |
| "policy_ms": 2.1152000408619642, | |
| "execution_ms": 2.374800038523972, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 34, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.98715740442276, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.753099996596575, | |
| "policy_ms": 3.1820000149309635, | |
| "execution_ms": 18.66950001567602, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 35, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.49889999255538, | |
| "policy_ms": 1.9891000119969249, | |
| "execution_ms": 30.96399991773069, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 36, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.855400029569864, | |
| "policy_ms": 1.9989999709650874, | |
| "execution_ms": 10.228500002995133, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 37, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.236199902370572, | |
| "policy_ms": 2.596600097604096, | |
| "execution_ms": 15.496000065468252, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 38, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9890561103820801, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.398599953390658, | |
| "policy_ms": 2.4811000330373645, | |
| "execution_ms": 10.626500006765127, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 39, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9998949766159058, | |
| "observation_ms": 12.702600099146366, | |
| "policy_ms": 2.3571999045088887, | |
| "execution_ms": 10.468199965544045, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 40, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9305520057678223, | |
| "observation_ms": 14.912900049239397, | |
| "policy_ms": 2.680000034160912, | |
| "execution_ms": 2.3592000361531973, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 41, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.922900030389428, | |
| "policy_ms": 2.2016000002622604, | |
| "execution_ms": 16.15399995353073, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 42, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.27980000153184, | |
| "policy_ms": 2.9817000031471252, | |
| "execution_ms": 20.689299912191927, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 43, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.495000010356307, | |
| "policy_ms": 1.7315000295639038, | |
| "execution_ms": 16.5631000418216, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 44, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.022499999962747, | |
| "policy_ms": 2.5951999705284834, | |
| "execution_ms": 28.8407999323681, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 45, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.994099990464747, | |
| "policy_ms": 1.922299969010055, | |
| "execution_ms": 15.168599900789559, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 46, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.887299992144108, | |
| "policy_ms": 2.1222999785095453, | |
| "execution_ms": 15.42199996765703, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 47, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.8051379323005676, | |
| "observation_ms": 15.364399994723499, | |
| "policy_ms": 2.498099929653108, | |
| "execution_ms": 2.7083000168204308, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 48, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.14170001912862, | |
| "policy_ms": 1.8744999542832375, | |
| "execution_ms": 9.740900015458465, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 49, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.013899977318943, | |
| "policy_ms": 2.550999983213842, | |
| "execution_ms": 11.068299994803965, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 50, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9796527624130249, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.284099939279258, | |
| "policy_ms": 1.7565001035109162, | |
| "execution_ms": 22.017300012521446, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 51, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.864399981684983, | |
| "policy_ms": 2.271600067615509, | |
| "execution_ms": 15.552200027741492, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 52, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.490600045770407, | |
| "policy_ms": 2.149399952031672, | |
| "execution_ms": 14.547700062394142, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 53, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9997931122779846, | |
| "observation_ms": 15.169100021012127, | |
| "policy_ms": 3.2730000093579292, | |
| "execution_ms": 2.885499969124794, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 54, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9972251653671265, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.367499970830977, | |
| "policy_ms": 3.09520005248487, | |
| "execution_ms": 16.815399983897805, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 55, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999972581863403, | |
| "observation_ms": 15.827800030820072, | |
| "policy_ms": 3.6145999329164624, | |
| "execution_ms": 17.35270000062883, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 56, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.40730004105717, | |
| "policy_ms": 2.7208999963477254, | |
| "execution_ms": 2.3763999342918396, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 57, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.356899959035218, | |
| "policy_ms": 2.341100014746189, | |
| "execution_ms": 48.094899975694716, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 58, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.218099997378886, | |
| "policy_ms": 3.1998000340536237, | |
| "execution_ms": 15.435200068168342, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 59, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.307899935171008, | |
| "policy_ms": 2.1096999989822507, | |
| "execution_ms": 15.629699919372797, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 60, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.890400015749037, | |
| "policy_ms": 2.027299953624606, | |
| "execution_ms": 15.748900012113154, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 61, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.705399935133755, | |
| "policy_ms": 3.4446000354364514, | |
| "execution_ms": 16.00020006299019, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 62, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.999985933303833, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.249400050379336, | |
| "policy_ms": 2.678500022739172, | |
| "execution_ms": 10.372799937613308, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 63, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9900351166725159, | |
| "observation_ms": 15.276299905963242, | |
| "policy_ms": 2.547200070694089, | |
| "execution_ms": 2.489199978299439, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 64, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999909400939941, | |
| "observation_ms": 14.591599931009114, | |
| "policy_ms": 2.7464000741019845, | |
| "execution_ms": 10.19070006441325, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 65, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.814099947921932, | |
| "policy_ms": 1.896600006148219, | |
| "execution_ms": 2.2869999520480633, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 66, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.999858021736145, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.144100063480437, | |
| "policy_ms": 2.86400003824383, | |
| "execution_ms": 34.816599916666746, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 67, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9981772899627686, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.527500046417117, | |
| "policy_ms": 1.9356999546289444, | |
| "execution_ms": 16.11970004159957, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 68, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 0.9811867475509644, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.66930003091693, | |
| "policy_ms": 1.8606000812724233, | |
| "execution_ms": 2.3653999669477344, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 69, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.11489993147552, | |
| "policy_ms": 1.9087999826297164, | |
| "execution_ms": 2.070999937132001, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 70, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.464600080624223, | |
| "policy_ms": 2.668699948117137, | |
| "execution_ms": 23.93499994650483, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 71, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.999996542930603, | |
| "observation_ms": 12.95220002066344, | |
| "policy_ms": 2.4679999332875013, | |
| "execution_ms": 16.157799982465804, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 72, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9893146753311157, | |
| "observation_ms": 15.717100002802908, | |
| "policy_ms": 3.7606999976560473, | |
| "execution_ms": 16.984299989417195, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 73, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.583000098355114, | |
| "policy_ms": 1.9235999789088964, | |
| "execution_ms": 15.615000040270388, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 74, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.8770361542701721, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.939899953082204, | |
| "policy_ms": 2.9320999747142196, | |
| "execution_ms": 10.032300022430718, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 75, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.446000007912517, | |
| "policy_ms": 1.9451999105513096, | |
| "execution_ms": 15.01169998664409, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 76, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.012399965897202, | |
| "policy_ms": 1.7288000090047717, | |
| "execution_ms": 16.6993000311777, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 77, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 12.958699953742325, | |
| "policy_ms": 2.9100000392645597, | |
| "execution_ms": 9.939000010490417, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 78, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.985300039872527, | |
| "policy_ms": 1.9712999928742647, | |
| "execution_ms": 14.669200056232512, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 79, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.078799962997437, | |
| "policy_ms": 3.023100085556507, | |
| "execution_ms": 15.87969996035099, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 80, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.870000024326146, | |
| "policy_ms": 2.5908000534400344, | |
| "execution_ms": 54.726800066418946, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 81, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999955892562866, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.03229997958988, | |
| "policy_ms": 2.6039000367745757, | |
| "execution_ms": 21.69069997034967, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 82, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9874667525291443, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.66629998665303, | |
| "policy_ms": 3.2644999446347356, | |
| "execution_ms": 22.565199993550777, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 83, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999885559082031, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.963400037027895, | |
| "policy_ms": 2.765199984423816, | |
| "execution_ms": 9.677199996076524, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 84, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.606799930334091, | |
| "policy_ms": 2.8017000295221806, | |
| "execution_ms": 10.312799946404994, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 85, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.567699981853366, | |
| "policy_ms": 3.14990000333637, | |
| "execution_ms": 16.22500002849847, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 86, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.63089999742806, | |
| "policy_ms": 1.9193000625818968, | |
| "execution_ms": 26.472300058230758, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 87, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9978881478309631, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.111700002104044, | |
| "policy_ms": 3.0320000369101763, | |
| "execution_ms": 22.802699939347804, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 88, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999955892562866, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.391700086183846, | |
| "policy_ms": 3.0305999098345637, | |
| "execution_ms": 39.677699911408126, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 89, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.580999945290387, | |
| "policy_ms": 3.050200059078634, | |
| "execution_ms": 16.045400057919323, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 90, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999955892562866, | |
| "target_confidence": 0.9998617172241211, | |
| "observation_ms": 16.759400023147464, | |
| "policy_ms": 3.5053000319749117, | |
| "execution_ms": 23.27140001580119, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 91, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.7773793935775757, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.848099905066192, | |
| "policy_ms": 2.7843000134453177, | |
| "execution_ms": 2.3981999838724732, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 92, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999998807907104, | |
| "observation_ms": 13.715100008994341, | |
| "policy_ms": 3.015700029209256, | |
| "execution_ms": 9.715899941511452, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 93, | |
| "template": "table", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.955100042745471, | |
| "policy_ms": 2.0581999560818076, | |
| "execution_ms": 12.647299910895526, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 94, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.999945878982544, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.752999995835125, | |
| "policy_ms": 2.7809999883174896, | |
| "execution_ms": 54.606499965302646, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 95, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9988864064216614, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.102500000968575, | |
| "policy_ms": 2.5030000833794475, | |
| "execution_ms": 23.343699984252453, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 96, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9808314442634583, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.83519999217242, | |
| "policy_ms": 2.760999952442944, | |
| "execution_ms": 9.711199905723333, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 97, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9050796627998352, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.939999996684492, | |
| "policy_ms": 1.9660999532788992, | |
| "execution_ms": 16.0264999140054, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 98, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.875799973495305, | |
| "policy_ms": 2.076099975965917, | |
| "execution_ms": 15.792699996381998, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 99, | |
| "template": "table", | |
| "success": true, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.462799929082394, | |
| "policy_ms": 3.6057999823242426, | |
| "execution_ms": 2.986599924042821, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 100, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9999902248382568, | |
| "observation_ms": 15.135299996472895, | |
| "policy_ms": 2.6094999630004168, | |
| "execution_ms": 2.2055000299587846, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 101, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.98309997562319, | |
| "policy_ms": 1.9260999979451299, | |
| "execution_ms": 2.0203000167384744, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 102, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.8218594193458557, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.545599977485836, | |
| "policy_ms": 3.12780006788671, | |
| "execution_ms": 2.2397999418899417, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 103, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9999995231628418, | |
| "target_confidence": 1.0, | |
| "observation_ms": 11.621500016190112, | |
| "policy_ms": 1.8804999999701977, | |
| "execution_ms": 18.729500006884336, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 104, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.6922178864479065, | |
| "observation_ms": 16.361800022423267, | |
| "policy_ms": 2.5900000473484397, | |
| "execution_ms": 2.6360000483691692, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 105, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.549699982628226, | |
| "policy_ms": 1.6466000815853477, | |
| "execution_ms": 17.236400046385825, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 106, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9999979734420776, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.705999939702451, | |
| "policy_ms": 1.84449995867908, | |
| "execution_ms": 12.095400015823543, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 107, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9943962097167969, | |
| "observation_ms": 15.221600071527064, | |
| "policy_ms": 3.771199961192906, | |
| "execution_ms": 15.91529999859631, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 108, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "C", | |
| "predicted_action": "A", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.304100070148706, | |
| "policy_ms": 2.8521999483928084, | |
| "execution_ms": 2.2107999539002776, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 109, | |
| "template": "nested", | |
| "success": false, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "O", | |
| "action_confidence": 0.9992850422859192, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.743900042958558, | |
| "policy_ms": 2.9267999343574047, | |
| "execution_ms": 10.188999935053289, | |
| "status": "rejected", | |
| "code": "SELECT_ROLE_MISMATCH" | |
| }, | |
| { | |
| "sample_id": 110, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.455100077204406, | |
| "policy_ms": 1.881500007584691, | |
| "execution_ms": 16.300100018270314, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 111, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 14.463100000284612, | |
| "policy_ms": 2.7065000031143427, | |
| "execution_ms": 15.26350004132837, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 112, | |
| "template": "table", | |
| "success": false, | |
| "abstained": true, | |
| "expected_action": "T", | |
| "predicted_action": "A", | |
| "action_confidence": 0.7560167908668518, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.202299945987761, | |
| "policy_ms": 2.3172999499365687, | |
| "execution_ms": 2.1214999724179506, | |
| "status": "ok", | |
| "code": "ASK_USER" | |
| }, | |
| { | |
| "sample_id": 113, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 12.698400067165494, | |
| "policy_ms": 2.3204999743029475, | |
| "execution_ms": 45.52929999772459, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 114, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 15.901500009931624, | |
| "policy_ms": 3.0267999973148108, | |
| "execution_ms": 17.193299951031804, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 115, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9989173412322998, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.38369995635003, | |
| "policy_ms": 2.64620000962168, | |
| "execution_ms": 22.608300088904798, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 116, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "T", | |
| "predicted_action": "T", | |
| "action_confidence": 0.9819120168685913, | |
| "target_confidence": 1.0, | |
| "observation_ms": 16.344000003300607, | |
| "policy_ms": 2.8233000775799155, | |
| "execution_ms": 22.272499976679683, | |
| "status": "ok", | |
| "code": "TYPE" | |
| }, | |
| { | |
| "sample_id": 117, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "C", | |
| "predicted_action": "C", | |
| "action_confidence": 0.9999998807907104, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.907600077800453, | |
| "policy_ms": 1.9036999437958002, | |
| "execution_ms": 25.179299991577864, | |
| "status": "ok", | |
| "code": "CLICK" | |
| }, | |
| { | |
| "sample_id": 118, | |
| "template": "table", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 1.0, | |
| "observation_ms": 13.256300007924438, | |
| "policy_ms": 2.396099967882037, | |
| "execution_ms": 15.387500054202974, | |
| "status": "ok", | |
| "code": "SELECT" | |
| }, | |
| { | |
| "sample_id": 119, | |
| "template": "nested", | |
| "success": true, | |
| "abstained": false, | |
| "expected_action": "O", | |
| "predicted_action": "O", | |
| "action_confidence": 1.0, | |
| "target_confidence": 0.9981478452682495, | |
| "observation_ms": 14.962199958972633, | |
| "policy_ms": 3.0468000331893563, | |
| "execution_ms": 15.09010000154376, | |
| "status": "ok", | |
| "code": "SELECT" | |
| } | |
| ] | |
| } |