{ "checkpoint": "models/v002-gru", "quantized": false, "platform": "Windows-11-10.0.26200-SP0", "torch_threads": 2, "model_load_ms": 8.020800072699785, "samples": 120, "success_rate": 0.39166666666666666, "abstention_rate": 0.4, "median_observation_ms": 14.639450004324317, "median_policy_ms": 5.642850010190159, "median_execution_ms": 15.774849976878613, "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites", "target_vps_validated": false, "results": [ { "sample_id": 0, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9106427431106567, "observation_ms": 44.24660000950098, "policy_ms": 7.870400091633201, "execution_ms": 3.6689999978989363, "status": "ok", "code": "ASK_USER" }, { "sample_id": 1, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9873443841934204, "target_confidence": 0.9999971389770508, "observation_ms": 14.778300072066486, "policy_ms": 6.43149996176362, "execution_ms": 43.75750001054257, "status": "ok", "code": "TYPE" }, { "sample_id": 2, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9888303875923157, "target_confidence": 1.0, "observation_ms": 36.495500011369586, "policy_ms": 5.358200054615736, "execution_ms": 22.587800049223006, "status": "ok", "code": "TYPE" }, { "sample_id": 3, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9998986721038818, "observation_ms": 15.025699976831675, "policy_ms": 4.462999990209937, "execution_ms": 3.001399920322001, "status": "ok", "code": "ASK_USER" }, { "sample_id": 4, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9679258465766907, "observation_ms": 16.171199968084693, "policy_ms": 5.426099989563227, "execution_ms": 2.8247000882402062, "status": "ok", "code": "ASK_USER" }, { "sample_id": 5, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9498040080070496, "observation_ms": 15.62059996649623, "policy_ms": 6.210600025951862, "execution_ms": 18.92640010919422, "status": "ok", "code": "SELECT" }, { "sample_id": 6, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.999823272228241, "observation_ms": 19.638199941255152, "policy_ms": 6.917600054293871, "execution_ms": 19.17179999873042, "status": "ok", "code": "SELECT" }, { "sample_id": 7, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9934561848640442, "observation_ms": 14.618700020946562, "policy_ms": 5.980800022371113, "execution_ms": 17.47959991917014, "status": "ok", "code": "SELECT" }, { "sample_id": 8, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9882636070251465, "observation_ms": 16.187299974262714, "policy_ms": 7.8931000316515565, "execution_ms": 18.755099968984723, "status": "ok", "code": "SELECT" }, { "sample_id": 9, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.6589809656143188, "target_confidence": 0.999976634979248, "observation_ms": 15.046400018036366, "policy_ms": 6.1913999961689115, "execution_ms": 2.5551000144332647, "status": "ok", "code": "ASK_USER" }, { "sample_id": 10, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9514384269714355, "observation_ms": 15.801900066435337, "policy_ms": 5.364800104871392, "execution_ms": 19.60490003693849, "status": "ok", "code": "SELECT" }, { "sample_id": 11, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9992782473564148, "observation_ms": 14.57150001078844, "policy_ms": 5.289900000207126, "execution_ms": 2.4638999020680785, "status": "ok", "code": "ASK_USER" }, { "sample_id": 12, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9710627198219299, "observation_ms": 14.585500000976026, "policy_ms": 5.454499972984195, "execution_ms": 17.143300035968423, "status": "ok", "code": "SELECT" }, { "sample_id": 13, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.6948939561843872, "observation_ms": 14.131800038740039, "policy_ms": 5.039899959228933, "execution_ms": 2.332399948500097, "status": "ok", "code": "ASK_USER" }, { "sample_id": 14, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.882300096563995, "policy_ms": 4.618899896740913, "execution_ms": 19.744499935768545, "status": "ok", "code": "SELECT" }, { "sample_id": 15, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.628600001335144, "policy_ms": 5.319400108419359, "execution_ms": 17.34090002719313, "status": "ok", "code": "SELECT" }, { "sample_id": 16, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9597891569137573, "observation_ms": 15.164899989031255, "policy_ms": 6.308699958026409, "execution_ms": 16.590299899689853, "status": "ok", "code": "SELECT" }, { "sample_id": 17, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 20.442000008188188, "policy_ms": 14.638499938882887, "execution_ms": 25.483200093731284, "status": "ok", "code": "SELECT" }, { "sample_id": 18, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9969860911369324, "observation_ms": 22.283100057393312, "policy_ms": 8.245299919508398, "execution_ms": 3.1480001052841544, "status": "ok", "code": "ASK_USER" }, { "sample_id": 19, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.50219995342195, "policy_ms": 5.641499999910593, "execution_ms": 20.024199970066547, "status": "ok", "code": "SELECT" }, { "sample_id": 20, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9956907629966736, "observation_ms": 21.485199918970466, "policy_ms": 10.731799993664026, "execution_ms": 3.246000036597252, "status": "ok", "code": "ASK_USER" }, { "sample_id": 21, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9999998807907104, "target_confidence": 0.9992986917495728, "observation_ms": 21.41929999925196, "policy_ms": 5.92360005248338, "execution_ms": 3.3164999913424253, "status": "ok", "code": "ASK_USER" }, { "sample_id": 22, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.8012701869010925, "observation_ms": 19.006700022146106, "policy_ms": 6.379799917340279, "execution_ms": 3.13510000705719, "status": "ok", "code": "ASK_USER" }, { "sample_id": 23, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.0764999454841, "policy_ms": 5.175700061954558, "execution_ms": 20.086499978788197, "status": "ok", "code": "SELECT" }, { "sample_id": 24, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.6497063636779785, "observation_ms": 17.871599993668497, "policy_ms": 5.860900040715933, "execution_ms": 2.8192000463604927, "status": "ok", "code": "ASK_USER" }, { "sample_id": 25, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.6710407137870789, "observation_ms": 15.921200043521821, "policy_ms": 4.068300011567771, "execution_ms": 3.2719000009819865, "status": "ok", "code": "ASK_USER" }, { "sample_id": 26, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9805163741111755, "observation_ms": 15.411500004120171, "policy_ms": 6.655100034549832, "execution_ms": 16.829799977131188, "status": "ok", "code": "SELECT" }, { "sample_id": 27, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9994151592254639, "observation_ms": 13.451500097289681, "policy_ms": 3.9328999118879437, "execution_ms": 17.0963000273332, "status": "ok", "code": "SELECT" }, { "sample_id": 28, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 13.573499978519976, "policy_ms": 4.711700021289289, "execution_ms": 16.50489994790405, "status": "ok", "code": "SELECT" }, { "sample_id": 29, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9940434098243713, "observation_ms": 14.351799967698753, "policy_ms": 4.563600057736039, "execution_ms": 2.69210000988096, "status": "ok", "code": "ASK_USER" }, { "sample_id": 30, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999980926513672, "observation_ms": 16.997999977320433, "policy_ms": 8.311899960972369, "execution_ms": 18.91580002848059, "status": "ok", "code": "SELECT" }, { "sample_id": 31, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.44539991300553, "policy_ms": 5.644200020469725, "execution_ms": 18.05759989656508, "status": "ok", "code": "SELECT" }, { "sample_id": 32, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999985694885254, "observation_ms": 15.494500054046512, "policy_ms": 4.54239989630878, "execution_ms": 16.404900001361966, "status": "ok", "code": "SELECT" }, { "sample_id": 33, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.381199911236763, "policy_ms": 4.269700031727552, "execution_ms": 2.302700071595609, "status": "ok", "code": "ASK_USER" }, { "sample_id": 34, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9988446235656738, "target_confidence": 0.999969482421875, "observation_ms": 15.084499958902597, "policy_ms": 6.35020004119724, "execution_ms": 11.644899961538613, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 35, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.976399972103536, "policy_ms": 6.781600066460669, "execution_ms": 2.4924000026658177, "status": "ok", "code": "ASK_USER" }, { "sample_id": 36, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.151700026355684, "policy_ms": 6.173499976284802, "execution_ms": 18.531200010329485, "status": "ok", "code": "SELECT" }, { "sample_id": 37, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.123800072818995, "policy_ms": 9.156499989330769, "execution_ms": 19.456699956208467, "status": "ok", "code": "SELECT" }, { "sample_id": 38, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9331730604171753, "target_confidence": 0.9992374181747437, "observation_ms": 16.19349990505725, "policy_ms": 5.165400099940598, "execution_ms": 11.042700032703578, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 39, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.94921875, "observation_ms": 13.69160006288439, "policy_ms": 5.2671999437734485, "execution_ms": 20.551699912175536, "status": "ok", "code": "SELECT" }, { "sample_id": 40, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.8621384501457214, "observation_ms": 14.68430005479604, "policy_ms": 5.860799923539162, "execution_ms": 2.53950001206249, "status": "ok", "code": "ASK_USER" }, { "sample_id": 41, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.169800047762692, "policy_ms": 5.166899994947016, "execution_ms": 19.90090007893741, "status": "ok", "code": "SELECT" }, { "sample_id": 42, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999990463256836, "observation_ms": 16.215500072576106, "policy_ms": 5.8868000051006675, "execution_ms": 15.761999995447695, "status": "ok", "code": "SELECT" }, { "sample_id": 43, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.84250000026077, "policy_ms": 4.4160999823361635, "execution_ms": 15.85169997997582, "status": "ok", "code": "SELECT" }, { "sample_id": 44, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9755481481552124, "observation_ms": 18.474900047294796, "policy_ms": 6.045799935236573, "execution_ms": 2.5333999656140804, "status": "ok", "code": "ASK_USER" }, { "sample_id": 45, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.774900067597628, "policy_ms": 4.94549993891269, "execution_ms": 15.691899927332997, "status": "ok", "code": "SELECT" }, { "sample_id": 46, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999706745147705, "observation_ms": 13.911699992604554, "policy_ms": 5.272399983368814, "execution_ms": 15.609099995344877, "status": "ok", "code": "SELECT" }, { "sample_id": 47, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9941600561141968, "observation_ms": 13.88399989809841, "policy_ms": 5.4524000734090805, "execution_ms": 16.107400064356625, "status": "ok", "code": "SELECT" }, { "sample_id": 48, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999527931213379, "observation_ms": 12.878799927420914, "policy_ms": 4.389000008814037, "execution_ms": 15.67819993942976, "status": "ok", "code": "SELECT" }, { "sample_id": 49, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999653100967407, "observation_ms": 12.603000039234757, "policy_ms": 4.5912000350654125, "execution_ms": 15.382300014607608, "status": "ok", "code": "SELECT" }, { "sample_id": 50, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.26609996613115, "policy_ms": 4.6440999722108245, "execution_ms": 2.288100076839328, "status": "ok", "code": "ASK_USER" }, { "sample_id": 51, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.426000019535422, "policy_ms": 5.947500001639128, "execution_ms": 16.165500041097403, "status": "ok", "code": "SELECT" }, { "sample_id": 52, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999997615814209, "observation_ms": 12.6547000836581, "policy_ms": 4.678700002841651, "execution_ms": 17.162400064989924, "status": "ok", "code": "SELECT" }, { "sample_id": 53, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9989532232284546, "observation_ms": 14.350200071930885, "policy_ms": 5.14120003208518, "execution_ms": 2.3252000100910664, "status": "ok", "code": "ASK_USER" }, { "sample_id": 54, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9810065031051636, "target_confidence": 0.998680055141449, "observation_ms": 14.452700037509203, "policy_ms": 6.582799949683249, "execution_ms": 18.378900014795363, "status": "ok", "code": "TYPE" }, { "sample_id": 55, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999997615814209, "observation_ms": 15.689300023950636, "policy_ms": 5.696299951523542, "execution_ms": 18.688600044697523, "status": "ok", "code": "SELECT" }, { "sample_id": 56, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999995231628418, "observation_ms": 15.179599984548986, "policy_ms": 7.325300015509129, "execution_ms": 2.467700047418475, "status": "ok", "code": "ASK_USER" }, { "sample_id": 57, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.71820003259927, "policy_ms": 6.719099939800799, "execution_ms": 2.520799986086786, "status": "ok", "code": "ASK_USER" }, { "sample_id": 58, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.816499944776297, "policy_ms": 5.723300040699542, "execution_ms": 16.337000066414475, "status": "ok", "code": "SELECT" }, { "sample_id": 59, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.891999959014356, "policy_ms": 5.145099945366383, "execution_ms": 15.593699994497001, "status": "ok", "code": "SELECT" }, { "sample_id": 60, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999799728393555, "observation_ms": 13.682099990546703, "policy_ms": 4.811599967069924, "execution_ms": 16.774000017903745, "status": "ok", "code": "SELECT" }, { "sample_id": 61, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.277099980972707, "policy_ms": 8.364200009964406, "execution_ms": 19.13380005862564, "status": "ok", "code": "SELECT" }, { "sample_id": 62, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.8035814762115479, "observation_ms": 15.815299935638905, "policy_ms": 5.672300001606345, "execution_ms": 2.436299924738705, "status": "ok", "code": "ASK_USER" }, { "sample_id": 63, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9838116765022278, "observation_ms": 14.660199987702072, "policy_ms": 5.406700074672699, "execution_ms": 2.404000028036535, "status": "ok", "code": "ASK_USER" }, { "sample_id": 64, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9968637228012085, "observation_ms": 14.243200072087348, "policy_ms": 5.423899972811341, "execution_ms": 15.442600008100271, "status": "ok", "code": "SELECT" }, { "sample_id": 65, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 13.283299980685115, "policy_ms": 4.714200040325522, "execution_ms": 2.179000061005354, "status": "ok", "code": "ASK_USER" }, { "sample_id": 66, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.8451939821243286, "observation_ms": 15.359399956651032, "policy_ms": 5.572100053541362, "execution_ms": 2.367699984461069, "status": "ok", "code": "ASK_USER" }, { "sample_id": 67, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.154100044630468, "policy_ms": 5.556099931709468, "execution_ms": 17.591699957847595, "status": "ok", "code": "TYPE" }, { "sample_id": 68, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.526800041086972, "policy_ms": 4.182599950581789, "execution_ms": 2.3926999419927597, "status": "ok", "code": "ASK_USER" }, { "sample_id": 69, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9994825124740601, "observation_ms": 14.410000061616302, "policy_ms": 5.191099946387112, "execution_ms": 2.831999910995364, "status": "ok", "code": "ASK_USER" }, { "sample_id": 70, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 0.9999996423721313, "target_confidence": 0.9999701976776123, "observation_ms": 14.026100048795342, "policy_ms": 5.702499998733401, "execution_ms": 2.484200056642294, "status": "ok", "code": "ASK_USER" }, { "sample_id": 71, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9580230712890625, "observation_ms": 12.456300086341798, "policy_ms": 4.1102000977844, "execution_ms": 15.295200049877167, "status": "ok", "code": "SELECT" }, { "sample_id": 72, "template": "table", "success": false, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.999990701675415, "observation_ms": 14.332700055092573, "policy_ms": 5.6705999886617064, "execution_ms": 16.608400037512183, "status": "ok", "code": "SELECT" }, { "sample_id": 73, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.999998927116394, "observation_ms": 11.805300018750131, "policy_ms": 5.159699940122664, "execution_ms": 15.969399944879115, "status": "ok", "code": "SELECT" }, { "sample_id": 74, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.9997367262840271, "target_confidence": 0.5425891280174255, "observation_ms": 15.75839996803552, "policy_ms": 5.910899955779314, "execution_ms": 2.4791999021545053, "status": "ok", "code": "ASK_USER" }, { "sample_id": 75, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999966621398926, "observation_ms": 12.371599907055497, "policy_ms": 4.641800071112812, "execution_ms": 16.78669999819249, "status": "ok", "code": "SELECT" }, { "sample_id": 76, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.665900052525103, "policy_ms": 4.241300048306584, "execution_ms": 15.841100015677512, "status": "ok", "code": "SELECT" }, { "sample_id": 77, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.91079998575151, "policy_ms": 7.136700092814863, "execution_ms": 16.29639999009669, "status": "ok", "code": "SELECT" }, { "sample_id": 78, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.315700042061508, "policy_ms": 5.353999906219542, "execution_ms": 16.416999977082014, "status": "ok", "code": "SELECT" }, { "sample_id": 79, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.127800034359097, "policy_ms": 8.579300018027425, "execution_ms": 16.487499931827188, "status": "ok", "code": "SELECT" }, { "sample_id": 80, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 14.218700001947582, "policy_ms": 5.369900027289987, "execution_ms": 2.751899999566376, "status": "ok", "code": "ASK_USER" }, { "sample_id": 81, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999608993530273, "observation_ms": 15.178900095634162, "policy_ms": 5.437299958430231, "execution_ms": 2.516100066713989, "status": "ok", "code": "ASK_USER" }, { "sample_id": 82, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9960330128669739, "observation_ms": 15.121700009331107, "policy_ms": 5.732999998144805, "execution_ms": 2.6709999656304717, "status": "ok", "code": "ASK_USER" }, { "sample_id": 83, "template": "nested", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.9999986886978149, "target_confidence": 0.8480029702186584, "observation_ms": 13.722400064580142, "policy_ms": 6.257299915887415, "execution_ms": 2.3250000085681677, "status": "ok", "code": "ASK_USER" }, { "sample_id": 84, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9913052916526794, "observation_ms": 15.474800020456314, "policy_ms": 5.90430002193898, "execution_ms": 16.301500028930604, "status": "ok", "code": "SELECT" }, { "sample_id": 85, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9209082722663879, "observation_ms": 14.292799984104931, "policy_ms": 6.63459999486804, "execution_ms": 16.415800084359944, "status": "ok", "code": "SELECT" }, { "sample_id": 86, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 12.9893000703305, "policy_ms": 4.4371000258252025, "execution_ms": 2.466200035996735, "status": "ok", "code": "ASK_USER" }, { "sample_id": 87, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.5589049458503723, "observation_ms": 15.525400056503713, "policy_ms": 7.321100099943578, "execution_ms": 2.8237999649718404, "status": "ok", "code": "ASK_USER" }, { "sample_id": 88, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9998763799667358, "observation_ms": 15.69139992352575, "policy_ms": 5.579799995757639, "execution_ms": 2.516699954867363, "status": "ok", "code": "ASK_USER" }, { "sample_id": 89, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9988917708396912, "observation_ms": 17.7850000327453, "policy_ms": 6.633299984969199, "execution_ms": 16.517199925146997, "status": "ok", "code": "SELECT" }, { "sample_id": 90, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999994039535522, "observation_ms": 15.42800001334399, "policy_ms": 6.517100031487644, "execution_ms": 2.6801000349223614, "status": "ok", "code": "ASK_USER" }, { "sample_id": 91, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999995231628418, "target_confidence": 0.999991774559021, "observation_ms": 18.885600031353533, "policy_ms": 7.807799964211881, "execution_ms": 21.415700088255107, "status": "ok", "code": "TYPE" }, { "sample_id": 92, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9968438148498535, "observation_ms": 15.483899973332882, "policy_ms": 6.061500054784119, "execution_ms": 16.276399954222143, "status": "ok", "code": "SELECT" }, { "sample_id": 93, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.5529961585998535, "observation_ms": 12.816700036637485, "policy_ms": 4.880900029093027, "execution_ms": 2.2583999671041965, "status": "ok", "code": "ASK_USER" }, { "sample_id": 94, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9746989607810974, "observation_ms": 14.847400016151369, "policy_ms": 8.308400050736964, "execution_ms": 2.891700016334653, "status": "ok", "code": "ASK_USER" }, { "sample_id": 95, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.6894709467887878, "observation_ms": 13.81010003387928, "policy_ms": 7.412100094370544, "execution_ms": 2.658099983818829, "status": "ok", "code": "ASK_USER" }, { "sample_id": 96, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9780493378639221, "target_confidence": 1.0, "observation_ms": 15.902300016023219, "policy_ms": 5.481399944983423, "execution_ms": 16.780099947936833, "status": "ok", "code": "TYPE" }, { "sample_id": 97, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.5665366649627686, "target_confidence": 0.9999916553497314, "observation_ms": 13.692499953322113, "policy_ms": 5.062200012616813, "execution_ms": 2.353199990466237, "status": "ok", "code": "ASK_USER" }, { "sample_id": 98, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.573700048960745, "policy_ms": 6.14269997458905, "execution_ms": 18.901500036008656, "status": "ok", "code": "SELECT" }, { "sample_id": 99, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999967813491821, "observation_ms": 19.381299964152277, "policy_ms": 5.829700035974383, "execution_ms": 2.478900016285479, "status": "ok", "code": "ASK_USER" }, { "sample_id": 100, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9519977569580078, "observation_ms": 14.208900043740869, "policy_ms": 5.3721999283879995, "execution_ms": 2.4892999790608883, "status": "ok", "code": "ASK_USER" }, { "sample_id": 101, "template": "table", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 11.710799997672439, "policy_ms": 4.391100024804473, "execution_ms": 2.3975000949576497, "status": "ok", "code": "ASK_USER" }, { "sample_id": 102, "template": "table", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 0.9999699592590332, "target_confidence": 0.9877586960792542, "observation_ms": 16.78659999743104, "policy_ms": 6.5367999486625195, "execution_ms": 16.190700000151992, "status": "ok", "code": "SELECT" }, { "sample_id": 103, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9960851669311523, "target_confidence": 0.9997296929359436, "observation_ms": 13.554899953305721, "policy_ms": 3.9613000117242336, "execution_ms": 17.097899923101068, "status": "ok", "code": "TYPE" }, { "sample_id": 104, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9994691014289856, "observation_ms": 14.564899960532784, "policy_ms": 5.859700031578541, "execution_ms": 15.829900046810508, "status": "ok", "code": "SELECT" }, { "sample_id": 105, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 1.0, "target_confidence": 0.9999922513961792, "observation_ms": 12.78019999153912, "policy_ms": 3.944299998693168, "execution_ms": 16.73899998422712, "status": "ok", "code": "TYPE" }, { "sample_id": 106, "template": "nested", "success": false, "abstained": false, "expected_action": "T", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999992847442627, "observation_ms": 15.06660005543381, "policy_ms": 5.027799983508885, "execution_ms": 11.834000004455447, "status": "rejected", "code": "SELECT_ROLE_MISMATCH" }, { "sample_id": 107, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 0.9999998807907104, "observation_ms": 16.96310006082058, "policy_ms": 6.54430000577122, "execution_ms": 17.723500030115247, "status": "ok", "code": "SELECT" }, { "sample_id": 108, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 17.147900070995092, "policy_ms": 6.6255999263376, "execution_ms": 2.570000011473894, "status": "ok", "code": "ASK_USER" }, { "sample_id": 109, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9680730104446411, "target_confidence": 0.9812092185020447, "observation_ms": 13.616799958981574, "policy_ms": 5.672800005413592, "execution_ms": 16.66189997922629, "status": "ok", "code": "TYPE" }, { "sample_id": 110, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.796399998478591, "policy_ms": 4.303400055505335, "execution_ms": 16.243300051428378, "status": "ok", "code": "SELECT" }, { "sample_id": 111, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 16.10790006816387, "policy_ms": 6.166999926790595, "execution_ms": 15.69439994636923, "status": "ok", "code": "SELECT" }, { "sample_id": 112, "template": "table", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9999850988388062, "target_confidence": 0.9864395260810852, "observation_ms": 15.109900035895407, "policy_ms": 4.6709999442100525, "execution_ms": 17.726799938827753, "status": "ok", "code": "TYPE" }, { "sample_id": 113, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 0.9999536275863647, "observation_ms": 14.065099996514618, "policy_ms": 5.889200023375452, "execution_ms": 2.464100020006299, "status": "ok", "code": "ASK_USER" }, { "sample_id": 114, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 19.09929991234094, "policy_ms": 6.019800086505711, "execution_ms": 15.787699958309531, "status": "ok", "code": "SELECT" }, { "sample_id": 115, "template": "nested", "success": true, "abstained": false, "expected_action": "T", "predicted_action": "T", "action_confidence": 0.9998906850814819, "target_confidence": 1.0, "observation_ms": 12.961399974301457, "policy_ms": 6.4616999588906765, "execution_ms": 17.88450009189546, "status": "ok", "code": "TYPE" }, { "sample_id": 116, "template": "table", "success": false, "abstained": true, "expected_action": "T", "predicted_action": "A", "action_confidence": 0.8286662697792053, "target_confidence": 0.9999352693557739, "observation_ms": 15.840500011108816, "policy_ms": 5.301999975927174, "execution_ms": 2.3964999709278345, "status": "ok", "code": "ASK_USER" }, { "sample_id": 117, "template": "nested", "success": false, "abstained": true, "expected_action": "C", "predicted_action": "A", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 12.605000054463744, "policy_ms": 4.79289994109422, "execution_ms": 2.412299974821508, "status": "ok", "code": "ASK_USER" }, { "sample_id": 118, "template": "table", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 13.805299997329712, "policy_ms": 5.196699989028275, "execution_ms": 16.23519998975098, "status": "ok", "code": "SELECT" }, { "sample_id": 119, "template": "nested", "success": true, "abstained": false, "expected_action": "O", "predicted_action": "O", "action_confidence": 1.0, "target_confidence": 1.0, "observation_ms": 15.033100033178926, "policy_ms": 6.443300051614642, "execution_ms": 15.90520003810525, "status": "ok", "code": "SELECT" } ] }