Hugging Face's logo

devildasdf
/
devils-agent

Safetensors
English
baim
browser-agent
cpu
experimental
custom-model
Model card Files
xet
 Community
devildasdf commited on
Commit
c8e5620
·
verified ·
1 Parent(s): c7d6933

Add trained contextual action candidate with browser evaluation and explicit real-web limits

Browse files
Files changed (14) hide show
  1. README.md +3 -1
  2. baim/model.py +7 -2
  3. baim/train.py +2 -1
  4. docs/CONTEXTUAL_V004.md +47 -0
  5. models/v004-contextual/README.md +13 -0
  6. models/v004-contextual/calibration.json +1 -0
  7. models/v004-contextual/config.json +7 -0
  8. models/v004-contextual/model.safetensors +3 -0
  9. models/v004-contextual/training-report.json +298 -0
  10. models/v004-contextual/vocab.json +1 -0
  11. reports/browser-v004-novel.json +1817 -0
  12. reports/cpu-v004.json +59 -0
  13. reports/mind2web-v004.json +26 -0
  14. tests/test_model.py +14 -0
README.md CHANGED
@@ -11,9 +11,11 @@ library_name: baim
11
 
12
  # Devils Agent / BAIM — experimental research checkpoints
13
 
 
 
14
  Runtime v0.0.2: **2.38x faster median synthetic CPU prediction**, bounded microbatch inference, and unchanged checkpoint weights. See [measured results and limits](docs/RUNTIME_V002.md).
15
 
16
- **Research prototype, not a production-ready general browser agent.** Four custom
17
  checkpoints are stored under `models/`. They require the accompanying Python code;
18
  this repository is not a standard Transformers `AutoModel` or hosted-inference
19
  package. No production champion has been selected.
 
11
 
12
  # Devils Agent / BAIM — experimental research checkpoints
13
 
14
+ New experimental **v004 contextual checkpoint**: novel-wording fixture match improved from 55.83% to 83.96%; actual browser fixtures from 74/120 to 98/120. Real-web diagnostic remains 1/46. [Results and limitations](docs/CONTEXTUAL_V004.md).
15
+
16
  Runtime v0.0.2: **2.38x faster median synthetic CPU prediction**, bounded microbatch inference, and unchanged checkpoint weights. See [measured results and limits](docs/RUNTIME_V002.md).
17
 
18
+ **Research prototype, not a production-ready general browser agent.** Five custom
19
  checkpoints are stored under `models/`. They require the accompanying Python code;
20
  this repository is not a standard Transformers `AutoModel` or hosted-inference
21
  package. No production champion has been selected.
baim/model.py CHANGED
@@ -5,10 +5,11 @@ from huggingface_hub import PyTorchModelHubMixin
5
 
6
 
7
  class PointerPolicy(nn.Module, PyTorchModelHubMixin, library_name='baim', tags=['browser-agent','cpu']):
8
- def __init__(self, vocab_size=4096, width=64, encoder='mean', lexical_features=True):
9
  super().__init__()
10
  self.encoder_kind = encoder
11
  self.lexical_features = lexical_features
 
12
  self.embedding = nn.Embedding(vocab_size,width,padding_idx=0)
13
  if encoder == 'gru':
14
  self.encoder = nn.GRU(width,width,batch_first=True)
@@ -18,7 +19,7 @@ class PointerPolicy(nn.Module, PyTorchModelHubMixin, library_name='baim', tags=[
18
  dropout=0.0,batch_first=True),num_layers=1,enable_nested_tensor=False)
19
  elif encoder != 'mean':
20
  raise ValueError('unknown encoder')
21
- self.action = nn.Sequential(nn.Linear(width,width),nn.ReLU(),nn.Linear(width,3))
22
  self.pointer = nn.Sequential(nn.Linear(width*4+5,width),nn.ReLU(),nn.Linear(width,1))
23
 
24
  def embed(self, ids):
@@ -42,4 +43,8 @@ class PointerPolicy(nn.Module, PyTorchModelHubMixin, library_name='baim', tags=[
42
  lexical = features if self.lexical_features else torch.zeros_like(features)
43
  pairs = torch.cat([expanded,e,expanded*e,torch.abs(expanded-e),lexical],dim=-1)
44
  pointers = self.pointer(pairs).squeeze(-1).masked_fill(~mask,-1e4)
 
 
 
 
45
  return self.action(g),pointers
 
5
 
6
 
7
  class PointerPolicy(nn.Module, PyTorchModelHubMixin, library_name='baim', tags=['browser-agent','cpu']):
8
+ def __init__(self, vocab_size=4096, width=64, encoder='mean', lexical_features=True, contextual_action=False):
9
  super().__init__()
10
  self.encoder_kind = encoder
11
  self.lexical_features = lexical_features
12
+ self.contextual_action = contextual_action
13
  self.embedding = nn.Embedding(vocab_size,width,padding_idx=0)
14
  if encoder == 'gru':
15
  self.encoder = nn.GRU(width,width,batch_first=True)
 
19
  dropout=0.0,batch_first=True),num_layers=1,enable_nested_tensor=False)
20
  elif encoder != 'mean':
21
  raise ValueError('unknown encoder')
22
+ self.action = nn.Sequential(nn.Linear(width*2 if contextual_action else width,width),nn.ReLU(),nn.Linear(width,3))
23
  self.pointer = nn.Sequential(nn.Linear(width*4+5,width),nn.ReLU(),nn.Linear(width,1))
24
 
25
  def embed(self, ids):
 
43
  lexical = features if self.lexical_features else torch.zeros_like(features)
44
  pairs = torch.cat([expanded,e,expanded*e,torch.abs(expanded-e),lexical],dim=-1)
45
  pointers = self.pointer(pairs).squeeze(-1).masked_fill(~mask,-1e4)
46
+ if self.contextual_action:
47
+ # Ground action classification in the learned target representation.
48
+ context = (pointers.softmax(-1).unsqueeze(-1)*e).sum(1)
49
+ g = torch.cat([g,context],dim=-1)
50
  return self.action(g),pointers
baim/train.py CHANGED
@@ -52,6 +52,7 @@ def main():
52
  parser.add_argument('--output',default='models/v000')
53
  parser.add_argument('--encoder',choices=['mean','gru','transformer'],default='mean')
54
  parser.add_argument('--no-lexical',action='store_true')
 
55
  parser.add_argument('--epochs',type=int,default=20)
56
  parser.add_argument('--width',type=int,default=64)
57
  parser.add_argument('--seed',type=int,default=1729)
@@ -71,7 +72,7 @@ def main():
71
  if (targets < 0).any():
72
  raise ValueError('training targets missing after retrieval')
73
  model = PointerPolicy(vocab_size=len(vocab),width=args.width,encoder=args.encoder,
74
- lexical_features=not args.no_lexical)
75
  optimizer = torch.optim.AdamW(model.parameters(),lr=.002,weight_decay=.01)
76
  history, best, best_state = [], -1, None
77
  started = time.perf_counter()
 
52
  parser.add_argument('--output',default='models/v000')
53
  parser.add_argument('--encoder',choices=['mean','gru','transformer'],default='mean')
54
  parser.add_argument('--no-lexical',action='store_true')
55
+ parser.add_argument('--contextual-action',action='store_true')
56
  parser.add_argument('--epochs',type=int,default=20)
57
  parser.add_argument('--width',type=int,default=64)
58
  parser.add_argument('--seed',type=int,default=1729)
 
72
  if (targets < 0).any():
73
  raise ValueError('training targets missing after retrieval')
74
  model = PointerPolicy(vocab_size=len(vocab),width=args.width,encoder=args.encoder,
75
+ lexical_features=not args.no_lexical,contextual_action=args.contextual_action)
76
  optimizer = torch.optim.AdamW(model.parameters(),lr=.002,weight_decay=.01)
77
  history, best, best_state = [], -1, None
78
  started = time.perf_counter()
docs/CONTEXTUAL_V004.md ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # v004: target-aware action classification
2
+
3
+ The previous action head read only the goal. The new optional `contextual_action`
4
+ head also reads a soft attention-weighted representation of the DOM candidates,
5
+ using the learned pointer logits. This is trained end-to-end, without runtime
6
+ phrase-to-action rules. Existing checkpoints load with the feature disabled.
7
+
8
+ Training: unchanged synthetic-v1 train split, 2,400 examples; 16 epochs; seed 1729;
9
+ mean encoder, width 64, 132,996 parameters. Selection uses validation joint accuracy
10
+ only (first best epoch wins ties). Temperatures fit validation only. The existing
11
+ novel-wording evaluation is not training data, but was previously inspected during
12
+ development, so it is not a fresh blind benchmark.
13
+
14
+ Results:
15
+
16
+ | Check | Previous mean | v004 |
17
+ |---|---:|---:|
18
+ | Novel-wording action + target match, 480 fixtures | 55.83% | 83.96% |
19
+ | Actual Chromium novel fixtures, 120 tasks | 74/120 | 98/120 |
20
+ | Standard synthetic test match | 100% | 100% |
21
+ | Small Mind2Web diagnostic | 1/46 | 1/46 |
22
+
23
+ CPU encoding plus neural forward pass: 1.103 ms median, 1.647 ms p95 on Windows
24
+ with two Torch threads. Browser policy prediction measured 2.046 ms median with
25
+ Chromium running. These different timing scopes must not be compared as equivalent.
26
+ Checkpoint size is 532,696 bytes. Reports include platform and measurement scope.
27
+
28
+ Usage:
29
+
30
+ ```python
31
+ from baim.policy import LearnedPolicy
32
+ policy = LearnedPolicy('models/v004-contextual')
33
+ decision = policy.predict(goal, observed_state, authority_ticket)
34
+ ```
35
+
36
+ Reproduce training:
37
+
38
+ ```sh
39
+ python -m baim.train --contextual-action --epochs 16 --output models/v004-contextual
40
+ ```
41
+
42
+ This candidate is **not production-promoted**. Real-site grounding, calibration
43
+ under distribution shift, long goals, and general planning remain unresolved.
44
+ It still predicts only click/type/select and copies one quoted literal for values.
45
+ The live AegisVision default remains unchanged. Independent review and approval
46
+ must remain in place. Reports: `cpu-v004.json`, `browser-v004-novel.json`, and
47
+ `mind2web-v004.json` under `reports/`, plus the checkpoint training report.
models/v004-contextual/README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: baim
3
+ tags:
4
+ - browser-agent
5
+ - cpu
6
+ - model_hub_mixin
7
+ - pytorch_model_hub_mixin
8
+ ---
9
+
10
+ This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
11
+ - Code: [More Information Needed]
12
+ - Paper: [More Information Needed]
13
+ - Docs: [More Information Needed]
models/v004-contextual/calibration.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"temperatures": [0.10000000149011612, 0.10000000149011612], "split": "validation"}
models/v004-contextual/config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "contextual_action": true,
3
+ "encoder": "mean",
4
+ "lexical_features": true,
5
+ "vocab_size": 1683,
6
+ "width": 64
7
+ }
models/v004-contextual/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bff26cf3b8cb4c76f971620cb281159cfe37743d953eaf68f7eace5b1534f66
3
+ size 532696
models/v004-contextual/training-report.json ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architecture": {
3
+ "data": "datasets/synthetic-v1",
4
+ "output": "models/v004-contextual",
5
+ "encoder": "mean",
6
+ "no_lexical": false,
7
+ "contextual_action": true,
8
+ "epochs": 16,
9
+ "width": 64,
10
+ "seed": 1729
11
+ },
12
+ "parameter_count": 132996,
13
+ "threads": 2,
14
+ "device": "cpu",
15
+ "training_seconds": 15.113418400054798,
16
+ "dataset_manifest": {
17
+ "train": {
18
+ "count": 2400,
19
+ "seed": 101,
20
+ "templates": [
21
+ "fieldset",
22
+ "grid",
23
+ "stack"
24
+ ],
25
+ "sha256": "3f24899383d914465232cfaac27f654eae7217ed3fb37894cf07e551120c56e5"
26
+ },
27
+ "validation": {
28
+ "count": 480,
29
+ "seed": 202,
30
+ "templates": [
31
+ "fieldset",
32
+ "grid",
33
+ "stack"
34
+ ],
35
+ "sha256": "1cf777e5703257e3471c3f67b2e421fd3de505de429fbf57223f452648c477a8"
36
+ },
37
+ "test": {
38
+ "count": 480,
39
+ "seed": 303,
40
+ "templates": [
41
+ "nested",
42
+ "table"
43
+ ],
44
+ "sha256": "849ccce76126296a01ee5ef756f2790470dea0e87fad94802d66c6e43fee20ac"
45
+ },
46
+ "novel_wording": {
47
+ "count": 480,
48
+ "seed": 404,
49
+ "templates": [
50
+ "nested",
51
+ "table"
52
+ ],
53
+ "sha256": "8b467567370321ad5fbe604b282b89de6ceaaa4b679621fa0f3685708fa6e1c2"
54
+ },
55
+ "limitations": "Single-step generated tasks; test layouts held out but vocabulary shared. No arbitrary-site claim."
56
+ },
57
+ "history": [
58
+ {
59
+ "epoch": 1,
60
+ "loss": 1.5690920323525603,
61
+ "validation": {
62
+ "samples": 480,
63
+ "action_accuracy": 1.0,
64
+ "target_accuracy": 1.0,
65
+ "joint_step_accuracy": 1.0,
66
+ "action_ece": 0.02158182067796588,
67
+ "target_ece": 0.05565767455846071,
68
+ "candidate_recall": 1.0
69
+ }
70
+ },
71
+ {
72
+ "epoch": 2,
73
+ "loss": 0.028715768277547078,
74
+ "validation": {
75
+ "samples": 480,
76
+ "action_accuracy": 1.0,
77
+ "target_accuracy": 1.0,
78
+ "joint_step_accuracy": 1.0,
79
+ "action_ece": 0.0017445683479309082,
80
+ "target_ece": 0.012464901199564338,
81
+ "candidate_recall": 1.0
82
+ }
83
+ },
84
+ {
85
+ "epoch": 3,
86
+ "loss": 0.010360222947048513,
87
+ "validation": {
88
+ "samples": 480,
89
+ "action_accuracy": 1.0,
90
+ "target_accuracy": 1.0,
91
+ "joint_step_accuracy": 1.0,
92
+ "action_ece": 0.0009952187538146973,
93
+ "target_ece": 0.008543595438823104,
94
+ "candidate_recall": 1.0
95
+ }
96
+ },
97
+ {
98
+ "epoch": 4,
99
+ "loss": 0.007286314448145659,
100
+ "validation": {
101
+ "samples": 480,
102
+ "action_accuracy": 1.0,
103
+ "target_accuracy": 1.0,
104
+ "joint_step_accuracy": 1.0,
105
+ "action_ece": 0.0007201433181762695,
106
+ "target_ece": 0.006550255697220564,
107
+ "candidate_recall": 1.0
108
+ }
109
+ },
110
+ {
111
+ "epoch": 5,
112
+ "loss": 0.005476677220461792,
113
+ "validation": {
114
+ "samples": 480,
115
+ "action_accuracy": 1.0,
116
+ "target_accuracy": 1.0,
117
+ "joint_step_accuracy": 1.0,
118
+ "action_ece": 0.0005319714546203613,
119
+ "target_ece": 0.0051317751640453935,
120
+ "candidate_recall": 1.0
121
+ }
122
+ },
123
+ {
124
+ "epoch": 6,
125
+ "loss": 0.004250973279244806,
126
+ "validation": {
127
+ "samples": 480,
128
+ "action_accuracy": 1.0,
129
+ "target_accuracy": 1.0,
130
+ "joint_step_accuracy": 1.0,
131
+ "action_ece": 0.00040030479431152344,
132
+ "target_ece": 0.004249494522809982,
133
+ "candidate_recall": 1.0
134
+ }
135
+ },
136
+ {
137
+ "epoch": 7,
138
+ "loss": 0.0033946395438090946,
139
+ "validation": {
140
+ "samples": 480,
141
+ "action_accuracy": 1.0,
142
+ "target_accuracy": 1.0,
143
+ "joint_step_accuracy": 1.0,
144
+ "action_ece": 0.0003128647804260254,
145
+ "target_ece": 0.003526005893945694,
146
+ "candidate_recall": 1.0
147
+ }
148
+ },
149
+ {
150
+ "epoch": 8,
151
+ "loss": 0.0028304545006616728,
152
+ "validation": {
153
+ "samples": 480,
154
+ "action_accuracy": 1.0,
155
+ "target_accuracy": 1.0,
156
+ "joint_step_accuracy": 1.0,
157
+ "action_ece": 0.0002523064613342285,
158
+ "target_ece": 0.00299576623365283,
159
+ "candidate_recall": 1.0
160
+ }
161
+ },
162
+ {
163
+ "epoch": 9,
164
+ "loss": 0.002464234003319258,
165
+ "validation": {
166
+ "samples": 480,
167
+ "action_accuracy": 1.0,
168
+ "target_accuracy": 1.0,
169
+ "joint_step_accuracy": 1.0,
170
+ "action_ece": 0.00020706653594970703,
171
+ "target_ece": 0.0025957798352465034,
172
+ "candidate_recall": 1.0
173
+ }
174
+ },
175
+ {
176
+ "epoch": 10,
177
+ "loss": 0.0021179942942628834,
178
+ "validation": {
179
+ "samples": 480,
180
+ "action_accuracy": 1.0,
181
+ "target_accuracy": 1.0,
182
+ "joint_step_accuracy": 1.0,
183
+ "action_ece": 0.00017279386520385742,
184
+ "target_ece": 0.002270008670166135,
185
+ "candidate_recall": 1.0
186
+ }
187
+ },
188
+ {
189
+ "epoch": 11,
190
+ "loss": 0.001785455371933303,
191
+ "validation": {
192
+ "samples": 480,
193
+ "action_accuracy": 1.0,
194
+ "target_accuracy": 1.0,
195
+ "joint_step_accuracy": 1.0,
196
+ "action_ece": 0.00014674663543701172,
197
+ "target_ece": 0.0019821266178041697,
198
+ "candidate_recall": 1.0
199
+ }
200
+ },
201
+ {
202
+ "epoch": 12,
203
+ "loss": 0.001567806809601423,
204
+ "validation": {
205
+ "samples": 480,
206
+ "action_accuracy": 1.0,
207
+ "target_accuracy": 1.0,
208
+ "joint_step_accuracy": 1.0,
209
+ "action_ece": 0.0001264810562133789,
210
+ "target_ece": 0.0017305880901403725,
211
+ "candidate_recall": 1.0
212
+ }
213
+ },
214
+ {
215
+ "epoch": 13,
216
+ "loss": 0.0013826081929820295,
217
+ "validation": {
218
+ "samples": 480,
219
+ "action_accuracy": 1.0,
220
+ "target_accuracy": 1.0,
221
+ "joint_step_accuracy": 1.0,
222
+ "action_ece": 0.00010889768600463867,
223
+ "target_ece": 0.0015634410083293915,
224
+ "candidate_recall": 1.0
225
+ }
226
+ },
227
+ {
228
+ "epoch": 14,
229
+ "loss": 0.0012740621605189517,
230
+ "validation": {
231
+ "samples": 480,
232
+ "action_accuracy": 1.0,
233
+ "target_accuracy": 1.0,
234
+ "joint_step_accuracy": 1.0,
235
+ "action_ece": 9.626150131225586e-05,
236
+ "target_ece": 0.0013960599899291992,
237
+ "candidate_recall": 1.0
238
+ }
239
+ },
240
+ {
241
+ "epoch": 15,
242
+ "loss": 0.001100011857679898,
243
+ "validation": {
244
+ "samples": 480,
245
+ "action_accuracy": 1.0,
246
+ "target_accuracy": 1.0,
247
+ "joint_step_accuracy": 1.0,
248
+ "action_ece": 8.493661880493164e-05,
249
+ "target_ece": 0.0012488961219787598,
250
+ "candidate_recall": 1.0
251
+ }
252
+ },
253
+ {
254
+ "epoch": 16,
255
+ "loss": 0.000987965263208791,
256
+ "validation": {
257
+ "samples": 480,
258
+ "action_accuracy": 1.0,
259
+ "target_accuracy": 1.0,
260
+ "joint_step_accuracy": 1.0,
261
+ "action_ece": 7.56978988647461e-05,
262
+ "target_ece": 0.0011227130889892578,
263
+ "candidate_recall": 1.0
264
+ }
265
+ }
266
+ ],
267
+ "evaluation": {
268
+ "validation": {
269
+ "samples": 480,
270
+ "action_accuracy": 1.0,
271
+ "target_accuracy": 1.0,
272
+ "joint_step_accuracy": 1.0,
273
+ "action_ece": 2.777576446533203e-05,
274
+ "target_ece": 0.0003707166906679049,
275
+ "candidate_recall": 1.0
276
+ },
277
+ "test": {
278
+ "samples": 480,
279
+ "action_accuracy": 1.0,
280
+ "target_accuracy": 1.0,
281
+ "joint_step_accuracy": 1.0,
282
+ "action_ece": 0.0,
283
+ "target_ece": 0.0005633262626361102,
284
+ "candidate_recall": 1.0
285
+ },
286
+ "novel_wording": {
287
+ "samples": 480,
288
+ "action_accuracy": 0.8520833253860474,
289
+ "target_accuracy": 0.9458333253860474,
290
+ "joint_step_accuracy": 0.8395833373069763,
291
+ "action_ece": 0.13770944606221747,
292
+ "target_ece": 0.046679925406351686,
293
+ "candidate_recall": 1.0
294
+ }
295
+ },
296
+ "limitations": "Synthetic single-step action/target prediction; not arbitrary-site task success.",
297
+ "production_promoted": false
298
+ }
models/v004-contextual/vocab.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"<pad>": 0, "<unk>": 1, "combobox": 2, "textbox": 3, "button": 4, "link": 5, "xenon": 6, "elm": 7, "umber": 8, "linen": 9, "pearl": 10, "fern": 11, "maple": 12, "amber": 13, "yellow": 14, "kelp": 15, "nectar": 16, "birch": 17, "jade": 18, "indigo": 19, "quartz": 20, "river": 21, "zinc": 22, "delta": 23, "cobalt": 24, "willow": 25, "olive": 26, "timber": 27, "silver": 28, "violet": 29, "harbor": 30, "granite": 31, "value": 32, "the": 33, "choose": 34, "in": 35, "set": 36, "dropdown": 37, "to": 38, "list": 39, "control": 40, "named": 41, "select": 42, "from": 43, "type": 44, "enter": 45, "into": 46, "fill": 47, "with": 48, "activate": 49, "click": 50, "115467": 51, "490435": 52, "770467": 53, "545721": 54, "389346": 55, "541416": 56, "799127": 57, "336171": 58, "798265": 59, "53461": 60, "248673": 61, "526604": 62, "433536": 63, "301809": 64, "977221": 65, "365736": 66, "754446": 67, "629256": 68, "712140": 69, "840862": 70, "511392": 71, "515028": 72, "373272": 73, "393319": 74, "814474": 75, "818735": 76, "854079": 77, "958648": 78, "237473": 79, "449968": 80, "915703": 81, "691498": 82, "14895": 83, "414787": 84, "32704": 85, "659207": 86, "664505": 87, "807394": 88, "890087": 89, "644485": 90, "280300": 91, "291296": 92, "99856": 93, "570239": 94, "71342": 95, "365646": 96, "586418": 97, "930407": 98, "272478": 99, "97841": 100, "619557": 101, "243905": 102, "148238": 103, "744035": 104, "79559": 105, "814297": 106, "151897": 107, "188718": 108, "342411": 109, "563850": 110, "896008": 111, "962749": 112, "565500": 113, "90282": 114, "77837": 115, "537278": 116, "786146": 117, "19916": 118, "277774": 119, "760933": 120, "953685": 121, "381965": 122, "783107": 123, "676785": 124, "225875": 125, "339015": 126, "285992": 127, "198614": 128, "254660": 129, "566242": 130, "901564": 131, "375058": 132, "397145": 133, "674987": 134, "642452": 135, "832244": 136, "110052": 137, "116054": 138, "392281": 139, "946952": 140, "221164": 141, "802121": 142, "263198": 143, "653517": 144, "384359": 145, "120313": 146, "44271": 147, "295039": 148, "100804": 149, "610338": 150, "607556": 151, "900566": 152, "462318": 153, "644067": 154, "780632": 155, "814972": 156, "470934": 157, "377889": 158, "924101": 159, "830126": 160, "826194": 161, "153526": 162, "213298": 163, "27501": 164, "330841": 165, "377668": 166, "473773": 167, "928145": 168, "227959": 169, "196473": 170, "222983": 171, "736243": 172, "652348": 173, "598144": 174, "46458": 175, "523351": 176, "619363": 177, "542431": 178, "88815": 179, "439494": 180, "253421": 181, "74583": 182, "485054": 183, "470266": 184, "631994": 185, "25713": 186, "529024": 187, "632645": 188, "212868": 189, "161131": 190, "88225": 191, "63850": 192, "392959": 193, "884559": 194, "688623": 195, "838806": 196, "151068": 197, "650738": 198, "797488": 199, "725913": 200, "795233": 201, "580574": 202, "156861": 203, "344933": 204, "654421": 205, "988034": 206, "691888": 207, "868035": 208, "211268": 209, "820030": 210, "616170": 211, "17453": 212, "537361": 213, "91336": 214, "160276": 215, "164911": 216, "579250": 217, "410077": 218, "255965": 219, "179547": 220, "597402": 221, "661782": 222, "162521": 223, "548426": 224, "956699": 225, "853226": 226, "992025": 227, "478652": 228, "439063": 229, "478815": 230, "630398": 231, "623173": 232, "252899": 233, "914715": 234, "454306": 235, "179712": 236, "626042": 237, "933653": 238, "912045": 239, "412217": 240, "121123": 241, "321155": 242, "110710": 243, "698704": 244, "50744": 245, "664084": 246, "282056": 247, "512406": 248, "442302": 249, "265811": 250, "737205": 251, "878974": 252, "132538": 253, "86818": 254, "775583": 255, "636356": 256, "365273": 257, "229787": 258, "116140": 259, "379995": 260, "882014": 261, "783397": 262, "768538": 263, "53880": 264, "336201": 265, "713025": 266, "331760": 267, "208508": 268, "672030": 269, "6606": 270, "547180": 271, "35886": 272, "984640": 273, "673947": 274, "743438": 275, "957700": 276, "392150": 277, "462781": 278, "201113": 279, "844685": 280, "564111": 281, "747507": 282, "932740": 283, "8727": 284, "636064": 285, "263259": 286, "738343": 287, "842848": 288, "515885": 289, "985830": 290, "646329": 291, "23735": 292, "931332": 293, "645012": 294, "106609": 295, "574910": 296, "75873": 297, "384549": 298, "122998": 299, "69861": 300, "725205": 301, "361220": 302, "394262": 303, "256583": 304, "193474": 305, "577374": 306, "72297": 307, "19755": 308, "570611": 309, "132902": 310, "764645": 311, "49236": 312, "310418": 313, "375052": 314, "823838": 315, "753571": 316, "613300": 317, "644265": 318, "811472": 319, "924491": 320, "311229": 321, "629062": 322, "599118": 323, "377922": 324, "529124": 325, "535279": 326, "183734": 327, "782438": 328, "171000": 329, "920994": 330, "914258": 331, "89195": 332, "149318": 333, "792449": 334, "412565": 335, "507202": 336, "711972": 337, "893427": 338, "895476": 339, "508480": 340, "977044": 341, "786458": 342, "31635": 343, "248699": 344, "536907": 345, "128688": 346, "846050": 347, "243004": 348, "95692": 349, "950949": 350, "703507": 351, "662825": 352, "703609": 353, "697811": 354, "729742": 355, "564845": 356, "150698": 357, "77189": 358, "975705": 359, "485189": 360, "497058": 361, "593845": 362, "718901": 363, "16700": 364, "555532": 365, "336041": 366, "186022": 367, "571109": 368, "985512": 369, "13494": 370, "621018": 371, "545978": 372, "340177": 373, "174925": 374, "322806": 375, "319176": 376, "219703": 377, "619393": 378, "444562": 379, "571890": 380, "886606": 381, "593294": 382, "970588": 383, "408188": 384, "43499": 385, "679468": 386, "905149": 387, "142280": 388, "34349": 389, "907598": 390, "512193": 391, "455431": 392, "396814": 393, "869296": 394, "465079": 395, "692060": 396, "165577": 397, "540210": 398, "195383": 399, "920643": 400, "854013": 401, "579011": 402, "796054": 403, "945123": 404, "833647": 405, "19172": 406, "965291": 407, "27543": 408, "775079": 409, "100388": 410, "603808": 411, "116427": 412, "890": 413, "710213": 414, "450223": 415, "39295": 416, "72496": 417, "304273": 418, "92746": 419, "312755": 420, "274929": 421, "895546": 422, "224859": 423, "809687": 424, "953553": 425, "280866": 426, "62587": 427, "578701": 428, "655927": 429, "681949": 430, "84898": 431, "735431": 432, "735809": 433, "589764": 434, "547470": 435, "398119": 436, "882821": 437, "340132": 438, "546925": 439, "541817": 440, "883312": 441, "429803": 442, "447996": 443, "831980": 444, "180088": 445, "203737": 446, "973314": 447, "734725": 448, "566657": 449, "681503": 450, "56593": 451, "390371": 452, "172372": 453, "961375": 454, "299322": 455, "351961": 456, "909260": 457, "56824": 458, "200798": 459, "626161": 460, "945878": 461, "443320": 462, "38504": 463, "910549": 464, "611316": 465, "187297": 466, "207387": 467, "462032": 468, "295780": 469, "325591": 470, "537591": 471, "390986": 472, "344402": 473, "70801": 474, "308331": 475, "213966": 476, "852721": 477, "96751": 478, "808731": 479, "879163": 480, "140271": 481, "823946": 482, "470438": 483, "436801": 484, "204411": 485, "701750": 486, "717027": 487, "196796": 488, "140126": 489, "897205": 490, "591094": 491, "491130": 492, "392171": 493, "555911": 494, "943728": 495, "159150": 496, "395034": 497, "514117": 498, "875584": 499, "779392": 500, "305117": 501, "439433": 502, "889262": 503, "800057": 504, "652629": 505, "527336": 506, "399867": 507, "337434": 508, "992460": 509, "175990": 510, "549246": 511, "649384": 512, "214711": 513, "368631": 514, "48889": 515, "526468": 516, "471523": 517, "160890": 518, "129062": 519, "249147": 520, "186490": 521, "479414": 522, "764442": 523, "139739": 524, "217501": 525, "622509": 526, "288141": 527, "521850": 528, "238036": 529, "65997": 530, "189087": 531, "191627": 532, "924480": 533, "494506": 534, "267510": 535, "760902": 536, "207814": 537, "734156": 538, "254422": 539, "641144": 540, "974312": 541, "18639": 542, "616790": 543, "132180": 544, "177187": 545, "267268": 546, "689805": 547, "397876": 548, "293963": 549, "185562": 550, "86397": 551, "904731": 552, "401149": 553, "106730": 554, "311503": 555, "849639": 556, "251110": 557, "146853": 558, "368852": 559, "958345": 560, "535825": 561, "752278": 562, "459433": 563, "817912": 564, "384726": 565, "997128": 566, "266211": 567, "493966": 568, "416827": 569, "924274": 570, "909789": 571, "527237": 572, "349957": 573, "314909": 574, "178470": 575, "783315": 576, "965811": 577, "415533": 578, "602712": 579, "69385": 580, "574028": 581, "615717": 582, "941130": 583, "385662": 584, "379380": 585, "358492": 586, "952478": 587, "702210": 588, "250978": 589, "429722": 590, "81371": 591, "835771": 592, "990149": 593, "76382": 594, "760490": 595, "771815": 596, "865511": 597, "998633": 598, "918669": 599, "662802": 600, "559625": 601, "864594": 602, "109796": 603, "300039": 604, "952237": 605, "969588": 606, "4729": 607, "445486": 608, "739965": 609, "257911": 610, "198875": 611, "245436": 612, "43536": 613, "481378": 614, "581619": 615, "135336": 616, "138168": 617, "103071": 618, "116687": 619, "775576": 620, "949525": 621, "85840": 622, "94205": 623, "837533": 624, "819965": 625, "392068": 626, "173793": 627, "343962": 628, "531792": 629, "94113": 630, "148081": 631, "669425": 632, "277195": 633, "26943": 634, "714386": 635, "252600": 636, "277710": 637, "736830": 638, "137222": 639, "698608": 640, "199935": 641, "684767": 642, "703870": 643, "668400": 644, "909089": 645, "320530": 646, "499189": 647, "193109": 648, "683680": 649, "241494": 650, "468028": 651, "195609": 652, "679144": 653, "937569": 654, "289013": 655, "374784": 656, "55076": 657, "669114": 658, "796930": 659, "446742": 660, "160057": 661, "532557": 662, "391161": 663, "822686": 664, "276383": 665, "187857": 666, "527760": 667, "188109": 668, "530745": 669, "413776": 670, "474143": 671, "449177": 672, "799914": 673, "606200": 674, "288209": 675, "21661": 676, "722874": 677, "783614": 678, "895727": 679, "453241": 680, "433947": 681, "946674": 682, "111336": 683, "813869": 684, "457280": 685, "592454": 686, "620730": 687, "612019": 688, "936820": 689, "73375": 690, "224998": 691, "600736": 692, "698640": 693, "398225": 694, "28634": 695, "906886": 696, "492933": 697, "404548": 698, "925760": 699, "561990": 700, "623552": 701, "63287": 702, "153303": 703, "140018": 704, "717458": 705, "120058": 706, "632351": 707, "444465": 708, "880061": 709, "219819": 710, "17184": 711, "131887": 712, "462241": 713, "467525": 714, "917284": 715, "138625": 716, "295819": 717, "484375": 718, "946955": 719, "45842": 720, "193468": 721, "286604": 722, "125068": 723, "575454": 724, "598146": 725, "871948": 726, "734985": 727, "398165": 728, "518746": 729, "60696": 730, "274778": 731, "555102": 732, "172433": 733, "628616": 734, "375223": 735, "341607": 736, "381217": 737, "467662": 738, "466493": 739, "408118": 740, "116641": 741, "808477": 742, "202762": 743, "533209": 744, "450695": 745, "721553": 746, "856364": 747, "422546": 748, "146187": 749, "102416": 750, "615533": 751, "735253": 752, "648006": 753, "71841": 754, "116355": 755, "201222": 756, "969866": 757, "816336": 758, "216915": 759, "95568": 760, "974964": 761, "587078": 762, "187873": 763, "379484": 764, "234951": 765, "921563": 766, "841673": 767, "525847": 768, "270704": 769, "541096": 770, "62286": 771, "751955": 772, "130166": 773, "925551": 774, "816381": 775, "467562": 776, "179853": 777, "125082": 778, "962421": 779, "538093": 780, "298569": 781, "269537": 782, "919307": 783, "532657": 784, "984688": 785, "899242": 786, "403231": 787, "923707": 788, "979095": 789, "820433": 790, "588557": 791, "884016": 792, "52671": 793, "766306": 794, "463916": 795, "376549": 796, "475044": 797, "113212": 798, "831301": 799, "740552": 800, "567039": 801, "282766": 802, "788975": 803, "803572": 804, "983933": 805, "879081": 806, "512717": 807, "591948": 808, "719709": 809, "804775": 810, "47360": 811, "915475": 812, "273703": 813, "850514": 814, "152837": 815, "658555": 816, "756374": 817, "24396": 818, "142430": 819, "457109": 820, "666326": 821, "6534": 822, "334534": 823, "269459": 824, "147033": 825, "622262": 826, "146499": 827, "864255": 828, "760865": 829, "759316": 830, "563733": 831, "4557": 832, "48700": 833, "154384": 834, "886699": 835, "539506": 836, "922314": 837, "373949": 838, "859186": 839, "285997": 840, "649413": 841, "12969": 842, "260758": 843, "289773": 844, "819259": 845, "178544": 846, "598906": 847, "528722": 848, "789263": 849, "100275": 850, "412737": 851, "155844": 852, "966361": 853, "514684": 854, "965266": 855, "746873": 856, "633288": 857, "518001": 858, "150213": 859, "403328": 860, "396832": 861, "186110": 862, "282051": 863, "710977": 864, "450484": 865, "847484": 866, "208479": 867, "699391": 868, "362121": 869, "466269": 870, "982435": 871, "313335": 872, "605400": 873, "951532": 874, "816931": 875, "757960": 876, "424768": 877, "897166": 878, "792211": 879, "895483": 880, "793918": 881, "363410": 882, "462010": 883, "104566": 884, "692426": 885, "852438": 886, "788557": 887, "853123": 888, "39654": 889, "313452": 890, "574154": 891, "796885": 892, "797657": 893, "43147": 894, "777064": 895, "376682": 896, "15695": 897, "243056": 898, "965840": 899, "134968": 900, "366751": 901, "543688": 902, "503238": 903, "875992": 904, "437070": 905, "302389": 906, "542747": 907, "360031": 908, "734133": 909, "118261": 910, "812237": 911, "74367": 912, "46042": 913, "960701": 914, "660700": 915, "178109": 916, "18777": 917, "840253": 918, "797974": 919, "446586": 920, "414043": 921, "452946": 922, "780043": 923, "319856": 924, "805320": 925, "360961": 926, "618168": 927, "636118": 928, "334479": 929, "238097": 930, "690682": 931, "886284": 932, "663947": 933, "731117": 934, "687373": 935, "331321": 936, "334726": 937, "526095": 938, "464211": 939, "315520": 940, "99965": 941, "12491": 942, "668782": 943, "899964": 944, "791624": 945, "475753": 946, "183843": 947, "353237": 948, "609901": 949, "243046": 950, "329345": 951, "7951": 952, "369912": 953, "932960": 954, "553106": 955, "339814": 956, "576039": 957, "610062": 958, "54151": 959, "117888": 960, "839511": 961, "271902": 962, "107854": 963, "921855": 964, "60468": 965, "423022": 966, "755737": 967, "72727": 968, "993841": 969, "796227": 970, "796808": 971, "945896": 972, "159193": 973, "133490": 974, "834779": 975, "742650": 976, "98815": 977, "997363": 978, "130903": 979, "146030": 980, "696941": 981, "147160": 982, "791522": 983, "49001": 984, "158778": 985, "797661": 986, "743450": 987, "35033": 988, "806416": 989, "599886": 990, "876102": 991, "706254": 992, "151683": 993, "293082": 994, "407303": 995, "159748": 996, "271101": 997, "191831": 998, "847637": 999, "81805": 1000, "109871": 1001, "501358": 1002, "425233": 1003, "347962": 1004, "500248": 1005, "530436": 1006, "571145": 1007, "581588": 1008, "134024": 1009, "41470": 1010, "693357": 1011, "240658": 1012, "713005": 1013, "507502": 1014, "44482": 1015, "216347": 1016, "286701": 1017, "830522": 1018, "860961": 1019, "985583": 1020, "980615": 1021, "636129": 1022, "658192": 1023, "909238": 1024, "455052": 1025, "696162": 1026, "410096": 1027, "407432": 1028, "573194": 1029, "451720": 1030, "435166": 1031, "389788": 1032, "210704": 1033, "403089": 1034, "604396": 1035, "148674": 1036, "106718": 1037, "794490": 1038, "151519": 1039, "2704": 1040, "399818": 1041, "580228": 1042, "532763": 1043, "85069": 1044, "396515": 1045, "984566": 1046, "477835": 1047, "530293": 1048, "599493": 1049, "976549": 1050, "980414": 1051, "754194": 1052, "445110": 1053, "935237": 1054, "934183": 1055, "830669": 1056, "275813": 1057, "460586": 1058, "28218": 1059, "773792": 1060, "910761": 1061, "662177": 1062, "871903": 1063, "929009": 1064, "193321": 1065, "922802": 1066, "383899": 1067, "656128": 1068, "645578": 1069, "475296": 1070, "93081": 1071, "208193": 1072, "897403": 1073, "364690": 1074, "62214": 1075, "191347": 1076, "87952": 1077, "263974": 1078, "545083": 1079, "578556": 1080, "686375": 1081, "490776": 1082, "120451": 1083, "558348": 1084, "369414": 1085, "846156": 1086, "819006": 1087, "535933": 1088, "241249": 1089, "804351": 1090, "284056": 1091, "636923": 1092, "89907": 1093, "430896": 1094, "100318": 1095, "846771": 1096, "470049": 1097, "975468": 1098, "488789": 1099, "95376": 1100, "595879": 1101, "578982": 1102, "83096": 1103, "959776": 1104, "371407": 1105, "387130": 1106, "894965": 1107, "252943": 1108, "81855": 1109, "244416": 1110, "2108": 1111, "413043": 1112, "718488": 1113, "868330": 1114, "611214": 1115, "954715": 1116, "526458": 1117, "585361": 1118, "337358": 1119, "896147": 1120, "320462": 1121, "227976": 1122, "61960": 1123, "371481": 1124, "76973": 1125, "668853": 1126, "464898": 1127, "83613": 1128, "10217": 1129, "31422": 1130, "270837": 1131, "722634": 1132, "983602": 1133, "19227": 1134, "47371": 1135, "991038": 1136, "139427": 1137, "334154": 1138, "701966": 1139, "733811": 1140, "959986": 1141, "693600": 1142, "17050": 1143, "654337": 1144, "478733": 1145, "911969": 1146, "429974": 1147, "421943": 1148, "521441": 1149, "756330": 1150, "213967": 1151, "553124": 1152, "410886": 1153, "937584": 1154, "897146": 1155, "825533": 1156, "386047": 1157, "146922": 1158, "304859": 1159, "967303": 1160, "413310": 1161, "945489": 1162, "277917": 1163, "623996": 1164, "705385": 1165, "809502": 1166, "44281": 1167, "741453": 1168, "809613": 1169, "118166": 1170, "253120": 1171, "118978": 1172, "573808": 1173, "179045": 1174, "852739": 1175, "824194": 1176, "363462": 1177, "781318": 1178, "30270": 1179, "426143": 1180, "509890": 1181, "613854": 1182, "367062": 1183, "22268": 1184, "820669": 1185, "736823": 1186, "401990": 1187, "189131": 1188, "998974": 1189, "745907": 1190, "83484": 1191, "336221": 1192, "573301": 1193, "331469": 1194, "710344": 1195, "637646": 1196, "701484": 1197, "910802": 1198, "921584": 1199, "332573": 1200, "333969": 1201, "969575": 1202, "671578": 1203, "771510": 1204, "40236": 1205, "427920": 1206, "775897": 1207, "704997": 1208, "653377": 1209, "532037": 1210, "474625": 1211, "841273": 1212, "374570": 1213, "727064": 1214, "626496": 1215, "60870": 1216, "758308": 1217, "663647": 1218, "647205": 1219, "751674": 1220, "326164": 1221, "397502": 1222, "562120": 1223, "943952": 1224, "37249": 1225, "444639": 1226, "109288": 1227, "246267": 1228, "958004": 1229, "173333": 1230, "781000": 1231, "464232": 1232, "369249": 1233, "74197": 1234, "523672": 1235, "224923": 1236, "575741": 1237, "622948": 1238, "712433": 1239, "709020": 1240, "322782": 1241, "158042": 1242, "731225": 1243, "375418": 1244, "901551": 1245, "371679": 1246, "29281": 1247, "882982": 1248, "826282": 1249, "627431": 1250, "865388": 1251, "780339": 1252, "474420": 1253, "802153": 1254, "949068": 1255, "496219": 1256, "713453": 1257, "5310": 1258, "265747": 1259, "368861": 1260, "177685": 1261, "530596": 1262, "354684": 1263, "235707": 1264, "546400": 1265, "837207": 1266, "278872": 1267, "827846": 1268, "641405": 1269, "932470": 1270, "771742": 1271, "811825": 1272, "776214": 1273, "50878": 1274, "779714": 1275, "729078": 1276, "409518": 1277, "415881": 1278, "541832": 1279, "8275": 1280, "604779": 1281, "516141": 1282, "263635": 1283, "588887": 1284, "598421": 1285, "974108": 1286, "41620": 1287, "16679": 1288, "420763": 1289, "587889": 1290, "506501": 1291, "78692": 1292, "526003": 1293, "509847": 1294, "282797": 1295, "995918": 1296, "817194": 1297, "569274": 1298, "368753": 1299, "185733": 1300, "922229": 1301, "164153": 1302, "168296": 1303, "972345": 1304, "457235": 1305, "730177": 1306, "976053": 1307, "245700": 1308, "306475": 1309, "630598": 1310, "154324": 1311, "17113": 1312, "138678": 1313, "406986": 1314, "431886": 1315, "725509": 1316, "717327": 1317, "850979": 1318, "71212": 1319, "381830": 1320, "259983": 1321, "607123": 1322, "439824": 1323, "276294": 1324, "991431": 1325, "109141": 1326, "328011": 1327, "705443": 1328, "521384": 1329, "645178": 1330, "572654": 1331, "116748": 1332, "847439": 1333, "298449": 1334, "671165": 1335, "832509": 1336, "244102": 1337, "721249": 1338, "973758": 1339, "515699": 1340, "934006": 1341, "204167": 1342, "490678": 1343, "679045": 1344, "803346": 1345, "327300": 1346, "867079": 1347, "889010": 1348, "324780": 1349, "564278": 1350, "370890": 1351, "531786": 1352, "517428": 1353, "911664": 1354, "797484": 1355, "721508": 1356, "738370": 1357, "554482": 1358, "58243": 1359, "910797": 1360, "100829": 1361, "282930": 1362, "783157": 1363, "592113": 1364, "10024": 1365, "669415": 1366, "907636": 1367, "146341": 1368, "796150": 1369, "962977": 1370, "798867": 1371, "149530": 1372, "606110": 1373, "550976": 1374, "782843": 1375, "41538": 1376, "856480": 1377, "229949": 1378, "846324": 1379, "318394": 1380, "338618": 1381, "838053": 1382, "128221": 1383, "3335": 1384, "976233": 1385, "841650": 1386, "852832": 1387, "467561": 1388, "136131": 1389, "232208": 1390, "143982": 1391, "859929": 1392, "269198": 1393, "447544": 1394, "776849": 1395, "491529": 1396, "415992": 1397, "15354": 1398, "223469": 1399, "574481": 1400, "512948": 1401, "649796": 1402, "366798": 1403, "589731": 1404, "461306": 1405, "171824": 1406, "423382": 1407, "417999": 1408, "324788": 1409, "930246": 1410, "888813": 1411, "395827": 1412, "702282": 1413, "842160": 1414, "672407": 1415, "758969": 1416, "339791": 1417, "56406": 1418, "937087": 1419, "22406": 1420, "740321": 1421, "86417": 1422, "403180": 1423, "368826": 1424, "348823": 1425, "559016": 1426, "382690": 1427, "610377": 1428, "92352": 1429, "792914": 1430, "147530": 1431, "828270": 1432, "770505": 1433, "307998": 1434, "805162": 1435, "228320": 1436, "219266": 1437, "62809": 1438, "108527": 1439, "508242": 1440, "818708": 1441, "617018": 1442, "683069": 1443, "430639": 1444, "475001": 1445, "602303": 1446, "891204": 1447, "776172": 1448, "151257": 1449, "454630": 1450, "562706": 1451, "348113": 1452, "922441": 1453, "261550": 1454, "679990": 1455, "897148": 1456, "561267": 1457, "164135": 1458, "836767": 1459, "286496": 1460, "521217": 1461, "814505": 1462, "418801": 1463, "37032": 1464, "857502": 1465, "893790": 1466, "835353": 1467, "32721": 1468, "725375": 1469, "579410": 1470, "898438": 1471, "310599": 1472, "318232": 1473, "735789": 1474, "593644": 1475, "657058": 1476, "566271": 1477, "931308": 1478, "11678": 1479, "419252": 1480, "977743": 1481, "899229": 1482, "529900": 1483, "66008": 1484, "333652": 1485, "638710": 1486, "330608": 1487, "362739": 1488, "562096": 1489, "193983": 1490, "255759": 1491, "389800": 1492, "484325": 1493, "326347": 1494, "536255": 1495, "190659": 1496, "53482": 1497, "46068": 1498, "454855": 1499, "990716": 1500, "29341": 1501, "313548": 1502, "810976": 1503, "382103": 1504, "593016": 1505, "197474": 1506, "304011": 1507, "229479": 1508, "3892": 1509, "726496": 1510, "419149": 1511, "896442": 1512, "688737": 1513, "301541": 1514, "484972": 1515, "620486": 1516, "161637": 1517, "703436": 1518, "43653": 1519, "934348": 1520, "950307": 1521, "50355": 1522, "172834": 1523, "706706": 1524, "67062": 1525, "332682": 1526, "725559": 1527, "500974": 1528, "973843": 1529, "838103": 1530, "696888": 1531, "985419": 1532, "325712": 1533, "480096": 1534, "956378": 1535, "973053": 1536, "563638": 1537, "462391": 1538, "129916": 1539, "443926": 1540, "381413": 1541, "823998": 1542, "360459": 1543, "540733": 1544, "960362": 1545, "931099": 1546, "98439": 1547, "382088": 1548, "529321": 1549, "29758": 1550, "801004": 1551, "849388": 1552, "430313": 1553, "547156": 1554, "446170": 1555, "71827": 1556, "452994": 1557, "371122": 1558, "915570": 1559, "212093": 1560, "355147": 1561, "251417": 1562, "423864": 1563, "962592": 1564, "900830": 1565, "917470": 1566, "486089": 1567, "607011": 1568, "514651": 1569, "547663": 1570, "60774": 1571, "430022": 1572, "63435": 1573, "14534": 1574, "965494": 1575, "810180": 1576, "754163": 1577, "537218": 1578, "667043": 1579, "416075": 1580, "174943": 1581, "463518": 1582, "126443": 1583, "353151": 1584, "133553": 1585, "452195": 1586, "613692": 1587, "160473": 1588, "403126": 1589, "126082": 1590, "555131": 1591, "258336": 1592, "573256": 1593, "523240": 1594, "129564": 1595, "342397": 1596, "938920": 1597, "482442": 1598, "360337": 1599, "623162": 1600, "990136": 1601, "852652": 1602, "22808": 1603, "787924": 1604, "637506": 1605, "276188": 1606, "757288": 1607, "254511": 1608, "841214": 1609, "329958": 1610, "231356": 1611, "783670": 1612, "288249": 1613, "192098": 1614, "377079": 1615, "33449": 1616, "609038": 1617, "887330": 1618, "984384": 1619, "941131": 1620, "717663": 1621, "757746": 1622, "64620": 1623, "805384": 1624, "183459": 1625, "807016": 1626, "738023": 1627, "316203": 1628, "982578": 1629, "699977": 1630, "557850": 1631, "496931": 1632, "66280": 1633, "50232": 1634, "548454": 1635, "224417": 1636, "129510": 1637, "55438": 1638, "309209": 1639, "582524": 1640, "286076": 1641, "414282": 1642, "413968": 1643, "865643": 1644, "439958": 1645, "854828": 1646, "450327": 1647, "569661": 1648, "977142": 1649, "659442": 1650, "884998": 1651, "367747": 1652, "740189": 1653, "552579": 1654, "564908": 1655, "423063": 1656, "87748": 1657, "318325": 1658, "592786": 1659, "73869": 1660, "695178": 1661, "311385": 1662, "591866": 1663, "584029": 1664, "507823": 1665, "156913": 1666, "426281": 1667, "702252": 1668, "853482": 1669, "578766": 1670, "578613": 1671, "185753": 1672, "420946": 1673, "218704": 1674, "318013": 1675, "279738": 1676, "185155": 1677, "875736": 1678, "430895": 1679, "71678": 1680, "317203": 1681, "11408": 1682}
reports/browser-v004-novel.json ADDED
@@ -0,0 +1,1817 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "models/v004-contextual",
3
+ "quantized": false,
4
+ "platform": "Windows-11-10.0.26200-SP0",
5
+ "torch_threads": 2,
6
+ "model_load_ms": 94.63169996161014,
7
+ "samples": 120,
8
+ "success_rate": 0.8166666666666667,
9
+ "abstention_rate": 0.10833333333333334,
10
+ "median_observation_ms": 19.925549917388707,
11
+ "median_policy_ms": 2.0463999826461077,
12
+ "median_execution_ms": 23.863700043875724,
13
+ "scope": "single-step generated browser fixtures, held-out layout templates; not arbitrary websites",
14
+ "target_vps_validated": false,
15
+ "results": [
16
+ {
17
+ "sample_id": 0,
18
+ "template": "table",
19
+ "success": false,
20
+ "abstained": true,
21
+ "expected_action": "C",
22
+ "predicted_action": "A",
23
+ "action_confidence": 0.997082531452179,
24
+ "target_confidence": 0.9630829095840454,
25
+ "observation_ms": 52.04129999037832,
26
+ "policy_ms": 3.6001999396830797,
27
+ "execution_ms": 24.167499970644712,
28
+ "status": "ok",
29
+ "code": "ASK_USER"
30
+ },
31
+ {
32
+ "sample_id": 1,
33
+ "template": "table",
34
+ "success": true,
35
+ "abstained": false,
36
+ "expected_action": "T",
37
+ "predicted_action": "T",
38
+ "action_confidence": 1.0,
39
+ "target_confidence": 1.0,
40
+ "observation_ms": 17.381199984811246,
41
+ "policy_ms": 1.756099984049797,
42
+ "execution_ms": 64.2642000457272,
43
+ "status": "ok",
44
+ "code": "TYPE"
45
+ },
46
+ {
47
+ "sample_id": 2,
48
+ "template": "nested",
49
+ "success": true,
50
+ "abstained": false,
51
+ "expected_action": "T",
52
+ "predicted_action": "T",
53
+ "action_confidence": 1.0,
54
+ "target_confidence": 1.0,
55
+ "observation_ms": 41.61069996189326,
56
+ "policy_ms": 2.2362000308930874,
57
+ "execution_ms": 25.761299999430776,
58
+ "status": "ok",
59
+ "code": "TYPE"
60
+ },
61
+ {
62
+ "sample_id": 3,
63
+ "template": "nested",
64
+ "success": true,
65
+ "abstained": false,
66
+ "expected_action": "C",
67
+ "predicted_action": "C",
68
+ "action_confidence": 1.0,
69
+ "target_confidence": 1.0,
70
+ "observation_ms": 18.043700023554265,
71
+ "policy_ms": 1.3657000381499529,
72
+ "execution_ms": 80.96819999627769,
73
+ "status": "ok",
74
+ "code": "CLICK"
75
+ },
76
+ {
77
+ "sample_id": 4,
78
+ "template": "nested",
79
+ "success": true,
80
+ "abstained": false,
81
+ "expected_action": "C",
82
+ "predicted_action": "C",
83
+ "action_confidence": 1.0,
84
+ "target_confidence": 1.0,
85
+ "observation_ms": 23.297299980185926,
86
+ "policy_ms": 1.9205999560654163,
87
+ "execution_ms": 62.41819995921105,
88
+ "status": "ok",
89
+ "code": "CLICK"
90
+ },
91
+ {
92
+ "sample_id": 5,
93
+ "template": "nested",
94
+ "success": false,
95
+ "abstained": false,
96
+ "expected_action": "T",
97
+ "predicted_action": "T",
98
+ "action_confidence": 1.0,
99
+ "target_confidence": 0.8784183263778687,
100
+ "observation_ms": 21.74580004066229,
101
+ "policy_ms": 2.3895999183878303,
102
+ "execution_ms": 20.609600003808737,
103
+ "status": "ok",
104
+ "code": "TYPE"
105
+ },
106
+ {
107
+ "sample_id": 6,
108
+ "template": "nested",
109
+ "success": true,
110
+ "abstained": false,
111
+ "expected_action": "O",
112
+ "predicted_action": "O",
113
+ "action_confidence": 1.0,
114
+ "target_confidence": 0.9999873638153076,
115
+ "observation_ms": 27.131599956192076,
116
+ "policy_ms": 2.516700071282685,
117
+ "execution_ms": 28.519999934360385,
118
+ "status": "ok",
119
+ "code": "SELECT"
120
+ },
121
+ {
122
+ "sample_id": 7,
123
+ "template": "table",
124
+ "success": true,
125
+ "abstained": false,
126
+ "expected_action": "O",
127
+ "predicted_action": "O",
128
+ "action_confidence": 1.0,
129
+ "target_confidence": 0.9832378625869751,
130
+ "observation_ms": 18.569599953480065,
131
+ "policy_ms": 1.7714999848976731,
132
+ "execution_ms": 20.5024000024423,
133
+ "status": "ok",
134
+ "code": "SELECT"
135
+ },
136
+ {
137
+ "sample_id": 8,
138
+ "template": "table",
139
+ "success": true,
140
+ "abstained": false,
141
+ "expected_action": "O",
142
+ "predicted_action": "O",
143
+ "action_confidence": 1.0,
144
+ "target_confidence": 1.0,
145
+ "observation_ms": 18.716900027357042,
146
+ "policy_ms": 2.2554000606760383,
147
+ "execution_ms": 20.574199967086315,
148
+ "status": "ok",
149
+ "code": "SELECT"
150
+ },
151
+ {
152
+ "sample_id": 9,
153
+ "template": "nested",
154
+ "success": true,
155
+ "abstained": false,
156
+ "expected_action": "T",
157
+ "predicted_action": "T",
158
+ "action_confidence": 1.0,
159
+ "target_confidence": 1.0,
160
+ "observation_ms": 16.976699931547046,
161
+ "policy_ms": 1.9339000573381782,
162
+ "execution_ms": 27.41740003693849,
163
+ "status": "ok",
164
+ "code": "TYPE"
165
+ },
166
+ {
167
+ "sample_id": 10,
168
+ "template": "nested",
169
+ "success": true,
170
+ "abstained": false,
171
+ "expected_action": "O",
172
+ "predicted_action": "O",
173
+ "action_confidence": 1.0,
174
+ "target_confidence": 1.0,
175
+ "observation_ms": 18.524899962358177,
176
+ "policy_ms": 1.7152000218629837,
177
+ "execution_ms": 22.37550006248057,
178
+ "status": "ok",
179
+ "code": "SELECT"
180
+ },
181
+ {
182
+ "sample_id": 11,
183
+ "template": "table",
184
+ "success": false,
185
+ "abstained": true,
186
+ "expected_action": "C",
187
+ "predicted_action": "A",
188
+ "action_confidence": 1.0,
189
+ "target_confidence": 0.5470258593559265,
190
+ "observation_ms": 18.770999973639846,
191
+ "policy_ms": 1.7818000633269548,
192
+ "execution_ms": 3.050200059078634,
193
+ "status": "ok",
194
+ "code": "ASK_USER"
195
+ },
196
+ {
197
+ "sample_id": 12,
198
+ "template": "table",
199
+ "success": false,
200
+ "abstained": false,
201
+ "expected_action": "T",
202
+ "predicted_action": "O",
203
+ "action_confidence": 0.9996076226234436,
204
+ "target_confidence": 0.9999998807907104,
205
+ "observation_ms": 18.240800010971725,
206
+ "policy_ms": 1.6741000581532717,
207
+ "execution_ms": 14.110499992966652,
208
+ "status": "rejected",
209
+ "code": "SELECT_ROLE_MISMATCH"
210
+ },
211
+ {
212
+ "sample_id": 13,
213
+ "template": "table",
214
+ "success": true,
215
+ "abstained": false,
216
+ "expected_action": "C",
217
+ "predicted_action": "C",
218
+ "action_confidence": 1.0,
219
+ "target_confidence": 1.0,
220
+ "observation_ms": 19.17189999949187,
221
+ "policy_ms": 2.1422000136226416,
222
+ "execution_ms": 50.70220003835857,
223
+ "status": "ok",
224
+ "code": "CLICK"
225
+ },
226
+ {
227
+ "sample_id": 14,
228
+ "template": "table",
229
+ "success": true,
230
+ "abstained": false,
231
+ "expected_action": "O",
232
+ "predicted_action": "O",
233
+ "action_confidence": 1.0,
234
+ "target_confidence": 1.0,
235
+ "observation_ms": 18.85320001747459,
236
+ "policy_ms": 2.1046000765636563,
237
+ "execution_ms": 25.676199933514,
238
+ "status": "ok",
239
+ "code": "SELECT"
240
+ },
241
+ {
242
+ "sample_id": 15,
243
+ "template": "nested",
244
+ "success": true,
245
+ "abstained": false,
246
+ "expected_action": "T",
247
+ "predicted_action": "T",
248
+ "action_confidence": 0.9997895359992981,
249
+ "target_confidence": 1.0,
250
+ "observation_ms": 15.526799950748682,
251
+ "policy_ms": 1.6744000604376197,
252
+ "execution_ms": 19.506199983879924,
253
+ "status": "ok",
254
+ "code": "TYPE"
255
+ },
256
+ {
257
+ "sample_id": 16,
258
+ "template": "nested",
259
+ "success": false,
260
+ "abstained": false,
261
+ "expected_action": "T",
262
+ "predicted_action": "O",
263
+ "action_confidence": 1.0,
264
+ "target_confidence": 0.9971709847450256,
265
+ "observation_ms": 19.791599945165217,
266
+ "policy_ms": 2.011900069192052,
267
+ "execution_ms": 21.349199931137264,
268
+ "status": "ok",
269
+ "code": "SELECT"
270
+ },
271
+ {
272
+ "sample_id": 17,
273
+ "template": "nested",
274
+ "success": true,
275
+ "abstained": false,
276
+ "expected_action": "O",
277
+ "predicted_action": "O",
278
+ "action_confidence": 1.0,
279
+ "target_confidence": 1.0,
280
+ "observation_ms": 19.411099958233535,
281
+ "policy_ms": 1.8891999498009682,
282
+ "execution_ms": 21.164899924769998,
283
+ "status": "ok",
284
+ "code": "SELECT"
285
+ },
286
+ {
287
+ "sample_id": 18,
288
+ "template": "nested",
289
+ "success": true,
290
+ "abstained": false,
291
+ "expected_action": "C",
292
+ "predicted_action": "C",
293
+ "action_confidence": 1.0,
294
+ "target_confidence": 1.0,
295
+ "observation_ms": 18.839700031094253,
296
+ "policy_ms": 1.5915000112727284,
297
+ "execution_ms": 70.20590000320226,
298
+ "status": "ok",
299
+ "code": "CLICK"
300
+ },
301
+ {
302
+ "sample_id": 19,
303
+ "template": "nested",
304
+ "success": true,
305
+ "abstained": false,
306
+ "expected_action": "O",
307
+ "predicted_action": "O",
308
+ "action_confidence": 1.0,
309
+ "target_confidence": 1.0,
310
+ "observation_ms": 22.31109992135316,
311
+ "policy_ms": 1.941800001077354,
312
+ "execution_ms": 21.803799900226295,
313
+ "status": "ok",
314
+ "code": "SELECT"
315
+ },
316
+ {
317
+ "sample_id": 20,
318
+ "template": "nested",
319
+ "success": true,
320
+ "abstained": false,
321
+ "expected_action": "C",
322
+ "predicted_action": "C",
323
+ "action_confidence": 1.0,
324
+ "target_confidence": 1.0,
325
+ "observation_ms": 18.659999943338335,
326
+ "policy_ms": 2.439300063997507,
327
+ "execution_ms": 51.54939997009933,
328
+ "status": "ok",
329
+ "code": "CLICK"
330
+ },
331
+ {
332
+ "sample_id": 21,
333
+ "template": "table",
334
+ "success": true,
335
+ "abstained": false,
336
+ "expected_action": "C",
337
+ "predicted_action": "C",
338
+ "action_confidence": 1.0,
339
+ "target_confidence": 1.0,
340
+ "observation_ms": 16.627000062726438,
341
+ "policy_ms": 2.043199958279729,
342
+ "execution_ms": 54.44460001308471,
343
+ "status": "ok",
344
+ "code": "CLICK"
345
+ },
346
+ {
347
+ "sample_id": 22,
348
+ "template": "table",
349
+ "success": true,
350
+ "abstained": false,
351
+ "expected_action": "C",
352
+ "predicted_action": "C",
353
+ "action_confidence": 1.0,
354
+ "target_confidence": 1.0,
355
+ "observation_ms": 19.43720004055649,
356
+ "policy_ms": 2.0836000330746174,
357
+ "execution_ms": 52.542699966579676,
358
+ "status": "ok",
359
+ "code": "CLICK"
360
+ },
361
+ {
362
+ "sample_id": 23,
363
+ "template": "nested",
364
+ "success": true,
365
+ "abstained": false,
366
+ "expected_action": "O",
367
+ "predicted_action": "O",
368
+ "action_confidence": 1.0,
369
+ "target_confidence": 1.0,
370
+ "observation_ms": 17.43700006045401,
371
+ "policy_ms": 1.8903999589383602,
372
+ "execution_ms": 22.533199982717633,
373
+ "status": "ok",
374
+ "code": "SELECT"
375
+ },
376
+ {
377
+ "sample_id": 24,
378
+ "template": "nested",
379
+ "success": true,
380
+ "abstained": false,
381
+ "expected_action": "C",
382
+ "predicted_action": "C",
383
+ "action_confidence": 1.0,
384
+ "target_confidence": 1.0,
385
+ "observation_ms": 19.37820005696267,
386
+ "policy_ms": 1.7327999230474234,
387
+ "execution_ms": 48.637299914844334,
388
+ "status": "ok",
389
+ "code": "CLICK"
390
+ },
391
+ {
392
+ "sample_id": 25,
393
+ "template": "table",
394
+ "success": true,
395
+ "abstained": false,
396
+ "expected_action": "C",
397
+ "predicted_action": "C",
398
+ "action_confidence": 1.0,
399
+ "target_confidence": 1.0,
400
+ "observation_ms": 19.0386000322178,
401
+ "policy_ms": 1.8211998976767063,
402
+ "execution_ms": 32.63259993400425,
403
+ "status": "ok",
404
+ "code": "CLICK"
405
+ },
406
+ {
407
+ "sample_id": 26,
408
+ "template": "nested",
409
+ "success": true,
410
+ "abstained": false,
411
+ "expected_action": "T",
412
+ "predicted_action": "T",
413
+ "action_confidence": 1.0,
414
+ "target_confidence": 1.0,
415
+ "observation_ms": 31.693299999460578,
416
+ "policy_ms": 4.196699941530824,
417
+ "execution_ms": 34.208000055514276,
418
+ "status": "ok",
419
+ "code": "TYPE"
420
+ },
421
+ {
422
+ "sample_id": 27,
423
+ "template": "nested",
424
+ "success": true,
425
+ "abstained": false,
426
+ "expected_action": "O",
427
+ "predicted_action": "O",
428
+ "action_confidence": 1.0,
429
+ "target_confidence": 1.0,
430
+ "observation_ms": 20.203899941407144,
431
+ "policy_ms": 1.6080000204965472,
432
+ "execution_ms": 23.110900074243546,
433
+ "status": "ok",
434
+ "code": "SELECT"
435
+ },
436
+ {
437
+ "sample_id": 28,
438
+ "template": "nested",
439
+ "success": true,
440
+ "abstained": false,
441
+ "expected_action": "T",
442
+ "predicted_action": "T",
443
+ "action_confidence": 1.0,
444
+ "target_confidence": 1.0,
445
+ "observation_ms": 19.319100072607398,
446
+ "policy_ms": 1.617099973373115,
447
+ "execution_ms": 27.02000003773719,
448
+ "status": "ok",
449
+ "code": "TYPE"
450
+ },
451
+ {
452
+ "sample_id": 29,
453
+ "template": "table",
454
+ "success": true,
455
+ "abstained": false,
456
+ "expected_action": "C",
457
+ "predicted_action": "C",
458
+ "action_confidence": 1.0,
459
+ "target_confidence": 1.0,
460
+ "observation_ms": 18.1315999943763,
461
+ "policy_ms": 1.984399976208806,
462
+ "execution_ms": 61.0787000041455,
463
+ "status": "ok",
464
+ "code": "CLICK"
465
+ },
466
+ {
467
+ "sample_id": 30,
468
+ "template": "table",
469
+ "success": false,
470
+ "abstained": false,
471
+ "expected_action": "T",
472
+ "predicted_action": "O",
473
+ "action_confidence": 0.9999998807907104,
474
+ "target_confidence": 0.9999998807907104,
475
+ "observation_ms": 24.716699961572886,
476
+ "policy_ms": 2.7118000434711576,
477
+ "execution_ms": 20.71119996253401,
478
+ "status": "rejected",
479
+ "code": "SELECT_ROLE_MISMATCH"
480
+ },
481
+ {
482
+ "sample_id": 31,
483
+ "template": "nested",
484
+ "success": true,
485
+ "abstained": false,
486
+ "expected_action": "O",
487
+ "predicted_action": "O",
488
+ "action_confidence": 1.0,
489
+ "target_confidence": 1.0,
490
+ "observation_ms": 19.431499997153878,
491
+ "policy_ms": 1.9639000529423356,
492
+ "execution_ms": 21.217299974523485,
493
+ "status": "ok",
494
+ "code": "SELECT"
495
+ },
496
+ {
497
+ "sample_id": 32,
498
+ "template": "nested",
499
+ "success": false,
500
+ "abstained": false,
501
+ "expected_action": "T",
502
+ "predicted_action": "O",
503
+ "action_confidence": 1.0,
504
+ "target_confidence": 0.999996542930603,
505
+ "observation_ms": 17.129699932411313,
506
+ "policy_ms": 1.640500035136938,
507
+ "execution_ms": 20.674900035373867,
508
+ "status": "ok",
509
+ "code": "SELECT"
510
+ },
511
+ {
512
+ "sample_id": 33,
513
+ "template": "table",
514
+ "success": false,
515
+ "abstained": true,
516
+ "expected_action": "C",
517
+ "predicted_action": "A",
518
+ "action_confidence": 0.9980350136756897,
519
+ "target_confidence": 0.99998939037323,
520
+ "observation_ms": 15.770700061693788,
521
+ "policy_ms": 1.789200003258884,
522
+ "execution_ms": 3.0932999216020107,
523
+ "status": "ok",
524
+ "code": "ASK_USER"
525
+ },
526
+ {
527
+ "sample_id": 34,
528
+ "template": "table",
529
+ "success": true,
530
+ "abstained": false,
531
+ "expected_action": "T",
532
+ "predicted_action": "T",
533
+ "action_confidence": 1.0,
534
+ "target_confidence": 1.0,
535
+ "observation_ms": 18.88559991493821,
536
+ "policy_ms": 2.4881999706849456,
537
+ "execution_ms": 21.22969995252788,
538
+ "status": "ok",
539
+ "code": "TYPE"
540
+ },
541
+ {
542
+ "sample_id": 35,
543
+ "template": "table",
544
+ "success": true,
545
+ "abstained": false,
546
+ "expected_action": "C",
547
+ "predicted_action": "C",
548
+ "action_confidence": 1.0,
549
+ "target_confidence": 1.0,
550
+ "observation_ms": 17.057199962437153,
551
+ "policy_ms": 2.530900062993169,
552
+ "execution_ms": 35.879299975931644,
553
+ "status": "ok",
554
+ "code": "CLICK"
555
+ },
556
+ {
557
+ "sample_id": 36,
558
+ "template": "nested",
559
+ "success": true,
560
+ "abstained": false,
561
+ "expected_action": "T",
562
+ "predicted_action": "T",
563
+ "action_confidence": 0.9625697731971741,
564
+ "target_confidence": 1.0,
565
+ "observation_ms": 19.18990002013743,
566
+ "policy_ms": 2.424499951303005,
567
+ "execution_ms": 25.121700018644333,
568
+ "status": "ok",
569
+ "code": "TYPE"
570
+ },
571
+ {
572
+ "sample_id": 37,
573
+ "template": "table",
574
+ "success": true,
575
+ "abstained": false,
576
+ "expected_action": "O",
577
+ "predicted_action": "O",
578
+ "action_confidence": 1.0,
579
+ "target_confidence": 1.0,
580
+ "observation_ms": 23.415900068357587,
581
+ "policy_ms": 2.4779000086709857,
582
+ "execution_ms": 23.03289994597435,
583
+ "status": "ok",
584
+ "code": "SELECT"
585
+ },
586
+ {
587
+ "sample_id": 38,
588
+ "template": "table",
589
+ "success": true,
590
+ "abstained": false,
591
+ "expected_action": "T",
592
+ "predicted_action": "T",
593
+ "action_confidence": 1.0,
594
+ "target_confidence": 1.0,
595
+ "observation_ms": 19.38509999308735,
596
+ "policy_ms": 1.792999915778637,
597
+ "execution_ms": 21.66259998921305,
598
+ "status": "ok",
599
+ "code": "TYPE"
600
+ },
601
+ {
602
+ "sample_id": 39,
603
+ "template": "nested",
604
+ "success": false,
605
+ "abstained": false,
606
+ "expected_action": "T",
607
+ "predicted_action": "O",
608
+ "action_confidence": 1.0,
609
+ "target_confidence": 0.9615044593811035,
610
+ "observation_ms": 17.784499912522733,
611
+ "policy_ms": 1.5483000315725803,
612
+ "execution_ms": 13.7020000256598,
613
+ "status": "rejected",
614
+ "code": "SELECT_ROLE_MISMATCH"
615
+ },
616
+ {
617
+ "sample_id": 40,
618
+ "template": "nested",
619
+ "success": false,
620
+ "abstained": true,
621
+ "expected_action": "C",
622
+ "predicted_action": "A",
623
+ "action_confidence": 0.9991171956062317,
624
+ "target_confidence": 0.8294482827186584,
625
+ "observation_ms": 18.787800101563334,
626
+ "policy_ms": 2.069000038318336,
627
+ "execution_ms": 3.284999984316528,
628
+ "status": "ok",
629
+ "code": "ASK_USER"
630
+ },
631
+ {
632
+ "sample_id": 41,
633
+ "template": "table",
634
+ "success": true,
635
+ "abstained": false,
636
+ "expected_action": "O",
637
+ "predicted_action": "O",
638
+ "action_confidence": 1.0,
639
+ "target_confidence": 1.0,
640
+ "observation_ms": 18.18359992466867,
641
+ "policy_ms": 1.8230000277981162,
642
+ "execution_ms": 23.239500005729496,
643
+ "status": "ok",
644
+ "code": "SELECT"
645
+ },
646
+ {
647
+ "sample_id": 42,
648
+ "template": "table",
649
+ "success": true,
650
+ "abstained": false,
651
+ "expected_action": "O",
652
+ "predicted_action": "O",
653
+ "action_confidence": 1.0,
654
+ "target_confidence": 1.0,
655
+ "observation_ms": 21.46099996753037,
656
+ "policy_ms": 2.3865000111982226,
657
+ "execution_ms": 26.55770001001656,
658
+ "status": "ok",
659
+ "code": "SELECT"
660
+ },
661
+ {
662
+ "sample_id": 43,
663
+ "template": "nested",
664
+ "success": true,
665
+ "abstained": false,
666
+ "expected_action": "O",
667
+ "predicted_action": "O",
668
+ "action_confidence": 1.0,
669
+ "target_confidence": 1.0,
670
+ "observation_ms": 20.84760006982833,
671
+ "policy_ms": 1.6396999126300216,
672
+ "execution_ms": 27.551600011065602,
673
+ "status": "ok",
674
+ "code": "SELECT"
675
+ },
676
+ {
677
+ "sample_id": 44,
678
+ "template": "nested",
679
+ "success": true,
680
+ "abstained": false,
681
+ "expected_action": "C",
682
+ "predicted_action": "C",
683
+ "action_confidence": 1.0,
684
+ "target_confidence": 1.0,
685
+ "observation_ms": 59.78959996718913,
686
+ "policy_ms": 2.021700027398765,
687
+ "execution_ms": 45.08690000511706,
688
+ "status": "ok",
689
+ "code": "CLICK"
690
+ },
691
+ {
692
+ "sample_id": 45,
693
+ "template": "table",
694
+ "success": true,
695
+ "abstained": false,
696
+ "expected_action": "O",
697
+ "predicted_action": "O",
698
+ "action_confidence": 1.0,
699
+ "target_confidence": 1.0,
700
+ "observation_ms": 18.197900033555925,
701
+ "policy_ms": 1.7804999370127916,
702
+ "execution_ms": 23.228899925015867,
703
+ "status": "ok",
704
+ "code": "SELECT"
705
+ },
706
+ {
707
+ "sample_id": 46,
708
+ "template": "table",
709
+ "success": true,
710
+ "abstained": false,
711
+ "expected_action": "O",
712
+ "predicted_action": "O",
713
+ "action_confidence": 1.0,
714
+ "target_confidence": 1.0,
715
+ "observation_ms": 20.208099973388016,
716
+ "policy_ms": 1.6938999760895967,
717
+ "execution_ms": 22.70880003925413,
718
+ "status": "ok",
719
+ "code": "SELECT"
720
+ },
721
+ {
722
+ "sample_id": 47,
723
+ "template": "nested",
724
+ "success": false,
725
+ "abstained": false,
726
+ "expected_action": "T",
727
+ "predicted_action": "O",
728
+ "action_confidence": 0.9346925020217896,
729
+ "target_confidence": 0.9996263980865479,
730
+ "observation_ms": 19.146199920214713,
731
+ "policy_ms": 1.9916000310331583,
732
+ "execution_ms": 14.529700041748583,
733
+ "status": "rejected",
734
+ "code": "SELECT_ROLE_MISMATCH"
735
+ },
736
+ {
737
+ "sample_id": 48,
738
+ "template": "nested",
739
+ "success": true,
740
+ "abstained": false,
741
+ "expected_action": "T",
742
+ "predicted_action": "T",
743
+ "action_confidence": 0.9997907280921936,
744
+ "target_confidence": 1.0,
745
+ "observation_ms": 18.851600005291402,
746
+ "policy_ms": 1.7500000540167093,
747
+ "execution_ms": 22.093100007623434,
748
+ "status": "ok",
749
+ "code": "TYPE"
750
+ },
751
+ {
752
+ "sample_id": 49,
753
+ "template": "table",
754
+ "success": true,
755
+ "abstained": false,
756
+ "expected_action": "T",
757
+ "predicted_action": "T",
758
+ "action_confidence": 0.9999998807907104,
759
+ "target_confidence": 1.0,
760
+ "observation_ms": 16.993399942293763,
761
+ "policy_ms": 1.2751000467687845,
762
+ "execution_ms": 22.90139999240637,
763
+ "status": "ok",
764
+ "code": "TYPE"
765
+ },
766
+ {
767
+ "sample_id": 50,
768
+ "template": "table",
769
+ "success": true,
770
+ "abstained": false,
771
+ "expected_action": "C",
772
+ "predicted_action": "C",
773
+ "action_confidence": 1.0,
774
+ "target_confidence": 1.0,
775
+ "observation_ms": 20.416999934241176,
776
+ "policy_ms": 1.7480000387877226,
777
+ "execution_ms": 55.009900010190904,
778
+ "status": "ok",
779
+ "code": "CLICK"
780
+ },
781
+ {
782
+ "sample_id": 51,
783
+ "template": "nested",
784
+ "success": true,
785
+ "abstained": false,
786
+ "expected_action": "O",
787
+ "predicted_action": "O",
788
+ "action_confidence": 1.0,
789
+ "target_confidence": 1.0,
790
+ "observation_ms": 19.035200006328523,
791
+ "policy_ms": 1.66229996830225,
792
+ "execution_ms": 22.664399934001267,
793
+ "status": "ok",
794
+ "code": "SELECT"
795
+ },
796
+ {
797
+ "sample_id": 52,
798
+ "template": "nested",
799
+ "success": true,
800
+ "abstained": false,
801
+ "expected_action": "O",
802
+ "predicted_action": "O",
803
+ "action_confidence": 1.0,
804
+ "target_confidence": 1.0,
805
+ "observation_ms": 20.522900042124093,
806
+ "policy_ms": 1.9642000552266836,
807
+ "execution_ms": 22.66699995379895,
808
+ "status": "ok",
809
+ "code": "SELECT"
810
+ },
811
+ {
812
+ "sample_id": 53,
813
+ "template": "nested",
814
+ "success": false,
815
+ "abstained": true,
816
+ "expected_action": "C",
817
+ "predicted_action": "A",
818
+ "action_confidence": 0.8216633796691895,
819
+ "target_confidence": 0.999998927116394,
820
+ "observation_ms": 18.218699959106743,
821
+ "policy_ms": 1.763500040397048,
822
+ "execution_ms": 2.8200000524520874,
823
+ "status": "ok",
824
+ "code": "ASK_USER"
825
+ },
826
+ {
827
+ "sample_id": 54,
828
+ "template": "nested",
829
+ "success": true,
830
+ "abstained": false,
831
+ "expected_action": "T",
832
+ "predicted_action": "T",
833
+ "action_confidence": 1.0,
834
+ "target_confidence": 1.0,
835
+ "observation_ms": 20.149499992839992,
836
+ "policy_ms": 2.4216000456362963,
837
+ "execution_ms": 23.797200061380863,
838
+ "status": "ok",
839
+ "code": "TYPE"
840
+ },
841
+ {
842
+ "sample_id": 55,
843
+ "template": "nested",
844
+ "success": true,
845
+ "abstained": false,
846
+ "expected_action": "O",
847
+ "predicted_action": "O",
848
+ "action_confidence": 1.0,
849
+ "target_confidence": 0.9997102618217468,
850
+ "observation_ms": 19.446899998001754,
851
+ "policy_ms": 2.3861000081524253,
852
+ "execution_ms": 22.44800003245473,
853
+ "status": "ok",
854
+ "code": "SELECT"
855
+ },
856
+ {
857
+ "sample_id": 56,
858
+ "template": "nested",
859
+ "success": false,
860
+ "abstained": true,
861
+ "expected_action": "C",
862
+ "predicted_action": "A",
863
+ "action_confidence": 1.0,
864
+ "target_confidence": 0.9994478821754456,
865
+ "observation_ms": 19.34529992286116,
866
+ "policy_ms": 1.6651999903842807,
867
+ "execution_ms": 2.9511998873203993,
868
+ "status": "ok",
869
+ "code": "ASK_USER"
870
+ },
871
+ {
872
+ "sample_id": 57,
873
+ "template": "table",
874
+ "success": true,
875
+ "abstained": false,
876
+ "expected_action": "C",
877
+ "predicted_action": "C",
878
+ "action_confidence": 1.0,
879
+ "target_confidence": 1.0,
880
+ "observation_ms": 20.44330001808703,
881
+ "policy_ms": 2.0528999157249928,
882
+ "execution_ms": 54.767899913713336,
883
+ "status": "ok",
884
+ "code": "CLICK"
885
+ },
886
+ {
887
+ "sample_id": 58,
888
+ "template": "nested",
889
+ "success": true,
890
+ "abstained": false,
891
+ "expected_action": "O",
892
+ "predicted_action": "O",
893
+ "action_confidence": 1.0,
894
+ "target_confidence": 1.0,
895
+ "observation_ms": 20.317499991506338,
896
+ "policy_ms": 2.5025999639183283,
897
+ "execution_ms": 22.964300005696714,
898
+ "status": "ok",
899
+ "code": "SELECT"
900
+ },
901
+ {
902
+ "sample_id": 59,
903
+ "template": "nested",
904
+ "success": true,
905
+ "abstained": false,
906
+ "expected_action": "O",
907
+ "predicted_action": "O",
908
+ "action_confidence": 1.0,
909
+ "target_confidence": 1.0,
910
+ "observation_ms": 18.604999990202487,
911
+ "policy_ms": 1.416000071913004,
912
+ "execution_ms": 21.546700038015842,
913
+ "status": "ok",
914
+ "code": "SELECT"
915
+ },
916
+ {
917
+ "sample_id": 60,
918
+ "template": "table",
919
+ "success": true,
920
+ "abstained": false,
921
+ "expected_action": "O",
922
+ "predicted_action": "O",
923
+ "action_confidence": 1.0,
924
+ "target_confidence": 1.0,
925
+ "observation_ms": 18.402900081127882,
926
+ "policy_ms": 2.1657999604940414,
927
+ "execution_ms": 23.002700065262616,
928
+ "status": "ok",
929
+ "code": "SELECT"
930
+ },
931
+ {
932
+ "sample_id": 61,
933
+ "template": "table",
934
+ "success": true,
935
+ "abstained": false,
936
+ "expected_action": "O",
937
+ "predicted_action": "O",
938
+ "action_confidence": 1.0,
939
+ "target_confidence": 0.9999998807907104,
940
+ "observation_ms": 19.693100010044873,
941
+ "policy_ms": 2.4681999348104,
942
+ "execution_ms": 23.930200026370585,
943
+ "status": "ok",
944
+ "code": "SELECT"
945
+ },
946
+ {
947
+ "sample_id": 62,
948
+ "template": "table",
949
+ "success": false,
950
+ "abstained": false,
951
+ "expected_action": "T",
952
+ "predicted_action": "O",
953
+ "action_confidence": 0.8983810544013977,
954
+ "target_confidence": 1.0,
955
+ "observation_ms": 23.263600072823465,
956
+ "policy_ms": 2.2478000028058887,
957
+ "execution_ms": 15.640300000086427,
958
+ "status": "rejected",
959
+ "code": "SELECT_ROLE_MISMATCH"
960
+ },
961
+ {
962
+ "sample_id": 63,
963
+ "template": "table",
964
+ "success": false,
965
+ "abstained": true,
966
+ "expected_action": "C",
967
+ "predicted_action": "A",
968
+ "action_confidence": 0.9983367919921875,
969
+ "target_confidence": 0.85873943567276,
970
+ "observation_ms": 21.32920001167804,
971
+ "policy_ms": 2.138499985449016,
972
+ "execution_ms": 3.9785000262781978,
973
+ "status": "ok",
974
+ "code": "ASK_USER"
975
+ },
976
+ {
977
+ "sample_id": 64,
978
+ "template": "table",
979
+ "success": true,
980
+ "abstained": false,
981
+ "expected_action": "T",
982
+ "predicted_action": "T",
983
+ "action_confidence": 0.9829460382461548,
984
+ "target_confidence": 1.0,
985
+ "observation_ms": 20.072199986316264,
986
+ "policy_ms": 1.874100067652762,
987
+ "execution_ms": 22.601000033318996,
988
+ "status": "ok",
989
+ "code": "TYPE"
990
+ },
991
+ {
992
+ "sample_id": 65,
993
+ "template": "nested",
994
+ "success": false,
995
+ "abstained": true,
996
+ "expected_action": "C",
997
+ "predicted_action": "A",
998
+ "action_confidence": 0.5237547159194946,
999
+ "target_confidence": 1.0,
1000
+ "observation_ms": 19.744299934245646,
1001
+ "policy_ms": 1.6528000123798847,
1002
+ "execution_ms": 3.1644999980926514,
1003
+ "status": "ok",
1004
+ "code": "ASK_USER"
1005
+ },
1006
+ {
1007
+ "sample_id": 66,
1008
+ "template": "table",
1009
+ "success": true,
1010
+ "abstained": false,
1011
+ "expected_action": "C",
1012
+ "predicted_action": "C",
1013
+ "action_confidence": 1.0,
1014
+ "target_confidence": 1.0,
1015
+ "observation_ms": 23.615000071004033,
1016
+ "policy_ms": 2.410199958831072,
1017
+ "execution_ms": 42.36810002475977,
1018
+ "status": "ok",
1019
+ "code": "CLICK"
1020
+ },
1021
+ {
1022
+ "sample_id": 67,
1023
+ "template": "nested",
1024
+ "success": true,
1025
+ "abstained": false,
1026
+ "expected_action": "T",
1027
+ "predicted_action": "T",
1028
+ "action_confidence": 1.0,
1029
+ "target_confidence": 1.0,
1030
+ "observation_ms": 18.37950001936406,
1031
+ "policy_ms": 1.804900006391108,
1032
+ "execution_ms": 23.62260001245886,
1033
+ "status": "ok",
1034
+ "code": "TYPE"
1035
+ },
1036
+ {
1037
+ "sample_id": 68,
1038
+ "template": "nested",
1039
+ "success": true,
1040
+ "abstained": false,
1041
+ "expected_action": "C",
1042
+ "predicted_action": "C",
1043
+ "action_confidence": 1.0,
1044
+ "target_confidence": 1.0,
1045
+ "observation_ms": 18.995500053279102,
1046
+ "policy_ms": 1.645199954509735,
1047
+ "execution_ms": 32.227500109001994,
1048
+ "status": "ok",
1049
+ "code": "CLICK"
1050
+ },
1051
+ {
1052
+ "sample_id": 69,
1053
+ "template": "nested",
1054
+ "success": false,
1055
+ "abstained": true,
1056
+ "expected_action": "C",
1057
+ "predicted_action": "A",
1058
+ "action_confidence": 0.9998936653137207,
1059
+ "target_confidence": 0.5836381912231445,
1060
+ "observation_ms": 18.146500107832253,
1061
+ "policy_ms": 1.6587999416515231,
1062
+ "execution_ms": 3.0242999782785773,
1063
+ "status": "ok",
1064
+ "code": "ASK_USER"
1065
+ },
1066
+ {
1067
+ "sample_id": 70,
1068
+ "template": "nested",
1069
+ "success": true,
1070
+ "abstained": false,
1071
+ "expected_action": "C",
1072
+ "predicted_action": "C",
1073
+ "action_confidence": 1.0,
1074
+ "target_confidence": 1.0,
1075
+ "observation_ms": 19.674799987114966,
1076
+ "policy_ms": 2.2829000372439623,
1077
+ "execution_ms": 58.3354999544099,
1078
+ "status": "ok",
1079
+ "code": "CLICK"
1080
+ },
1081
+ {
1082
+ "sample_id": 71,
1083
+ "template": "table",
1084
+ "success": true,
1085
+ "abstained": false,
1086
+ "expected_action": "O",
1087
+ "predicted_action": "O",
1088
+ "action_confidence": 1.0,
1089
+ "target_confidence": 1.0,
1090
+ "observation_ms": 20.07989992853254,
1091
+ "policy_ms": 1.7679999582469463,
1092
+ "execution_ms": 22.723899921402335,
1093
+ "status": "ok",
1094
+ "code": "SELECT"
1095
+ },
1096
+ {
1097
+ "sample_id": 72,
1098
+ "template": "table",
1099
+ "success": false,
1100
+ "abstained": true,
1101
+ "expected_action": "O",
1102
+ "predicted_action": "A",
1103
+ "action_confidence": 1.0,
1104
+ "target_confidence": 0.6719118356704712,
1105
+ "observation_ms": 20.61340003274381,
1106
+ "policy_ms": 2.3207999765872955,
1107
+ "execution_ms": 3.163499990478158,
1108
+ "status": "ok",
1109
+ "code": "ASK_USER"
1110
+ },
1111
+ {
1112
+ "sample_id": 73,
1113
+ "template": "nested",
1114
+ "success": false,
1115
+ "abstained": false,
1116
+ "expected_action": "O",
1117
+ "predicted_action": "O",
1118
+ "action_confidence": 1.0,
1119
+ "target_confidence": 0.9052738547325134,
1120
+ "observation_ms": 16.669899923726916,
1121
+ "policy_ms": 1.638100016862154,
1122
+ "execution_ms": 22.68329996149987,
1123
+ "status": "ok",
1124
+ "code": "SELECT"
1125
+ },
1126
+ {
1127
+ "sample_id": 74,
1128
+ "template": "nested",
1129
+ "success": true,
1130
+ "abstained": false,
1131
+ "expected_action": "T",
1132
+ "predicted_action": "T",
1133
+ "action_confidence": 1.0,
1134
+ "target_confidence": 1.0,
1135
+ "observation_ms": 21.329900017008185,
1136
+ "policy_ms": 1.9255999941378832,
1137
+ "execution_ms": 23.794899927452207,
1138
+ "status": "ok",
1139
+ "code": "TYPE"
1140
+ },
1141
+ {
1142
+ "sample_id": 75,
1143
+ "template": "table",
1144
+ "success": true,
1145
+ "abstained": false,
1146
+ "expected_action": "O",
1147
+ "predicted_action": "O",
1148
+ "action_confidence": 1.0,
1149
+ "target_confidence": 1.0,
1150
+ "observation_ms": 43.32369996700436,
1151
+ "policy_ms": 1.9292000215500593,
1152
+ "execution_ms": 25.223999982699752,
1153
+ "status": "ok",
1154
+ "code": "SELECT"
1155
+ },
1156
+ {
1157
+ "sample_id": 76,
1158
+ "template": "table",
1159
+ "success": true,
1160
+ "abstained": false,
1161
+ "expected_action": "O",
1162
+ "predicted_action": "O",
1163
+ "action_confidence": 1.0,
1164
+ "target_confidence": 1.0,
1165
+ "observation_ms": 16.537899966351688,
1166
+ "policy_ms": 1.6442000633105636,
1167
+ "execution_ms": 22.20050001051277,
1168
+ "status": "ok",
1169
+ "code": "SELECT"
1170
+ },
1171
+ {
1172
+ "sample_id": 77,
1173
+ "template": "table",
1174
+ "success": true,
1175
+ "abstained": false,
1176
+ "expected_action": "T",
1177
+ "predicted_action": "T",
1178
+ "action_confidence": 0.999998927116394,
1179
+ "target_confidence": 1.0,
1180
+ "observation_ms": 19.53789999242872,
1181
+ "policy_ms": 2.0939999958500266,
1182
+ "execution_ms": 21.898900042288005,
1183
+ "status": "ok",
1184
+ "code": "TYPE"
1185
+ },
1186
+ {
1187
+ "sample_id": 78,
1188
+ "template": "nested",
1189
+ "success": true,
1190
+ "abstained": false,
1191
+ "expected_action": "O",
1192
+ "predicted_action": "O",
1193
+ "action_confidence": 1.0,
1194
+ "target_confidence": 1.0,
1195
+ "observation_ms": 17.23720005247742,
1196
+ "policy_ms": 1.4942999696359038,
1197
+ "execution_ms": 20.16330009792,
1198
+ "status": "ok",
1199
+ "code": "SELECT"
1200
+ },
1201
+ {
1202
+ "sample_id": 79,
1203
+ "template": "table",
1204
+ "success": true,
1205
+ "abstained": false,
1206
+ "expected_action": "O",
1207
+ "predicted_action": "O",
1208
+ "action_confidence": 1.0,
1209
+ "target_confidence": 1.0,
1210
+ "observation_ms": 20.72490006685257,
1211
+ "policy_ms": 2.668099943548441,
1212
+ "execution_ms": 20.48149995971471,
1213
+ "status": "ok",
1214
+ "code": "SELECT"
1215
+ },
1216
+ {
1217
+ "sample_id": 80,
1218
+ "template": "table",
1219
+ "success": true,
1220
+ "abstained": false,
1221
+ "expected_action": "C",
1222
+ "predicted_action": "C",
1223
+ "action_confidence": 1.0,
1224
+ "target_confidence": 1.0,
1225
+ "observation_ms": 19.801099901087582,
1226
+ "policy_ms": 2.180800074711442,
1227
+ "execution_ms": 39.21910002827644,
1228
+ "status": "ok",
1229
+ "code": "CLICK"
1230
+ },
1231
+ {
1232
+ "sample_id": 81,
1233
+ "template": "nested",
1234
+ "success": true,
1235
+ "abstained": false,
1236
+ "expected_action": "C",
1237
+ "predicted_action": "C",
1238
+ "action_confidence": 0.9999997615814209,
1239
+ "target_confidence": 0.9999921321868896,
1240
+ "observation_ms": 17.265999922528863,
1241
+ "policy_ms": 1.5931000234559178,
1242
+ "execution_ms": 39.03109999373555,
1243
+ "status": "ok",
1244
+ "code": "CLICK"
1245
+ },
1246
+ {
1247
+ "sample_id": 82,
1248
+ "template": "nested",
1249
+ "success": true,
1250
+ "abstained": false,
1251
+ "expected_action": "C",
1252
+ "predicted_action": "C",
1253
+ "action_confidence": 1.0,
1254
+ "target_confidence": 1.0,
1255
+ "observation_ms": 20.587499951943755,
1256
+ "policy_ms": 1.7686000792309642,
1257
+ "execution_ms": 54.260200005955994,
1258
+ "status": "ok",
1259
+ "code": "CLICK"
1260
+ },
1261
+ {
1262
+ "sample_id": 83,
1263
+ "template": "nested",
1264
+ "success": true,
1265
+ "abstained": false,
1266
+ "expected_action": "T",
1267
+ "predicted_action": "T",
1268
+ "action_confidence": 0.9128974080085754,
1269
+ "target_confidence": 1.0,
1270
+ "observation_ms": 21.60480001475662,
1271
+ "policy_ms": 1.6494999872520566,
1272
+ "execution_ms": 26.27339994069189,
1273
+ "status": "ok",
1274
+ "code": "TYPE"
1275
+ },
1276
+ {
1277
+ "sample_id": 84,
1278
+ "template": "table",
1279
+ "success": true,
1280
+ "abstained": false,
1281
+ "expected_action": "T",
1282
+ "predicted_action": "T",
1283
+ "action_confidence": 1.0,
1284
+ "target_confidence": 1.0,
1285
+ "observation_ms": 22.096799919381738,
1286
+ "policy_ms": 2.284600050188601,
1287
+ "execution_ms": 23.54269998613745,
1288
+ "status": "ok",
1289
+ "code": "TYPE"
1290
+ },
1291
+ {
1292
+ "sample_id": 85,
1293
+ "template": "nested",
1294
+ "success": true,
1295
+ "abstained": false,
1296
+ "expected_action": "O",
1297
+ "predicted_action": "O",
1298
+ "action_confidence": 1.0,
1299
+ "target_confidence": 1.0,
1300
+ "observation_ms": 20.568500040099025,
1301
+ "policy_ms": 2.6114999782294035,
1302
+ "execution_ms": 22.038600058294833,
1303
+ "status": "ok",
1304
+ "code": "SELECT"
1305
+ },
1306
+ {
1307
+ "sample_id": 86,
1308
+ "template": "nested",
1309
+ "success": true,
1310
+ "abstained": false,
1311
+ "expected_action": "C",
1312
+ "predicted_action": "C",
1313
+ "action_confidence": 1.0,
1314
+ "target_confidence": 1.0,
1315
+ "observation_ms": 20.049999933689833,
1316
+ "policy_ms": 1.8260000506415963,
1317
+ "execution_ms": 33.05269998963922,
1318
+ "status": "ok",
1319
+ "code": "CLICK"
1320
+ },
1321
+ {
1322
+ "sample_id": 87,
1323
+ "template": "nested",
1324
+ "success": true,
1325
+ "abstained": false,
1326
+ "expected_action": "C",
1327
+ "predicted_action": "C",
1328
+ "action_confidence": 1.0,
1329
+ "target_confidence": 1.0,
1330
+ "observation_ms": 21.442299941554666,
1331
+ "policy_ms": 1.7699000891298056,
1332
+ "execution_ms": 46.82319995481521,
1333
+ "status": "ok",
1334
+ "code": "CLICK"
1335
+ },
1336
+ {
1337
+ "sample_id": 88,
1338
+ "template": "table",
1339
+ "success": true,
1340
+ "abstained": false,
1341
+ "expected_action": "C",
1342
+ "predicted_action": "C",
1343
+ "action_confidence": 1.0,
1344
+ "target_confidence": 1.0,
1345
+ "observation_ms": 25.167400017380714,
1346
+ "policy_ms": 2.3444999242201447,
1347
+ "execution_ms": 60.75640011113137,
1348
+ "status": "ok",
1349
+ "code": "CLICK"
1350
+ },
1351
+ {
1352
+ "sample_id": 89,
1353
+ "template": "nested",
1354
+ "success": true,
1355
+ "abstained": false,
1356
+ "expected_action": "O",
1357
+ "predicted_action": "O",
1358
+ "action_confidence": 1.0,
1359
+ "target_confidence": 1.0,
1360
+ "observation_ms": 20.64460003748536,
1361
+ "policy_ms": 2.152100089006126,
1362
+ "execution_ms": 22.202300024218857,
1363
+ "status": "ok",
1364
+ "code": "SELECT"
1365
+ },
1366
+ {
1367
+ "sample_id": 90,
1368
+ "template": "table",
1369
+ "success": true,
1370
+ "abstained": false,
1371
+ "expected_action": "C",
1372
+ "predicted_action": "C",
1373
+ "action_confidence": 0.9992997646331787,
1374
+ "target_confidence": 0.9984721541404724,
1375
+ "observation_ms": 23.232600069604814,
1376
+ "policy_ms": 2.1249999990686774,
1377
+ "execution_ms": 53.32009994890541,
1378
+ "status": "ok",
1379
+ "code": "CLICK"
1380
+ },
1381
+ {
1382
+ "sample_id": 91,
1383
+ "template": "nested",
1384
+ "success": true,
1385
+ "abstained": false,
1386
+ "expected_action": "T",
1387
+ "predicted_action": "T",
1388
+ "action_confidence": 1.0,
1389
+ "target_confidence": 1.0,
1390
+ "observation_ms": 24.84650001861155,
1391
+ "policy_ms": 2.548099961131811,
1392
+ "execution_ms": 27.10760000627488,
1393
+ "status": "ok",
1394
+ "code": "TYPE"
1395
+ },
1396
+ {
1397
+ "sample_id": 92,
1398
+ "template": "table",
1399
+ "success": true,
1400
+ "abstained": false,
1401
+ "expected_action": "T",
1402
+ "predicted_action": "T",
1403
+ "action_confidence": 0.9999769926071167,
1404
+ "target_confidence": 0.9999783039093018,
1405
+ "observation_ms": 21.894300007261336,
1406
+ "policy_ms": 2.4238000623881817,
1407
+ "execution_ms": 24.46879993658513,
1408
+ "status": "ok",
1409
+ "code": "TYPE"
1410
+ },
1411
+ {
1412
+ "sample_id": 93,
1413
+ "template": "table",
1414
+ "success": true,
1415
+ "abstained": false,
1416
+ "expected_action": "T",
1417
+ "predicted_action": "T",
1418
+ "action_confidence": 1.0,
1419
+ "target_confidence": 1.0,
1420
+ "observation_ms": 19.086599932052195,
1421
+ "policy_ms": 1.950700068846345,
1422
+ "execution_ms": 26.81089995894581,
1423
+ "status": "ok",
1424
+ "code": "TYPE"
1425
+ },
1426
+ {
1427
+ "sample_id": 94,
1428
+ "template": "nested",
1429
+ "success": true,
1430
+ "abstained": false,
1431
+ "expected_action": "C",
1432
+ "predicted_action": "C",
1433
+ "action_confidence": 1.0,
1434
+ "target_confidence": 1.0,
1435
+ "observation_ms": 27.7623999863863,
1436
+ "policy_ms": 1.9869999960064888,
1437
+ "execution_ms": 55.17740000505,
1438
+ "status": "ok",
1439
+ "code": "CLICK"
1440
+ },
1441
+ {
1442
+ "sample_id": 95,
1443
+ "template": "nested",
1444
+ "success": true,
1445
+ "abstained": false,
1446
+ "expected_action": "C",
1447
+ "predicted_action": "C",
1448
+ "action_confidence": 1.0,
1449
+ "target_confidence": 1.0,
1450
+ "observation_ms": 24.448600015603006,
1451
+ "policy_ms": 1.9808000652119517,
1452
+ "execution_ms": 56.22260004747659,
1453
+ "status": "ok",
1454
+ "code": "CLICK"
1455
+ },
1456
+ {
1457
+ "sample_id": 96,
1458
+ "template": "nested",
1459
+ "success": true,
1460
+ "abstained": false,
1461
+ "expected_action": "T",
1462
+ "predicted_action": "T",
1463
+ "action_confidence": 1.0,
1464
+ "target_confidence": 1.0,
1465
+ "observation_ms": 27.35049999319017,
1466
+ "policy_ms": 2.7284000534564257,
1467
+ "execution_ms": 56.82669999077916,
1468
+ "status": "ok",
1469
+ "code": "TYPE"
1470
+ },
1471
+ {
1472
+ "sample_id": 97,
1473
+ "template": "table",
1474
+ "success": true,
1475
+ "abstained": false,
1476
+ "expected_action": "T",
1477
+ "predicted_action": "T",
1478
+ "action_confidence": 1.0,
1479
+ "target_confidence": 1.0,
1480
+ "observation_ms": 26.53720008675009,
1481
+ "policy_ms": 2.753300010226667,
1482
+ "execution_ms": 28.886200045235455,
1483
+ "status": "ok",
1484
+ "code": "TYPE"
1485
+ },
1486
+ {
1487
+ "sample_id": 98,
1488
+ "template": "nested",
1489
+ "success": true,
1490
+ "abstained": false,
1491
+ "expected_action": "O",
1492
+ "predicted_action": "O",
1493
+ "action_confidence": 1.0,
1494
+ "target_confidence": 1.0,
1495
+ "observation_ms": 19.75300000049174,
1496
+ "policy_ms": 1.8582999473437667,
1497
+ "execution_ms": 27.006000047549605,
1498
+ "status": "ok",
1499
+ "code": "SELECT"
1500
+ },
1501
+ {
1502
+ "sample_id": 99,
1503
+ "template": "table",
1504
+ "success": true,
1505
+ "abstained": true,
1506
+ "expected_action": "C",
1507
+ "predicted_action": "A",
1508
+ "action_confidence": 0.5633274912834167,
1509
+ "target_confidence": 1.0,
1510
+ "observation_ms": 26.3063000747934,
1511
+ "policy_ms": 2.529999939724803,
1512
+ "execution_ms": 4.016100079752505,
1513
+ "status": "ok",
1514
+ "code": "ASK_USER"
1515
+ },
1516
+ {
1517
+ "sample_id": 100,
1518
+ "template": "table",
1519
+ "success": false,
1520
+ "abstained": true,
1521
+ "expected_action": "C",
1522
+ "predicted_action": "A",
1523
+ "action_confidence": 0.9986655712127686,
1524
+ "target_confidence": 0.9999961853027344,
1525
+ "observation_ms": 24.024399928748608,
1526
+ "policy_ms": 2.307699993252754,
1527
+ "execution_ms": 3.3103000605478883,
1528
+ "status": "ok",
1529
+ "code": "ASK_USER"
1530
+ },
1531
+ {
1532
+ "sample_id": 101,
1533
+ "template": "table",
1534
+ "success": true,
1535
+ "abstained": false,
1536
+ "expected_action": "C",
1537
+ "predicted_action": "C",
1538
+ "action_confidence": 0.9980510473251343,
1539
+ "target_confidence": 1.0,
1540
+ "observation_ms": 20.079100038856268,
1541
+ "policy_ms": 1.4865999110043049,
1542
+ "execution_ms": 37.717900006100535,
1543
+ "status": "ok",
1544
+ "code": "CLICK"
1545
+ },
1546
+ {
1547
+ "sample_id": 102,
1548
+ "template": "table",
1549
+ "success": true,
1550
+ "abstained": false,
1551
+ "expected_action": "T",
1552
+ "predicted_action": "T",
1553
+ "action_confidence": 0.9999996423721313,
1554
+ "target_confidence": 1.0,
1555
+ "observation_ms": 23.466199985705316,
1556
+ "policy_ms": 2.661900012753904,
1557
+ "execution_ms": 25.023799971677363,
1558
+ "status": "ok",
1559
+ "code": "TYPE"
1560
+ },
1561
+ {
1562
+ "sample_id": 103,
1563
+ "template": "table",
1564
+ "success": true,
1565
+ "abstained": false,
1566
+ "expected_action": "T",
1567
+ "predicted_action": "T",
1568
+ "action_confidence": 1.0,
1569
+ "target_confidence": 1.0,
1570
+ "observation_ms": 19.584399997256696,
1571
+ "policy_ms": 2.478900016285479,
1572
+ "execution_ms": 24.177800049073994,
1573
+ "status": "ok",
1574
+ "code": "TYPE"
1575
+ },
1576
+ {
1577
+ "sample_id": 104,
1578
+ "template": "nested",
1579
+ "success": false,
1580
+ "abstained": false,
1581
+ "expected_action": "T",
1582
+ "predicted_action": "O",
1583
+ "action_confidence": 0.9814552664756775,
1584
+ "target_confidence": 0.9991796612739563,
1585
+ "observation_ms": 21.240900037810206,
1586
+ "policy_ms": 2.6496999198570848,
1587
+ "execution_ms": 15.518000000156462,
1588
+ "status": "rejected",
1589
+ "code": "SELECT_ROLE_MISMATCH"
1590
+ },
1591
+ {
1592
+ "sample_id": 105,
1593
+ "template": "table",
1594
+ "success": true,
1595
+ "abstained": false,
1596
+ "expected_action": "T",
1597
+ "predicted_action": "T",
1598
+ "action_confidence": 1.0,
1599
+ "target_confidence": 1.0,
1600
+ "observation_ms": 18.418700085021555,
1601
+ "policy_ms": 2.1063999738544226,
1602
+ "execution_ms": 24.387600016780198,
1603
+ "status": "ok",
1604
+ "code": "TYPE"
1605
+ },
1606
+ {
1607
+ "sample_id": 106,
1608
+ "template": "nested",
1609
+ "success": true,
1610
+ "abstained": false,
1611
+ "expected_action": "T",
1612
+ "predicted_action": "T",
1613
+ "action_confidence": 1.0,
1614
+ "target_confidence": 1.0,
1615
+ "observation_ms": 19.328000023961067,
1616
+ "policy_ms": 2.288699964992702,
1617
+ "execution_ms": 24.971900042146444,
1618
+ "status": "ok",
1619
+ "code": "TYPE"
1620
+ },
1621
+ {
1622
+ "sample_id": 107,
1623
+ "template": "nested",
1624
+ "success": true,
1625
+ "abstained": false,
1626
+ "expected_action": "O",
1627
+ "predicted_action": "O",
1628
+ "action_confidence": 1.0,
1629
+ "target_confidence": 0.9998756647109985,
1630
+ "observation_ms": 24.564399966038764,
1631
+ "policy_ms": 2.544500050134957,
1632
+ "execution_ms": 22.7174999890849,
1633
+ "status": "ok",
1634
+ "code": "SELECT"
1635
+ },
1636
+ {
1637
+ "sample_id": 108,
1638
+ "template": "nested",
1639
+ "success": false,
1640
+ "abstained": true,
1641
+ "expected_action": "C",
1642
+ "predicted_action": "A",
1643
+ "action_confidence": 0.9161800742149353,
1644
+ "target_confidence": 0.9998284578323364,
1645
+ "observation_ms": 22.466100053861737,
1646
+ "policy_ms": 2.624599961563945,
1647
+ "execution_ms": 3.638099995441735,
1648
+ "status": "ok",
1649
+ "code": "ASK_USER"
1650
+ },
1651
+ {
1652
+ "sample_id": 109,
1653
+ "template": "nested",
1654
+ "success": true,
1655
+ "abstained": false,
1656
+ "expected_action": "T",
1657
+ "predicted_action": "T",
1658
+ "action_confidence": 1.0,
1659
+ "target_confidence": 1.0,
1660
+ "observation_ms": 31.51220001745969,
1661
+ "policy_ms": 2.2133999736979604,
1662
+ "execution_ms": 27.044099988415837,
1663
+ "status": "ok",
1664
+ "code": "TYPE"
1665
+ },
1666
+ {
1667
+ "sample_id": 110,
1668
+ "template": "nested",
1669
+ "success": true,
1670
+ "abstained": false,
1671
+ "expected_action": "O",
1672
+ "predicted_action": "O",
1673
+ "action_confidence": 1.0,
1674
+ "target_confidence": 1.0,
1675
+ "observation_ms": 18.84640008211136,
1676
+ "policy_ms": 2.81360000371933,
1677
+ "execution_ms": 23.55599997099489,
1678
+ "status": "ok",
1679
+ "code": "SELECT"
1680
+ },
1681
+ {
1682
+ "sample_id": 111,
1683
+ "template": "nested",
1684
+ "success": true,
1685
+ "abstained": false,
1686
+ "expected_action": "O",
1687
+ "predicted_action": "O",
1688
+ "action_confidence": 1.0,
1689
+ "target_confidence": 1.0,
1690
+ "observation_ms": 23.418699973262846,
1691
+ "policy_ms": 2.1739000221714377,
1692
+ "execution_ms": 25.060700019821525,
1693
+ "status": "ok",
1694
+ "code": "SELECT"
1695
+ },
1696
+ {
1697
+ "sample_id": 112,
1698
+ "template": "table",
1699
+ "success": true,
1700
+ "abstained": false,
1701
+ "expected_action": "T",
1702
+ "predicted_action": "T",
1703
+ "action_confidence": 1.0,
1704
+ "target_confidence": 1.0,
1705
+ "observation_ms": 23.06119992863387,
1706
+ "policy_ms": 2.1838999819010496,
1707
+ "execution_ms": 27.73279999382794,
1708
+ "status": "ok",
1709
+ "code": "TYPE"
1710
+ },
1711
+ {
1712
+ "sample_id": 113,
1713
+ "template": "nested",
1714
+ "success": true,
1715
+ "abstained": false,
1716
+ "expected_action": "C",
1717
+ "predicted_action": "C",
1718
+ "action_confidence": 1.0,
1719
+ "target_confidence": 1.0,
1720
+ "observation_ms": 26.08530002180487,
1721
+ "policy_ms": 2.0496000070124865,
1722
+ "execution_ms": 52.54499998409301,
1723
+ "status": "ok",
1724
+ "code": "CLICK"
1725
+ },
1726
+ {
1727
+ "sample_id": 114,
1728
+ "template": "table",
1729
+ "success": true,
1730
+ "abstained": false,
1731
+ "expected_action": "O",
1732
+ "predicted_action": "O",
1733
+ "action_confidence": 1.0,
1734
+ "target_confidence": 1.0,
1735
+ "observation_ms": 21.78979991003871,
1736
+ "policy_ms": 2.9105000430718064,
1737
+ "execution_ms": 27.780500007793307,
1738
+ "status": "ok",
1739
+ "code": "SELECT"
1740
+ },
1741
+ {
1742
+ "sample_id": 115,
1743
+ "template": "nested",
1744
+ "success": true,
1745
+ "abstained": false,
1746
+ "expected_action": "T",
1747
+ "predicted_action": "T",
1748
+ "action_confidence": 0.9999645948410034,
1749
+ "target_confidence": 1.0,
1750
+ "observation_ms": 21.985100000165403,
1751
+ "policy_ms": 2.178400056436658,
1752
+ "execution_ms": 25.093400035984814,
1753
+ "status": "ok",
1754
+ "code": "TYPE"
1755
+ },
1756
+ {
1757
+ "sample_id": 116,
1758
+ "template": "table",
1759
+ "success": true,
1760
+ "abstained": false,
1761
+ "expected_action": "T",
1762
+ "predicted_action": "T",
1763
+ "action_confidence": 1.0,
1764
+ "target_confidence": 1.0,
1765
+ "observation_ms": 21.745900041423738,
1766
+ "policy_ms": 2.273899968713522,
1767
+ "execution_ms": 25.786099955439568,
1768
+ "status": "ok",
1769
+ "code": "TYPE"
1770
+ },
1771
+ {
1772
+ "sample_id": 117,
1773
+ "template": "nested",
1774
+ "success": true,
1775
+ "abstained": false,
1776
+ "expected_action": "C",
1777
+ "predicted_action": "C",
1778
+ "action_confidence": 1.0,
1779
+ "target_confidence": 1.0,
1780
+ "observation_ms": 22.26999995764345,
1781
+ "policy_ms": 2.4512000381946564,
1782
+ "execution_ms": 59.026199975050986,
1783
+ "status": "ok",
1784
+ "code": "CLICK"
1785
+ },
1786
+ {
1787
+ "sample_id": 118,
1788
+ "template": "table",
1789
+ "success": true,
1790
+ "abstained": false,
1791
+ "expected_action": "O",
1792
+ "predicted_action": "O",
1793
+ "action_confidence": 1.0,
1794
+ "target_confidence": 1.0,
1795
+ "observation_ms": 21.8062000349164,
1796
+ "policy_ms": 2.3586000315845013,
1797
+ "execution_ms": 23.766899947077036,
1798
+ "status": "ok",
1799
+ "code": "SELECT"
1800
+ },
1801
+ {
1802
+ "sample_id": 119,
1803
+ "template": "nested",
1804
+ "success": true,
1805
+ "abstained": false,
1806
+ "expected_action": "O",
1807
+ "predicted_action": "O",
1808
+ "action_confidence": 1.0,
1809
+ "target_confidence": 0.9999450445175171,
1810
+ "observation_ms": 25.91029996983707,
1811
+ "policy_ms": 2.724300022237003,
1812
+ "execution_ms": 26.5594000229612,
1813
+ "status": "ok",
1814
+ "code": "SELECT"
1815
+ }
1816
+ ]
1817
+ }
reports/cpu-v004.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "models/v004-contextual",
3
+ "quantization": "FP32",
4
+ "platform": "Windows-11-10.0.26200-SP0",
5
+ "torch_version": "2.14.0+cpu",
6
+ "threads": 2,
7
+ "load_ms": 11.063799960538745,
8
+ "disk_bytes": 532696,
9
+ "artifact_sha256": "8bff26cf3b8cb4c76f971620cb281159cfe37743d953eaf68f7eace5b1534f66",
10
+ "observed_process_rss_bytes": 281284608,
11
+ "memory_scope": "Observed Python RSS including training-library imports and evaluation tensors, not browser or exact peak",
12
+ "end_to_end_policy_ms": {
13
+ "median": 1.1031999601982534,
14
+ "p95": 1.6467999666929245
15
+ },
16
+ "neural_forward_ms": {
17
+ "median": 0.6952500552870333,
18
+ "p95": 1.0463000508025289
19
+ },
20
+ "feature_encoding_ms": {
21
+ "median": 0.4278999986127019,
22
+ "p95": 0.7296999683603644
23
+ },
24
+ "python_cpu_ms": {
25
+ "median": 0.0,
26
+ "p95": 15.625
27
+ },
28
+ "evaluation": {
29
+ "validation": {
30
+ "samples": 480,
31
+ "action_accuracy": 1.0,
32
+ "target_accuracy": 1.0,
33
+ "joint_step_accuracy": 1.0,
34
+ "action_ece": 2.777576446533203e-05,
35
+ "target_ece": 0.0003707166906679049,
36
+ "candidate_recall": 1.0
37
+ },
38
+ "test": {
39
+ "samples": 480,
40
+ "action_accuracy": 1.0,
41
+ "target_accuracy": 1.0,
42
+ "joint_step_accuracy": 1.0,
43
+ "action_ece": 0.0,
44
+ "target_ece": 0.0005633262626361102,
45
+ "candidate_recall": 1.0
46
+ },
47
+ "novel_wording": {
48
+ "samples": 480,
49
+ "action_accuracy": 0.8520833253860474,
50
+ "target_accuracy": 0.9458333253860474,
51
+ "joint_step_accuracy": 0.8395833373069763,
52
+ "action_ece": 0.13770944606221747,
53
+ "target_ece": 0.046679925406351686,
54
+ "candidate_recall": 1.0
55
+ }
56
+ },
57
+ "target_vps_validated": false,
58
+ "scope": "Synthetic single-step benchmark. Two torch threads, not two-vCPU CPU affinity or target EPYC."
59
+ }
reports/mind2web-v004.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "source": "osunlp/Mind2Web",
3
+ "revision": "6314166657eec4aa0e22c00f8d801e609ce8e80f",
4
+ "file": "train_10.json",
5
+ "source_sha256": "182542d7947b3fa9e90fc57a3d82d4d8f2997ca5a06664217720d7a78a956e33",
6
+ "license": "CC-BY-4.0",
7
+ "attribution": "Deng et al., Mind2Web: Towards a Generalist Agent for the Web, 2023, arXiv:2306.06070",
8
+ "checkpoint": "models/v004-contextual",
9
+ "tasks": 9,
10
+ "websites": 3,
11
+ "total_steps": 49,
12
+ "scorable_steps": 46,
13
+ "unscorable_steps": 3,
14
+ "operations": {
15
+ "C": 42,
16
+ "T": 3,
17
+ "O": 1
18
+ },
19
+ "candidate_recall": 0.47826087474823,
20
+ "action_accuracy": 0.021739130839705467,
21
+ "target_accuracy": 0.021739130839705467,
22
+ "joint_accuracy": 0.021739130839705467,
23
+ "browser_execution": false,
24
+ "training_on_source": false,
25
+ "limitations": "Small training-shard diagnostic, NOT official held-out benchmark. Approximate HTML names/roles; no history; 24-token goal truncation. No raw page or goal text persisted in report."
26
+ }
tests/test_model.py CHANGED
@@ -42,3 +42,17 @@ class ModelTests(unittest.TestCase):
42
  a,t = model(*x)
43
  self.assertTrue(torch.isfinite(a).all())
44
  self.assertTrue(torch.isfinite(t).all())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  a,t = model(*x)
43
  self.assertTrue(torch.isfinite(a).all())
44
  self.assertTrue(torch.isfinite(t).all())
45
+
46
+ def test_contextual_action_is_order_equivariant_and_trainable(self):
47
+ rows = list(generate('train',3,29))
48
+ vocab = fit_vocab(rows)
49
+ x,*_ = encode(rows,vocab)
50
+ model = PointerPolicy(vocab_size=len(vocab),contextual_action=True).eval()
51
+ a,t = model(*x)
52
+ permutation = torch.arange(x[1].shape[1]-1,-1,-1)
53
+ other_a,other_t = model(x[0],x[1][:,permutation],x[2][:,permutation],x[3][:,permutation])
54
+ torch.testing.assert_close(a,other_a)
55
+ torch.testing.assert_close(t[:,permutation],other_t)
56
+ a.sum().backward()
57
+ self.assertIsNotNone(model.pointer[0].weight.grad)
58
+ self.assertTrue(torch.isfinite(model.pointer[0].weight.grad).all())