diff --git a/ScreenExplorer-7B-E1/episode_0/recording_env_rank_0_348c7.h5 b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_0_348c7.h5 new file mode 100644 index 0000000000000000000000000000000000000000..ba5683b26db2ebefb0c77a5a0b343ceed351f473 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_0_348c7.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87af36313af5008dedb762ca61ae954bcdce535ab2920735593ef77e6c5829f4 +size 19151928 diff --git a/ScreenExplorer-7B-E1/episode_0/recording_env_rank_1_2cd0f.h5 b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_1_2cd0f.h5 new file mode 100644 index 0000000000000000000000000000000000000000..800e24ea88dfa1337a39059423a8f21927351c81 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_1_2cd0f.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b10174900f5785e089db014d8e408163593f146c23293804ec2ca65007a12fd4 +size 19599358 diff --git a/ScreenExplorer-7B-E1/episode_0/recording_env_rank_2_b3912.h5 b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_2_b3912.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8adc51a4bb0ce71d671a3f05fe1812b566227a43 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_2_b3912.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbe0ed21acdd341108690853ee01ed86d705ae58c03c3b2a5183b12daa90e842 +size 4633155 diff --git a/ScreenExplorer-7B-E1/episode_0/recording_env_rank_3_3be1d.h5 b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_3_3be1d.h5 new file mode 100644 index 0000000000000000000000000000000000000000..98f0cb68db623b7ee894eb54b8ee79e0316bb094 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_3_3be1d.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c7ef257d3ecd54104d1e39bda8077ed101dfc4fceeba38c807ddba08a271253 +size 11224316 diff --git a/ScreenExplorer-7B-E1/episode_0/recording_env_rank_4_8c05b.h5 b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_4_8c05b.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8d1e0c6c1121e439816bc10f99e287679e308264 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_4_8c05b.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2f90c3e511459fb96fbe8ea1df53761ecc147986a408ba439c72ba418b15fc6 +size 11957975 diff --git a/ScreenExplorer-7B-E1/episode_0/recording_env_rank_5_662f0.h5 b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_5_662f0.h5 new file mode 100644 index 0000000000000000000000000000000000000000..7cc7eae9667e2e07acdd94646906c48406f718cc --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_5_662f0.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32a97ef7ff27c6e64177f12482afe4c38971b04202f04adf562cea44f99f5387 +size 6201332 diff --git a/ScreenExplorer-7B-E1/episode_0/recording_env_rank_6_2e31e.h5 b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_6_2e31e.h5 new file mode 100644 index 0000000000000000000000000000000000000000..e98054520287fd31dad6579385bbffa288c47ccc --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_6_2e31e.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b80a7209dbf18633640c00e933fd4f68891dcd226088268e9ea5eb8b320facfa +size 6026329 diff --git a/ScreenExplorer-7B-E1/episode_0/recording_env_rank_7_e719f.h5 b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_7_e719f.h5 new file mode 100644 index 0000000000000000000000000000000000000000..b96a51d56e0be18cbb5a7acdc6f6dbc6373d5607 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_0/recording_env_rank_7_e719f.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:433685767e010b81cf2ca68e56a958e130ef2a766eeeea0fa162f0016691cb50 +size 7805366 diff --git a/ScreenExplorer-7B-E1/episode_100/actor_model_100/adapter_config.json b/ScreenExplorer-7B-E1/episode_100/actor_model_100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c138106acf79edf195f0e28248c173bb4c71f7 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/actor_model_100/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen2.5-VL-7B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "k_proj", + "v_proj", + "o_proj", + "q_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_100/recording_env_rank_0_b796d.h5 b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_0_b796d.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8f6c6a20c1569ed333a418f736e8157a9ff73f4c --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_0_b796d.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22823e6f17a4c1bfd1bb9ad0a5e903083d55b1cd3186eb73aa1d44d0945eb6b3 +size 12664953 diff --git a/ScreenExplorer-7B-E1/episode_100/recording_env_rank_1_2eb1a.h5 b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_1_2eb1a.h5 new file mode 100644 index 0000000000000000000000000000000000000000..5b934ee2e2e71f45a892e3b98fbb6c68eed96654 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_1_2eb1a.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4413e3e48777adc29272bf127b1453d1c83931114dcd40316bfb5fec847669d +size 3492518 diff --git a/ScreenExplorer-7B-E1/episode_100/recording_env_rank_2_6590c.h5 b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_2_6590c.h5 new file mode 100644 index 0000000000000000000000000000000000000000..2e178f16b459c1ec89fe8ff34ee3d159b8120467 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_2_6590c.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31347fabc565bcb726df5917630514d88f3ac879d368df3fd8ac03f439371699 +size 28174958 diff --git a/ScreenExplorer-7B-E1/episode_100/recording_env_rank_3_4c8b9.h5 b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_3_4c8b9.h5 new file mode 100644 index 0000000000000000000000000000000000000000..9f9f556db3b31440a910c6c4220622c274687a31 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_3_4c8b9.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b479904cea28643a7290a45d48de53a75b5973c9074064166bed094add4df95 +size 19235951 diff --git a/ScreenExplorer-7B-E1/episode_100/recording_env_rank_4_9a9da.h5 b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_4_9a9da.h5 new file mode 100644 index 0000000000000000000000000000000000000000..9e8c0f26fd0c1a522690c9d3608dd1b6046b8d3f --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_4_9a9da.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d0464561ab2a4c54d16226e732a59892ecfb0d6ffd48c8d2ab7cea99a31fec +size 12632833 diff --git a/ScreenExplorer-7B-E1/episode_100/recording_env_rank_5_8d425.h5 b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_5_8d425.h5 new file mode 100644 index 0000000000000000000000000000000000000000..de238d5ecc9ad3be1c113144d66ecd14cd3d8d39 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_5_8d425.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f586a2c26c7c157b152b06edb31e34bdb1892199b5af389f8f33ae593777dc16 +size 5791183 diff --git a/ScreenExplorer-7B-E1/episode_100/recording_env_rank_6_165d1.h5 b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_6_165d1.h5 new file mode 100644 index 0000000000000000000000000000000000000000..e223efa8ed554ac3c8236324aa7494058068d510 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_6_165d1.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad0f5164ef55c72a7cd75c0af4fd72a22083459eb3a64e38bd1303418dff03a3 +size 15725474 diff --git a/ScreenExplorer-7B-E1/episode_100/recording_env_rank_7_a6d3e.h5 b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_7_a6d3e.h5 new file mode 100644 index 0000000000000000000000000000000000000000..64c3430b4d126c9a2cfdcf28263bfa2e6899855f --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_100/recording_env_rank_7_a6d3e.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e981b659f068a571506393a2433d6a9dbbbf5db28b2260a02128406606cf49b +size 9602116 diff --git a/ScreenExplorer-7B-E1/episode_120/game_record_20250518_125823.json b/ScreenExplorer-7B-E1/episode_120/game_record_20250518_125823.json new file mode 100644 index 0000000000000000000000000000000000000000..f72154124ef37d1fa695fcfa913f65a37e4e0e46 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_120/game_record_20250518_125823.json @@ -0,0 +1,48186 @@ +{ + "start_time": "2025-05-18T12:58:23.056926", + "end_time": "2025-05-18T13:06:11.468011", + "buffer_size": 80, + "all_records": [ + { + "frame_id": "120_0_0", + "episode_id": 120, + "env_id": 0, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.278512001037598, + "advantages": 1.76486074924469, + "generated_text": "{\n \"intent\": \"To start browsing through available applications, clicking Firefox seems like an action that will set the environment dynamic.\",\n \"action\": \"DoubleClick(52, 41)\"\n}", + "action_string": "DoubleClick(52, 41)", + "intent_string": "To start browsing through available applications, clicking Firefox seems like an action that will set the environment dynamic.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C", + "boxes": [ + 74, + 69, + 124, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "120_1_0", + "episode_id": 120, + "env_id": 1, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.447542190551758, + "advantages": 2.0042507648468018, + "generated_text": "{\n \"intent\": \"To start by clicking on Firefox to see if there's an interesting or navigable content inside the browser.\",\n \"action\": \"DoubleClick(57, 33)\"\n}", + "action_string": "DoubleClick(57, 33)", + "intent_string": "To start by clicking on Firefox to see if there's an interesting or navigable content inside the browser.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C", + "boxes": [ + 54, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "120_2_0", + "episode_id": 120, + "env_id": 2, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.73668098449707, + "advantages": 2.4137449264526367, + "generated_text": "{\n \"intent\": \"To initiate the exploration by selecting an application from the desktop icons.\",\n \"action\": \"DoubleClick(50, 78)\"\n}", + "action_string": "DoubleClick(50, 78)", + "intent_string": "To initiate the exploration by selecting an application from the desktop icons.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "", + "boxes": [ + 56, + 68, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "120_3_0", + "episode_id": 120, + "env_id": 3, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.418434143066406, + "advantages": 1.9630262851715088, + "generated_text": "{\n \"intent\": \"My first step would be to open an application to see how it works or access files directly from the desktop.\",\n \"action\": \"DoubleClick(40, 38)\"\n}", + "action_string": "DoubleClick(40, 38)", + "intent_string": "My first step would be to open an application to see how it works or access files directly from the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C", + "boxes": [ + 54, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "120_4_0", + "episode_id": 120, + "env_id": 4, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.25889778137207, + "advantages": 1.7370820045471191, + "generated_text": "{\n \"intent\": \"Starting by clicking on an icon such as Firefox will allow access to internet capabilities which could provide further exploration about the system or even the environment itself.\",\n \"action\": \"DoubleClick(58, 37)\"\n}", + "action_string": "DoubleClick(58, 37)", + "intent_string": "Starting by clicking on an icon such as Firefox will allow access to internet capabilities which could provide further exploration about the system or even the environment itself.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "120_5_0", + "episode_id": 120, + "env_id": 5, + "step": 0, + "format_rewards": 1.0, + "rewards": 9.627967834472656, + "advantages": -0.5727306604385376, + "generated_text": "{\n \"intent\": \"I will click on one of the available applications on the desktop to explore its functionality.\",\n \"action\": \"DoubleClick(98, 710)\"\n}", + "action_string": "DoubleClick(98, 710)", + "intent_string": "I will click on one of the available applications on the desktop to explore its functionality.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + } + ] + }, + { + "frame_id": "120_6_0", + "episode_id": 120, + "env_id": 6, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.400667190551758, + "advantages": 1.9378637075424194, + "generated_text": "{\n \"intent\": \"The goal is to explore as much of the environment as possible within a limited number of steps. To start this exploration, clicking an application will open it, allowing access to more features and content. Firefox seems like a popular web browser and thus could provide interesting content.\",\n \"action\": \"DoubleClick(57, 40)\"\n}", + "action_string": "DoubleClick(57, 40)", + "intent_string": "The goal is to explore as much of the environment as possible within a limited number of steps. To start this exploration, clicking an application will open it, allowing access to more features and content. Firefox seems like a popular web browser and thus could provide interesting content.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C", + "boxes": [ + 70, + 69, + 124, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "120_7_0", + "episode_id": 120, + "env_id": 7, + "step": 0, + "format_rewards": 1.0, + "rewards": 9.630913734436035, + "advantages": -0.5685585141181946, + "generated_text": "{\n \"intent\": \"To start by exploring the available applications on the desktop, which could lead to various interesting functionalities or opened applications you might not have noticed initially.\",\n \"action\": \"DoubleClick(29, 152)\"\n}", + "action_string": "DoubleClick(29, 152)", + "intent_string": "To start by exploring the available applications on the desktop, which could lead to various interesting functionalities or opened applications you might not have noticed initially.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 34, + 129, + 78, + 155 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 3, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 6, + 385, + 118, + 423 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "120_0_1", + "episode_id": 120, + "env_id": 0, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.596433639526367, + "advantages": 0.7988638877868652, + "generated_text": "{\n \"intent\": \"The goal is to explore different applications and websites sequentially. Clicking on 'Getting Started' will likely provide an introduction to常用 tools or resources available in this environment.\",\n \"action\": \"DoubleClick(261, 112)\"\n}", + "action_string": "DoubleClick(261, 112)", + "intent_string": "The goal is to explore different applications and websites sequentially. Clicking on 'Getting Started' will likely provide an introduction to常用 tools or resources available in this environment.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 251, + 103, + 377, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "120_1_1", + "episode_id": 120, + "env_id": 1, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.758622169494629, + "advantages": 1.028564214706421, + "generated_text": "{\n \"intent\": \"The goal is to explore the computer desktop environment thoroughly. The current screen is a browser tab open to a new tab page, which already provides several useful options like accessing frequently visited sites or highlights. Clicking one of these icons will quickly lead us to the selected website, allowing us to gather information about it.\",\n \"action\": \"DoubleClick(1071, 695)\"\n}", + "action_string": "DoubleClick(1071, 695)", + "intent_string": "The goal is to explore the computer desktop environment thoroughly. The current screen is a browser tab open to a new tab page, which already provides several useful options like accessing frequently visited sites or highlights. Clicking one of these icons will quickly lead us to the selected website, allowing us to gather information about it.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→e", + "boxes": [ + 46, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 121, + 1474, + 135 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "You are on amazon.com.", + "boxes": [ + 510, + 461, + 662, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 530, + 1288, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1534, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1462, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1555, + 996, + 1647, + 1025 + ] + }, + { + "text": "af9b-4a25-b67a", + "boxes": [ + 1514, + 1020, + 1620, + 1038 + ] + } + ] + }, + { + "frame_id": "120_2_1", + "episode_id": 120, + "env_id": 2, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.56350040435791, + "advantages": 0.7522220611572266, + "generated_text": "{\n \"intent\": \"Opening an app icon like 'Getting Started' will help in accessing various resources and information related to the environment or the user's goals.\",\n \"action\": \"DoubleClick(275, 112)\"\n}", + "action_string": "DoubleClick(275, 112)", + "intent_string": "Opening an app icon like 'Getting Started' will help in accessing various resources and information related to the environment or the user's goals.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "NC白", + "boxes": [ + 46, + 66, + 130, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "D", + "boxes": [ + 260, + 69, + 274, + 85 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 258, + 105, + 376, + 149 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "120_3_1", + "episode_id": 120, + "env_id": 3, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.797456741333008, + "advantages": 1.0835639238357544, + "generated_text": "{\n \"intent\": \"To continue exploring different web pages or settings, clicking on one of the thumbnails under Highlights will provide new content.\",\n \"action\": \"DoubleClick(1078, 712)\"\n}", + "action_string": "DoubleClick(1078, 712)", + "intent_string": "To continue exploring different web pages or settings, clicking on one of the thumbnails under Highlights will provide new content.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 48, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 121, + 1474, + 135 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1336, + 486, + 1416, + 510 + ] + }, + { + "text": "", + "boxes": [ + 1446, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 982, + 530, + 1288, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1534, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1118, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1462, + 1024 + ] + }, + { + "text": "2", + "boxes": [ + 1553, + 994, + 1647, + 1027 + ] + } + ] + }, + { + "frame_id": "120_4_1", + "episode_id": 120, + "env_id": 4, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.468661308288574, + "advantages": 0.6179056763648987, + "generated_text": "{\"intent\": \"To explore different pages on the internet.\", \"action\": \"DoubleClick(735, 485)\"}", + "action_string": "DoubleClick(735, 485)", + "intent_string": "To explore different pages on the internet.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1336, + 486, + 1416, + 510 + ] + }, + { + "text": "", + "boxes": [ + 1446, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 530, + 1288, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1028 + ] + } + ] + }, + { + "frame_id": "120_5_1", + "episode_id": 120, + "env_id": 5, + "step": 1, + "format_rewards": 1.0, + "rewards": 11.783341407775879, + "advantages": 2.479827880859375, + "generated_text": "{\n \"intent\": \"To begin exploring the desktop environment by opening an application to provide functionality and familiarity.\",\n \"action\": \"DoubleClick(82, 710)\"\n}", + "action_string": "DoubleClick(82, 710)", + "intent_string": "To begin exploring the desktop environment by opening an application to provide functionality and familiarity.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 94, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "·口·目·目@自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "白·>、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "=[", + "boxes": [ + 148, + 125, + 260, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "46", + "boxes": [ + 16, + 943, + 28, + 955 + ] + }, + { + "text": "→→+", + "boxes": [ + 14, + 995, + 132, + 1017 + ] + } + ] + }, + { + "frame_id": "120_6_1", + "episode_id": 120, + "env_id": 6, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.40079116821289, + "advantages": 0.521784245967865, + "generated_text": "{\n \"intent\": \"I will start by clicking on one of the top sites or a highlight to begin navigating the web and explore various resources.\",\n \"action\": \"DoubleClick(766, 523)\"\n}", + "action_string": "DoubleClick(766, 523)", + "intent_string": "I will start by clicking on one of the top sites or a highlight to begin navigating the web and explore various resources.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "You are on ama", + "boxes": [ + 510, + 461, + 606, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1336, + 486, + 1416, + 510 + ] + }, + { + "text": "", + "boxes": [ + 1446, + 486, + 1542, + 510 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 606, + 520, + 836, + 564 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 530, + 1288, + 560 + ] + }, + { + "text": "Get your game on ", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1028 + ] + } + ] + }, + { + "frame_id": "120_7_1", + "episode_id": 120, + "env_id": 7, + "step": 1, + "format_rewards": 1.0, + "rewards": 11.484484672546387, + "advantages": 2.056570529937744, + "generated_text": "{\n \"intent\": \"To start exploring more applications by selecting one from the desktop.\",\n \"action\": \"DoubleClick(55, 163)\"\n}", + "action_string": "DoubleClick(55, 163)", + "intent_string": "To start exploring more applications by selecting one from the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 34, + 129, + 78, + 155 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 3, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit Search View Document Help", + "boxes": [ + 4, + 22, + 312, + 52 + ] + } + ] + }, + { + "frame_id": "120_0_2", + "episode_id": 120, + "env_id": 0, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.280435562133789, + "advantages": 0.351330041885376, + "generated_text": "{\n \"intent\": \"The current page is about Firefox browser, which suggests potential interest in exploring web browsers. Clicking 'Firefox browsers' might provide additional information or options related to Firefox. This action will allow me to gather more insights about Firefox's features and its latest version.\",\n \"action\": \"Click(844, 129)\"\n}", + "action_string": "Click(844, 129)", + "intent_string": "The current page is about Firefox browser, which suggests potential interest in exploring web browsers. Clicking 'Firefox browsers' might provide additional information or options related to Firefox. This action will allow me to gather more insights about Firefox's features and its latest version.", + "before_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 251, + 103, + 377, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1872, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 236, + 171, + 328, + 197 + ] + }, + { + "text": "Mobile", + "boxes": [ + 1340, + 171, + 1396, + 191 + ] + }, + { + "text": "More", + "boxes": [ + 1622, + 173, + 1670, + 191 + ] + }, + { + "text": " Firefox for Desktop", + "boxes": [ + 636, + 191, + 836, + 220 + ] + }, + { + "text": " Firefox for ios", + "boxes": [ + 636, + 228, + 794, + 258 + ] + }, + { + "text": " Firefox for Android", + "boxes": [ + 636, + 266, + 832, + 296 + ] + }, + { + "text": " Firefox Focus", + "boxes": [ + 634, + 302, + 786, + 332 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 400, + 316, + 608, + 351 + ] + }, + { + "text": " Firefox blog", + "boxes": [ + 640, + 341, + 772, + 371 + ] + }, + { + "text": "Take controI OI", + "boxes": [ + 338, + 387, + 686, + 445 + ] + }, + { + "text": "your internet", + "boxes": [ + 340, + 442, + 644, + 497 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 346, + 528, + 670, + 552 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 689, + 492, + 707 + ] + }, + { + "text": "?", + "boxes": [ + 474, + 1016, + 508, + 1038 + ] + } + ] + }, + { + "frame_id": "120_1_2", + "episode_id": 120, + "env_id": 1, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.405905723571777, + "advantages": 0.5290277600288391, + "generated_text": "{\n \"intent\": \"The goal here is to explore different parts of the Amazon website quick. Since Account & Lists dropdown contains options like 'Hello, sign in' and 'Returns & Orders', clicking on either could lead to further exploration specific to account management or browsing product returns, which are important features for users. Clicking 'Sign in' might also indicate user presence and interaction potential.\",\n \"action\": \"Click(1675, 133)\"\n}", + "action_string": "Click(1675, 133)", + "intent_string": "The goal here is to explore different parts of the Amazon website quick. Since Account & Lists dropdown contains options like 'Hello, sign in' and 'Returns & Orders', clicking on either could lead to further exploration specific to account management or browsing product returns, which are important features for users. Clicking 'Sign in' might also indicate user presence and interaction potential.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": " Signin", + "boxes": [ + 1622, + 175, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1226, + 488, + 1276, + 508 + ] + }, + { + "text": "1", + "boxes": [ + 1332, + 488, + 1364, + 508 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 528, + 1288, + 558 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": " See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1462, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1555, + 996, + 1647, + 1025 + ] + }, + { + "text": "=UTF8&k=", + "boxes": [ + 264, + 1020, + 334, + 1042 + ] + }, + { + "text": "af9b-4a25-b67a", + "boxes": [ + 1512, + 1020, + 1620, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 902, + 4, + 1020, + 22 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 806, + 434, + 930, + 471 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "120_2_2", + "episode_id": 120, + "env_id": 2, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.709403991699219, + "advantages": 0.9588587284088135, + "generated_text": "{\n \"intent\": \"The goal is to explore the available options on the browser page. Clicking on 'Download Firefox' will initiate downloading the browser, providing access to more internet exploration capabilities.\",\n \"action\": \"DoubleClick(426, 595)\"\n}", + "action_string": "DoubleClick(426, 595)", + "intent_string": "The goal is to explore the available options on the browser page. Clicking on 'Download Firefox' will initiate downloading the browser, providing access to more internet exploration capabilities.", + "before_screen_ocr_results": [ + { + "text": "D", + "boxes": [ + 260, + 69, + 274, + 85 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 258, + 105, + 376, + 149 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_3_2", + "episode_id": 120, + "env_id": 3, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.379045486450195, + "advantages": 0.4909868538379669, + "generated_text": "{\n \"intent\": \"The current page is on Amazon's Home page. To continue exploring the site, clicking on an element such as 'Shop for your home essentials' would provide more diverse products which might lead to further exploration. This action will likely show more specific categories related to home essentials.\",\n \"action\": \"DoubleClick(727, 642)\"\n}", + "action_string": "DoubleClick(727, 642)", + "intent_string": "The current page is on Amazon's Home page. To continue exploring the site, clicking on an element such as 'Shop for your home essentials' would provide more diverse products which might lead to further exploration. This action will likely show more specific categories related to home essentials.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": " Signin", + "boxes": [ + 1622, + 175, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1226, + 488, + 1276, + 508 + ] + }, + { + "text": "1", + "boxes": [ + 1332, + 488, + 1364, + 508 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 528, + 1288, + 558 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": " See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 967, + 1546, + 1003 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1358, + 1001, + 1462, + 1024 + ] + }, + { + "text": "2", + "boxes": [ + 1553, + 994, + 1647, + 1027 + ] + }, + { + "text": "=UTF8&k=", + "boxes": [ + 264, + 1020, + 334, + 1042 + ] + }, + { + "text": "20bb-4616-9468-71", + "boxes": [ + 1514, + 1020, + 1652, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 370, + 117, + 406, + 137 + ] + }, + { + "text": "EN", + "boxes": [ + 1400, + 125, + 1444, + 143 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三 AIl", + "boxes": [ + 22, + 167, + 68, + 195 + ] + }, + { + "text": "Gender", + "boxes": [ + 10, + 256, + 68, + 288 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 378, + 290 + ] + }, + { + "text": "women", + "boxes": [ + 9, + 275, + 85, + 309 + ] + }, + { + "text": "unisex", + "boxes": [ + 7, + 297, + 81, + 330 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1876, + 322 + ] + }, + { + "text": "35", + "boxes": [ + 18, + 425, + 52, + 447 + ] + }, + { + "text": "00P", + "boxes": [ + 18, + 494, + 52, + 516 + ] + }, + { + "text": "oP（", + "boxes": [ + 56, + 492, + 114, + 518 + ] + }, + { + "text": "2P", + "boxes": [ + 102, + 492, + 142, + 518 + ] + }, + { + "text": "4P", + "boxes": [ + 154, + 494, + 188, + 514 + ] + }, + { + "text": "6P", + "boxes": [ + 194, + 494, + 228, + 516 + ] + }, + { + "text": "8P", + "boxes": [ + 242, + 494, + 276, + 516 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "10P12P14P(", + "boxes": [ + 14, + 520, + 152, + 550 + ] + }, + { + "text": "16P", + "boxes": [ + 144, + 522, + 190, + 548 + ] + }, + { + "text": "neric Clothing Size", + "boxes": [ + 104, + 562, + 224, + 590 + ] + }, + { + "text": "1-2", + "boxes": [ + 180, + 596, + 212, + 615 + ] + }, + { + "text": "Womens Jeans High Waisted ", + "boxes": [ + 314, + 613, + 530, + 643 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit", + "boxes": [ + 608, + 611, + 858, + 641 + ] + }, + { + "text": " Sidefeel Women's Wide Leg Jeans", + "boxes": [ + 898, + 609, + 1152, + 643 + ] + }, + { + "text": "5-6", + "boxes": [ + 102, + 619, + 142, + 647 + ] + }, + { + "text": "7-8", + "boxes": [ + 180, + 621, + 216, + 649 + ] + }, + { + "text": "KDF Straight Leg Jeans for Women", + "boxes": [ + 1196, + 613, + 1452, + 643 + ] + }, + { + "text": "Lee Women's Legendary Mid Rise", + "boxes": [ + 1488, + 613, + 1736, + 643 + ] + }, + { + "text": "3-4", + "boxes": [ + 34, + 623, + 66, + 643 + ] + }, + { + "text": "Straight Leg Loose Stretchy", + "boxes": [ + 312, + 635, + 518, + 665 + ] + }, + { + "text": "Straight Leg Jea", + "boxes": [ + 606, + 635, + 746, + 665 + ] + }, + { + "text": "High Waisted Strechy Raw Hem", + "boxes": [ + 900, + 637, + 1138, + 667 + ] + }, + { + "text": "High Waisted Jeans Frayed Baggy", + "boxes": [ + 1194, + 637, + 1444, + 667 + ] + }, + { + "text": "Bootcut Jean", + "boxes": [ + 1490, + 639, + 1590, + 665 + ] + }, + { + "text": "9-10", + "boxes": [ + 24, + 653, + 70, + 679 + ] + }, + { + "text": "11-12", + "boxes": [ + 98, + 651, + 152, + 681 + ] + }, + { + "text": "13-14", + "boxes": [ + 170, + 653, + 226, + 679 + ] + }, + { + "text": "Lightweight Tummy Control Trendy", + "boxes": [ + 312, + 659, + 576, + 689 + ] + }, + { + "text": "★★★★★10.791", + "boxes": [ + 608, + 663, + 758, + 693 + ] + }, + { + "text": "Zimbaplatinum Denim Pants ", + "boxes": [ + 902, + 661, + 1116, + 691 + ] + }, + { + "text": "Pants Raw Hem Loose Stretchy", + "boxes": [ + 1194, + 661, + 1428, + 691 + ] + }, + { + "text": "★★★★☆5,246", + "boxes": [ + 1486, + 661, + 1632, + 695 + ] + }, + { + "text": "16", + "boxes": [ + 36, + 689, + 58, + 703 + ] + }, + { + "text": "Jeans for Women 2025", + "boxes": [ + 318, + 687, + 486, + 711 + ] + }, + { + "text": "★★★★☆1.,177", + "boxes": [ + 904, + 687, + 1046, + 715 + ] + }, + { + "text": " Cropped Wide Leg Jeans", + "boxes": [ + 1192, + 683, + 1380, + 717 + ] + }, + { + "text": "★★★★☆2,581", + "boxes": [ + 320, + 715, + 454, + 739 + ] + }, + { + "text": "$28% List: 542.90", + "boxes": [ + 604, + 712, + 746, + 755 + ] + }, + { + "text": "★★★★☆414", + "boxes": [ + 1196, + 713, + 1328, + 742 + ] + }, + { + "text": "$299 ist: $42.98", + "boxes": [ + 1484, + 712, + 1626, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "$3229 Typlcal: 539.99", + "boxes": [ + 900, + 738, + 1064, + 781 + ] + }, + { + "text": "500+ bought in past month", + "boxes": [ + 1198, + 735, + 1374, + 758 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 608, + 750, + 748, + 782 + ] + }, + { + "text": "$19*9 Typical: 37.99", + "boxes": [ + 307, + 760, + 475, + 807 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 1488, + 752, + 1636, + 782 + ] + }, + { + "text": "Petit", + "boxes": [ + 33, + 768, + 67, + 791 + ] + }, + { + "text": "$3888", + "boxes": [ + 1194, + 764, + 1260, + 798 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 900, + 776, + 1038, + 806 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 312, + 800, + 456, + 830 + ] + }, + { + "text": "Ships to Canad", + "boxes": [ + 906, + 804, + 984, + 822 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1192, + 800, + 1336, + 830 + ] + }, + { + "text": "JUNBAOSS Men's Slim Fit Stretch Jeans Ripped Skinny Jeans for Men,", + "boxes": [ + 1194, + 901, + 1764, + 931 + ] + }, + { + "text": " Distressed Straight Leg Fashion Comfort Flex Waist Pants", + "boxes": [ + 1190, + 923, + 1666, + 957 + ] + }, + { + "text": "★★★★☆6.792", + "boxes": [ + 1198, + 955, + 1334, + 979 + ] + }, + { + "text": "2", + "boxes": [ + 1820, + 969, + 1850, + 983 + ] + }, + { + "text": "$3099", + "boxes": [ + 1194, + 989, + 1254, + 1014 + ] + } + ] + }, + { + "frame_id": "120_4_2", + "episode_id": 120, + "env_id": 4, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.718118667602539, + "advantages": 0.9712009429931641, + "generated_text": "{\n \"intent\": \"The goal is to explore more of the Amazon product categories. Clicking 'See all' next to 'Shop deals in Fashion' will allow us to view a wider selection of fashion deals.\",\n \"action\": \"DoubleClick(653, 905)\"\n}", + "action_string": "DoubleClick(653, 905)", + "intent_string": "The goal is to explore more of the Amazon product categories. Clicking 'See all' next to 'Shop deals in Fashion' will allow us to view a wider selection of fashion deals.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 249, + 63, + 313, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "is of pre", + "boxes": [ + 974, + 461, + 1020, + 478 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1226, + 488, + 1276, + 508 + ] + }, + { + "text": "1", + "boxes": [ + 1328, + 485, + 1366, + 513 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 530, + 1288, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": " See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 910, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "oday's Deals", + "boxes": [ + 40, + 32, + 124, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 66, + 130, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " canarta", + "boxes": [ + 132, + 107, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": " Search Amazon", + "boxes": [ + 302, + 115, + 412, + 139 + ] + }, + { + "text": "", + "boxes": [ + 1696, + 115, + 1730, + 135 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 199, + 166, + 245, + 189 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Memorial Day Sale:", + "boxes": [ + 658, + 226, + 1124, + 294 + ] + }, + { + "text": "Up to 35% off", + "boxes": [ + 728, + 290, + 1056, + 351 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": " Shop deals by category", + "boxes": [ + 150, + 437, + 376, + 471 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "JBL", + "boxes": [ + 364, + 528, + 424, + 570 + ] + }, + { + "text": "Shop All", + "boxes": [ + 200, + 619, + 256, + 645 + ] + }, + { + "text": "Top 100", + "boxes": [ + 354, + 619, + 410, + 645 + ] + }, + { + "text": "Sports & Outdoors", + "boxes": [ + 623, + 613, + 745, + 646 + ] + }, + { + "text": "Fashior", + "boxes": [ + 816, + 621, + 860, + 641 + ] + }, + { + "text": "& Kitche", + "boxes": [ + 1134, + 621, + 1188, + 639 + ] + }, + { + "text": "Electronic", + "boxes": [ + 1264, + 621, + 1328, + 641 + ] + }, + { + "text": "Outlet Deal", + "boxes": [ + 1410, + 619, + 1486, + 643 + ] + }, + { + "text": "Today's featured deals", + "boxes": [ + 152, + 713, + 370, + 742 + ] + }, + { + "text": "15% ff", + "boxes": [ + 162, + 979, + 222, + 1005 + ] + }, + { + "text": "imited tlme deal", + "boxes": [ + 224, + 980, + 320, + 1003 + ] + }, + { + "text": "21% offEnds in 02:01:09", + "boxes": [ + 414, + 981, + 570, + 1005 + ] + }, + { + "text": "15% off", + "boxes": [ + 664, + 979, + 722, + 1005 + ] + }, + { + "text": " 49% off", + "boxes": [ + 916, + 979, + 976, + 1005 + ] + }, + { + "text": "Limited time d", + "boxes": [ + 976, + 980, + 1058, + 1003 + ] + }, + { + "text": " 20% off", + "boxes": [ + 1168, + 979, + 1228, + 1005 + ] + }, + { + "text": " 50% off", + "boxes": [ + 1420, + 979, + 1478, + 1005 + ] + }, + { + "text": "Limited time deal", + "boxes": [ + 1480, + 980, + 1578, + 1003 + ] + }, + { + "text": "$3612 Typical: $42:49 ", + "boxes": [ + 156, + 1007, + 290, + 1036 + ] + }, + { + "text": "$29995 ist:$5379.95", + "boxes": [ + 408, + 1007, + 540, + 1036 + ] + }, + { + "text": "$5865 Lit 369:06", + "boxes": [ + 660, + 1007, + 776, + 1038 + ] + }, + { + "text": "$1518 List: $30.00", + "boxes": [ + 910, + 1007, + 1028, + 1038 + ] + }, + { + "text": "$3599 Lit 44.99 ", + "boxes": [ + 1164, + 1007, + 1280, + 1038 + ] + }, + { + "text": "$4999 it $99.9", + "boxes": [ + 1416, + 1007, + 1530, + 1036 + ] + }, + { + "text": "ay's Dea", + "boxes": [ + 242, + 1050, + 300, + 1070 + ] + } + ] + }, + { + "frame_id": "120_5_2", + "episode_id": 120, + "env_id": 5, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.165824890136719, + "advantages": -1.2272429466247559, + "generated_text": "{\n \"intent\": \"This action will restart the process by returning to the desktop environment where we can explore other applications or files.\",\n \"action\": \"DoubleClick(7, 18)\"\n}", + "action_string": "DoubleClick(7, 18)", + "intent_string": "This action will restart the process by returning to the desktop environment where we can explore other applications or files.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "-·目- 自. ", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "勺-→、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa要目目-% 00t三图-口目", + "boxes": [ + 12, + 93, + 1206, + 123 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 258, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 16, + 943, + 28, + 955 + ] + }, + { + "text": "一→二", + "boxes": [ + 14, + 997, + 104, + 1014 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "画·口·目·目@%血-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "G-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "、", + "boxes": [ + 130, + 129, + 160, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 258, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 234, + 154, + 256, + 176 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "120_6_2", + "episode_id": 120, + "env_id": 6, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.8118314743042, + "advantages": 1.1039221286773682, + "generated_text": "{\n \"intent\": \"The goal here is to explore this environment as much as possible by interacting with various sections of the Amazon homepage. Clicking the 'Shop Books' section seems like a strategic choice as it could lead to a different set of products or features not immediately visible from the main page. This action is valuable because it increases the exploration depth within the Amazon environment.\",\n \"action\": \"DoubleClick(737, 300)\"\n}", + "action_string": "DoubleClick(737, 300)", + "intent_string": "The goal here is to explore this environment as much as possible by interacting with various sections of the Amazon homepage. Clicking the 'Shop Books' section seems like a strategic choice as it could lead to a different set of products or features not immediately visible from the main page. This action is valuable because it increases the exploration depth within the Amazon environment.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 249, + 63, + 313, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "ofpr", + "boxes": [ + 982, + 463, + 1018, + 476 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1226, + 488, + 1276, + 508 + ] + }, + { + "text": "1", + "boxes": [ + 1328, + 485, + 1366, + 513 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 606, + 520, + 836, + 564 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 530, + 1288, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": " See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 910, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "1.com : beauty ", + "boxes": [ + 82, + 30, + 182, + 54 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "d_rd_r=b3031735-6", + "boxes": [ + 1432, + 66, + 1566, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1578, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "beauty", + "boxes": [ + 270, + 113, + 358, + 145 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1618, + 117, + 1644, + 137 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 All", + "boxes": [ + 20, + 167, + 70, + 195 + ] + }, + { + "text": " Gift Cards", + "boxes": [ + 354, + 167, + 422, + 193 + ] + }, + { + "text": "1-48 of over 60,000 res", + "boxes": [ + 10, + 206, + 158, + 230 + ] + }, + { + "text": "ults for", + "boxes": [ + 158, + 210, + 204, + 228 + ] + }, + { + "text": "Explore Related Products", + "boxes": [ + 10, + 256, + 184, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 304, + 264, + 380, + 290 + ] + }, + { + "text": "laser hair removal", + "boxes": [ + 10, + 280, + 130, + 310 + ] + }, + { + "text": "Checke", + "boxes": [ + 312, + 294, + 352, + 310 + ] + }, + { + "text": " each product page for other buying optio", + "boxes": [ + 346, + 292, + 610, + 310 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1798, + 294, + 1878, + 326 + ] + }, + { + "text": "cerave", + "boxes": [ + 14, + 306, + 60, + 326 + ] + }, + { + "text": "pimple patches ", + "boxes": [ + 10, + 320, + 116, + 351 + ] + }, + { + "text": "beauty blender", + "boxes": [ + 14, + 341, + 112, + 367 + ] + }, + { + "text": "Biodance", + "boxes": [ + 1580, + 343, + 1644, + 363 + ] + }, + { + "text": "See", + "boxes": [ + 20, + 367, + 50, + 381 + ] + }, + { + "text": "BROALCO", + "boxes": [ + 1516, + 367, + 1544, + 391 + ] + }, + { + "text": "Biodance", + "boxes": [ + 1614, + 375, + 1676, + 395 + ] + }, + { + "text": "REAOLLAGE", + "boxes": [ + 1568, + 399, + 1612, + 419 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 411, + 134, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 361, + 436, + 535, + 481 + ] + }, + { + "text": "MIGHTY", + "boxes": [ + 1260, + 443, + 1388, + 480 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 10, + 469, + 134, + 498 + ] + }, + { + "text": "PATCH°", + "boxes": [ + 1258, + 472, + 1352, + 510 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 12, + 494, + 98, + 518 + ] + }, + { + "text": "TheOriginaol", + "boxes": [ + 1261, + 499, + 1321, + 531 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 322, + 510, + 524, + 540 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 516, + 102, + 540 + ] + }, + { + "text": "SINGTOWELETTES", + "boxes": [ + 420, + 534, + 510, + 552 + ] + }, + { + "text": "", + "boxes": [ + 1264, + 534, + 1334, + 556 + ] + }, + { + "text": "36", + "boxes": [ + 1354, + 526, + 1382, + 548 + ] + }, + { + "text": " Price", + "boxes": [ + 8, + 546, + 54, + 574 + ] + }, + { + "text": "50", + "boxes": [ + 326, + 550, + 358, + 580 + ] + }, + { + "text": "lant-BasedCompostableTowelettes", + "boxes": [ + 368, + 554, + 566, + 578 + ] + }, + { + "text": "$0 -$2,450+", + "boxes": [ + 10, + 572, + 102, + 598 + ] + }, + { + "text": "Neutrogena Makeup Remover Wipes,", + "boxes": [ + 314, + 590, + 588, + 619 + ] + }, + { + "text": "Clean Skin Club Clean Towels XLTM,", + "boxes": [ + 606, + 590, + 868, + 619 + ] + }, + { + "text": " 6Pcs 5inch Makeup Scoops 5 Color", + "boxes": [ + 898, + 586, + 1160, + 620 + ] + }, + { + "text": "Mighty PatchTM Original patch from ", + "boxes": [ + 1194, + 590, + 1460, + 619 + ] + }, + { + "text": " BIODANCE Bio-Collagen Real Deep ", + "boxes": [ + 1486, + 590, + 1752, + 619 + ] + }, + { + "text": "?", + "boxes": [ + 18, + 611, + 34, + 627 + ] + }, + { + "text": "Daily Facial Cleanser Towelettes,", + "boxes": [ + 312, + 609, + 558, + 643 + ] + }, + { + "text": "100% USDA Biobased Face Towel,", + "boxes": [ + 608, + 613, + 862, + 643 + ] + }, + { + "text": "Plastic Beauty Spoon Facial Cream", + "boxes": [ + 900, + 613, + 1160, + 643 + ] + }, + { + "text": "Hero Cosmetics - Hydrocolloid Acne ", + "boxes": [ + 1194, + 613, + 1464, + 643 + ] + }, + { + "text": " Mask, Hydrating Overnight Hydrogel", + "boxes": [ + 1484, + 611, + 1758, + 641 + ] + }, + { + "text": "Gently Cleanse and Remove Oil&.", + "boxes": [ + 312, + 635, + 574, + 665 + ] + }, + { + "text": "Disposable Face Towelete, Eczem...", + "boxes": [ + 606, + 637, + 872, + 667 + ] + }, + { + "text": "Brush Cosmetic Spatula Mask Mix...", + "boxes": [ + 900, + 637, + 1168, + 667 + ] + }, + { + "text": " Pimple Patch for Covering Zits and...", + "boxes": [ + 1192, + 637, + 1464, + 667 + ] + }, + { + "text": "Mask, Pore Minimizing, Elasticity...", + "boxes": [ + 1486, + 637, + 1744, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 8, + 653, + 102, + 685 + ] + }, + { + "text": "Wipes 25 Count (Pack of2)", + "boxes": [ + 316, + 663, + 518, + 699 + ] + }, + { + "text": "Towel 50 Count (Pack of 1)", + "boxes": [ + 608, + 663, + 810, + 699 + ] + }, + { + "text": "All Skin Type1 Count (Pack of 1)", + "boxes": [ + 902, + 663, + 1142, + 697 + ] + }, + { + "text": "36 Count (Pack of 1)", + "boxes": [ + 1196, + 661, + 1340, + 697 + ] + }, + { + "text": "4 Count (Pack of 1)", + "boxes": [ + 1488, + 661, + 1626, + 697 + ] + }, + { + "text": "Beauty& Personal Care", + "boxes": [ + 10, + 677, + 168, + 707 + ] + }, + { + "text": "#1 Top Rated", + "boxes": [ + 316, + 697, + 406, + 723 + ] + }, + { + "text": "★★★★★34.096", + "boxes": [ + 608, + 697, + 760, + 727 + ] + }, + { + "text": "$439 ($4.89/count)", + "boxes": [ + 895, + 700, + 1039, + 747 + ] + }, + { + "text": "Options: 3 sizes", + "boxes": [ + 1198, + 699, + 1286, + 725 + ] + }, + { + "text": "Options: 2 sizes", + "boxes": [ + 1490, + 699, + 1578, + 723 + ] + }, + { + "text": "Facial Skin Care Products", + "boxes": [ + 20, + 713, + 190, + 748 + ] + }, + { + "text": "★★★★★109.879", + "boxes": [ + 314, + 715, + 474, + 744 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 606, + 710, + 794, + 749 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "★★★★☆173,304", + "boxes": [ + 1196, + 723, + 1352, + 752 + ] + }, + { + "text": "★★★★★15,952", + "boxes": [ + 1488, + 721, + 1642, + 754 + ] + }, + { + "text": "Makeup Brush Sets & Kits", + "boxes": [ + 22, + 737, + 192, + 766 + ] + }, + { + "text": "$179 $0.36count)", + "boxes": [ + 601, + 744, + 759, + 791 + ] + }, + { + "text": "Delivery Sat, May 24", + "boxes": [ + 900, + 737, + 1042, + 766 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 1196, + 744, + 1384, + 774 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 1490, + 746, + 1676, + 770 + ] + }, + { + "text": "$97 60.20/cout Lst: 43569", + "boxes": [ + 305, + 762, + 527, + 817 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 908, + 764, + 988, + 782 + ] + }, + { + "text": " Face Makeup", + "boxes": [ + 20, + 778, + 110, + 804 + ] + }, + { + "text": "Only2left in stock-order soon", + "boxes": [ + 902, + 778, + 1106, + 808 + ] + }, + { + "text": "$129 ($.36/count)", + "boxes": [ + 1187, + 772, + 1347, + 817 + ] + }, + { + "text": "$190 ($4.75/count)", + "boxes": [ + 1483, + 772, + 1641, + 817 + ] + }, + { + "text": "Delivery Tue, May 20 ", + "boxes": [ + 606, + 784, + 750, + 816 + ] + }, + { + "text": "Movies & TV ", + "boxes": [ + 10, + 796, + 102, + 828 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 314, + 804, + 456, + 834 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1194, + 812, + 1334, + 840 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1484, + 808, + 1632, + 842 + ] + }, + { + "text": "Movies & Films", + "boxes": [ + 24, + 818, + 122, + 844 + ] + }, + { + "text": " 3 sustainabilityfeatures ", + "boxes": [ + 610, + 828, + 800, + 858 + ] + }, + { + "text": "Prime ", + "boxes": [ + 26, + 842, + 68, + 862 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1198, + 836, + 1286, + 860 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1490, + 836, + 1580, + 860 + ] + }, + { + "text": "FSA or HSA eligible", + "boxes": [ + 1198, + 858, + 1328, + 881 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 874, + 60, + 893 + ] + }, + { + "text": "FSA or HSA eligible", + "boxes": [ + 316, + 876, + 440, + 899 + ] + }, + { + "text": "e.lf.", + "boxes": [ + 8, + 889, + 66, + 921 + ] + }, + { + "text": " wet n wild", + "boxes": [ + 5, + 909, + 105, + 946 + ] + }, + { + "text": "conair", + "boxes": [ + 9, + 930, + 79, + 962 + ] + }, + { + "text": "VALUE4PACK", + "boxes": [ + 1244, + 923, + 1464, + 965 + ] + }, + { + "text": "cOVERGIRL", + "boxes": [ + 10, + 951, + 110, + 981 + ] + }, + { + "text": "secret", + "boxes": [ + 12, + 973, + 78, + 999 + ] + }, + { + "text": "rest", + "boxes": [ + 1328, + 985, + 1412, + 1011 + ] + }, + { + "text": "eos", + "boxes": [ + 1024, + 1003, + 1054, + 1022 + ] + }, + { + "text": "3DWHITE", + "boxes": [ + 1302, + 999, + 1406, + 1030 + ] + } + ] + }, + { + "frame_id": "120_7_2", + "episode_id": 120, + "env_id": 7, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.431715965270996, + "advantages": -0.8506733179092407, + "generated_text": "{\n \"intent\": \"The explorer wants to continue exploring the computer desktop environment. The most efficient way to do this would be to open an icon on the desktop, which requires a double-click. Currently, the only visible icon is the flextreme icon. Clicking this icon will allow the explorer to explore its content.\",\n \"action\": \"DoubleClick(41, 1046)\"\n}", + "action_string": "DoubleClick(41, 1046)", + "intent_string": "The explorer wants to continue exploring the computer desktop environment. The most efficient way to do this would be to open an icon on the desktop, which requires a double-click. Currently, the only visible icon is the flextreme icon. Clicking this icon will allow the explorer to explore its content.", + "before_screen_ocr_results": [ + { + "text": " Untitled - M..", + "boxes": [ + 139, + 1038, + 319, + 1080 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit Search View Document Help", + "boxes": [ + 4, + 22, + 312, + 52 + ] + }, + { + "text": " Untitled 1 - M.", + "boxes": [ + 54, + 1044, + 318, + 1074 + ] + } + ] + }, + { + "frame_id": "120_0_3", + "episode_id": 120, + "env_id": 0, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.582942008972168, + "advantages": -0.6364986896514893, + "generated_text": "{\n \"intent\": \"The user is interested in exploring different Firefox products. By clicking on 'Firefox browsers', they want to gather information about available browser options for download. This action helps in discovering more about the browser and its features.\",\n \"action\": \"DoubleClick(76, 210)\"\n}", + "action_string": "DoubleClick(76, 210)", + "intent_string": "The user is interested in exploring different Firefox products. By clicking on 'Firefox browsers', they want to gather information about available browser options for download. This action helps in discovering more about the browser and its features.", + "before_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1872, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 236, + 171, + 328, + 197 + ] + }, + { + "text": "Mobile", + "boxes": [ + 1340, + 171, + 1396, + 191 + ] + }, + { + "text": "More", + "boxes": [ + 1622, + 173, + 1670, + 191 + ] + }, + { + "text": " Firefox for Desktop", + "boxes": [ + 636, + 191, + 836, + 220 + ] + }, + { + "text": " Firefox for ios", + "boxes": [ + 636, + 228, + 794, + 258 + ] + }, + { + "text": " Firefox for Android", + "boxes": [ + 636, + 266, + 832, + 296 + ] + }, + { + "text": " Firefox Focus", + "boxes": [ + 634, + 302, + 786, + 332 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 400, + 316, + 608, + 351 + ] + }, + { + "text": " Firefox blog", + "boxes": [ + 640, + 341, + 772, + 371 + ] + }, + { + "text": "Take controI OI", + "boxes": [ + 338, + 387, + 686, + 445 + ] + }, + { + "text": "your internet", + "boxes": [ + 340, + 442, + 644, + 497 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 346, + 528, + 670, + 552 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 689, + 492, + 707 + ] + }, + { + "text": "?", + "boxes": [ + 474, + 1016, + 508, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 242, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1622, + 193 + ] + }, + { + "text": "More", + "boxes": [ + 1638, + 175, + 1672, + 189 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "120_1_3", + "episode_id": 120, + "env_id": 1, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.797186851501465, + "advantages": -0.33307334780693054, + "generated_text": "{\n \"intent\": \"Since I'm currently exploring a browser environment with an Amazon login page, clicking on the Home icon will help me return to the desktop view, which will allow me to browse files, folders, or application shortcuts more effectively.\",\n \"action\": \"DoubleClick(297, 47)\"\n}", + "action_string": "DoubleClick(297, 47)", + "intent_string": "Since I'm currently exploring a browser environment with an Amazon login page, clicking on the Home icon will help me return to the desktop view, which will allow me to browse files, folders, or application shortcuts more effectively.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 902, + 4, + 1020, + 22 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 806, + 434, + 930, + 471 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q earch with Bing or enter address", + "boxes": [ + 246, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 658, + 290, + 766, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "120_2_3", + "episode_id": 120, + "env_id": 2, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.181463241577148, + "advantages": 0.21116000413894653, + "generated_text": "{\n \"intent\": \"The current screen shows the download page for Firefox. The goal is to continue exploring the desktop environment by interacting with the available options. Clicking the 'Download for Linux' button will help explore the download process.\",\n \"action\": \"DoubleClick(960, 605)\"\n}", + "action_string": "DoubleClick(960, 605)", + "intent_string": "The current screen shows the download page for Firefox. The goal is to continue exploring the desktop environment by interacting with the available options. Clicking the 'Download for Linux' button will help explore the download process.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三四?", + "boxes": [ + 1788, + 66, + 1880, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "pn", + "boxes": [ + 1184, + 427, + 1238, + 463 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_3_3", + "episode_id": 120, + "env_id": 3, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.836775779724121, + "advantages": -0.2770053446292877, + "generated_text": "{\n \"intent\": \"Selecting an item from the Amazon search results allows me to view more details about the product such as reviews, specifications, and purchase options. This will help in making an informed decision if buying the jeans or further browsing the website.\",\n \"action\": \"DoubleClick(724, 528)\"\n}", + "action_string": "DoubleClick(724, 528)", + "intent_string": "Selecting an item from the Amazon search results allows me to view more details about the product such as reviews, specifications, and purchase options. This will help in making an informed decision if buying the jeans or further browsing the website.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 38, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 370, + 117, + 406, + 137 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三 AIl", + "boxes": [ + 22, + 167, + 68, + 195 + ] + }, + { + "text": "Gender", + "boxes": [ + 10, + 256, + 68, + 288 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 378, + 290 + ] + }, + { + "text": "women", + "boxes": [ + 9, + 275, + 85, + 309 + ] + }, + { + "text": "unisex", + "boxes": [ + 7, + 297, + 81, + 330 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1876, + 322 + ] + }, + { + "text": "35", + "boxes": [ + 18, + 425, + 52, + 447 + ] + }, + { + "text": "00P", + "boxes": [ + 18, + 494, + 52, + 516 + ] + }, + { + "text": "oP（", + "boxes": [ + 56, + 492, + 114, + 518 + ] + }, + { + "text": "2P", + "boxes": [ + 102, + 492, + 142, + 518 + ] + }, + { + "text": "4P", + "boxes": [ + 154, + 494, + 188, + 514 + ] + }, + { + "text": "6P", + "boxes": [ + 194, + 494, + 228, + 516 + ] + }, + { + "text": "8P", + "boxes": [ + 242, + 494, + 276, + 516 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "10P12P14P(", + "boxes": [ + 14, + 520, + 152, + 550 + ] + }, + { + "text": "16P", + "boxes": [ + 144, + 522, + 190, + 548 + ] + }, + { + "text": "neric Clothing Size", + "boxes": [ + 104, + 562, + 224, + 590 + ] + }, + { + "text": "1-2", + "boxes": [ + 180, + 596, + 212, + 615 + ] + }, + { + "text": "Womens Jeans High Waisted", + "boxes": [ + 314, + 613, + 530, + 643 + ] + }, + { + "text": "Lee Women's Pus Size Relaxed it", + "boxes": [ + 606, + 609, + 860, + 645 + ] + }, + { + "text": "Sidefeel Women's Wide Leg Jeans", + "boxes": [ + 898, + 609, + 1152, + 643 + ] + }, + { + "text": "5-6", + "boxes": [ + 102, + 619, + 142, + 647 + ] + }, + { + "text": "7-8", + "boxes": [ + 180, + 621, + 216, + 649 + ] + }, + { + "text": "KDF Straight Leg Jeans for Women", + "boxes": [ + 1196, + 613, + 1452, + 643 + ] + }, + { + "text": "Lee Women's Legendary Mid Rise", + "boxes": [ + 1488, + 613, + 1736, + 643 + ] + }, + { + "text": "3-4", + "boxes": [ + 34, + 623, + 66, + 643 + ] + }, + { + "text": "Straight Leg Loose Stretchy", + "boxes": [ + 312, + 635, + 518, + 665 + ] + }, + { + "text": "Straight Leg Jea", + "boxes": [ + 606, + 635, + 746, + 665 + ] + }, + { + "text": "High Waisted Strechy Raw Hem", + "boxes": [ + 900, + 637, + 1138, + 667 + ] + }, + { + "text": "High Waisted Jeans Frayed Baggy", + "boxes": [ + 1194, + 637, + 1444, + 667 + ] + }, + { + "text": "Bootcut Jean", + "boxes": [ + 1490, + 639, + 1590, + 665 + ] + }, + { + "text": "9-10", + "boxes": [ + 24, + 653, + 70, + 679 + ] + }, + { + "text": "11-12", + "boxes": [ + 98, + 651, + 152, + 681 + ] + }, + { + "text": "13-14", + "boxes": [ + 170, + 653, + 226, + 679 + ] + }, + { + "text": "Lightweight Tummy Control Trendy", + "boxes": [ + 314, + 661, + 578, + 691 + ] + }, + { + "text": "★★★★★10.791", + "boxes": [ + 608, + 663, + 758, + 693 + ] + }, + { + "text": "Zimbaplatinum Denim Pants ", + "boxes": [ + 902, + 661, + 1116, + 691 + ] + }, + { + "text": "Pants Raw Hem Loose Stretchy", + "boxes": [ + 1194, + 661, + 1428, + 691 + ] + }, + { + "text": "★★★★☆5,246", + "boxes": [ + 1486, + 661, + 1632, + 695 + ] + }, + { + "text": "16", + "boxes": [ + 36, + 689, + 58, + 703 + ] + }, + { + "text": "Jeans for Women 2025", + "boxes": [ + 318, + 687, + 486, + 711 + ] + }, + { + "text": "★★★★☆1.,177", + "boxes": [ + 904, + 687, + 1046, + 715 + ] + }, + { + "text": " Cropped Wide Leg Jeans", + "boxes": [ + 1192, + 683, + 1380, + 717 + ] + }, + { + "text": "Special Clothing Size", + "boxes": [ + 12, + 719, + 150, + 748 + ] + }, + { + "text": "★★★★☆2,581", + "boxes": [ + 320, + 715, + 454, + 739 + ] + }, + { + "text": "$28% List: 542.90", + "boxes": [ + 604, + 712, + 746, + 755 + ] + }, + { + "text": "★★★★☆414", + "boxes": [ + 1196, + 713, + 1328, + 742 + ] + }, + { + "text": "$299 ist: $42.98", + "boxes": [ + 1484, + 712, + 1626, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "$3229 Typcal: 39.99 ", + "boxes": [ + 898, + 738, + 1064, + 781 + ] + }, + { + "text": "500+ bought in past month", + "boxes": [ + 1198, + 735, + 1374, + 758 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 608, + 750, + 748, + 782 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 1488, + 752, + 1636, + 782 + ] + }, + { + "text": "Petit", + "boxes": [ + 33, + 768, + 67, + 791 + ] + }, + { + "text": "$199 Typical: 37.99", + "boxes": [ + 309, + 762, + 475, + 805 + ] + }, + { + "text": "$3888", + "boxes": [ + 1194, + 764, + 1260, + 798 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 900, + 776, + 1038, + 806 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 314, + 800, + 456, + 830 + ] + }, + { + "text": "Ships to Canad", + "boxes": [ + 906, + 804, + 984, + 822 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1192, + 800, + 1336, + 830 + ] + }, + { + "text": "ponsored ", + "boxes": [ + 1204, + 883, + 1270, + 903 + ] + }, + { + "text": "JUNBAOSS Men's Slim Fit Stretch Jeans Ripped Skinny Jeans for Men,", + "boxes": [ + 1194, + 901, + 1764, + 931 + ] + }, + { + "text": " Distressed Straight Leg Fashion Comfort Flex Waist Pants", + "boxes": [ + 1190, + 923, + 1666, + 957 + ] + }, + { + "text": "★★★★☆6.792", + "boxes": [ + 1198, + 955, + 1334, + 979 + ] + }, + { + "text": "2", + "boxes": [ + 1820, + 969, + 1850, + 983 + ] + }, + { + "text": "$3099", + "boxes": [ + 1194, + 989, + 1254, + 1014 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jean", + "boxes": [ + 652, + 286, + 1224, + 316 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings I Search this page", + "boxes": [ + 792, + 316, + 1020, + 339 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1462, + 312, + 1536, + 338 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Selet v", + "boxes": [ + 655, + 523, + 739, + 562 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": "Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spandex", + "boxes": [ + 826, + 639, + 1088, + 663 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise,straight leg, Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "· Rise: 12 inches, leg opening: 17 inches, inseam: petite: 29 inches, medum: 31, long: 33 inchs", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "·ProductMeasurements were takenusing size 2owmedlum.leasenote that Measurements mayvary", + "boxes": [ + 658, + 897, + 1316, + 929 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials xSofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ] + }, + { + "frame_id": "120_4_3", + "episode_id": 120, + "env_id": 4, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.073972702026367, + "advantages": 0.05892598628997803, + "generated_text": "{\n \"intent\": \"To explore the Amazon page further, the best action would be to click on an item or category to view more details.\",\n \"action\": \"DoubleClick(275, 0)\"\n}", + "action_string": "DoubleClick(275, 0)", + "intent_string": "To explore the Amazon page further, the best action would be to click on an item or category to view more details.", + "before_screen_ocr_results": [ + { + "text": "oday's Deals", + "boxes": [ + 40, + 32, + 124, + 50 + ] + }, + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " canarta", + "boxes": [ + 132, + 107, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": " Search Amazon", + "boxes": [ + 300, + 115, + 412, + 139 + ] + }, + { + "text": "", + "boxes": [ + 1696, + 115, + 1730, + 135 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 199, + 166, + 245, + 189 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Memorial Day Sale:", + "boxes": [ + 658, + 226, + 1124, + 294 + ] + }, + { + "text": "Up to 35% off", + "boxes": [ + 728, + 290, + 1056, + 351 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1798, + 294, + 1880, + 326 + ] + }, + { + "text": " Shop deals by category", + "boxes": [ + 150, + 437, + 376, + 471 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "JBL", + "boxes": [ + 364, + 528, + 424, + 570 + ] + }, + { + "text": "Shop All", + "boxes": [ + 200, + 619, + 256, + 645 + ] + }, + { + "text": "Top 100", + "boxes": [ + 354, + 619, + 410, + 645 + ] + }, + { + "text": "Sports & Outdoors", + "boxes": [ + 623, + 613, + 745, + 646 + ] + }, + { + "text": "Fashior", + "boxes": [ + 816, + 621, + 860, + 641 + ] + }, + { + "text": "& Kitche", + "boxes": [ + 1134, + 621, + 1188, + 639 + ] + }, + { + "text": "Electronic", + "boxes": [ + 1264, + 621, + 1328, + 641 + ] + }, + { + "text": "Outlet Deal", + "boxes": [ + 1410, + 619, + 1486, + 643 + ] + }, + { + "text": "Today's featured deals", + "boxes": [ + 152, + 713, + 370, + 742 + ] + }, + { + "text": "Liss", + "boxes": [ + 1016, + 862, + 1048, + 876 + ] + }, + { + "text": "15% ff", + "boxes": [ + 162, + 979, + 222, + 1005 + ] + }, + { + "text": "imited tlme deal", + "boxes": [ + 224, + 980, + 320, + 1003 + ] + }, + { + "text": "21% offEnds in 02:00:54", + "boxes": [ + 414, + 981, + 568, + 1005 + ] + }, + { + "text": "15% off", + "boxes": [ + 664, + 979, + 722, + 1005 + ] + }, + { + "text": "imited time", + "boxes": [ + 730, + 985, + 802, + 1003 + ] + }, + { + "text": " 49% off", + "boxes": [ + 916, + 979, + 976, + 1005 + ] + }, + { + "text": "Limited time d", + "boxes": [ + 976, + 980, + 1058, + 1003 + ] + }, + { + "text": " 20% off", + "boxes": [ + 1168, + 979, + 1228, + 1005 + ] + }, + { + "text": " 50% off", + "boxes": [ + 1420, + 979, + 1478, + 1005 + ] + }, + { + "text": "Llmited time deal", + "boxes": [ + 1479, + 976, + 1579, + 1007 + ] + }, + { + "text": "$3612 Typical: $42:49 ", + "boxes": [ + 156, + 1007, + 290, + 1036 + ] + }, + { + "text": "$29995 ist:$5379.95", + "boxes": [ + 408, + 1007, + 540, + 1036 + ] + }, + { + "text": "$5865 Lit 369:06", + "boxes": [ + 660, + 1007, + 776, + 1038 + ] + }, + { + "text": "$1518 List: $30.00", + "boxes": [ + 910, + 1007, + 1028, + 1038 + ] + }, + { + "text": "$3599 Lit 44.99 ", + "boxes": [ + 1164, + 1007, + 1280, + 1038 + ] + }, + { + "text": "$4999 it $99.9", + "boxes": [ + 1416, + 1007, + 1530, + 1036 + ] + }, + { + "text": "ay's Dea", + "boxes": [ + 242, + 1050, + 300, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Mozilla Firefox", + "boxes": [ + 966, + 42, + 1088, + 60 + ] + }, + { + "text": "Firefox", + "boxes": [ + 19, + 61, + 97, + 98 + ] + }, + { + "text": "a, Today's Deals", + "boxes": [ + 326, + 65, + 446, + 93 + ] + }, + { + "text": "→白", + "boxes": [ + 326, + 101, + 454, + 133 + ] + }, + { + "text": "ersion%2522%", + "boxes": [ + 1178, + 103, + 1278, + 127 + ] + }, + { + "text": "·☆", + "boxes": [ + 1284, + 101, + 1368, + 133 + ] + }, + { + "text": "三四?", + "boxes": [ + 1466, + 103, + 1558, + 129 + ] + }, + { + "text": "amazon", + "boxes": [ + 336, + 149, + 448, + 181 + ] + }, + { + "text": "Search Amazor", + "boxes": [ + 602, + 155, + 710, + 179 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1496, + 147, + 1550, + 167 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1484, + 163, + 1564, + 189 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 110, + 199 + ] + }, + { + "text": "Memorial Day Sale:", + "boxes": [ + 712, + 256, + 1070, + 310 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 284, + 102, + 308 + ] + }, + { + "text": "Up to 35% off", + "boxes": [ + 770, + 310, + 1016, + 353 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1480, + 336, + 1558, + 361 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": "18+", + "boxes": [ + 1472, + 385, + 1570, + 409 + ] + }, + { + "text": "Shop deals by category", + "boxes": [ + 332, + 425, + 562, + 461 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 16, + 498, + 92, + 530 + ] + }, + { + "text": "JBL", + "boxes": [ + 546, + 513, + 613, + 561 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1488, + 554, + 1550, + 580 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "Shop All", + "boxes": [ + 382, + 613, + 436, + 633 + ] + }, + { + "text": "Top100", + "boxes": [ + 538, + 613, + 590, + 633 + ] + }, + { + "text": "Sports & Outdoors", + "boxes": [ + 806, + 607, + 926, + 636 + ] + }, + { + "text": "國", + "boxes": [ + 38, + 681, + 80, + 719 + ] + }, + { + "text": " Today's featured deals", + "boxes": [ + 330, + 699, + 554, + 735 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1492, + 770, + 1546, + 796 + ] + }, + { + "text": "口", + "boxes": [ + 36, + 782, + 82, + 830 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 31, + 936, + 87, + 974 + ] + } + ] + }, + { + "frame_id": "120_5_3", + "episode_id": 120, + "env_id": 5, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.418879508972168, + "advantages": -0.8688530325889587, + "generated_text": "{\n \"intent\": \"Since the objective is to explore the desktop environment and this is an open application, clicking the 'File' menu at coordinates (11, 43) will allow us to view potential actions such as opening a new file, saving the current file, or quitting the application. This will help us understand the available options and functionalities.\",\n \"action\": \"Click(11, 43)\"\n}", + "action_string": "Click(11, 43)", + "intent_string": "Since the objective is to explore the desktop environment and this is an open application, clicking the 'File' menu at coordinates (11, 43) will allow us to view potential actions such as opening a new file, saving the current file, or quitting the application. This will help us understand the available options and functionalities.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "画·口·目·目@%血-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "G-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "、", + "boxes": [ + 130, + 129, + 160, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 258, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 234, + 154, + 256, + 176 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Edit", + "boxes": [ + 48, + 30, + 72, + 44 + ] + }, + { + "text": "Data", + "boxes": [ + 366, + 30, + 388, + 44 + ] + }, + { + "text": "晶自", + "boxes": [ + 292, + 54, + 370, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 378, + 54, + 444, + 85 + ] + }, + { + "text": "G-.", + "boxes": [ + 452, + 56, + 550, + 81 + ] + }, + { + "text": "v田·I·子I", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Open.", + "boxes": [ + 28, + 81, + 68, + 101 + ] + }, + { + "text": "Ctrl+C", + "boxes": [ + 252, + 79, + 290, + 99 + ] + }, + { + "text": "a", + "boxes": [ + 312, + 99, + 332, + 119 + ] + }, + { + "text": "a要日日%00三田二口", + "boxes": [ + 342, + 93, + 1204, + 123 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 26, + 191, + 84, + 216 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 270 + ] + }, + { + "text": "Reloao", + "boxes": [ + 32, + 254, + 72, + 270 + ] + }, + { + "text": "Save", + "boxes": [ + 26, + 312, + 60, + 332 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 296, + 359 + ] + }, + { + "text": "Save Remote", + "boxes": [ + 28, + 367, + 114, + 385 + ] + }, + { + "text": "Save a Copy...", + "boxes": [ + 24, + 387, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 21, + 444, + 83, + 477 + ] + }, + { + "text": "Export as PDF", + "boxes": [ + 26, + 476, + 118, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 68, + 528 + ] + }, + { + "text": "Preview in Web Brows", + "boxes": [ + 26, + 534, + 172, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 296, + 584 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 250, + 590, + 292, + 609 + ] + }, + { + "text": "Printer Settings.", + "boxes": [ + 26, + 619, + 132, + 637 + ] + }, + { + "text": " Properties.", + "boxes": [ + 24, + 649, + 100, + 673 + ] + }, + { + "text": "Digital :", + "boxes": [ + 26, + 679, + 78, + 697 + ] + }, + { + "text": " Exit LibreOffice", + "boxes": [ + 24, + 707, + 126, + 731 + ] + }, + { + "text": "Ctrl+(", + "boxes": [ + 254, + 711, + 288, + 725 + ] + }, + { + "text": "35", + "boxes": [ + 12, + 750, + 32, + 766 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "一→二", + "boxes": [ + 20, + 997, + 92, + 1014 + ] + } + ] + }, + { + "frame_id": "120_6_3", + "episode_id": 120, + "env_id": 6, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.724989891052246, + "advantages": -0.43532267212867737, + "generated_text": "{\n \"intent\": \"The user is currently browsing Amazon's beauty section. The goal is to explore different products and potentially make purchases. To do this, clicking on one of the product thumbnails will allow the user to view more details about that item.\",\n \"action\": \"DoubleClick(450, 664)\"\n}", + "action_string": "DoubleClick(450, 664)", + "intent_string": "The user is currently browsing Amazon's beauty section. The goal is to explore different products and potentially make purchases. To do this, clicking on one of the product thumbnails will allow the user to view more details about that item.", + "before_screen_ocr_results": [ + { + "text": "1.com : beauty ", + "boxes": [ + 82, + 30, + 182, + 54 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "d_rd_r=b3031735-6", + "boxes": [ + 1432, + 66, + 1566, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1578, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "beauty", + "boxes": [ + 270, + 113, + 358, + 145 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1618, + 117, + 1644, + 137 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 All", + "boxes": [ + 20, + 167, + 70, + 195 + ] + }, + { + "text": " Gift Cards", + "boxes": [ + 354, + 167, + 422, + 193 + ] + }, + { + "text": "1-48 of over 60,000 res", + "boxes": [ + 10, + 206, + 158, + 230 + ] + }, + { + "text": "ults for", + "boxes": [ + 158, + 210, + 204, + 228 + ] + }, + { + "text": "Explore Related Products", + "boxes": [ + 10, + 256, + 184, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 304, + 264, + 380, + 290 + ] + }, + { + "text": "laser hair removal", + "boxes": [ + 10, + 280, + 130, + 310 + ] + }, + { + "text": "Checke", + "boxes": [ + 312, + 294, + 352, + 310 + ] + }, + { + "text": " each product page for other buying optio", + "boxes": [ + 346, + 292, + 610, + 310 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1798, + 294, + 1878, + 326 + ] + }, + { + "text": "cerave", + "boxes": [ + 14, + 306, + 60, + 326 + ] + }, + { + "text": "pimple patches ", + "boxes": [ + 10, + 320, + 116, + 351 + ] + }, + { + "text": "beauty blender", + "boxes": [ + 14, + 341, + 112, + 367 + ] + }, + { + "text": "Biodance", + "boxes": [ + 1580, + 343, + 1644, + 363 + ] + }, + { + "text": "See", + "boxes": [ + 20, + 367, + 50, + 381 + ] + }, + { + "text": "BROALCO", + "boxes": [ + 1516, + 367, + 1544, + 391 + ] + }, + { + "text": "Biodance", + "boxes": [ + 1614, + 375, + 1676, + 395 + ] + }, + { + "text": "REAOLLAGE", + "boxes": [ + 1568, + 399, + 1612, + 419 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 411, + 134, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 361, + 436, + 535, + 481 + ] + }, + { + "text": "MIGHTY", + "boxes": [ + 1260, + 443, + 1388, + 480 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 10, + 469, + 134, + 498 + ] + }, + { + "text": "PATCH°", + "boxes": [ + 1258, + 472, + 1352, + 510 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 12, + 494, + 98, + 518 + ] + }, + { + "text": "TheOriginaol", + "boxes": [ + 1261, + 499, + 1321, + 531 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 322, + 510, + 524, + 540 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 516, + 102, + 540 + ] + }, + { + "text": "SINGTOWELETTES", + "boxes": [ + 420, + 534, + 510, + 552 + ] + }, + { + "text": "", + "boxes": [ + 1264, + 534, + 1334, + 556 + ] + }, + { + "text": "36", + "boxes": [ + 1354, + 526, + 1382, + 548 + ] + }, + { + "text": " Price", + "boxes": [ + 8, + 546, + 54, + 574 + ] + }, + { + "text": "50", + "boxes": [ + 326, + 550, + 358, + 580 + ] + }, + { + "text": "lant-BasedCompostableTowelettes", + "boxes": [ + 368, + 554, + 566, + 578 + ] + }, + { + "text": "$0 -$2,450+", + "boxes": [ + 10, + 572, + 102, + 598 + ] + }, + { + "text": "Neutrogena Makeup Remover Wipes,", + "boxes": [ + 314, + 590, + 588, + 619 + ] + }, + { + "text": "Clean Skin Club Clean Towels XLTM,", + "boxes": [ + 606, + 590, + 868, + 619 + ] + }, + { + "text": " 6Pcs 5inch Makeup Scoops 5 Color", + "boxes": [ + 898, + 586, + 1160, + 620 + ] + }, + { + "text": "Mighty PatchTM Original patch from ", + "boxes": [ + 1194, + 590, + 1460, + 619 + ] + }, + { + "text": " BIODANCE Bio-Collagen Real Deep ", + "boxes": [ + 1486, + 590, + 1752, + 619 + ] + }, + { + "text": "?", + "boxes": [ + 18, + 611, + 34, + 627 + ] + }, + { + "text": "Daily Facial Cleanser Towelettes,", + "boxes": [ + 312, + 609, + 558, + 643 + ] + }, + { + "text": "100% USDA Biobased Face Towel,", + "boxes": [ + 608, + 613, + 862, + 643 + ] + }, + { + "text": "Plastic Beauty Spoon Facial Cream", + "boxes": [ + 900, + 613, + 1160, + 643 + ] + }, + { + "text": "Hero Cosmetics - Hydrocolloid Acne ", + "boxes": [ + 1194, + 613, + 1464, + 643 + ] + }, + { + "text": " Mask, Hydrating Overnight Hydrogel", + "boxes": [ + 1484, + 611, + 1758, + 641 + ] + }, + { + "text": "Gently Cleanse and Remove Oil&.", + "boxes": [ + 312, + 635, + 574, + 665 + ] + }, + { + "text": "Disposable Face Towelete, Eczem...", + "boxes": [ + 606, + 637, + 872, + 667 + ] + }, + { + "text": "Brush Cosmetic Spatula Mask Mix...", + "boxes": [ + 900, + 637, + 1168, + 667 + ] + }, + { + "text": " Pimple Patch for Covering Zits and...", + "boxes": [ + 1192, + 637, + 1464, + 667 + ] + }, + { + "text": "Mask, Pore Minimizing, Elasticity...", + "boxes": [ + 1486, + 637, + 1744, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 8, + 653, + 102, + 685 + ] + }, + { + "text": "Wipes 25 Count (Pack of2)", + "boxes": [ + 316, + 663, + 518, + 699 + ] + }, + { + "text": "Towel 50 Count (Pack of 1)", + "boxes": [ + 608, + 663, + 810, + 699 + ] + }, + { + "text": "All Skin Type1 Count (Pack of 1)", + "boxes": [ + 902, + 663, + 1142, + 697 + ] + }, + { + "text": "36 Count (Pack of 1)", + "boxes": [ + 1196, + 661, + 1340, + 697 + ] + }, + { + "text": "4 Count (Pack of 1)", + "boxes": [ + 1488, + 661, + 1626, + 697 + ] + }, + { + "text": "Beauty& Personal Care", + "boxes": [ + 10, + 677, + 168, + 707 + ] + }, + { + "text": "#1 Top Rated", + "boxes": [ + 316, + 697, + 406, + 723 + ] + }, + { + "text": "★★★★★34.096", + "boxes": [ + 608, + 697, + 760, + 727 + ] + }, + { + "text": "$439 ($4.89/count)", + "boxes": [ + 895, + 700, + 1039, + 747 + ] + }, + { + "text": "Options: 3 sizes", + "boxes": [ + 1198, + 699, + 1286, + 725 + ] + }, + { + "text": "Options: 2 sizes", + "boxes": [ + 1490, + 699, + 1578, + 723 + ] + }, + { + "text": "Facial Skin Care Products", + "boxes": [ + 20, + 713, + 190, + 748 + ] + }, + { + "text": "★★★★★109.879", + "boxes": [ + 314, + 715, + 474, + 744 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 606, + 710, + 794, + 749 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "★★★★☆173,304", + "boxes": [ + 1196, + 723, + 1352, + 752 + ] + }, + { + "text": "★★★★★15,952", + "boxes": [ + 1488, + 721, + 1642, + 754 + ] + }, + { + "text": "Makeup Brush Sets & Kits", + "boxes": [ + 22, + 737, + 192, + 766 + ] + }, + { + "text": "$179 $0.36count)", + "boxes": [ + 601, + 744, + 759, + 791 + ] + }, + { + "text": "Delivery Sat, May 24", + "boxes": [ + 900, + 737, + 1042, + 766 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 1196, + 744, + 1384, + 774 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 1490, + 746, + 1676, + 770 + ] + }, + { + "text": "$97 60.20/cout Lst: 43569", + "boxes": [ + 305, + 762, + 527, + 817 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 908, + 764, + 988, + 782 + ] + }, + { + "text": " Face Makeup", + "boxes": [ + 20, + 778, + 110, + 804 + ] + }, + { + "text": "Only2left in stock-order soon", + "boxes": [ + 902, + 778, + 1106, + 808 + ] + }, + { + "text": "$129 ($.36/count)", + "boxes": [ + 1187, + 772, + 1347, + 817 + ] + }, + { + "text": "$190 ($4.75/count)", + "boxes": [ + 1483, + 772, + 1641, + 817 + ] + }, + { + "text": "Delivery Tue, May 20 ", + "boxes": [ + 606, + 784, + 750, + 816 + ] + }, + { + "text": "Movies & TV ", + "boxes": [ + 10, + 796, + 102, + 828 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 314, + 804, + 456, + 834 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1194, + 812, + 1334, + 840 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1484, + 808, + 1632, + 842 + ] + }, + { + "text": "Movies & Films", + "boxes": [ + 24, + 818, + 122, + 844 + ] + }, + { + "text": " 3 sustainabilityfeatures ", + "boxes": [ + 610, + 828, + 800, + 858 + ] + }, + { + "text": "Prime ", + "boxes": [ + 26, + 842, + 68, + 862 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1198, + 836, + 1286, + 860 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1490, + 836, + 1580, + 860 + ] + }, + { + "text": "FSA or HSA eligible", + "boxes": [ + 1198, + 858, + 1328, + 881 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 874, + 60, + 893 + ] + }, + { + "text": "FSA or HSA eligible", + "boxes": [ + 316, + 876, + 440, + 899 + ] + }, + { + "text": "e.lf.", + "boxes": [ + 8, + 889, + 66, + 921 + ] + }, + { + "text": " wet n wild", + "boxes": [ + 5, + 909, + 105, + 946 + ] + }, + { + "text": "conair", + "boxes": [ + 9, + 930, + 79, + 962 + ] + }, + { + "text": "VALUE4PACK", + "boxes": [ + 1244, + 923, + 1464, + 965 + ] + }, + { + "text": "cOVERGIRL", + "boxes": [ + 10, + 951, + 110, + 981 + ] + }, + { + "text": "secret", + "boxes": [ + 12, + 973, + 78, + 999 + ] + }, + { + "text": "rest", + "boxes": [ + 1328, + 985, + 1412, + 1011 + ] + }, + { + "text": "eos", + "boxes": [ + 1024, + 1003, + 1054, + 1022 + ] + }, + { + "text": "3DWHITE", + "boxes": [ + 1302, + 999, + 1406, + 1030 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "1.com : beauty ", + "boxes": [ + 82, + 30, + 182, + 54 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "d_rd_r=b3031735-6", + "boxes": [ + 1432, + 66, + 1568, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1576, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "beauty", + "boxes": [ + 270, + 113, + 360, + 145 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1618, + 117, + 1644, + 137 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 All", + "boxes": [ + 20, + 167, + 70, + 195 + ] + }, + { + "text": " Gift Cards", + "boxes": [ + 354, + 167, + 424, + 193 + ] + }, + { + "text": "1-48 of over 60,000 res", + "boxes": [ + 10, + 206, + 158, + 230 + ] + }, + { + "text": "ults for", + "boxes": [ + 158, + 210, + 204, + 228 + ] + }, + { + "text": "Explore Related Products", + "boxes": [ + 10, + 256, + 184, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 304, + 264, + 380, + 290 + ] + }, + { + "text": "laser hair removal", + "boxes": [ + 10, + 280, + 130, + 310 + ] + }, + { + "text": "Checke", + "boxes": [ + 312, + 294, + 352, + 310 + ] + }, + { + "text": " each product page for other buying optio", + "boxes": [ + 346, + 292, + 610, + 310 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1798, + 294, + 1878, + 326 + ] + }, + { + "text": "cerave", + "boxes": [ + 14, + 306, + 60, + 326 + ] + }, + { + "text": "pimple patches ", + "boxes": [ + 10, + 320, + 116, + 351 + ] + }, + { + "text": "beauty blender", + "boxes": [ + 14, + 341, + 112, + 367 + ] + }, + { + "text": "Biodance", + "boxes": [ + 1580, + 343, + 1644, + 363 + ] + }, + { + "text": "See", + "boxes": [ + 20, + 367, + 50, + 381 + ] + }, + { + "text": "BROALCO", + "boxes": [ + 1516, + 367, + 1544, + 391 + ] + }, + { + "text": "Biodance", + "boxes": [ + 1614, + 375, + 1676, + 395 + ] + }, + { + "text": "REAOLLAGE", + "boxes": [ + 1568, + 399, + 1612, + 419 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 411, + 134, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 361, + 436, + 535, + 481 + ] + }, + { + "text": "MIGHTY", + "boxes": [ + 1260, + 443, + 1388, + 480 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 10, + 469, + 136, + 498 + ] + }, + { + "text": "PATCH°", + "boxes": [ + 1257, + 468, + 1351, + 511 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 12, + 494, + 98, + 518 + ] + }, + { + "text": "TheOriginaol", + "boxes": [ + 1261, + 499, + 1321, + 531 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 322, + 510, + 524, + 540 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 516, + 102, + 540 + ] + }, + { + "text": "SOFTCLEAN", + "boxes": [ + 362, + 534, + 420, + 552 + ] + }, + { + "text": "NSINGTOWELETTES", + "boxes": [ + 414, + 534, + 510, + 552 + ] + }, + { + "text": "", + "boxes": [ + 1264, + 534, + 1334, + 556 + ] + }, + { + "text": "36", + "boxes": [ + 1354, + 526, + 1382, + 548 + ] + }, + { + "text": "Price", + "boxes": [ + 8, + 548, + 54, + 574 + ] + }, + { + "text": "50", + "boxes": [ + 326, + 548, + 358, + 580 + ] + }, + { + "text": "Plant-BasedCompostableTowelettes", + "boxes": [ + 366, + 554, + 566, + 578 + ] + }, + { + "text": "口", + "boxes": [ + 1796, + 556, + 1810, + 572 + ] + }, + { + "text": "$0 -$2,450+", + "boxes": [ + 10, + 572, + 102, + 598 + ] + }, + { + "text": "Neutrogena Makeup Remover Wipes,", + "boxes": [ + 314, + 590, + 588, + 619 + ] + }, + { + "text": "Clean Skin Club Clean Towels XLTM,", + "boxes": [ + 606, + 590, + 868, + 619 + ] + }, + { + "text": " 6Pcs 5inch Makeup Scoops 5 Color", + "boxes": [ + 898, + 586, + 1160, + 620 + ] + }, + { + "text": "Mighty PatchTM Original patch from ", + "boxes": [ + 1194, + 590, + 1460, + 619 + ] + }, + { + "text": " BIODANCE Bio-Collagen Real Deep ", + "boxes": [ + 1486, + 590, + 1752, + 619 + ] + }, + { + "text": "?", + "boxes": [ + 18, + 611, + 34, + 627 + ] + }, + { + "text": "Daily Facial Cleanser Towelettes,", + "boxes": [ + 312, + 609, + 556, + 643 + ] + }, + { + "text": "100% USDA Biobased Face Towel, ", + "boxes": [ + 608, + 613, + 862, + 643 + ] + }, + { + "text": "Plastic Beauty Spoon Facial Cream", + "boxes": [ + 900, + 613, + 1160, + 643 + ] + }, + { + "text": "Hero Cosmetics - Hydrocolloid Acne ", + "boxes": [ + 1194, + 613, + 1464, + 643 + ] + }, + { + "text": " Mask, Hydrating Overnight Hydrogel", + "boxes": [ + 1484, + 611, + 1758, + 641 + ] + }, + { + "text": "Gently Cleanse and Remove Oil&..", + "boxes": [ + 312, + 635, + 572, + 665 + ] + }, + { + "text": "Disposable Face Towelette, Eczem..", + "boxes": [ + 606, + 637, + 872, + 667 + ] + }, + { + "text": "Brush Cosmetic Spatula Mask Mixi..", + "boxes": [ + 900, + 637, + 1168, + 667 + ] + }, + { + "text": " Pimple Patch for Covering Zits and...", + "boxes": [ + 1192, + 637, + 1464, + 667 + ] + }, + { + "text": "Mask, Pore Minimizing, Elasticity...", + "boxes": [ + 1486, + 637, + 1744, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 8, + 653, + 102, + 685 + ] + }, + { + "text": "Wipes 25 Count (Pakof 2)", + "boxes": [ + 314, + 661, + 522, + 701 + ] + }, + { + "text": "Towel 50 Count (Pack of 1)", + "boxes": [ + 608, + 663, + 810, + 699 + ] + }, + { + "text": "AllSkin Type1 Count (Pack of 1)", + "boxes": [ + 902, + 663, + 1142, + 697 + ] + }, + { + "text": "36 Count (Pack of 1)", + "boxes": [ + 1196, + 661, + 1340, + 697 + ] + }, + { + "text": "4 Count (Pack of 1)", + "boxes": [ + 1488, + 661, + 1626, + 697 + ] + }, + { + "text": "Beauty& Personal Care", + "boxes": [ + 10, + 677, + 168, + 707 + ] + }, + { + "text": "#1 Top Rated", + "boxes": [ + 316, + 697, + 406, + 723 + ] + }, + { + "text": "★★★★★34.096", + "boxes": [ + 608, + 697, + 760, + 727 + ] + }, + { + "text": "$439 ($4.89/count)", + "boxes": [ + 895, + 700, + 1039, + 747 + ] + }, + { + "text": "Options: 3 sizes", + "boxes": [ + 1198, + 699, + 1286, + 725 + ] + }, + { + "text": "Options: 2 sizes", + "boxes": [ + 1490, + 699, + 1578, + 723 + ] + }, + { + "text": "Facial Skin Care Products", + "boxes": [ + 20, + 713, + 190, + 748 + ] + }, + { + "text": "★★★★★109.879", + "boxes": [ + 314, + 715, + 472, + 744 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 606, + 710, + 794, + 749 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "★★★★☆173,304", + "boxes": [ + 1196, + 723, + 1352, + 752 + ] + }, + { + "text": "★★★★★15,952", + "boxes": [ + 1488, + 721, + 1642, + 754 + ] + }, + { + "text": "Makeup Brush Sets & Kits", + "boxes": [ + 22, + 737, + 192, + 766 + ] + }, + { + "text": "onth", + "boxes": [ + 470, + 742, + 498, + 756 + ] + }, + { + "text": "$179 $0.36count)", + "boxes": [ + 601, + 744, + 759, + 791 + ] + }, + { + "text": "Delivery Sat, May 24", + "boxes": [ + 900, + 737, + 1042, + 766 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 1196, + 744, + 1384, + 774 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 1488, + 744, + 1674, + 768 + ] + }, + { + "text": "$97 60.20/cout Lst: 43569", + "boxes": [ + 305, + 762, + 527, + 817 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 908, + 764, + 988, + 782 + ] + }, + { + "text": " Face Makeup", + "boxes": [ + 20, + 778, + 110, + 804 + ] + }, + { + "text": "Only 2 left in stock - order soon", + "boxes": [ + 902, + 778, + 1106, + 808 + ] + }, + { + "text": "$129 ($.36/count)", + "boxes": [ + 1187, + 772, + 1347, + 817 + ] + }, + { + "text": "$190 ($4.75/count)", + "boxes": [ + 1483, + 772, + 1641, + 817 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 784, + 750, + 814 + ] + }, + { + "text": "Movies & TV ", + "boxes": [ + 10, + 796, + 102, + 828 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 314, + 804, + 456, + 834 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1194, + 812, + 1334, + 840 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1484, + 808, + 1632, + 842 + ] + }, + { + "text": "Movies & Films", + "boxes": [ + 24, + 818, + 122, + 844 + ] + }, + { + "text": " 3 sustainabilityfeatures ", + "boxes": [ + 610, + 828, + 800, + 858 + ] + }, + { + "text": "Prime ", + "boxes": [ + 26, + 842, + 68, + 862 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1198, + 836, + 1286, + 860 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1490, + 836, + 1580, + 860 + ] + }, + { + "text": "FSA or HSA eligible", + "boxes": [ + 1198, + 858, + 1328, + 881 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 874, + 60, + 893 + ] + }, + { + "text": "FSA or HSA eligible", + "boxes": [ + 316, + 876, + 442, + 899 + ] + }, + { + "text": "e.lf.", + "boxes": [ + 8, + 889, + 66, + 921 + ] + }, + { + "text": " wet n wild", + "boxes": [ + 5, + 909, + 105, + 946 + ] + }, + { + "text": "conair", + "boxes": [ + 9, + 930, + 79, + 962 + ] + }, + { + "text": "VALUE4PACK", + "boxes": [ + 1244, + 923, + 1464, + 965 + ] + }, + { + "text": "cOVERGIRL", + "boxes": [ + 10, + 951, + 110, + 981 + ] + }, + { + "text": "secret", + "boxes": [ + 12, + 973, + 78, + 999 + ] + }, + { + "text": "rest", + "boxes": [ + 1328, + 985, + 1412, + 1011 + ] + }, + { + "text": "eos", + "boxes": [ + 1024, + 1003, + 1054, + 1022 + ] + }, + { + "text": "3DWHITE", + "boxes": [ + 1302, + 999, + 1406, + 1030 + ] + } + ] + }, + { + "frame_id": "120_7_3", + "episode_id": 120, + "env_id": 7, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.409074783325195, + "advantages": 0.533515989780426, + "generated_text": "{\n \"intent\": \"The current screen shows a text editor, which seems like an application. Clicking the 'File' menu could lead to exploring file management options or accessing tools related to file operations.\",\n \"action\": \"Click(16, 34)\"\n}", + "action_string": "Click(16, 34)", + "intent_string": "The current screen shows a text editor, which seems like an application. Clicking the 'File' menu could lead to exploring file management options or accessing tools related to file operations.", + "before_screen_ocr_results": [ + { + "text": "File Edit Search View Document Help", + "boxes": [ + 4, + 22, + 312, + 52 + ] + }, + { + "text": " Untitled 1 - M.", + "boxes": [ + 54, + 1044, + 318, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "D", + "boxes": [ + 3, + 2, + 25, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 28, + 36, + 52 + ] + }, + { + "text": "Edit", + "boxes": [ + 42, + 26, + 76, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 274, + 26, + 308, + 48 + ] + }, + { + "text": "Nev", + "boxes": [ + 10, + 54, + 50, + 73 + ] + }, + { + "text": " Ctrl+N", + "boxes": [ + 210, + 54, + 264, + 73 + ] + }, + { + "text": "New Window", + "boxes": [ + 16, + 75, + 112, + 103 + ] + }, + { + "text": "Shift+Ctrl+N", + "boxes": [ + 172, + 77, + 264, + 103 + ] + }, + { + "text": "I open..", + "boxes": [ + 4, + 109, + 68, + 135 + ] + }, + { + "text": "Ctrl+o", + "boxes": [ + 212, + 109, + 264, + 135 + ] + }, + { + "text": "Open Recent", + "boxes": [ + 24, + 137, + 110, + 163 + ] + }, + { + "text": " save", + "boxes": [ + 0, + 165, + 64, + 197 + ] + }, + { + "text": "Ctrl+s", + "boxes": [ + 216, + 167, + 266, + 195 + ] + }, + { + "text": " save As.", + "boxes": [ + 0, + 193, + 84, + 224 + ] + }, + { + "text": "Shift+Ctrl+s", + "boxes": [ + 176, + 195, + 266, + 220 + ] + }, + { + "text": "Save All ", + "boxes": [ + 18, + 220, + 86, + 252 + ] + }, + { + "text": " Revert", + "boxes": [ + 2, + 250, + 72, + 276 + ] + }, + { + "text": " Print.", + "boxes": [ + 2, + 282, + 62, + 308 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 216, + 280, + 266, + 308 + ] + }, + { + "text": " Detach Tab", + "boxes": [ + 20, + 314, + 100, + 339 + ] + }, + { + "text": " Close Tab", + "boxes": [ + 4, + 345, + 94, + 369 + ] + }, + { + "text": "Ctrl+w", + "boxes": [ + 208, + 341, + 266, + 373 + ] + }, + { + "text": " close Window", + "boxes": [ + 2, + 371, + 120, + 401 + ] + }, + { + "text": "Ctr+o", + "boxes": [ + 210, + 367, + 268, + 401 + ] + } + ] + }, + { + "frame_id": "120_0_4", + "episode_id": 120, + "env_id": 0, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.42155933380127, + "advantages": 0.5511972904205322, + "generated_text": "{\n \"intent\": \"Clicking the 'Download Firefox' button will progress towards achieving the goal of exploring the internet by downloading a web browser. This will allow further exploration into various online resources.\",\n \"action\": \"DoubleClick(447, 595)\"\n}", + "action_string": "DoubleClick(447, 595)", + "intent_string": "Clicking the 'Download Firefox' button will progress towards achieving the goal of exploring the internet by downloading a web browser. This will allow further exploration into various online resources.", + "before_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 242, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1622, + 193 + ] + }, + { + "text": "More", + "boxes": [ + 1638, + 175, + 1672, + 189 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_1_4", + "episode_id": 120, + "env_id": 1, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.17859172821045, + "advantages": 0.20709320902824402, + "generated_text": "{\n \"intent\": \"Clicking on one of the top sites or highlights will direct us to specific websites that might provide more information about various topics. Selecting one will help us gather knowledge or insights.\",\n \"action\": \"DoubleClick(776, 487)\"\n}", + "action_string": "DoubleClick(776, 487)", + "intent_string": "Clicking on one of the top sites or highlights will direct us to specific websites that might provide more information about various topics. Selecting one will help us gather knowledge or insights.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 658, + 290, + 766, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caivarta", + "boxes": [ + 134, + 109, + 218, + 149 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 302, + 119, + 410, + 137 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "You are on amazon.com.", + "boxes": [ + 510, + 461, + 662, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1336, + 486, + 1416, + 510 + ] + }, + { + "text": "", + "boxes": [ + 1446, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 530, + 1288, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1028 + ] + } + ] + }, + { + "frame_id": "120_2_4", + "episode_id": 120, + "env_id": 2, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.68713092803955, + "advantages": -0.48894062638282776, + "generated_text": "{\n \"intent\": \"The action will click on 'Cancel' to close the file dialog window and return to the browser. This allows for the exploration of the web page content.\",\n \"action\": \"Click(1041, 668)\"\n}", + "action_string": "Click(1041, 668)", + "intent_string": "The action will click on 'Cancel' to close the file dialog window and return to the browser. This allows for the exploration of the web page content.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三四?", + "boxes": [ + 1788, + 66, + 1880, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "pn", + "boxes": [ + 1184, + 427, + 1238, + 463 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository insteadl.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system", + "boxes": [ + 704, + 889, + 1198, + 913 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": "Download for Linux 32-bit", + "boxes": [ + 848, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompany", + "boxes": [ + 738, + 1028, + 794, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "Get Firefox fo.", + "boxes": [ + 220, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "120_3_4", + "episode_id": 120, + "env_id": 3, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.476516723632812, + "advantages": -0.7872240543365479, + "generated_text": "{\n \"intent\": \"The user is currently viewing a product page on Amazon. To continue exploring the environment, it's useful to click on the 'Add to Cart' button to potentially add the item to the shopping cart. This will help the user in their exploratory goal.\",\n \"action\": \"DoubleClick(1502, 324)\"\n}", + "action_string": "DoubleClick(1502, 324)", + "intent_string": "The user is currently viewing a product page on Amazon. To continue exploring the environment, it's useful to click on the 'Add to Cart' button to potentially add the item to the shopping cart. This will help the user in their exploratory goal.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jean", + "boxes": [ + 652, + 286, + 1224, + 316 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings I Search this page", + "boxes": [ + 792, + 316, + 1020, + 339 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1462, + 312, + 1536, + 338 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Selet v", + "boxes": [ + 655, + 523, + 739, + 562 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": "Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spandex", + "boxes": [ + 826, + 639, + 1088, + 663 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise,straight leg, Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "· Rise: 12 inches, leg opening: 17 inches, inseam: petite: 29 inches, medum: 31, long: 33 inchs", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "·ProductMeasurements were takenusing size 2owmedlum.leasenote that Measurements mayvary", + "boxes": [ + 658, + 897, + 1316, + 929 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials xSofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 276, + 1560, + 300 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Add ta Cart", + "boxes": [ + 1457, + 309, + 1537, + 348 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": " Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spande", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise straight leg,Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ] + }, + { + "frame_id": "120_4_4", + "episode_id": 120, + "env_id": 4, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.630661010742188, + "advantages": 0.8473386168479919, + "generated_text": "{\n \"intent\": \"The current goal is to explore the computer desktop environment. One way to do this is to open an application from the desktop icons or start searching for more applications by expanding the search bar in Firefox. In this case, clicking on one of the desktop icons will provide an opportunity to explore that application.\",\n \"action\": \"DoubleClick(73, 605)\"\n}", + "action_string": "DoubleClick(73, 605)", + "intent_string": "The current goal is to explore the computer desktop environment. One way to do this is to open an application from the desktop icons or start searching for more applications by expanding the search bar in Firefox. In this case, clicking on one of the desktop icons will provide an opportunity to explore that application.", + "before_screen_ocr_results": [ + { + "text": "Mozilla Firefox", + "boxes": [ + 966, + 42, + 1088, + 60 + ] + }, + { + "text": "Firefox", + "boxes": [ + 19, + 61, + 97, + 98 + ] + }, + { + "text": "a, Today's Deals", + "boxes": [ + 326, + 65, + 446, + 93 + ] + }, + { + "text": "→白", + "boxes": [ + 326, + 101, + 454, + 133 + ] + }, + { + "text": "ersion%2522%", + "boxes": [ + 1178, + 103, + 1278, + 127 + ] + }, + { + "text": "·☆", + "boxes": [ + 1284, + 101, + 1368, + 133 + ] + }, + { + "text": "三四?", + "boxes": [ + 1466, + 103, + 1558, + 129 + ] + }, + { + "text": "amazon", + "boxes": [ + 336, + 149, + 448, + 181 + ] + }, + { + "text": "Search Amazor", + "boxes": [ + 602, + 155, + 710, + 179 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1496, + 147, + 1550, + 167 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1484, + 163, + 1564, + 189 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 110, + 199 + ] + }, + { + "text": "Memorial Day Sale:", + "boxes": [ + 712, + 256, + 1070, + 310 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 284, + 102, + 308 + ] + }, + { + "text": "Up to 35% off", + "boxes": [ + 770, + 310, + 1016, + 353 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1480, + 336, + 1558, + 361 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": "18+", + "boxes": [ + 1472, + 385, + 1570, + 409 + ] + }, + { + "text": "Shop deals by category", + "boxes": [ + 332, + 425, + 562, + 461 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 16, + 498, + 92, + 530 + ] + }, + { + "text": "JBL", + "boxes": [ + 546, + 513, + 613, + 561 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1488, + 554, + 1550, + 580 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "Shop All", + "boxes": [ + 382, + 613, + 436, + 633 + ] + }, + { + "text": "Top100", + "boxes": [ + 538, + 613, + 590, + 633 + ] + }, + { + "text": "Sports & Outdoors", + "boxes": [ + 806, + 607, + 926, + 636 + ] + }, + { + "text": "國", + "boxes": [ + 38, + 681, + 80, + 719 + ] + }, + { + "text": " Today's featured deals", + "boxes": [ + 330, + 699, + 554, + 735 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1492, + 770, + 1546, + 796 + ] + }, + { + "text": "口", + "boxes": [ + 36, + 782, + 82, + 830 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 31, + 936, + 87, + 974 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 4, + 22, + 38, + 47 + ] + }, + { + "text": "Edit", + "boxes": [ + 40, + 28, + 74, + 42 + ] + }, + { + "text": "--- @x自", + "boxes": [ + 10, + 52, + 358, + 87 + ] + }, + { + "text": "中園口", + "boxes": [ + 1002, + 56, + 1150, + 79 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 240 + ] + }, + { + "text": "", + "boxes": [ + 1886, + 234, + 1912, + 306 + ] + }, + { + "text": "O", + "boxes": [ + 1884, + 308, + 1914, + 342 + ] + } + ] + }, + { + "frame_id": "120_5_4", + "episode_id": 120, + "env_id": 5, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.361945152282715, + "advantages": -0.9494866132736206, + "generated_text": "{\n \"intent\": \"Selecting 'File' from the menu will allow me to perform various actions such as saving the current document or accessing recent documents. This choice helps in exploring the software's functionalities and user interface.\",\n \"action\": \"DoubleClick(37, 18)\"\n}", + "action_string": "DoubleClick(37, 18)", + "intent_string": "Selecting 'File' from the menu will allow me to perform various actions such as saving the current document or accessing recent documents. This choice helps in exploring the software's functionalities and user interface.", + "before_screen_ocr_results": [ + { + "text": "Edit", + "boxes": [ + 48, + 30, + 72, + 44 + ] + }, + { + "text": "Data", + "boxes": [ + 366, + 30, + 388, + 44 + ] + }, + { + "text": "晶自", + "boxes": [ + 292, + 54, + 370, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 378, + 54, + 444, + 85 + ] + }, + { + "text": "G-.", + "boxes": [ + 452, + 56, + 550, + 81 + ] + }, + { + "text": "v田·I·子I", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Open.", + "boxes": [ + 28, + 81, + 68, + 101 + ] + }, + { + "text": "Ctrl+C", + "boxes": [ + 252, + 79, + 290, + 99 + ] + }, + { + "text": "a", + "boxes": [ + 312, + 99, + 332, + 119 + ] + }, + { + "text": "a要日日%00三田二口", + "boxes": [ + 342, + 93, + 1204, + 123 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 26, + 191, + 84, + 216 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 270 + ] + }, + { + "text": "Reloao", + "boxes": [ + 32, + 254, + 72, + 270 + ] + }, + { + "text": "Save", + "boxes": [ + 26, + 312, + 60, + 332 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 296, + 359 + ] + }, + { + "text": "Save Remote", + "boxes": [ + 28, + 367, + 114, + 385 + ] + }, + { + "text": "Save a Copy...", + "boxes": [ + 24, + 387, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 21, + 444, + 83, + 477 + ] + }, + { + "text": "Export as PDF", + "boxes": [ + 26, + 476, + 118, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 68, + 528 + ] + }, + { + "text": "Preview in Web Brows", + "boxes": [ + 26, + 534, + 172, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 296, + 584 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 250, + 590, + 292, + 609 + ] + }, + { + "text": "Printer Settings.", + "boxes": [ + 26, + 619, + 132, + 637 + ] + }, + { + "text": " Properties.", + "boxes": [ + 24, + 649, + 100, + 673 + ] + }, + { + "text": "Digital :", + "boxes": [ + 26, + 679, + 78, + 697 + ] + }, + { + "text": " Exit LibreOffice", + "boxes": [ + 24, + 707, + 126, + 731 + ] + }, + { + "text": "Ctrl+(", + "boxes": [ + 254, + 711, + 288, + 725 + ] + }, + { + "text": "35", + "boxes": [ + 12, + 750, + 32, + 766 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "一→二", + "boxes": [ + 20, + 997, + 92, + 1014 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "--目·@%自-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "勺·、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberation san10aaaa旦号盈日目% 00三至田口目", + "boxes": [ + 10, + 93, + 1208, + 123 + ] + }, + { + "text": "2、 =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "120_6_4", + "episode_id": 120, + "env_id": 6, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.340734481811523, + "advantages": 0.43672868609428406, + "generated_text": "{\n \"intent\": \"To continue exploring the beauty section of Amazon, I will choose a product to view more details.\",\n \"action\": \"DoubleClick(392, 654)\"\n}", + "action_string": "DoubleClick(392, 654)", + "intent_string": "To continue exploring the beauty section of Amazon, I will choose a product to view more details.", + "before_screen_ocr_results": [ + { + "text": "1.com : beauty ", + "boxes": [ + 82, + 30, + 182, + 54 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "d_rd_r=b3031735-6", + "boxes": [ + 1432, + 66, + 1568, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1576, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "beauty", + "boxes": [ + 270, + 113, + 360, + 145 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1618, + 117, + 1644, + 137 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 All", + "boxes": [ + 20, + 167, + 70, + 195 + ] + }, + { + "text": " Gift Cards", + "boxes": [ + 354, + 167, + 424, + 193 + ] + }, + { + "text": "1-48 of over 60,000 res", + "boxes": [ + 10, + 206, + 158, + 230 + ] + }, + { + "text": "ults for", + "boxes": [ + 158, + 210, + 204, + 228 + ] + }, + { + "text": "Explore Related Products", + "boxes": [ + 10, + 256, + 184, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 304, + 264, + 380, + 290 + ] + }, + { + "text": "laser hair removal", + "boxes": [ + 10, + 280, + 130, + 310 + ] + }, + { + "text": "Checke", + "boxes": [ + 312, + 294, + 352, + 310 + ] + }, + { + "text": " each product page for other buying optio", + "boxes": [ + 346, + 292, + 610, + 310 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1798, + 294, + 1878, + 326 + ] + }, + { + "text": "cerave", + "boxes": [ + 14, + 306, + 60, + 326 + ] + }, + { + "text": "pimple patches ", + "boxes": [ + 10, + 320, + 116, + 351 + ] + }, + { + "text": "beauty blender", + "boxes": [ + 14, + 341, + 112, + 367 + ] + }, + { + "text": "Biodance", + "boxes": [ + 1580, + 343, + 1644, + 363 + ] + }, + { + "text": "See", + "boxes": [ + 20, + 367, + 50, + 381 + ] + }, + { + "text": "BROALCO", + "boxes": [ + 1516, + 367, + 1544, + 391 + ] + }, + { + "text": "Biodance", + "boxes": [ + 1614, + 375, + 1676, + 395 + ] + }, + { + "text": "REAOLLAGE", + "boxes": [ + 1568, + 399, + 1612, + 419 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 411, + 134, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 361, + 436, + 535, + 481 + ] + }, + { + "text": "MIGHTY", + "boxes": [ + 1260, + 443, + 1388, + 480 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 10, + 469, + 136, + 498 + ] + }, + { + "text": "PATCH°", + "boxes": [ + 1258, + 472, + 1352, + 510 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 12, + 494, + 98, + 518 + ] + }, + { + "text": "TheOriginaol", + "boxes": [ + 1261, + 499, + 1321, + 531 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 322, + 510, + 524, + 540 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 516, + 102, + 540 + ] + }, + { + "text": "SOFTCLEAN", + "boxes": [ + 362, + 534, + 420, + 552 + ] + }, + { + "text": "NSINGTOWELETTES", + "boxes": [ + 414, + 534, + 510, + 552 + ] + }, + { + "text": "", + "boxes": [ + 1264, + 534, + 1334, + 556 + ] + }, + { + "text": "36", + "boxes": [ + 1354, + 526, + 1382, + 548 + ] + }, + { + "text": "Price", + "boxes": [ + 8, + 548, + 54, + 574 + ] + }, + { + "text": "50", + "boxes": [ + 326, + 548, + 358, + 580 + ] + }, + { + "text": "Plant-BasedCompostableTowelettes", + "boxes": [ + 366, + 554, + 566, + 578 + ] + }, + { + "text": "口", + "boxes": [ + 1796, + 556, + 1810, + 572 + ] + }, + { + "text": "$0 -$2,450+", + "boxes": [ + 10, + 572, + 102, + 598 + ] + }, + { + "text": "Neutrogena Makeup Remover Wipes,", + "boxes": [ + 314, + 590, + 588, + 619 + ] + }, + { + "text": "Clean Skin Club Clean Towels XLTM,", + "boxes": [ + 606, + 590, + 868, + 619 + ] + }, + { + "text": " 6Pcs 5inch Makeup Scoops 5 Color", + "boxes": [ + 898, + 586, + 1160, + 620 + ] + }, + { + "text": "Mighty PatchTM Original patch from ", + "boxes": [ + 1194, + 590, + 1460, + 619 + ] + }, + { + "text": " BIODANCE Bio-Collagen Real Deep ", + "boxes": [ + 1486, + 590, + 1752, + 619 + ] + }, + { + "text": "?", + "boxes": [ + 18, + 611, + 34, + 627 + ] + }, + { + "text": "Daily Facial Cleanser Towelettes,", + "boxes": [ + 312, + 609, + 556, + 643 + ] + }, + { + "text": "100% USDA Biobased Face Towel, ", + "boxes": [ + 608, + 613, + 862, + 643 + ] + }, + { + "text": "Plastic Beauty Spoon Facial Cream", + "boxes": [ + 900, + 613, + 1160, + 643 + ] + }, + { + "text": "Hero Cosmetics - Hydrocolloid Acne ", + "boxes": [ + 1194, + 613, + 1464, + 643 + ] + }, + { + "text": " Mask, Hydrating Overnight Hydrogel", + "boxes": [ + 1484, + 611, + 1758, + 641 + ] + }, + { + "text": "Gently Cleanse and Remove Oil&..", + "boxes": [ + 312, + 635, + 572, + 665 + ] + }, + { + "text": "Disposable Face Towelette, Eczem..", + "boxes": [ + 606, + 637, + 872, + 667 + ] + }, + { + "text": "Brush Cosmetic Spatula Mask Mixi..", + "boxes": [ + 900, + 637, + 1168, + 667 + ] + }, + { + "text": " Pimple Patch for Covering Zits and...", + "boxes": [ + 1192, + 637, + 1464, + 667 + ] + }, + { + "text": "Mask, Pore Minimizing, Elasticity...", + "boxes": [ + 1486, + 637, + 1744, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 8, + 653, + 102, + 685 + ] + }, + { + "text": "Wipes 25 Count (Pakof 2)", + "boxes": [ + 314, + 661, + 522, + 701 + ] + }, + { + "text": "Towel 50 Count (Pack of 1)", + "boxes": [ + 608, + 663, + 810, + 699 + ] + }, + { + "text": "AllSkin Type1 Count (Pack of 1)", + "boxes": [ + 902, + 663, + 1142, + 697 + ] + }, + { + "text": "36 Count (Pack of 1)", + "boxes": [ + 1196, + 661, + 1340, + 697 + ] + }, + { + "text": "4 Count (Pack of 1)", + "boxes": [ + 1488, + 661, + 1626, + 697 + ] + }, + { + "text": "Beauty& Personal Care", + "boxes": [ + 10, + 677, + 168, + 707 + ] + }, + { + "text": "#1 Top Rated", + "boxes": [ + 316, + 697, + 406, + 723 + ] + }, + { + "text": "★★★★★34.096", + "boxes": [ + 608, + 697, + 760, + 727 + ] + }, + { + "text": "$439 ($4.89/count)", + "boxes": [ + 895, + 700, + 1039, + 747 + ] + }, + { + "text": "Options: 3 sizes", + "boxes": [ + 1198, + 699, + 1286, + 725 + ] + }, + { + "text": "Options: 2 sizes", + "boxes": [ + 1490, + 699, + 1578, + 723 + ] + }, + { + "text": "Facial Skin Care Products", + "boxes": [ + 20, + 713, + 190, + 748 + ] + }, + { + "text": "★★★★★109.879", + "boxes": [ + 314, + 715, + 472, + 744 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 606, + 710, + 794, + 749 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "★★★★☆173,304", + "boxes": [ + 1196, + 723, + 1352, + 752 + ] + }, + { + "text": "★★★★★15,952", + "boxes": [ + 1488, + 721, + 1642, + 754 + ] + }, + { + "text": "Makeup Brush Sets & Kits", + "boxes": [ + 22, + 737, + 192, + 766 + ] + }, + { + "text": "onth", + "boxes": [ + 470, + 742, + 498, + 756 + ] + }, + { + "text": "$179 $0.36count)", + "boxes": [ + 601, + 744, + 759, + 791 + ] + }, + { + "text": "Delivery Sat, May 24", + "boxes": [ + 900, + 737, + 1042, + 766 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 1196, + 744, + 1384, + 774 + ] + }, + { + "text": "100K+ bought in past month", + "boxes": [ + 1488, + 744, + 1674, + 768 + ] + }, + { + "text": "$97 60.20/cout Lst: 43569", + "boxes": [ + 305, + 762, + 527, + 817 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 908, + 764, + 988, + 782 + ] + }, + { + "text": " Face Makeup", + "boxes": [ + 20, + 778, + 110, + 804 + ] + }, + { + "text": "Only 2 left in stock - order soon", + "boxes": [ + 902, + 778, + 1106, + 808 + ] + }, + { + "text": "$129 ($.36/count)", + "boxes": [ + 1187, + 772, + 1347, + 817 + ] + }, + { + "text": "$190 ($4.75/count)", + "boxes": [ + 1483, + 772, + 1641, + 817 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 784, + 750, + 814 + ] + }, + { + "text": "Movies & TV ", + "boxes": [ + 10, + 796, + 102, + 828 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 314, + 804, + 456, + 834 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1194, + 812, + 1334, + 840 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1484, + 808, + 1632, + 842 + ] + }, + { + "text": "Movies & Films", + "boxes": [ + 24, + 818, + 122, + 844 + ] + }, + { + "text": " 3 sustainabilityfeatures ", + "boxes": [ + 610, + 828, + 800, + 858 + ] + }, + { + "text": "Prime ", + "boxes": [ + 26, + 842, + 68, + 862 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1198, + 836, + 1286, + 860 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1490, + 836, + 1580, + 860 + ] + }, + { + "text": "FSA or HSA eligible", + "boxes": [ + 1198, + 858, + 1328, + 881 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 874, + 60, + 893 + ] + }, + { + "text": "FSA or HSA eligible", + "boxes": [ + 316, + 876, + 442, + 899 + ] + }, + { + "text": "e.lf.", + "boxes": [ + 8, + 889, + 66, + 921 + ] + }, + { + "text": " wet n wild", + "boxes": [ + 5, + 909, + 105, + 946 + ] + }, + { + "text": "conair", + "boxes": [ + 9, + 930, + 79, + 962 + ] + }, + { + "text": "VALUE4PACK", + "boxes": [ + 1244, + 923, + 1464, + 965 + ] + }, + { + "text": "cOVERGIRL", + "boxes": [ + 10, + 951, + 110, + 981 + ] + }, + { + "text": "secret", + "boxes": [ + 12, + 973, + 78, + 999 + ] + }, + { + "text": "rest", + "boxes": [ + 1328, + 985, + 1412, + 1011 + ] + }, + { + "text": "eos", + "boxes": [ + 1024, + 1003, + 1054, + 1022 + ] + }, + { + "text": "3DWHITE", + "boxes": [ + 1302, + 999, + 1406, + 1030 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 38, + 66, + 130, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1782, + 62, + 1878, + 93 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "beauty", + "boxes": [ + 291, + 108, + 359, + 146 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Neutrogena Makeup Remover Wipes, Daily Facial Cleanser", + "boxes": [ + 680, + 258, + 1318, + 294 + ] + }, + { + "text": "↑", + "boxes": [ + 628, + 276, + 642, + 292 + ] + }, + { + "text": "$997 ($0.20 /count)", + "boxes": [ + 1394, + 276, + 1524, + 308 + ] + }, + { + "text": "Towelettes, Gently Cleanse and Remove Oil & Makeup,", + "boxes": [ + 676, + 286, + 1284, + 332 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1876, + 322 + ] + }, + { + "text": "日", + "boxes": [ + 164, + 318, + 196, + 353 + ] + }, + { + "text": "Alcohol-Free Makeup Wipes, 2 x 25 ct ", + "boxes": [ + 680, + 324, + 1098, + 359 + ] + }, + { + "text": "FREE International Retul", + "boxes": [ + 1396, + 316, + 1548, + 334 + ] + }, + { + "text": "No Import Charges & $6.99", + "boxes": [ + 1394, + 343, + 1574, + 367 + ] + }, + { + "text": "Visit the Neutrogena Store", + "boxes": [ + 684, + 355, + 856, + 379 + ] + }, + { + "text": "4.8★★★★★~", + "boxes": [ + 684, + 371, + 808, + 401 + ] + }, + { + "text": "Shipping to Canada Details ", + "boxes": [ + 1394, + 365, + 1578, + 389 + ] + }, + { + "text": "Delivery Tuesday, May 20.", + "boxes": [ + 1394, + 381, + 1564, + 411 + ] + }, + { + "text": " Order within 1 hr 45 mins", + "boxes": [ + 1392, + 399, + 1560, + 431 + ] + }, + { + "text": " 1 sustainabilty feature ", + "boxes": [ + 680, + 429, + 878, + 463 + ] + }, + { + "text": "@ Deliver to Canada", + "boxes": [ + 1392, + 433, + 1514, + 461 + ] + }, + { + "text": "Neutrogenao", + "boxes": [ + 380, + 443, + 480, + 469 + ] + }, + { + "text": "In Stock", + "boxes": [ + 1392, + 474, + 1466, + 500 + ] + }, + { + "text": "MAKEUPREMOVER", + "boxes": [ + 322, + 508, + 532, + 544 + ] + }, + { + "text": "-27% $997 ($0.20 /cout)", + "boxes": [ + 680, + 504, + 884, + 540 + ] + }, + { + "text": "Quantity: 1", + "boxes": [ + 1399, + 510, + 1485, + 541 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "List Price: $43.69 0", + "boxes": [ + 678, + 534, + 794, + 564 + ] + }, + { + "text": "Neutrogena\"", + "boxes": [ + 369, + 557, + 489, + 600 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1462, + 556, + 1536, + 582 + ] + }, + { + "text": "立", + "boxes": [ + 1791, + 551, + 1822, + 578 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 268, + 590, + 528, + 629 + ] + }, + { + "text": "No Import Charges & $6.99 Shipping to C", + "boxes": [ + 682, + 588, + 946, + 611 + ] + }, + { + "text": " Buy Now", + "boxes": [ + 1466, + 598, + 1528, + 629 + ] + }, + { + "text": "Brand", + "boxes": [ + 682, + 613, + 728, + 639 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 860, + 613, + 934, + 639 + ] + }, + { + "text": " Skin Type", + "boxes": [ + 678, + 637, + 756, + 669 + ] + }, + { + "text": "Plant-BaseCompostableTowelettes", + "boxes": [ + 318, + 649, + 586, + 679 + ] + }, + { + "text": "All", + "boxes": [ + 858, + 641, + 884, + 665 + ] + }, + { + "text": "Material Feature", + "boxes": [ + 680, + 663, + 794, + 693 + ] + }, + { + "text": "Compostable", + "boxes": [ + 858, + 667, + 944, + 691 + ] + }, + { + "text": "Sold by", + "boxes": [ + 1402, + 661, + 1446, + 681 + ] + }, + { + "text": "Amazon.com", + "boxes": [ + 1466, + 659, + 1540, + 683 + ] + }, + { + "text": "50 Count", + "boxes": [ + 857, + 688, + 923, + 717 + ] + }, + { + "text": "Returns", + "boxes": [ + 1400, + 679, + 1450, + 705 + ] + }, + { + "text": "Unit Count", + "boxes": [ + 682, + 691, + 758, + 717 + ] + }, + { + "text": "Packaging ", + "boxes": [ + 1398, + 715, + 1462, + 741 + ] + }, + { + "text": "Ships in proc", + "boxes": [ + 1468, + 717, + 1538, + 739 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1858, + 731 + ] + }, + { + "text": "rofItem", + "boxes": [ + 730, + 721, + 788, + 739 + ] + }, + { + "text": " packaging", + "boxes": [ + 1465, + 730, + 1529, + 755 + ] + }, + { + "text": " About this item", + "boxes": [ + 678, + 758, + 838, + 794 + ] + }, + { + "text": "VSeemore", + "boxes": [ + 1400, + 752, + 1466, + 772 + ] + }, + { + "text": "Twin Pack with 25 count of ultra-soft, pre-mo", + "boxes": [ + 692, + 788, + 990, + 814 + ] + }, + { + "text": "d Neutrogena Makeup Remover Wipes with a", + "boxes": [ + 1030, + 790, + 1318, + 814 + ] + }, + { + "text": "FSA or HSA eligible Details", + "boxes": [ + 1392, + 786, + 1540, + 810 + ] + }, + { + "text": "口", + "boxes": [ + 1396, + 820, + 1420, + 842 + ] + }, + { + "text": "it feeling refreshed, soft, sm", + "boxes": [ + 700, + 830, + 878, + 854 + ] + }, + { + "text": "Add a gift receipt for easy", + "boxes": [ + 1414, + 838, + 1586, + 868 + ] + }, + { + "text": "returns", + "boxes": [ + 1413, + 857, + 1469, + 888 + ] + }, + { + "text": "er at your fingertips", + "boxes": [ + 942, + 895, + 1064, + 917 + ] + }, + { + "text": "Facial cleansing wet wipes are ophthalmologist, dermatologist-, and allergy-tested and gentle around", + "boxes": [ + 698, + 917, + 1354, + 941 + ] + }, + { + "text": "Add to List ", + "boxes": [ + 1400, + 911, + 1478, + 937 + ] + }, + { + "text": "without tugging", + "boxes": [ + 768, + 959, + 866, + 981 + ] + }, + { + "text": "Other sellers on Amazon", + "boxes": [ + 1386, + 981, + 1578, + 1011 + ] + } + ] + }, + { + "frame_id": "120_7_4", + "episode_id": 120, + "env_id": 7, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.5255126953125, + "advantages": 0.6984217762947083, + "generated_text": "{\n \"intent\": \"To continue exploring the environment, clicking the 'File' menu will allow us to access various file operations such as creating a new document or opening an existing one.\",\n \"action\": \"Click(40, 119)\"\n}", + "action_string": "Click(40, 119)", + "intent_string": "To continue exploring the environment, clicking the 'File' menu will allow us to access various file operations such as creating a new document or opening an existing one.", + "before_screen_ocr_results": [ + { + "text": "D", + "boxes": [ + 3, + 2, + 25, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 28, + 36, + 52 + ] + }, + { + "text": "Edit", + "boxes": [ + 42, + 26, + 76, + 46 + ] + }, + { + "text": "Nev", + "boxes": [ + 10, + 54, + 50, + 73 + ] + }, + { + "text": " Ctrl+N", + "boxes": [ + 210, + 54, + 264, + 73 + ] + }, + { + "text": "New Window", + "boxes": [ + 16, + 75, + 112, + 103 + ] + }, + { + "text": "Shift+Ctrl+N", + "boxes": [ + 172, + 77, + 264, + 103 + ] + }, + { + "text": "I open..", + "boxes": [ + 4, + 109, + 68, + 135 + ] + }, + { + "text": "Ctrl+o", + "boxes": [ + 212, + 109, + 264, + 135 + ] + }, + { + "text": "Open Recent", + "boxes": [ + 24, + 137, + 110, + 163 + ] + }, + { + "text": " save", + "boxes": [ + 0, + 165, + 64, + 197 + ] + }, + { + "text": "Ctrl+s", + "boxes": [ + 216, + 167, + 266, + 195 + ] + }, + { + "text": " save As.", + "boxes": [ + 0, + 193, + 84, + 224 + ] + }, + { + "text": "Shift+Ctrl+s", + "boxes": [ + 176, + 195, + 266, + 220 + ] + }, + { + "text": "Save All ", + "boxes": [ + 18, + 220, + 86, + 252 + ] + }, + { + "text": " Revert", + "boxes": [ + 2, + 250, + 72, + 276 + ] + }, + { + "text": " Print.", + "boxes": [ + 2, + 282, + 62, + 308 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 216, + 280, + 266, + 308 + ] + }, + { + "text": " Detach Tab", + "boxes": [ + 20, + 314, + 100, + 339 + ] + }, + { + "text": " Close Tab", + "boxes": [ + 4, + 345, + 94, + 369 + ] + }, + { + "text": "Ctrl+w", + "boxes": [ + 208, + 341, + 266, + 373 + ] + }, + { + "text": " close Window", + "boxes": [ + 2, + 371, + 120, + 401 + ] + }, + { + "text": "Ctr+o", + "boxes": [ + 210, + 367, + 268, + 401 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "A", + "boxes": [ + 0, + 0, + 32, + 30 + ] + }, + { + "text": "Untitled 1- Mousepad", + "boxes": [ + 872, + 4, + 1048, + 22 + ] + }, + { + "text": "回×", + "boxes": [ + 1860, + 4, + 1914, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 30, + 34, + 44 + ] + }, + { + "text": "Edit", + "boxes": [ + 44, + 30, + 74, + 44 + ] + }, + { + "text": "Open File", + "boxes": [ + 920, + 103, + 1006, + 127 + ] + }, + { + "text": "日可", + "boxes": [ + 1442, + 105, + 1500, + 125 + ] + }, + { + "text": " Recent", + "boxes": [ + 418, + 131, + 496, + 157 + ] + }, + { + "text": "Location", + "boxes": [ + 930, + 129, + 996, + 147 + ] + }, + { + "text": "Size", + "boxes": [ + 1274, + 131, + 1306, + 145 + ] + }, + { + "text": "Type", + "boxes": [ + 1348, + 127, + 1398, + 153 + ] + }, + { + "text": "|Accessed", + "boxes": [ + 1410, + 129, + 1490, + 147 + ] + }, + { + "text": " pcmanfm.conf", + "boxes": [ + 565, + 141, + 692, + 177 + ] + }, + { + "text": ".config/pcmanfm/LXDE", + "boxes": [ + 930, + 147, + 1086, + 177 + ] + }, + { + "text": "416 bytes", + "boxes": [ + 1270, + 145, + 1352, + 177 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 147, + 1390, + 175 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 147, + 1464, + 175 + ] + }, + { + "text": "仓 Home", + "boxes": [ + 413, + 160, + 491, + 191 + ] + }, + { + "text": " libfm.conf ", + "boxes": [ + 563, + 162, + 667, + 199 + ] + }, + { + "text": ".config/ibfm", + "boxes": [ + 929, + 168, + 1015, + 199 + ] + }, + { + "text": "853 bytes", + "boxes": [ + 1272, + 171, + 1346, + 197 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 173, + 1386, + 195 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 171, + 1462, + 197 + ] + }, + { + "text": " Desktop", + "boxes": [ + 420, + 199, + 506, + 224 + ] + }, + { + "text": " desktop-items-0.conf ", + "boxes": [ + 566, + 193, + 740, + 222 + ] + }, + { + "text": ".config/pcmanfm/LXDE", + "boxes": [ + 930, + 193, + 1086, + 222 + ] + }, + { + "text": "381 bytes", + "boxes": [ + 1274, + 195, + 1348, + 220 + ] + }, + { + "text": "Text ", + "boxes": [ + 1350, + 193, + 1390, + 222 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 193, + 1464, + 220 + ] + }, + { + "text": " panel", + "boxes": [ + 570, + 216, + 632, + 242 + ] + }, + { + "text": ".config/lxpanel/LXDE/panels", + "boxes": [ + 932, + 218, + 1114, + 242 + ] + }, + { + "text": "1.4 kB", + "boxes": [ + 1274, + 214, + 1324, + 242 + ] + }, + { + "text": "Text", + "boxes": [ + 1349, + 209, + 1390, + 246 + ] + }, + { + "text": "7 May", + "boxes": [ + 1413, + 211, + 1463, + 245 + ] + }, + { + "text": "Other Locati", + "boxes": [ + 432, + 238, + 526, + 262 + ] + }, + { + "text": " config", + "boxes": [ + 569, + 231, + 639, + 271 + ] + }, + { + "text": ".config/lxpanel/LXDE", + "boxes": [ + 932, + 242, + 1070, + 266 + ] + }, + { + "text": "29 bytes ", + "boxes": [ + 1274, + 240, + 1338, + 266 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 242, + 1388, + 264 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 240, + 1462, + 268 + ] + }, + { + "text": " rc.xml", + "boxes": [ + 574, + 262, + 638, + 288 + ] + }, + { + "text": ".config/openbox", + "boxes": [ + 932, + 264, + 1038, + 288 + ] + }, + { + "text": "26.1 kB", + "boxes": [ + 1272, + 260, + 1334, + 292 + ] + }, + { + "text": "Markup", + "boxes": [ + 1350, + 260, + 1412, + 292 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 264, + 1462, + 290 + ] + }, + { + "text": "firefox.desktop", + "boxes": [ + 572, + 286, + 690, + 310 + ] + }, + { + "text": "Desktop", + "boxes": [ + 932, + 286, + 992, + 312 + ] + }, + { + "text": "96 bytes", + "boxes": [ + 1276, + 286, + 1334, + 312 + ] + }, + { + "text": "Text", + "boxes": [ + 1356, + 288, + 1384, + 310 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 290, + 1460, + 310 + ] + }, + { + "text": "0 rc.xml", + "boxes": [ + 572, + 310, + 638, + 336 + ] + }, + { + "text": "/etc/xdg/openbox/LXDE ", + "boxes": [ + 930, + 308, + 1090, + 338 + ] + }, + { + "text": "22.3 kB", + "boxes": [ + 1272, + 306, + 1336, + 338 + ] + }, + { + "text": "Markup ", + "boxes": [ + 1352, + 310, + 1410, + 336 + ] + }, + { + "text": "7 May", + "boxes": [ + 1414, + 310, + 1462, + 336 + ] + }, + { + "text": "desktop-it", + "boxes": [ + 570, + 334, + 662, + 357 + ] + }, + { + "text": "/usr/local/share/doro-lxde-", + "boxes": [ + 932, + 334, + 1102, + 357 + ] + }, + { + "text": "298 bytes", + "boxes": [ + 1276, + 336, + 1340, + 355 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 336, + 1382, + 357 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 334, + 1462, + 359 + ] + }, + { + "text": " mimeapps.list ", + "boxes": [ + 568, + 353, + 696, + 383 + ] + }, + { + "text": ".config", + "boxes": [ + 932, + 353, + 982, + 381 + ] + }, + { + "text": "79 bytes", + "boxes": [ + 1270, + 351, + 1340, + 383 + ] + }, + { + "text": "Text ", + "boxes": [ + 1352, + 353, + 1390, + 381 + ] + }, + { + "text": "7 May", + "boxes": [ + 1414, + 353, + 1462, + 381 + ] + }, + { + "text": "Ixterminal.conf", + "boxes": [ + 570, + 379, + 694, + 403 + ] + }, + { + "text": ".config/lxterminal", + "boxes": [ + 932, + 379, + 1048, + 403 + ] + }, + { + "text": "1.4 kB", + "boxes": [ + 1274, + 377, + 1328, + 407 + ] + }, + { + "text": "Text", + "boxes": [ + 1352, + 377, + 1388, + 405 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1409, + 371, + 1499, + 408 + ] + }, + { + "text": "launchtaskbar.cfg", + "boxes": [ + 572, + 399, + 712, + 427 + ] + }, + { + "text": ".config/lxpanel", + "boxes": [ + 932, + 401, + 1030, + 427 + ] + }, + { + "text": "96 bytes", + "boxes": [ + 1272, + 399, + 1342, + 431 + ] + }, + { + "text": "Text", + "boxes": [ + 1349, + 395, + 1391, + 430 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1414, + 401, + 1496, + 427 + ] + }, + { + "text": "bg1.jpg", + "boxes": [ + 570, + 425, + 646, + 451 + ] + }, + { + "text": "/usr/local/share/doro-lxe-wallpapers", + "boxes": [ + 926, + 419, + 1176, + 453 + ] + }, + { + "text": "347.4 kB", + "boxes": [ + 1270, + 417, + 1346, + 455 + ] + }, + { + "text": "Image ", + "boxes": [ + 1350, + 421, + 1408, + 453 + ] + }, + { + "text": " 4 Apr 2021", + "boxes": [ + 1410, + 425, + 1496, + 451 + ] + }, + { + "text": " rc.xml ", + "boxes": [ + 570, + 449, + 640, + 474 + ] + }, + { + "text": "/etc/xdg/openbox", + "boxes": [ + 928, + 447, + 1050, + 476 + ] + }, + { + "text": "24.2 kB", + "boxes": [ + 1274, + 447, + 1334, + 472 + ] + }, + { + "text": "Markup", + "boxes": [ + 1350, + 445, + 1416, + 476 + ] + }, + { + "text": "10 Mar 2021", + "boxes": [ + 1408, + 447, + 1504, + 472 + ] + }, + { + "text": " menu.xml", + "boxes": [ + 570, + 472, + 662, + 496 + ] + }, + { + "text": "/etc/xdg/openbox", + "boxes": [ + 928, + 469, + 1050, + 498 + ] + }, + { + "text": "1.1 kB", + "boxes": [ + 1274, + 471, + 1322, + 498 + ] + }, + { + "text": "Markup", + "boxes": [ + 1354, + 471, + 1414, + 496 + ] + }, + { + "text": "15 Mar 2020", + "boxes": [ + 1404, + 471, + 1506, + 496 + ] + }, + { + "text": "Open File", + "boxes": [ + 370, + 1050, + 432, + 1070 + ] + } + ] + }, + { + "frame_id": "120_0_5", + "episode_id": 120, + "env_id": 0, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.030243873596191, + "advantages": -0.0030051874928176403, + "generated_text": "{\n \"intent\": \"To continue exploring the Firefox download options and possibly downloading it. The current page provides different download options based on the operating system the user might be using.\",\n \"action\": \"Click(958, 658)\"\n}", + "action_string": "Click(958, 658)", + "intent_string": "To continue exploring the Firefox download options and possibly downloading it. The current page provides different download options based on the operating system the user might be using.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口", + "boxes": [ + 1118, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Whichis: TAR Archive (69.8 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilenet ", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for files like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": "+", + "boxes": [ + 958, + 661, + 970, + 673 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 702, + 887, + 1200, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_1_5", + "episode_id": 120, + "env_id": 1, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.438146591186523, + "advantages": 0.5746890902519226, + "generated_text": "{\n \"intent\": \"Clicking the search bar will allow us to explore different products by inputting keywords or phrases.\",\n \"action\": \"DoubleClick(316, 125)\"\n}", + "action_string": "DoubleClick(316, 125)", + "intent_string": "Clicking the search bar will allow us to explore different products by inputting keywords or phrases.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1586, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " canarta", + "boxes": [ + 132, + 107, + 220, + 153 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 117, + 1476, + 137 + ] + }, + { + "text": "MEN+", + "boxes": [ + 1519, + 116, + 1579, + 150 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "New arrivals in Toys", + "boxes": [ + 770, + 252, + 1192, + 314 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 606, + 524, + 832, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 528, + 1288, + 558 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1534, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "Seemore", + "boxes": [ + 240, + 891, + 300, + 911 + ] + }, + { + "text": " See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 910, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": " Refresh your space", + "boxes": [ + 1346, + 965, + 1544, + 1003 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Canara", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "SEarch Amazon", + "boxes": [ + 296, + 115, + 414, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1452, + 117, + 1480, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "19", + "boxes": [ + 1822, + 113, + 1862, + 139 + ] + }, + { + "text": "jeans", + "boxes": [ + 302, + 155, + 350, + 181 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 86, + 169, + 172, + 187 + ] + }, + { + "text": "beauty", + "boxes": [ + 302, + 189, + 364, + 220 + ] + }, + { + "text": " gaming", + "boxes": [ + 300, + 222, + 370, + 254 + ] + }, + { + "text": " gaming headsets", + "boxes": [ + 302, + 258, + 440, + 288 + ] + }, + { + "text": " gaming keyboard", + "boxes": [ + 302, + 294, + 444, + 324 + ] + }, + { + "text": "home decor", + "boxes": [ + 306, + 365, + 398, + 389 + ] + }, + { + "text": "×", + "boxes": [ + 1454, + 367, + 1472, + 383 + ] + }, + { + "text": "fou are on amazon.com. You can also shop", + "boxes": [ + 514, + 461, + 780, + 478 + ] + }, + { + "text": "精上", + "boxes": [ + 464, + 486, + 576, + 510 + ] + }, + { + "text": "", + "boxes": [ + 626, + 488, + 696, + 510 + ] + }, + { + "text": "", + "boxes": [ + 1350, + 482, + 1536, + 512 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 610, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 984, + 532, + 1286, + 556 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1534, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 612, + 889, + 690, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 242, + 969, + 498, + 999 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 354, + 1026 + ] + } + ] + }, + { + "frame_id": "120_2_5", + "episode_id": 120, + "env_id": 2, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.056482315063477, + "advantages": 0.034155137836933136, + "generated_text": "{\n \"intent\": \"Clicking 'Download for Linux 64-bit' will allow me to download the Firefox browser for my workstation. This will help in exploring more of the internet and potentially finding new information or applications.\",\n \"action\": \"Click(959, 658)\"\n}", + "action_string": "Click(959, 658)", + "intent_string": "Clicking 'Download for Linux 64-bit' will allow me to download the Firefox browser for my workstation. This will help in exploring more of the internet and potentially finding new information or applications.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": "Download for Linux 32-bit", + "boxes": [ + 848, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompany", + "boxes": [ + 738, + 1028, + 794, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "Get Firefox fo.", + "boxes": [ + 220, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口", + "boxes": [ + 1118, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Whichis: TAR Archive (69.8 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilenet ", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for files like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": "", + "boxes": [ + 958, + 659, + 972, + 675 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_3_5", + "episode_id": 120, + "env_id": 3, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.843515396118164, + "advantages": -0.26746031641960144, + "generated_text": "{\n \"intent\": \"The user is exploring the Amazon product page for Lee Women's Plus Size Relaxed Fit Straight Leg Jean. A logical next step is to select the 'Add to Cart' button to add the item to the shopping cart, continuing the user's shopping exploration.\",\n \"action\": \"DoubleClick(1497, 328)\"\n}", + "action_string": "DoubleClick(1497, 328)", + "intent_string": "The user is exploring the Amazon product page for Lee Women's Plus Size Relaxed Fit Straight Leg Jean. A logical next step is to select the 'Add to Cart' button to add the item to the shopping cart, continuing the user's shopping exploration.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 276, + 1560, + 300 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Add ta Cart", + "boxes": [ + 1457, + 309, + 1537, + 348 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": " Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spande", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise straight leg,Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1462, + 312, + 1534, + 338 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": " Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spandex", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise,straight leg, Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ] + }, + { + "frame_id": "120_4_5", + "episode_id": 120, + "env_id": 4, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.240915298461914, + "advantages": 0.2953592836856842, + "generated_text": "{\n \"intent\": \"Since the current screen shows an editor application, accessing various tools and options within the app is instrumental for exploration. Clicking on the 'File' menu will provide us access to file management options such as saving, opening, and exporting, which can aid in understanding how to manage files within this environment.\",\n \"action\": \"Click(10, 35)\"\n}", + "action_string": "Click(10, 35)", + "intent_string": "Since the current screen shows an editor application, accessing various tools and options within the app is instrumental for exploration. Clicking on the 'File' menu will provide us access to file management options such as saving, opening, and exporting, which can aid in understanding how to manage files within this environment.", + "before_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 4, + 22, + 38, + 47 + ] + }, + { + "text": "Edit", + "boxes": [ + 40, + 28, + 74, + 42 + ] + }, + { + "text": "--- @x自", + "boxes": [ + 10, + 52, + 358, + 87 + ] + }, + { + "text": "-2", + "boxes": [ + 836, + 62, + 876, + 77 + ] + }, + { + "text": "園口", + "boxes": [ + 1002, + 56, + 1150, + 79 + ] + }, + { + "text": "Q?", + "boxes": [ + 154, + 99, + 212, + 117 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 240 + ] + }, + { + "text": "", + "boxes": [ + 1886, + 234, + 1912, + 306 + ] + }, + { + "text": "O", + "boxes": [ + 1884, + 308, + 1914, + 342 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New", + "boxes": [ + 28, + 54, + 58, + 73 + ] + }, + { + "text": "T曼·Ω", + "boxes": [ + 726, + 54, + 888, + 83 + ] + }, + { + "text": "園一→田", + "boxes": [ + 992, + 54, + 1252, + 83 + ] + }, + { + "text": "Open..", + "boxes": [ + 24, + 77, + 72, + 103 + ] + }, + { + "text": "Ctrl+(", + "boxes": [ + 254, + 81, + 288, + 95 + ] + }, + { + "text": "Open Remc", + "boxes": [ + 30, + 109, + 100, + 127 + ] + }, + { + "text": "a", + "boxes": [ + 520, + 101, + 538, + 115 + ] + }, + { + "text": "Recent Docu", + "boxes": [ + 30, + 135, + 106, + 153 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 24, + 191, + 84, + 216 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 238 + ] + }, + { + "text": "Template", + "boxes": [ + 30, + 222, + 90, + 240 + ] + }, + { + "text": "Reload", + "boxes": [ + 30, + 252, + 74, + 272 + ] + }, + { + "text": "Save", + "boxes": [ + 26, + 312, + 60, + 332 + ] + }, + { + "text": "Ctrl+s", + "boxes": [ + 250, + 312, + 294, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1874, + 303, + 1920, + 350 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 298, + 359 + ] + }, + { + "text": " Save Remote", + "boxes": [ + 24, + 365, + 116, + 389 + ] + }, + { + "text": "Save a Copy..", + "boxes": [ + 26, + 389, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 22, + 447, + 84, + 478 + ] + }, + { + "text": "Export As", + "boxes": [ + 24, + 476, + 94, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 68, + 528 + ] + }, + { + "text": "Preview in Web Bro", + "boxes": [ + 26, + 534, + 156, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 298, + 586 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 248, + 588, + 296, + 613 + ] + }, + { + "text": " Printer Settings.", + "boxes": [ + 24, + 617, + 134, + 641 + ] + }, + { + "text": " Properties..", + "boxes": [ + 24, + 649, + 102, + 673 + ] + }, + { + "text": " Exit Libreoffice", + "boxes": [ + 24, + 707, + 132, + 731 + ] + }, + { + "text": "Ctrl+Q", + "boxes": [ + 250, + 709, + 294, + 729 + ] + } + ] + }, + { + "frame_id": "120_5_5", + "episode_id": 120, + "env_id": 5, + "step": 5, + "format_rewards": 1.0, + "rewards": 8.843624114990234, + "advantages": -1.6835613250732422, + "generated_text": "{\n \"intent\": \"The current action will double-click the close button of the LibreOffice Calc application to exit the spreadsheet, allowing me to return to the desktop environment and explore further. This step is necessary to continue exploring the desktop and other applications.\",\n \"action\": \"DoubleClick(1722, 35)\"\n}", + "action_string": "DoubleClick(1722, 35)", + "intent_string": "The current action will double-click the close button of the LibreOffice Calc application to exit the spreadsheet, allowing me to return to the desktop environment and explore further. This step is necessary to continue exploring the desktop and other applications.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "--目-@电自-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "勺-→、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 258, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "·口·目·目@自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "G-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa要目目-% 00t三图-口目", + "boxes": [ + 12, + 93, + 1206, + 123 + ] + }, + { + "text": "·", + "boxes": [ + 126, + 126, + 162, + 155 + ] + }, + { + "text": "=[", + "boxes": [ + 148, + 125, + 260, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K\\→→+", + "boxes": [ + 14, + 995, + 132, + 1017 + ] + } + ] + }, + { + "frame_id": "120_6_5", + "episode_id": 120, + "env_id": 6, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.681946754455566, + "advantages": 0.91997230052948, + "generated_text": "{\n \"intent\": \"The current goal is to explore the environment as much as possible. Here, clicking on the 'Neutrogena' link will likely take me to another page about Neutrogena products or their official store, which might provide more insights into other items available for purchase or product details.\",\n \"action\": \"DoubleClick(781, 366)\"\n}", + "action_string": "DoubleClick(781, 366)", + "intent_string": "The current goal is to explore the environment as much as possible. Here, clicking on the 'Neutrogena' link will likely take me to another page about Neutrogena products or their official store, which might provide more insights into other items available for purchase or product details.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 38, + 66, + 130, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1782, + 62, + 1878, + 93 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "beauty", + "boxes": [ + 291, + 108, + 359, + 146 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Neutrogena Makeup Remover Wipes, Daily Facial Cleanser", + "boxes": [ + 680, + 258, + 1318, + 294 + ] + }, + { + "text": "↑", + "boxes": [ + 628, + 276, + 642, + 292 + ] + }, + { + "text": "$997 ($0.20 /count)", + "boxes": [ + 1394, + 276, + 1524, + 308 + ] + }, + { + "text": "Towelettes, Gently Cleanse and Remove Oil & Makeup,", + "boxes": [ + 676, + 286, + 1284, + 332 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1876, + 322 + ] + }, + { + "text": "日", + "boxes": [ + 164, + 318, + 196, + 353 + ] + }, + { + "text": "Alcohol-Free Makeup Wipes, 2 x 25 ct ", + "boxes": [ + 680, + 324, + 1098, + 359 + ] + }, + { + "text": "FREE International Retul", + "boxes": [ + 1396, + 316, + 1548, + 334 + ] + }, + { + "text": "No Import Charges & $6.99", + "boxes": [ + 1394, + 343, + 1574, + 367 + ] + }, + { + "text": "Visit the Neutrogena Store", + "boxes": [ + 684, + 355, + 856, + 379 + ] + }, + { + "text": "4.8★★★★★~", + "boxes": [ + 684, + 371, + 808, + 401 + ] + }, + { + "text": "Shipping to Canada Details ", + "boxes": [ + 1394, + 365, + 1578, + 389 + ] + }, + { + "text": "Delivery Tuesday, May 20.", + "boxes": [ + 1394, + 381, + 1564, + 411 + ] + }, + { + "text": " Order within 1 hr 45 mins", + "boxes": [ + 1392, + 399, + 1560, + 431 + ] + }, + { + "text": " 1 sustainabilty feature ", + "boxes": [ + 680, + 429, + 878, + 463 + ] + }, + { + "text": "@ Deliver to Canada", + "boxes": [ + 1392, + 433, + 1514, + 461 + ] + }, + { + "text": "Neutrogenao", + "boxes": [ + 380, + 443, + 480, + 469 + ] + }, + { + "text": "In Stock", + "boxes": [ + 1392, + 474, + 1466, + 500 + ] + }, + { + "text": "MAKEUPREMOVER", + "boxes": [ + 322, + 508, + 532, + 544 + ] + }, + { + "text": "-27% $997 ($0.20 /cout)", + "boxes": [ + 680, + 504, + 884, + 540 + ] + }, + { + "text": "Quantity: 1", + "boxes": [ + 1399, + 510, + 1485, + 541 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "List Price: $43.69 0", + "boxes": [ + 678, + 534, + 794, + 564 + ] + }, + { + "text": "Neutrogena\"", + "boxes": [ + 369, + 557, + 489, + 600 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1462, + 556, + 1536, + 582 + ] + }, + { + "text": "立", + "boxes": [ + 1791, + 551, + 1822, + 578 + ] + }, + { + "text": "MAKEUP REMOVER", + "boxes": [ + 268, + 590, + 528, + 629 + ] + }, + { + "text": "No Import Charges & $6.99 Shipping to C", + "boxes": [ + 682, + 588, + 946, + 611 + ] + }, + { + "text": " Buy Now", + "boxes": [ + 1466, + 598, + 1528, + 629 + ] + }, + { + "text": "Brand", + "boxes": [ + 682, + 613, + 728, + 639 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 860, + 613, + 934, + 639 + ] + }, + { + "text": " Skin Type", + "boxes": [ + 678, + 637, + 756, + 669 + ] + }, + { + "text": "Plant-BaseCompostableTowelettes", + "boxes": [ + 318, + 649, + 586, + 679 + ] + }, + { + "text": "All", + "boxes": [ + 858, + 641, + 884, + 665 + ] + }, + { + "text": "Material Feature", + "boxes": [ + 680, + 663, + 794, + 693 + ] + }, + { + "text": "Compostable", + "boxes": [ + 858, + 667, + 944, + 691 + ] + }, + { + "text": "Sold by", + "boxes": [ + 1402, + 661, + 1446, + 681 + ] + }, + { + "text": "Amazon.com", + "boxes": [ + 1466, + 659, + 1540, + 683 + ] + }, + { + "text": "50 Count", + "boxes": [ + 857, + 688, + 923, + 717 + ] + }, + { + "text": "Returns", + "boxes": [ + 1400, + 679, + 1450, + 705 + ] + }, + { + "text": "Unit Count", + "boxes": [ + 682, + 691, + 758, + 717 + ] + }, + { + "text": "Packaging ", + "boxes": [ + 1398, + 715, + 1462, + 741 + ] + }, + { + "text": "Ships in proc", + "boxes": [ + 1468, + 717, + 1538, + 739 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1858, + 731 + ] + }, + { + "text": "rofItem", + "boxes": [ + 730, + 721, + 788, + 739 + ] + }, + { + "text": " packaging", + "boxes": [ + 1465, + 730, + 1529, + 755 + ] + }, + { + "text": " About this item", + "boxes": [ + 678, + 758, + 838, + 794 + ] + }, + { + "text": "VSeemore", + "boxes": [ + 1400, + 752, + 1466, + 772 + ] + }, + { + "text": "Twin Pack with 25 count of ultra-soft, pre-mo", + "boxes": [ + 692, + 788, + 990, + 814 + ] + }, + { + "text": "d Neutrogena Makeup Remover Wipes with a", + "boxes": [ + 1030, + 790, + 1318, + 814 + ] + }, + { + "text": "FSA or HSA eligible Details", + "boxes": [ + 1392, + 786, + 1540, + 810 + ] + }, + { + "text": "口", + "boxes": [ + 1396, + 820, + 1420, + 842 + ] + }, + { + "text": "it feeling refreshed, soft, sm", + "boxes": [ + 700, + 830, + 878, + 854 + ] + }, + { + "text": "Add a gift receipt for easy", + "boxes": [ + 1414, + 838, + 1586, + 868 + ] + }, + { + "text": "returns", + "boxes": [ + 1413, + 857, + 1469, + 888 + ] + }, + { + "text": "er at your fingertips", + "boxes": [ + 942, + 895, + 1064, + 917 + ] + }, + { + "text": "Facial cleansing wet wipes are ophthalmologist, dermatologist-, and allergy-tested and gentle around", + "boxes": [ + 698, + 917, + 1354, + 941 + ] + }, + { + "text": "Add to List ", + "boxes": [ + 1400, + 911, + 1478, + 937 + ] + }, + { + "text": "without tugging", + "boxes": [ + 768, + 959, + 866, + 981 + ] + }, + { + "text": "Other sellers on Amazon", + "boxes": [ + 1386, + 981, + 1578, + 1011 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 6, + 59, + 132, + 96 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 136, + 143 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 298, + 115, + 410, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1580, + 147 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 632, + 223, + 1278, + 381 + ] + }, + { + "text": "BEAUTY TOA SCIENCE", + "boxes": [ + 746, + 427, + 1162, + 463 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 338, + 512, + 402, + 532 + ] + }, + { + "text": "Follow", + "boxes": [ + 358, + 548, + 404, + 574 + ] + }, + { + "text": "凸", + "boxes": [ + 450, + 550, + 474, + 574 + ] + }, + { + "text": " Q Search all Neutrogena", + "boxes": [ + 1406, + 544, + 1606, + 580 + ] + }, + { + "text": "HOME", + "boxes": [ + 494, + 554, + 542, + 574 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1004, + 655, + 1372, + 697 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 647, + 670, + 881, + 729 + ] + }, + { + "text": "Sheer Berry", + "boxes": [ + 999, + 696, + 1195, + 747 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 1407, + 698, + 1531, + 737 + ] + }, + { + "text": "BEAUTYTOASCIENCE", + "boxes": [ + 692, + 741, + 834, + 758 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and ", + "boxes": [ + 1002, + 752, + 1366, + 782 + ] + }, + { + "text": "DRO", + "boxes": [ + 1426, + 744, + 1460, + 764 + ] + }, + { + "text": " shea butter to instantly nourish dry lips ", + "boxes": [ + 998, + 772, + 1336, + 812 + ] + }, + { + "text": "$970 ($13.86/ounce) ", + "boxes": [ + 999, + 813, + 1135, + 857 + ] + }, + { + "text": "Pre-order. This item will be released on May 20,", + "boxes": [ + 1004, + 850, + 1332, + 874 + ] + }, + { + "text": "Nourishing", + "boxes": [ + 617, + 863, + 909, + 942 + ] + }, + { + "text": "2025.", + "boxes": [ + 1008, + 868, + 1044, + 887 + ] + }, + { + "text": "hydration", + "boxes": [ + 638, + 931, + 888, + 993 + ] + }, + { + "text": "for iuicv lips", + "boxes": [ + 602, + 982, + 924, + 1055 + ] + } + ] + }, + { + "frame_id": "120_7_5", + "episode_id": 120, + "env_id": 7, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.816093444824219, + "advantages": -0.30629679560661316, + "generated_text": "{\n \"intent\": \"To continue exploring the file system by accessing more directories or files. The 'Desktop' option in the left sidebar seems like a logical next step.\",\n \"action\": \"Click(485, 210)\"\n}", + "action_string": "Click(485, 210)", + "intent_string": "To continue exploring the file system by accessing more directories or files. The 'Desktop' option in the left sidebar seems like a logical next step.", + "before_screen_ocr_results": [ + { + "text": "A", + "boxes": [ + 0, + 0, + 32, + 30 + ] + }, + { + "text": "Untitled 1- Mousepad", + "boxes": [ + 872, + 4, + 1048, + 22 + ] + }, + { + "text": "回×", + "boxes": [ + 1860, + 4, + 1914, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 30, + 34, + 44 + ] + }, + { + "text": "Edit", + "boxes": [ + 44, + 30, + 74, + 44 + ] + }, + { + "text": "Open File", + "boxes": [ + 920, + 103, + 1006, + 127 + ] + }, + { + "text": "日可", + "boxes": [ + 1442, + 105, + 1500, + 125 + ] + }, + { + "text": " Recent", + "boxes": [ + 418, + 131, + 496, + 157 + ] + }, + { + "text": "Location", + "boxes": [ + 930, + 129, + 996, + 147 + ] + }, + { + "text": "Size", + "boxes": [ + 1274, + 131, + 1306, + 145 + ] + }, + { + "text": "Type", + "boxes": [ + 1348, + 127, + 1398, + 153 + ] + }, + { + "text": "|Accessed", + "boxes": [ + 1410, + 129, + 1490, + 147 + ] + }, + { + "text": " pcmanfm.conf", + "boxes": [ + 565, + 141, + 692, + 177 + ] + }, + { + "text": ".config/pcmanfm/LXDE", + "boxes": [ + 930, + 147, + 1086, + 177 + ] + }, + { + "text": "416 bytes", + "boxes": [ + 1270, + 145, + 1352, + 177 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 147, + 1390, + 175 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 147, + 1464, + 175 + ] + }, + { + "text": "仓 Home", + "boxes": [ + 413, + 160, + 491, + 191 + ] + }, + { + "text": " libfm.conf ", + "boxes": [ + 563, + 162, + 667, + 199 + ] + }, + { + "text": ".config/ibfm", + "boxes": [ + 929, + 168, + 1015, + 199 + ] + }, + { + "text": "853 bytes", + "boxes": [ + 1272, + 171, + 1346, + 197 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 173, + 1386, + 195 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 171, + 1462, + 197 + ] + }, + { + "text": " Desktop", + "boxes": [ + 420, + 199, + 506, + 224 + ] + }, + { + "text": " desktop-items-0.conf ", + "boxes": [ + 566, + 193, + 740, + 222 + ] + }, + { + "text": ".config/pcmanfm/LXDE", + "boxes": [ + 930, + 193, + 1086, + 222 + ] + }, + { + "text": "381 bytes", + "boxes": [ + 1274, + 195, + 1348, + 220 + ] + }, + { + "text": "Text ", + "boxes": [ + 1350, + 193, + 1390, + 222 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 193, + 1464, + 220 + ] + }, + { + "text": " panel", + "boxes": [ + 570, + 216, + 632, + 242 + ] + }, + { + "text": ".config/lxpanel/LXDE/panels", + "boxes": [ + 932, + 218, + 1114, + 242 + ] + }, + { + "text": "1.4 kB", + "boxes": [ + 1274, + 214, + 1324, + 242 + ] + }, + { + "text": "Text", + "boxes": [ + 1349, + 209, + 1390, + 246 + ] + }, + { + "text": "7 May", + "boxes": [ + 1413, + 211, + 1463, + 245 + ] + }, + { + "text": "Other Locati", + "boxes": [ + 432, + 238, + 526, + 262 + ] + }, + { + "text": " config", + "boxes": [ + 569, + 231, + 639, + 271 + ] + }, + { + "text": ".config/lxpanel/LXDE", + "boxes": [ + 932, + 242, + 1070, + 266 + ] + }, + { + "text": "29 bytes ", + "boxes": [ + 1274, + 240, + 1338, + 266 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 242, + 1388, + 264 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 240, + 1462, + 268 + ] + }, + { + "text": " rc.xml", + "boxes": [ + 574, + 262, + 638, + 288 + ] + }, + { + "text": ".config/openbox", + "boxes": [ + 932, + 264, + 1038, + 288 + ] + }, + { + "text": "26.1 kB", + "boxes": [ + 1272, + 260, + 1334, + 292 + ] + }, + { + "text": "Markup", + "boxes": [ + 1350, + 260, + 1412, + 292 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 264, + 1462, + 290 + ] + }, + { + "text": "firefox.desktop", + "boxes": [ + 572, + 286, + 690, + 310 + ] + }, + { + "text": "Desktop", + "boxes": [ + 932, + 286, + 992, + 312 + ] + }, + { + "text": "96 bytes", + "boxes": [ + 1276, + 286, + 1334, + 312 + ] + }, + { + "text": "Text", + "boxes": [ + 1356, + 288, + 1384, + 310 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 290, + 1460, + 310 + ] + }, + { + "text": "0 rc.xml", + "boxes": [ + 572, + 310, + 638, + 336 + ] + }, + { + "text": "/etc/xdg/openbox/LXDE ", + "boxes": [ + 930, + 308, + 1090, + 338 + ] + }, + { + "text": "22.3 kB", + "boxes": [ + 1272, + 306, + 1336, + 338 + ] + }, + { + "text": "Markup ", + "boxes": [ + 1352, + 310, + 1410, + 336 + ] + }, + { + "text": "7 May", + "boxes": [ + 1414, + 310, + 1462, + 336 + ] + }, + { + "text": "desktop-it", + "boxes": [ + 570, + 334, + 662, + 357 + ] + }, + { + "text": "/usr/local/share/doro-lxde-", + "boxes": [ + 932, + 334, + 1102, + 357 + ] + }, + { + "text": "298 bytes", + "boxes": [ + 1276, + 336, + 1340, + 355 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 336, + 1382, + 357 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 334, + 1462, + 359 + ] + }, + { + "text": " mimeapps.list ", + "boxes": [ + 568, + 353, + 696, + 383 + ] + }, + { + "text": ".config", + "boxes": [ + 932, + 353, + 982, + 381 + ] + }, + { + "text": "79 bytes", + "boxes": [ + 1270, + 351, + 1340, + 383 + ] + }, + { + "text": "Text ", + "boxes": [ + 1352, + 353, + 1390, + 381 + ] + }, + { + "text": "7 May", + "boxes": [ + 1414, + 353, + 1462, + 381 + ] + }, + { + "text": "Ixterminal.conf", + "boxes": [ + 570, + 379, + 694, + 403 + ] + }, + { + "text": ".config/lxterminal", + "boxes": [ + 932, + 379, + 1048, + 403 + ] + }, + { + "text": "1.4 kB", + "boxes": [ + 1274, + 377, + 1328, + 407 + ] + }, + { + "text": "Text", + "boxes": [ + 1352, + 377, + 1388, + 405 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1409, + 371, + 1499, + 408 + ] + }, + { + "text": "launchtaskbar.cfg", + "boxes": [ + 572, + 399, + 712, + 427 + ] + }, + { + "text": ".config/lxpanel", + "boxes": [ + 932, + 401, + 1030, + 427 + ] + }, + { + "text": "96 bytes", + "boxes": [ + 1272, + 399, + 1342, + 431 + ] + }, + { + "text": "Text", + "boxes": [ + 1349, + 395, + 1391, + 430 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1414, + 401, + 1496, + 427 + ] + }, + { + "text": "bg1.jpg", + "boxes": [ + 570, + 425, + 646, + 451 + ] + }, + { + "text": "/usr/local/share/doro-lxe-wallpapers", + "boxes": [ + 926, + 419, + 1176, + 453 + ] + }, + { + "text": "347.4 kB", + "boxes": [ + 1270, + 417, + 1346, + 455 + ] + }, + { + "text": "Image ", + "boxes": [ + 1350, + 421, + 1408, + 453 + ] + }, + { + "text": " 4 Apr 2021", + "boxes": [ + 1410, + 425, + 1496, + 451 + ] + }, + { + "text": " rc.xml ", + "boxes": [ + 570, + 449, + 640, + 474 + ] + }, + { + "text": "/etc/xdg/openbox", + "boxes": [ + 928, + 447, + 1050, + 476 + ] + }, + { + "text": "24.2 kB", + "boxes": [ + 1274, + 447, + 1334, + 472 + ] + }, + { + "text": "Markup", + "boxes": [ + 1350, + 445, + 1416, + 476 + ] + }, + { + "text": "10 Mar 2021", + "boxes": [ + 1408, + 447, + 1504, + 472 + ] + }, + { + "text": " menu.xml", + "boxes": [ + 570, + 472, + 662, + 496 + ] + }, + { + "text": "/etc/xdg/openbox", + "boxes": [ + 928, + 469, + 1050, + 498 + ] + }, + { + "text": "1.1 kB", + "boxes": [ + 1274, + 471, + 1322, + 498 + ] + }, + { + "text": "Markup", + "boxes": [ + 1354, + 471, + 1414, + 496 + ] + }, + { + "text": "15 Mar 2020", + "boxes": [ + 1404, + 471, + 1506, + 496 + ] + }, + { + "text": "OpenFile", + "boxes": [ + 370, + 1050, + 430, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "D", + "boxes": [ + 0, + 0, + 30, + 28 + ] + }, + { + "text": "Untitled 1- Mousepad", + "boxes": [ + 872, + 4, + 1048, + 22 + ] + }, + { + "text": "@", + "boxes": [ + 1860, + 6, + 1914, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 30, + 34, + 44 + ] + }, + { + "text": " Edit ", + "boxes": [ + 40, + 28, + 84, + 46 + ] + }, + { + "text": "Open File", + "boxes": [ + 920, + 103, + 1004, + 127 + ] + }, + { + "text": " Recent", + "boxes": [ + 418, + 131, + 494, + 157 + ] + }, + { + "text": "仑root", + "boxes": [ + 566, + 131, + 652, + 163 + ] + }, + { + "text": " Desktop", + "boxes": [ + 654, + 131, + 748, + 163 + ] + }, + { + "text": "Name", + "boxes": [ + 568, + 169, + 612, + 189 + ] + }, + { + "text": "Size", + "boxes": [ + 1268, + 173, + 1302, + 187 + ] + }, + { + "text": "Type", + "boxes": [ + 1356, + 173, + 1386, + 187 + ] + }, + { + "text": "[ Modified", + "boxes": [ + 1410, + 167, + 1486, + 191 + ] + }, + { + "text": " Desktop", + "boxes": [ + 426, + 189, + 506, + 232 + ] + }, + { + "text": "96 bytes", + "boxes": [ + 1274, + 193, + 1330, + 210 + ] + }, + { + "text": "7 May", + "boxes": [ + 1420, + 195, + 1460, + 210 + ] + }, + { + "text": " galculator.desktop", + "boxes": [ + 582, + 208, + 718, + 240 + ] + }, + { + "text": "105 bytes", + "boxes": [ + 1270, + 210, + 1344, + 242 + ] + }, + { + "text": "Text", + "boxes": [ + 1346, + 216, + 1382, + 236 + ] + }, + { + "text": "7 May", + "boxes": [ + 1413, + 209, + 1463, + 239 + ] + }, + { + "text": "e-calc.desktop", + "boxes": [ + 650, + 236, + 746, + 258 + ] + }, + { + "text": "112 bytes", + "boxes": [ + 1270, + 232, + 1344, + 264 + ] + }, + { + "text": "Text", + "boxes": [ + 1352, + 238, + 1382, + 258 + ] + }, + { + "text": "7 May", + "boxes": [ + 1417, + 233, + 1463, + 261 + ] + }, + { + "text": "draw.desktop", + "boxes": [ + 668, + 262, + 750, + 280 + ] + }, + { + "text": "11lbytes", + "boxes": [ + 1270, + 256, + 1344, + 288 + ] + }, + { + "text": "Text", + "boxes": [ + 1348, + 256, + 1386, + 286 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 258, + 1464, + 286 + ] + }, + { + "text": "132 bytes", + "boxes": [ + 1270, + 278, + 1344, + 310 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 280, + 1386, + 310 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 280, + 1464, + 308 + ] + }, + { + "text": "117 bytes", + "boxes": [ + 1270, + 302, + 1344, + 334 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 306, + 1382, + 330 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 304, + 1464, + 332 + ] + }, + { + "text": " Ixterminal.desktop", + "boxes": [ + 572, + 328, + 720, + 351 + ] + }, + { + "text": "103 bytes", + "boxes": [ + 1268, + 322, + 1346, + 359 + ] + }, + { + "text": "Text", + "boxes": [ + 1344, + 319, + 1390, + 360 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 324, + 1468, + 357 + ] + }, + { + "text": "114 bytes", + "boxes": [ + 1270, + 347, + 1344, + 379 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 349, + 1384, + 377 + ] + }, + { + "text": "19 Feb 2", + "boxes": [ + 1418, + 349, + 1472, + 379 + ] + }, + { + "text": "2024", + "boxes": [ + 1462, + 353, + 1502, + 373 + ] + }, + { + "text": " pcmanfm.desktop", + "boxes": [ + 568, + 371, + 716, + 401 + ] + }, + { + "text": "106 bytes", + "boxes": [ + 1270, + 371, + 1344, + 403 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 373, + 1386, + 401 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 373, + 1462, + 401 + ] + }, + { + "text": "OpenFile", + "boxes": [ + 370, + 1052, + 428, + 1070 + ] + } + ] + }, + { + "frame_id": "120_0_6", + "episode_id": 120, + "env_id": 0, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.623228073120117, + "advantages": -0.5794433355331421, + "generated_text": "{\n \"intent\": \"The action chosen is to click the 'Cancel' button to dismiss the file opening dialog and continue exploring the environment. This is valuable because it allows me to return to the original page or continue to different parts of the website.\",\n \"action\": \"Click(1045, 668)\"\n}", + "action_string": "Click(1045, 668)", + "intent_string": "The action chosen is to click the 'Cancel' button to dismiss the file opening dialog and continue exploring the environment. This is valuable because it allows me to return to the original page or continue to different parts of the website.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口", + "boxes": [ + 1118, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Whichis: TAR Archive (69.8 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilenet ", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for files like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": "+", + "boxes": [ + 958, + 661, + 970, + 673 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 702, + 887, + 1200, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1605, + 65, + 1659, + 88 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1086, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1238, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 873, + 466, + 1031, + 523 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system", + "boxes": [ + 704, + 887, + 1198, + 917 + ] + }, + { + "text": "ow @Mozill:a", + "boxes": [ + 1546, + 1024, + 1638, + 1048 + ] + }, + { + "text": "et Firefox fo", + "boxes": [ + 226, + 1052, + 308, + 1070 + ] + } + ] + }, + { + "frame_id": "120_1_6", + "episode_id": 120, + "env_id": 1, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.408445358276367, + "advantages": 0.5326245427131653, + "generated_text": "{\n \"intent\": \"The screenshot shows the Amazon website open, indicating that previous navigation actions led here. The click action will be performed on one of the available options or categories to explore further, which is a common everyday task on this platform.\",\n \"action\": \"DoubleClick(391, 161)\"\n}", + "action_string": "DoubleClick(391, 161)", + "intent_string": "The screenshot shows the Amazon website open, indicating that previous navigation actions led here. The click action will be performed on one of the available options or categories to explore further, which is a common everyday task on this platform.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四@", + "boxes": [ + 1779, + 59, + 1881, + 96 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Canara", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "SEarch Amazon", + "boxes": [ + 296, + 115, + 414, + 139 + ] + }, + { + "text": "α", + "boxes": [ + 1450, + 115, + 1480, + 147 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "+", + "boxes": [ + 1826, + 115, + 1860, + 137 + ] + }, + { + "text": "jeans", + "boxes": [ + 300, + 155, + 350, + 181 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 151, + 1478, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Today'sDeals", + "boxes": [ + 88, + 169, + 170, + 187 + ] + }, + { + "text": "×××××", + "boxes": [ + 1446, + 179, + 1482, + 361 + ] + }, + { + "text": "beauty", + "boxes": [ + 302, + 189, + 364, + 220 + ] + }, + { + "text": " gaming", + "boxes": [ + 299, + 218, + 371, + 257 + ] + }, + { + "text": " gaming headsets", + "boxes": [ + 302, + 258, + 440, + 288 + ] + }, + { + "text": " gaming keyboard", + "boxes": [ + 300, + 290, + 444, + 326 + ] + }, + { + "text": "shoes in Women's Shoes", + "boxes": [ + 304, + 328, + 486, + 357 + ] + }, + { + "text": "home decor", + "boxes": [ + 306, + 365, + 398, + 391 + ] + }, + { + "text": "+", + "boxes": [ + 1450, + 363, + 1474, + 386 + ] + }, + { + "text": "fou are on a", + "boxes": [ + 514, + 461, + 588, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1348, + 492, + 1370, + 506 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 610, + 526, + 832, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 984, + 532, + 1286, + 556 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1534, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 612, + 889, + 690, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 242, + 969, + 498, + 999 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 354, + 1026 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "", + "boxes": [ + 251, + 59, + 311, + 96 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "jeans", + "boxes": [ + 294, + 115, + 348, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": " Gift Cards ", + "boxes": [ + 354, + 163, + 428, + 187 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 172, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Thin", + "boxes": [ + 14, + 284, + 44, + 306 + ] + }, + { + "text": "High Waist", + "boxes": [ + 12, + 302, + 86, + 328 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1876, + 322 + ] + }, + { + "text": "Straight ", + "boxes": [ + 10, + 320, + 74, + 351 + ] + }, + { + "text": "Bootcut", + "boxes": [ + 14, + 343, + 68, + 363 + ] + }, + { + "text": "Gender", + "boxes": [ + 10, + 409, + 64, + 435 + ] + }, + { + "text": "□Men", + "boxes": [ + 9, + 428, + 71, + 461 + ] + }, + { + "text": "women", + "boxes": [ + 10, + 453, + 86, + 478 + ] + }, + { + "text": " Boys", + "boxes": [ + 9, + 467, + 71, + 502 + ] + }, + { + "text": "Girls", + "boxes": [ + 14, + 494, + 68, + 514 + ] + }, + { + "text": "Babies", + "boxes": [ + 34, + 514, + 76, + 534 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "26", + "boxes": [ + 108, + 600, + 138, + 613 + ] + }, + { + "text": "Men's Classic Relaxed Fit Jean ", + "boxes": [ + 608, + 613, + 830, + 643 + ] + }, + { + "text": "Wrangler Authentics Men's Regular", + "boxes": [ + 900, + 613, + 1160, + 643 + ] + }, + { + "text": "Gloria Vanderbilt Women's Amanda", + "boxes": [ + 1194, + 611, + 1460, + 641 + ] + }, + { + "text": " Trendy Queen Womens Baggy Jeans", + "boxes": [ + 1486, + 613, + 1758, + 641 + ] + }, + { + "text": "33", + "boxes": [ + 156, + 627, + 186, + 641 + ] + }, + { + "text": "Amazon Essentials Men's Slim-Fit ", + "boxes": [ + 314, + 641, + 562, + 665 + ] + }, + { + "text": "★★★★☆28.090", + "boxes": [ + 610, + 639, + 758, + 667 + ] + }, + { + "text": "Fit Comfort Flex Waist Jean", + "boxes": [ + 900, + 635, + 1108, + 667 + ] + }, + { + "text": "Classic High Rise Tapered Jean ", + "boxes": [ + 1196, + 635, + 1422, + 667 + ] + }, + { + "text": "Wide Leg Denim Pants Boyfriend", + "boxes": [ + 1486, + 635, + 1734, + 665 + ] + }, + { + "text": "Jeans", + "boxes": [ + 314, + 665, + 360, + 691 + ] + }, + { + "text": "★★★★★ 83.863", + "boxes": [ + 904, + 663, + 1056, + 693 + ] + }, + { + "text": "★★★★☆75.800", + "boxes": [ + 1198, + 667, + 1346, + 691 + ] + }, + { + "text": "Loose Streetwear Trousers Fall", + "boxes": [ + 1486, + 661, + 1718, + 689 + ] + }, + { + "text": "$1398 List: $4.99", + "boxes": [ + 603, + 688, + 749, + 735 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 902, + 679, + 1080, + 717 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 1194, + 679, + 1376, + 717 + ] + }, + { + "text": "★★★★☆42.605", + "boxes": [ + 318, + 691, + 456, + 715 + ] + }, + { + "text": "Fashion y2k Outfits 2025", + "boxes": [ + 1486, + 685, + 1680, + 715 + ] + }, + { + "text": "$3155", + "boxes": [ + 898, + 713, + 970, + 752 + ] + }, + { + "text": "$29 st: 4-00", + "boxes": [ + 1187, + 708, + 1339, + 759 + ] + }, + { + "text": "★★★★☆91", + "boxes": [ + 1490, + 715, + 1612, + 739 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1810, + 707, + 1862, + 733 + ] + }, + { + "text": "$2660", + "boxes": [ + 312, + 719, + 378, + 758 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 608, + 728, + 748, + 757 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 1486, + 728, + 1670, + 765 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 314, + 760, + 444, + 784 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 608, + 752, + 698, + 776 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 900, + 752, + 1048, + 782 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 1194, + 752, + 1342, + 782 + ] + }, + { + "text": "$3299", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "hips to Canada", + "boxes": [ + 322, + 784, + 404, + 802 + ] + }, + { + "text": "1-2", + "boxes": [ + 114, + 798, + 134, + 812 + ] + }, + { + "text": " 1 sustainability feature ", + "boxes": [ + 318, + 800, + 506, + 828 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 1484, + 796, + 1636, + 832 + ] + }, + { + "text": "11-12", + "boxes": [ + 30, + 860, + 66, + 874 + ] + }, + { + "text": "sored", + "boxes": [ + 1222, + 883, + 1270, + 903 + ] + }, + { + "text": "Lysse Women's Denim Straight Leg", + "boxes": [ + 1192, + 899, + 1484, + 933 + ] + }, + { + "text": "★★★★☆324", + "boxes": [ + 1196, + 927, + 1326, + 957 + ] + }, + { + "text": "18", + "boxes": [ + 116, + 959, + 134, + 973 + ] + }, + { + "text": "八22 24", + "boxes": [ + 184, + 957, + 272, + 975 + ] + }, + { + "text": "$8800", + "boxes": [ + 1189, + 958, + 1257, + 997 + ] + }, + { + "text": "Delivery May 27 - 31", + "boxes": [ + 1196, + 999, + 1330, + 1022 + ] + } + ] + }, + { + "frame_id": "120_2_6", + "episode_id": 120, + "env_id": 2, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.217788696289062, + "advantages": -1.153648853302002, + "generated_text": "{\n \"intent\": \"To continue exploring the browser environment, the first meaningful action is to close the download window. This will allow us to access the browser tab and potentially navigate elsewhere or download more content.\",\n \"action\": \"DoubleClick(1038, 700)\"\n}", + "action_string": "DoubleClick(1038, 700)", + "intent_string": "To continue exploring the browser environment, the first meaningful action is to close the download window. This will allow us to access the browser tab and potentially navigate elsewhere or download more content.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口", + "boxes": [ + 1118, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Whichis: TAR Archive (69.8 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilenet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now", + "boxes": [ + 764, + 594, + 1102, + 617 + ] + }, + { + "text": "", + "boxes": [ + 958, + 659, + 972, + 675 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1882, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 351, + 1088, + 381 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 383, + 922, + 411 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (69.8 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwnload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now", + "boxes": [ + 764, + 594, + 1102, + 617 + ] + }, + { + "text": "Cancel", + "boxes": [ + 1010, + 653, + 1060, + 673 + ] + }, + { + "text": "Using Debian,Ubuntu or any Deban-Daed distruton?", + "boxes": [ + 728, + 677, + 1178, + 717 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1122, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompany", + "boxes": [ + 738, + 1030, + 792, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "120_3_6", + "episode_id": 120, + "env_id": 3, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.557122230529785, + "advantages": -0.6730660796165466, + "generated_text": "{\n \"intent\": \"The user wants to add the jeans to their shopping cart. This action will allow them to purchase the item after selecting the desired size.\",\n \"action\": \"DoubleClick(1512, 322)\"\n}", + "action_string": "DoubleClick(1512, 322)", + "intent_string": "The user wants to add the jeans to their shopping cart. This action will allow them to purchase the item after selecting the desired size.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1462, + 312, + 1534, + 338 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": " Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spande", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise straight leg,Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "Add o Rart", + "boxes": [ + 1455, + 305, + 1541, + 348 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": "Product details", + "boxes": [ + 652, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spandex", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise,straight leg, Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ] + }, + { + "frame_id": "120_4_6", + "episode_id": 120, + "env_id": 4, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.705574035644531, + "advantages": -0.46282047033309937, + "generated_text": "{\n \"intent\": \"To begin exploring the LibreOffice Writer environment, clicking 'File' will allow access to various options such as Save, New Document, Recent Documents, etc.\",\n \"action\": \"DoubleClick(25, 17)\"\n}", + "action_string": "DoubleClick(25, 17)", + "intent_string": "To begin exploring the LibreOffice Writer environment, clicking 'File' will allow access to various options such as Save, New Document, Recent Documents, etc.", + "before_screen_ocr_results": [ + { + "text": "New", + "boxes": [ + 28, + 54, + 58, + 73 + ] + }, + { + "text": "園→-", + "boxes": [ + 994, + 54, + 1252, + 83 + ] + }, + { + "text": "Open..", + "boxes": [ + 24, + 77, + 72, + 103 + ] + }, + { + "text": "Ctrl+(", + "boxes": [ + 254, + 81, + 288, + 95 + ] + }, + { + "text": "Open Remc", + "boxes": [ + 30, + 109, + 100, + 127 + ] + }, + { + "text": "RecentDocu", + "boxes": [ + 30, + 135, + 106, + 153 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 24, + 191, + 84, + 216 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 238 + ] + }, + { + "text": "Template", + "boxes": [ + 30, + 222, + 90, + 240 + ] + }, + { + "text": "Reload", + "boxes": [ + 28, + 252, + 74, + 272 + ] + }, + { + "text": "Save", + "boxes": [ + 26, + 312, + 60, + 332 + ] + }, + { + "text": "Ctrl+s", + "boxes": [ + 250, + 312, + 294, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1874, + 303, + 1920, + 350 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 298, + 359 + ] + }, + { + "text": " Save Remote", + "boxes": [ + 24, + 365, + 116, + 389 + ] + }, + { + "text": "Save a Copy..", + "boxes": [ + 26, + 389, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 22, + 447, + 84, + 478 + ] + }, + { + "text": "Export As", + "boxes": [ + 24, + 476, + 94, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 68, + 528 + ] + }, + { + "text": "Preview in Web Bro", + "boxes": [ + 26, + 534, + 156, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 298, + 586 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 248, + 588, + 296, + 613 + ] + }, + { + "text": " Printer Settings.", + "boxes": [ + 24, + 617, + 134, + 641 + ] + }, + { + "text": " Properties..", + "boxes": [ + 24, + 649, + 102, + 673 + ] + }, + { + "text": " Exit LibreOffice", + "boxes": [ + 22, + 705, + 134, + 735 + ] + }, + { + "text": "Ctrl+Q", + "boxes": [ + 250, + 709, + 294, + 729 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 10, + 22, + 36, + 42 + ] + }, + { + "text": "--- @x自", + "boxes": [ + 10, + 52, + 358, + 87 + ] + }, + { + "text": "中園口园", + "boxes": [ + 1002, + 56, + 1140, + 79 + ] + }, + { + "text": "Q?", + "boxes": [ + 154, + 99, + 210, + 117 + ] + }, + { + "text": "口", + "boxes": [ + 1890, + 204, + 1912, + 238 + ] + }, + { + "text": "", + "boxes": [ + 1886, + 234, + 1912, + 304 + ] + }, + { + "text": "O", + "boxes": [ + 1884, + 308, + 1914, + 342 + ] + } + ] + }, + { + "frame_id": "120_5_6", + "episode_id": 120, + "env_id": 5, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.131204605102539, + "advantages": -1.2762740850448608, + "generated_text": "{\n \"intent\": \"To explore by moving to a location typically associated with accessing the file menu options. Typically, file operations such as saving, opening files, or exiting an application need to be performed from here.\",\n \"action\": \"Click(14, 39)\"\n}", + "action_string": "Click(14, 39)", + "intent_string": "To explore by moving to a location typically associated with accessing the file menu options. Typically, file operations such as saving, opening files, or exiting an application need to be performed from here.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "·口·目·目@自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "白·>、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa要目目-% 00t三图-口目", + "boxes": [ + 12, + 93, + 1206, + 123 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 258, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "→→+", + "boxes": [ + 14, + 995, + 132, + 1017 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "ata", + "boxes": [ + 368, + 30, + 388, + 44 + ] + }, + { + "text": "晶自", + "boxes": [ + 292, + 54, + 370, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 378, + 54, + 444, + 85 + ] + }, + { + "text": "G-.", + "boxes": [ + 454, + 56, + 550, + 81 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Open.", + "boxes": [ + 28, + 81, + 68, + 101 + ] + }, + { + "text": "Ctrl+C", + "boxes": [ + 252, + 79, + 290, + 99 + ] + }, + { + "text": "a", + "boxes": [ + 312, + 99, + 332, + 119 + ] + }, + { + "text": "a要日日%00三田二口", + "boxes": [ + 342, + 93, + 1204, + 123 + ] + }, + { + "text": "Open Ren", + "boxes": [ + 30, + 109, + 92, + 127 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 26, + 191, + 84, + 216 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 270 + ] + }, + { + "text": "Reloao", + "boxes": [ + 32, + 254, + 72, + 270 + ] + }, + { + "text": "Save", + "boxes": [ + 28, + 312, + 60, + 332 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 296, + 359 + ] + }, + { + "text": "SaveRemote", + "boxes": [ + 28, + 367, + 114, + 385 + ] + }, + { + "text": "Save a Copy...", + "boxes": [ + 24, + 387, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 21, + 444, + 83, + 477 + ] + }, + { + "text": "Export as PDF", + "boxes": [ + 26, + 476, + 118, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 66, + 528 + ] + }, + { + "text": "Preview in Web Brows", + "boxes": [ + 26, + 534, + 172, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 296, + 584 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 250, + 590, + 292, + 609 + ] + }, + { + "text": "Printer Settings.", + "boxes": [ + 26, + 619, + 132, + 637 + ] + }, + { + "text": " Properties.", + "boxes": [ + 24, + 649, + 100, + 673 + ] + }, + { + "text": "Digital :", + "boxes": [ + 26, + 679, + 78, + 697 + ] + }, + { + "text": "Exit LibreOffice", + "boxes": [ + 24, + 707, + 126, + 731 + ] + }, + { + "text": "Ctrl+(", + "boxes": [ + 254, + 711, + 288, + 725 + ] + }, + { + "text": "35", + "boxes": [ + 12, + 750, + 32, + 766 + ] + }, + { + "text": "1→一", + "boxes": [ + 22, + 997, + 92, + 1014 + ] + } + ] + }, + { + "frame_id": "120_6_6", + "episode_id": 120, + "env_id": 6, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.094701766967773, + "advantages": 0.08828362822532654, + "generated_text": "{\n \"intent\": \"To continue exploring the Neutrogena brand page and potentially find more products or information about the product displayed.\",\n \"action\": \"DoubleClick(654, 702)\"\n}", + "action_string": "DoubleClick(654, 702)", + "intent_string": "To continue exploring the Neutrogena brand page and potentially find more products or information about the product displayed.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 6, + 59, + 132, + 96 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 136, + 143 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 298, + 115, + 410, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1580, + 147 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 632, + 223, + 1278, + 381 + ] + }, + { + "text": "BEAUTY TOA SCIENCE", + "boxes": [ + 746, + 427, + 1162, + 463 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 338, + 512, + 402, + 532 + ] + }, + { + "text": "Follow", + "boxes": [ + 358, + 548, + 404, + 574 + ] + }, + { + "text": "凸", + "boxes": [ + 450, + 550, + 474, + 574 + ] + }, + { + "text": " Q Search all Neutrogena", + "boxes": [ + 1406, + 544, + 1606, + 580 + ] + }, + { + "text": "HOME", + "boxes": [ + 494, + 554, + 542, + 574 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1004, + 655, + 1372, + 697 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 647, + 670, + 881, + 729 + ] + }, + { + "text": "Sheer Berry", + "boxes": [ + 999, + 696, + 1195, + 747 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 1407, + 698, + 1531, + 737 + ] + }, + { + "text": "BEAUTYTOASCIENCE", + "boxes": [ + 692, + 741, + 834, + 758 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and ", + "boxes": [ + 1002, + 752, + 1366, + 782 + ] + }, + { + "text": "DRO", + "boxes": [ + 1426, + 744, + 1460, + 764 + ] + }, + { + "text": " shea butter to instantly nourish dry lips ", + "boxes": [ + 998, + 772, + 1336, + 812 + ] + }, + { + "text": "$970 ($13.86/ounce) ", + "boxes": [ + 999, + 813, + 1135, + 857 + ] + }, + { + "text": "Pre-order. This item will be released on May 20,", + "boxes": [ + 1004, + 850, + 1332, + 874 + ] + }, + { + "text": "Nourishing", + "boxes": [ + 617, + 863, + 909, + 942 + ] + }, + { + "text": "2025.", + "boxes": [ + 1008, + 868, + 1044, + 887 + ] + }, + { + "text": "hydration", + "boxes": [ + 638, + 931, + 888, + 993 + ] + }, + { + "text": "for iuicv lips", + "boxes": [ + 602, + 982, + 924, + 1055 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 6, + 59, + 132, + 96 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 136, + 143 + ] + }, + { + "text": " a", + "boxes": [ + 130, + 103, + 224, + 155 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 298, + 115, + 410, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1452, + 115, + 1476, + 137 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1580, + 147 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "day's Deals", + "boxes": [ + 106, + 169, + 172, + 187 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 634, + 225, + 1276, + 377 + ] + }, + { + "text": "BEAUTYTOA SCIENCE", + "boxes": [ + 746, + 427, + 1162, + 463 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 338, + 512, + 402, + 532 + ] + }, + { + "text": "Follow", + "boxes": [ + 358, + 548, + 404, + 574 + ] + }, + { + "text": "凸", + "boxes": [ + 450, + 550, + 474, + 574 + ] + }, + { + "text": "HOME", + "boxes": [ + 492, + 552, + 544, + 578 + ] + }, + { + "text": "Q Search all Neutrogena", + "boxes": [ + 1406, + 544, + 1606, + 580 + ] + }, + { + "text": "FESTIVAL", + "boxes": [ + 568, + 556, + 638, + 574 + ] + }, + { + "text": "COLLAGEN", + "boxes": [ + 930, + 556, + 1012, + 574 + ] + }, + { + "text": "MORE", + "boxes": [ + 1278, + 556, + 1324, + 574 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1004, + 655, + 1372, + 697 + ] + }, + { + "text": "Neutrogena:", + "boxes": [ + 647, + 670, + 881, + 729 + ] + }, + { + "text": "Sheer Berry", + "boxes": [ + 999, + 696, + 1195, + 747 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 1407, + 698, + 1531, + 737 + ] + }, + { + "text": "BEAUTYTOASCIENCE", + "boxes": [ + 694, + 741, + 834, + 758 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and ", + "boxes": [ + 1002, + 752, + 1366, + 782 + ] + }, + { + "text": "DRO", + "boxes": [ + 1426, + 744, + 1460, + 764 + ] + }, + { + "text": " shea butter to instantly nourish dry ips ", + "boxes": [ + 998, + 772, + 1336, + 812 + ] + }, + { + "text": "$970 ($13.86/ounce) ", + "boxes": [ + 999, + 813, + 1135, + 857 + ] + }, + { + "text": "Pre-order. This item will be released on May 20,", + "boxes": [ + 1004, + 850, + 1332, + 874 + ] + }, + { + "text": "Nourishing", + "boxes": [ + 617, + 863, + 909, + 942 + ] + }, + { + "text": "2025.", + "boxes": [ + 1006, + 866, + 1046, + 891 + ] + }, + { + "text": "hydration", + "boxes": [ + 644, + 935, + 886, + 991 + ] + }, + { + "text": "for iuicv lips", + "boxes": [ + 604, + 982, + 924, + 1055 + ] + } + ] + }, + { + "frame_id": "120_7_6", + "episode_id": 120, + "env_id": 7, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.573829650878906, + "advantages": -0.6494041085243225, + "generated_text": "{\n \"intent\": \"The goal is to continue exploring the file system from the home directory to the 'Desktop' folder, where various files and shortcuts are located. By clicking on the 'Desktop' icon, we can view the contents of the 'Desktop' folder. This will help us understand what files and shortcuts are available.\",\n \"action\": \"DoubleClick(501, 211)\"\n}", + "action_string": "DoubleClick(501, 211)", + "intent_string": "The goal is to continue exploring the file system from the home directory to the 'Desktop' folder, where various files and shortcuts are located. By clicking on the 'Desktop' icon, we can view the contents of the 'Desktop' folder. This will help us understand what files and shortcuts are available.", + "before_screen_ocr_results": [ + { + "text": "D", + "boxes": [ + 0, + 0, + 30, + 28 + ] + }, + { + "text": "Untitled 1- Mousepad", + "boxes": [ + 872, + 4, + 1048, + 22 + ] + }, + { + "text": "@", + "boxes": [ + 1860, + 6, + 1914, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 30, + 34, + 44 + ] + }, + { + "text": " Edit ", + "boxes": [ + 40, + 28, + 84, + 46 + ] + }, + { + "text": "Open File", + "boxes": [ + 920, + 103, + 1004, + 127 + ] + }, + { + "text": " Recent", + "boxes": [ + 418, + 131, + 494, + 157 + ] + }, + { + "text": "仑root", + "boxes": [ + 566, + 131, + 652, + 163 + ] + }, + { + "text": " Desktop", + "boxes": [ + 654, + 131, + 748, + 163 + ] + }, + { + "text": "Name", + "boxes": [ + 568, + 169, + 612, + 189 + ] + }, + { + "text": "Size", + "boxes": [ + 1268, + 173, + 1302, + 187 + ] + }, + { + "text": "Type", + "boxes": [ + 1356, + 173, + 1386, + 187 + ] + }, + { + "text": "[ Modified", + "boxes": [ + 1410, + 167, + 1486, + 191 + ] + }, + { + "text": " Desktop", + "boxes": [ + 426, + 189, + 506, + 232 + ] + }, + { + "text": "96 bytes", + "boxes": [ + 1274, + 193, + 1330, + 210 + ] + }, + { + "text": "7 May", + "boxes": [ + 1420, + 195, + 1460, + 210 + ] + }, + { + "text": " galculator.desktop", + "boxes": [ + 582, + 208, + 718, + 240 + ] + }, + { + "text": "105 bytes", + "boxes": [ + 1270, + 210, + 1344, + 242 + ] + }, + { + "text": "Text", + "boxes": [ + 1346, + 216, + 1382, + 236 + ] + }, + { + "text": "7 May", + "boxes": [ + 1413, + 209, + 1463, + 239 + ] + }, + { + "text": "e-calc.desktop", + "boxes": [ + 650, + 236, + 746, + 258 + ] + }, + { + "text": "112 bytes", + "boxes": [ + 1270, + 232, + 1344, + 264 + ] + }, + { + "text": "Text", + "boxes": [ + 1352, + 238, + 1382, + 258 + ] + }, + { + "text": "7 May", + "boxes": [ + 1417, + 233, + 1463, + 261 + ] + }, + { + "text": "draw.desktop", + "boxes": [ + 668, + 262, + 750, + 280 + ] + }, + { + "text": "11lbytes", + "boxes": [ + 1270, + 256, + 1344, + 288 + ] + }, + { + "text": "Text", + "boxes": [ + 1348, + 256, + 1386, + 286 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 258, + 1464, + 286 + ] + }, + { + "text": "132 bytes", + "boxes": [ + 1270, + 278, + 1344, + 310 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 280, + 1386, + 310 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 280, + 1464, + 308 + ] + }, + { + "text": "117 bytes", + "boxes": [ + 1270, + 302, + 1344, + 334 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 306, + 1382, + 330 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 304, + 1464, + 332 + ] + }, + { + "text": " Ixterminal.desktop", + "boxes": [ + 572, + 328, + 720, + 351 + ] + }, + { + "text": "103 bytes", + "boxes": [ + 1268, + 322, + 1346, + 359 + ] + }, + { + "text": "Text", + "boxes": [ + 1344, + 319, + 1390, + 360 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 324, + 1468, + 357 + ] + }, + { + "text": "114 bytes", + "boxes": [ + 1270, + 347, + 1344, + 379 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 349, + 1384, + 377 + ] + }, + { + "text": "19 Feb 2", + "boxes": [ + 1418, + 349, + 1472, + 379 + ] + }, + { + "text": "2024", + "boxes": [ + 1462, + 353, + 1502, + 373 + ] + }, + { + "text": " pcmanfm.desktop", + "boxes": [ + 568, + 371, + 716, + 401 + ] + }, + { + "text": "106 bytes", + "boxes": [ + 1270, + 371, + 1344, + 403 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 373, + 1386, + 401 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 373, + 1462, + 401 + ] + }, + { + "text": "OpenFile", + "boxes": [ + 370, + 1052, + 428, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "D", + "boxes": [ + 0, + 0, + 30, + 28 + ] + }, + { + "text": "Untitled 1- Mousepad", + "boxes": [ + 872, + 4, + 1048, + 22 + ] + }, + { + "text": "@", + "boxes": [ + 1860, + 6, + 1914, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 30, + 34, + 44 + ] + }, + { + "text": " Edit ", + "boxes": [ + 40, + 28, + 84, + 46 + ] + }, + { + "text": "View", + "boxes": [ + 134, + 30, + 184, + 48 + ] + }, + { + "text": "Open File", + "boxes": [ + 920, + 103, + 1004, + 127 + ] + }, + { + "text": " Recent", + "boxes": [ + 418, + 131, + 494, + 157 + ] + }, + { + "text": "仑root", + "boxes": [ + 566, + 131, + 652, + 163 + ] + }, + { + "text": " Desktop", + "boxes": [ + 654, + 131, + 748, + 163 + ] + }, + { + "text": " Home", + "boxes": [ + 418, + 165, + 486, + 191 + ] + }, + { + "text": "Name", + "boxes": [ + 568, + 169, + 612, + 189 + ] + }, + { + "text": "Size", + "boxes": [ + 1268, + 173, + 1302, + 187 + ] + }, + { + "text": "Type", + "boxes": [ + 1356, + 173, + 1386, + 187 + ] + }, + { + "text": "[ Modified", + "boxes": [ + 1410, + 167, + 1486, + 191 + ] + }, + { + "text": "96 bytes", + "boxes": [ + 1274, + 193, + 1330, + 210 + ] + }, + { + "text": "7 May", + "boxes": [ + 1420, + 195, + 1460, + 210 + ] + }, + { + "text": " galculator.desktop", + "boxes": [ + 582, + 208, + 718, + 240 + ] + }, + { + "text": "105 bytes", + "boxes": [ + 1270, + 210, + 1344, + 242 + ] + }, + { + "text": "Text", + "boxes": [ + 1346, + 216, + 1382, + 236 + ] + }, + { + "text": "7 May", + "boxes": [ + 1413, + 209, + 1463, + 239 + ] + }, + { + "text": "e-calc.desktop", + "boxes": [ + 652, + 236, + 746, + 258 + ] + }, + { + "text": "112 bytes", + "boxes": [ + 1270, + 232, + 1344, + 264 + ] + }, + { + "text": "Text", + "boxes": [ + 1347, + 235, + 1383, + 262 + ] + }, + { + "text": "7 May", + "boxes": [ + 1417, + 233, + 1463, + 261 + ] + }, + { + "text": "draw.desktop", + "boxes": [ + 668, + 262, + 750, + 280 + ] + }, + { + "text": "11lbytes", + "boxes": [ + 1270, + 256, + 1344, + 288 + ] + }, + { + "text": "Text", + "boxes": [ + 1348, + 256, + 1386, + 286 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 258, + 1464, + 286 + ] + }, + { + "text": "132 bytes", + "boxes": [ + 1270, + 278, + 1344, + 310 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 280, + 1386, + 310 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 280, + 1464, + 308 + ] + }, + { + "text": "117 bytes", + "boxes": [ + 1270, + 302, + 1344, + 334 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 306, + 1382, + 330 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 304, + 1464, + 332 + ] + }, + { + "text": " Ixterminal.desktop", + "boxes": [ + 572, + 328, + 720, + 351 + ] + }, + { + "text": "103 bytes", + "boxes": [ + 1268, + 322, + 1346, + 359 + ] + }, + { + "text": "Text", + "boxes": [ + 1344, + 319, + 1390, + 360 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 324, + 1468, + 357 + ] + }, + { + "text": "114 bytes", + "boxes": [ + 1270, + 347, + 1344, + 379 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 349, + 1384, + 377 + ] + }, + { + "text": "19 Feb 2", + "boxes": [ + 1418, + 349, + 1472, + 379 + ] + }, + { + "text": "2024", + "boxes": [ + 1462, + 353, + 1502, + 373 + ] + }, + { + "text": " pcmanfm.desktop", + "boxes": [ + 568, + 371, + 716, + 401 + ] + }, + { + "text": "106 bytes", + "boxes": [ + 1270, + 371, + 1344, + 403 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 373, + 1386, + 401 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 373, + 1462, + 401 + ] + }, + { + "text": "OpenFile", + "boxes": [ + 370, + 1052, + 428, + 1070 + ] + } + ] + }, + { + "frame_id": "120_0_7", + "episode_id": 120, + "env_id": 0, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.072770118713379, + "advantages": 0.057222820818424225, + "generated_text": "{\n \"intent\": \"The current task is to explore the internet by downloading Firefox. Clicking 'Download for Linux 64-bit' will help me progress towards achieving this goal.\",\n \"action\": \"Click(960, 660)\"\n}", + "action_string": "Click(960, 660)", + "intent_string": "The current task is to explore the internet by downloading Firefox. Clicking 'Download for Linux 64-bit' will help me progress towards achieving this goal.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1605, + 65, + 1659, + 88 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1086, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1238, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 873, + 466, + 1031, + 523 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system", + "boxes": [ + 704, + 887, + 1198, + 917 + ] + }, + { + "text": "ow @Mozill", + "boxes": [ + 1546, + 1024, + 1636, + 1048 + ] + }, + { + "text": "et Firefox fo", + "boxes": [ + 226, + 1052, + 308, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口", + "boxes": [ + 1118, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Whichis: TAR Archive (69.8 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now", + "boxes": [ + 764, + 594, + 1102, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system", + "boxes": [ + 702, + 887, + 1200, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_1_7", + "episode_id": 120, + "env_id": 1, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.690788269042969, + "advantages": -0.48376089334487915, + "generated_text": "{\n \"intent\": \"The goal is to continue exploring the environment, so it's beneficial to scroll down and view more product listings.\",\n \"action\": \"DoubleClick(330, 813)\"\n}", + "action_string": "DoubleClick(330, 813)", + "intent_string": "The goal is to continue exploring the environment, so it's beneficial to scroll down and view more product listings.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 251, + 61, + 311, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "jeans", + "boxes": [ + 294, + 115, + 348, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 165, + 422, + 185 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 172, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Thin ", + "boxes": [ + 12, + 280, + 46, + 308 + ] + }, + { + "text": "High Waist", + "boxes": [ + 12, + 302, + 86, + 328 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1876, + 322 + ] + }, + { + "text": "Straight ", + "boxes": [ + 10, + 320, + 74, + 351 + ] + }, + { + "text": "Bootcut", + "boxes": [ + 14, + 343, + 68, + 363 + ] + }, + { + "text": "Gender", + "boxes": [ + 10, + 409, + 64, + 435 + ] + }, + { + "text": "Men ", + "boxes": [ + 9, + 426, + 71, + 462 + ] + }, + { + "text": "women", + "boxes": [ + 10, + 453, + 86, + 478 + ] + }, + { + "text": " Boys", + "boxes": [ + 9, + 467, + 71, + 502 + ] + }, + { + "text": "Girls", + "boxes": [ + 14, + 494, + 68, + 514 + ] + }, + { + "text": "Babies", + "boxes": [ + 34, + 514, + 76, + 534 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "25", + "boxes": [ + 64, + 600, + 98, + 613 + ] + }, + { + "text": "26", + "boxes": [ + 110, + 600, + 138, + 613 + ] + }, + { + "text": "Men's Classic Relaxed Fit Jean ", + "boxes": [ + 608, + 613, + 830, + 643 + ] + }, + { + "text": "Wrangler Authentics Men's Regular", + "boxes": [ + 900, + 613, + 1160, + 643 + ] + }, + { + "text": "Gloria Vanderbilt Women's Amanda", + "boxes": [ + 1196, + 615, + 1458, + 639 + ] + }, + { + "text": " Trendy Queen Womens Baggy Jeans ", + "boxes": [ + 1486, + 613, + 1760, + 643 + ] + }, + { + "text": "Amazon Essentials Men's Slim-Fit ", + "boxes": [ + 314, + 641, + 562, + 665 + ] + }, + { + "text": "★★★★☆28.090", + "boxes": [ + 612, + 641, + 758, + 665 + ] + }, + { + "text": "Fit Comfort Flex Waist Jean", + "boxes": [ + 900, + 635, + 1108, + 667 + ] + }, + { + "text": "Classic High Rise Tapered Jean", + "boxes": [ + 1196, + 635, + 1422, + 667 + ] + }, + { + "text": "Wide Leg Denim Pants Boyfriend", + "boxes": [ + 1486, + 635, + 1734, + 665 + ] + }, + { + "text": "Jeans", + "boxes": [ + 314, + 665, + 360, + 691 + ] + }, + { + "text": "★★★★★ 83.863", + "boxes": [ + 904, + 663, + 1056, + 693 + ] + }, + { + "text": "★★★★☆75.800", + "boxes": [ + 1198, + 667, + 1346, + 691 + ] + }, + { + "text": "Loose Streetwear Trousers Fall ", + "boxes": [ + 1486, + 661, + 1720, + 691 + ] + }, + { + "text": "$1398 List: $4.99", + "boxes": [ + 603, + 688, + 749, + 735 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 902, + 679, + 1080, + 717 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 1194, + 679, + 1376, + 717 + ] + }, + { + "text": "★★★★☆42.605", + "boxes": [ + 318, + 691, + 456, + 715 + ] + }, + { + "text": "Fashion y2k Outfits 2025", + "boxes": [ + 1486, + 685, + 1680, + 715 + ] + }, + { + "text": "$3155", + "boxes": [ + 898, + 713, + 970, + 752 + ] + }, + { + "text": "$29 st: 4-00", + "boxes": [ + 1187, + 708, + 1339, + 759 + ] + }, + { + "text": "★★★★☆91", + "boxes": [ + 1490, + 715, + 1612, + 739 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1810, + 707, + 1862, + 733 + ] + }, + { + "text": "$2660", + "boxes": [ + 312, + 719, + 378, + 758 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 608, + 728, + 748, + 757 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 1486, + 728, + 1670, + 765 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 314, + 760, + 444, + 784 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 608, + 752, + 696, + 776 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 900, + 752, + 1048, + 782 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 1194, + 752, + 1342, + 782 + ] + }, + { + "text": "$3299", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "hips to Canada", + "boxes": [ + 322, + 784, + 404, + 802 + ] + }, + { + "text": "1-2", + "boxes": [ + 114, + 798, + 134, + 812 + ] + }, + { + "text": " 1 sustainability feature ", + "boxes": [ + 318, + 800, + 508, + 828 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 1486, + 800, + 1636, + 832 + ] + }, + { + "text": "sored", + "boxes": [ + 1222, + 883, + 1270, + 903 + ] + }, + { + "text": " Lysse Women's Denim Straight Leg", + "boxes": [ + 1190, + 899, + 1484, + 933 + ] + }, + { + "text": "★★★★☆324", + "boxes": [ + 1196, + 927, + 1326, + 957 + ] + }, + { + "text": "2224", + "boxes": [ + 186, + 957, + 272, + 975 + ] + }, + { + "text": "$8800", + "boxes": [ + 1189, + 958, + 1257, + 997 + ] + }, + { + "text": "Delivery May 27 - 31", + "boxes": [ + 1196, + 999, + 1330, + 1022 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 251, + 61, + 311, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "、", + "boxes": [ + 276, + 121, + 298, + 135 + ] + }, + { + "text": "jeans", + "boxes": [ + 294, + 115, + 348, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 420, + 189 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 172, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Thin", + "boxes": [ + 14, + 284, + 44, + 306 + ] + }, + { + "text": "High Waist", + "boxes": [ + 12, + 302, + 86, + 328 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1876, + 322 + ] + }, + { + "text": "Straight ", + "boxes": [ + 10, + 320, + 74, + 351 + ] + }, + { + "text": "Bootcut", + "boxes": [ + 14, + 343, + 68, + 363 + ] + }, + { + "text": "Gender", + "boxes": [ + 10, + 409, + 64, + 435 + ] + }, + { + "text": "□Men", + "boxes": [ + 9, + 428, + 71, + 461 + ] + }, + { + "text": "women", + "boxes": [ + 7, + 446, + 87, + 483 + ] + }, + { + "text": " Boys", + "boxes": [ + 9, + 467, + 71, + 502 + ] + }, + { + "text": "Girls", + "boxes": [ + 14, + 494, + 68, + 514 + ] + }, + { + "text": "Babies", + "boxes": [ + 34, + 514, + 76, + 534 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "26", + "boxes": [ + 108, + 600, + 138, + 613 + ] + }, + { + "text": "Men's Classic Relaxed Fit Jean ", + "boxes": [ + 608, + 613, + 830, + 643 + ] + }, + { + "text": "Wrangler Authentics Men's Regular", + "boxes": [ + 900, + 613, + 1160, + 643 + ] + }, + { + "text": "Gloria Vanderbilt Women's Amanda", + "boxes": [ + 1194, + 611, + 1460, + 641 + ] + }, + { + "text": " Trendy Queen Womens Baggy Jeans", + "boxes": [ + 1486, + 613, + 1758, + 641 + ] + }, + { + "text": "33", + "boxes": [ + 156, + 627, + 186, + 641 + ] + }, + { + "text": "Amazon Essentials Men's Slim-Fit ", + "boxes": [ + 314, + 641, + 562, + 665 + ] + }, + { + "text": "★★★★☆28.090", + "boxes": [ + 612, + 641, + 758, + 665 + ] + }, + { + "text": "Fit Comfort Flex Waist Jean", + "boxes": [ + 900, + 635, + 1108, + 667 + ] + }, + { + "text": "Classic High Rise Tapered Jean", + "boxes": [ + 1196, + 635, + 1422, + 667 + ] + }, + { + "text": "Wide Leg Denim Pants Boyfriend", + "boxes": [ + 1486, + 635, + 1734, + 665 + ] + }, + { + "text": "Jeans", + "boxes": [ + 314, + 665, + 360, + 691 + ] + }, + { + "text": "★★★★★ 83.863", + "boxes": [ + 904, + 663, + 1056, + 693 + ] + }, + { + "text": "★★★★☆75.800", + "boxes": [ + 1198, + 667, + 1346, + 691 + ] + }, + { + "text": "Loose Streetwear Trousers Fall", + "boxes": [ + 1488, + 661, + 1718, + 689 + ] + }, + { + "text": "$1398 List: $14.99", + "boxes": [ + 603, + 688, + 749, + 733 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 902, + 679, + 1080, + 717 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 1194, + 679, + 1376, + 717 + ] + }, + { + "text": "★★★★☆42.605", + "boxes": [ + 318, + 691, + 458, + 715 + ] + }, + { + "text": "Fashion y2k Outfits 2025", + "boxes": [ + 1486, + 685, + 1680, + 715 + ] + }, + { + "text": "12P", + "boxes": [ + 246, + 697, + 274, + 711 + ] + }, + { + "text": "$2660", + "boxes": [ + 309, + 715, + 379, + 761 + ] + }, + { + "text": "$3155", + "boxes": [ + 898, + 713, + 970, + 752 + ] + }, + { + "text": "$229 Ist: 4:00", + "boxes": [ + 1187, + 710, + 1339, + 757 + ] + }, + { + "text": "★★★★☆91", + "boxes": [ + 1490, + 715, + 1612, + 739 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1810, + 707, + 1862, + 733 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 608, + 729, + 746, + 752 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 1486, + 728, + 1670, + 765 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 314, + 760, + 444, + 784 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 900, + 752, + 1048, + 782 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 1194, + 752, + 1342, + 782 + ] + }, + { + "text": "$3299", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 320, + 784, + 404, + 802 + ] + }, + { + "text": "1-2", + "boxes": [ + 114, + 798, + 134, + 812 + ] + }, + { + "text": "3-4", + "boxes": [ + 188, + 798, + 210, + 812 + ] + }, + { + "text": "1 sustainability feature", + "boxes": [ + 318, + 802, + 488, + 832 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 1484, + 796, + 1636, + 832 + ] + }, + { + "text": "11-12", + "boxes": [ + 26, + 858, + 70, + 876 + ] + }, + { + "text": "13-14", + "boxes": [ + 101, + 853, + 149, + 876 + ] + }, + { + "text": "sored", + "boxes": [ + 1222, + 883, + 1270, + 903 + ] + }, + { + "text": " Lysse Women's Denim Straight Leg", + "boxes": [ + 1190, + 899, + 1484, + 933 + ] + }, + { + "text": "★★★★☆324", + "boxes": [ + 1196, + 927, + 1326, + 957 + ] + }, + { + "text": "18", + "boxes": [ + 116, + 959, + 134, + 973 + ] + }, + { + "text": "22 24", + "boxes": [ + 182, + 957, + 272, + 975 + ] + }, + { + "text": "$8800", + "boxes": [ + 1189, + 958, + 1257, + 997 + ] + }, + { + "text": "Delivery May 27 - 31", + "boxes": [ + 1196, + 999, + 1330, + 1022 + ] + } + ] + }, + { + "frame_id": "120_2_7", + "episode_id": 120, + "env_id": 2, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.015665054321289, + "advantages": -1.4399075508117676, + "generated_text": "{\n \"intent\": \"The goal is to continue exploring the computer desktop environment by accessing more files or applications. The current context suggests that we are interacting with a file related to Firefox installation. Clicking 'Save File' will download the Firefox installation archive, which could be useful for further exploration of system capabilities or software installation.\",\n \"action\": \"DoubleClick(901, 576)\"\n}", + "action_string": "DoubleClick(901, 576)", + "intent_string": "The goal is to continue exploring the computer desktop environment by accessing more files or applications. The current context suggests that we are interacting with a file related to Firefox installation. Clicking 'Save File' will download the Firefox installation archive, which could be useful for further exploration of system capabilities or software installation.", + "before_screen_ocr_results": [ + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1882, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 351, + 1088, + 381 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 383, + 922, + 411 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (69.8 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwnload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now", + "boxes": [ + 764, + 594, + 1102, + 617 + ] + }, + { + "text": "Cancel", + "boxes": [ + 1010, + 653, + 1060, + 673 + ] + }, + { + "text": "Using Debian,Ubuntu or any Deban-Daed distruton?", + "boxes": [ + 728, + 677, + 1178, + 717 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1122, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TARArchive(69.8 MB)", + "boxes": [ + 772, + 433, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1036, + 532 + ] + }, + { + "text": " Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "OSave File", + "boxes": [ + 764, + 558, + 846, + 584 + ] + }, + { + "text": "+", + "boxes": [ + 902, + 580, + 914, + 592 + ] + }, + { + "text": " Do this automatically for files like this from now on.", + "boxes": [ + 762, + 594, + 1124, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 732, + 681, + 1176, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1122, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "firet", + "boxes": [ + 422, + 1054, + 456, + 1068 + ] + } + ] + }, + { + "frame_id": "120_3_7", + "episode_id": 120, + "env_id": 3, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.202627182006836, + "advantages": -1.1751214265823364, + "generated_text": "{\n \"intent\": \"Clicking the 'Add to Cart' button will add the jeans to the shopping cart, achieving the goal of exploring the purchasing process within the Amazon storefront.\",\n \"action\": \"DoubleClick(1510, 325)\"\n}", + "action_string": "DoubleClick(1510, 325)", + "intent_string": "Clicking the 'Add to Cart' button will add the jeans to the shopping cart, achieving the goal of exploring the purchasing process within the Amazon storefront.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "Add o Rart", + "boxes": [ + 1455, + 305, + 1541, + 348 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": "Product details", + "boxes": [ + 652, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spandex", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise,straight leg, Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1457, + 309, + 1537, + 346 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": " Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spande", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise straight leg,Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ] + }, + { + "frame_id": "120_4_7", + "episode_id": 120, + "env_id": 4, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.624886512756348, + "advantages": -0.577094554901123, + "generated_text": "{\n \"intent\": \"The user is currently in LibreOffice Writer, which suggests they might want to explore this application further or prepare to use it. Clicking on the File menu at coordinates (10, 53) would provide access to the different file options, allowing the user to save the document, open existing files, or create new documents.\",\n \"action\": \"Click(10, 53)\"\n}", + "action_string": "Click(10, 53)", + "intent_string": "The user is currently in LibreOffice Writer, which suggests they might want to explore this application further or prepare to use it. Clicking on the File menu at coordinates (10, 53) would provide access to the different file options, allowing the user to save the document, open existing files, or create new documents.", + "before_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 10, + 22, + 36, + 42 + ] + }, + { + "text": "--- @x自", + "boxes": [ + 10, + 52, + 358, + 87 + ] + }, + { + "text": "中園口", + "boxes": [ + 1004, + 56, + 1142, + 79 + ] + }, + { + "text": "Q?", + "boxes": [ + 154, + 99, + 210, + 117 + ] + }, + { + "text": "口", + "boxes": [ + 1890, + 204, + 1912, + 238 + ] + }, + { + "text": "", + "boxes": [ + 1886, + 234, + 1912, + 304 + ] + }, + { + "text": "O", + "boxes": [ + 1884, + 308, + 1914, + 342 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "---@", + "boxes": [ + 4, + 47, + 270, + 88 + ] + }, + { + "text": "X电旨", + "boxes": [ + 258, + 60, + 348, + 77 + ] + }, + { + "text": "園口", + "boxes": [ + 996, + 56, + 1146, + 79 + ] + }, + { + "text": "C", + "boxes": [ + 184, + 101, + 206, + 115 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 240 + ] + }, + { + "text": "", + "boxes": [ + 1886, + 234, + 1912, + 304 + ] + }, + { + "text": "O", + "boxes": [ + 1884, + 308, + 1914, + 342 + ] + } + ] + }, + { + "frame_id": "120_5_7", + "episode_id": 120, + "env_id": 5, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.149853706359863, + "advantages": -1.2498621940612793, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment, I will click on the 'File' menu to access options related to file management such as creating or opening new files, saving documents, or exporting them as PDFs.\",\n \"action\": \"DoubleClick(40, 31)\"\n}", + "action_string": "DoubleClick(40, 31)", + "intent_string": "To continue exploring the desktop environment, I will click on the 'File' menu to access options related to file management such as creating or opening new files, saving documents, or exporting them as PDFs.", + "before_screen_ocr_results": [ + { + "text": "ata", + "boxes": [ + 368, + 30, + 388, + 44 + ] + }, + { + "text": "晶自", + "boxes": [ + 292, + 54, + 370, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 378, + 54, + 444, + 85 + ] + }, + { + "text": "G-.", + "boxes": [ + 454, + 56, + 550, + 81 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Open.", + "boxes": [ + 28, + 81, + 68, + 101 + ] + }, + { + "text": "Ctrl+C", + "boxes": [ + 252, + 79, + 290, + 99 + ] + }, + { + "text": "a", + "boxes": [ + 312, + 99, + 332, + 119 + ] + }, + { + "text": "a要日日%00三田二口", + "boxes": [ + 342, + 93, + 1204, + 123 + ] + }, + { + "text": "Open Ren", + "boxes": [ + 30, + 109, + 92, + 127 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 26, + 191, + 84, + 216 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 270 + ] + }, + { + "text": "Reloao", + "boxes": [ + 32, + 254, + 72, + 270 + ] + }, + { + "text": "Save", + "boxes": [ + 26, + 312, + 60, + 332 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 296, + 359 + ] + }, + { + "text": "SaveRemote", + "boxes": [ + 28, + 367, + 114, + 385 + ] + }, + { + "text": "Save a Copy...", + "boxes": [ + 24, + 387, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 21, + 444, + 83, + 477 + ] + }, + { + "text": "Export as PDF", + "boxes": [ + 26, + 476, + 118, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 66, + 528 + ] + }, + { + "text": "Preview in Web Brows", + "boxes": [ + 26, + 534, + 172, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 296, + 584 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 250, + 590, + 292, + 609 + ] + }, + { + "text": "Printer Settings.", + "boxes": [ + 26, + 619, + 132, + 637 + ] + }, + { + "text": " Properties.", + "boxes": [ + 24, + 649, + 100, + 673 + ] + }, + { + "text": "Digital :", + "boxes": [ + 26, + 679, + 78, + 697 + ] + }, + { + "text": "Exit LibreOffice", + "boxes": [ + 24, + 707, + 126, + 731 + ] + }, + { + "text": "Ctrl+(", + "boxes": [ + 254, + 711, + 288, + 725 + ] + }, + { + "text": "35", + "boxes": [ + 12, + 750, + 32, + 766 + ] + }, + { + "text": "1→一", + "boxes": [ + 22, + 997, + 92, + 1014 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "FileEditView", + "boxes": [ + 6, + 24, + 124, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "画·口·目·目@%血-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "G-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "、", + "boxes": [ + 130, + 129, + 160, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 258, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "120_6_7", + "episode_id": 120, + "env_id": 6, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.949712753295898, + "advantages": -0.11705779284238815, + "generated_text": "{\n \"intent\": \"The current page is showing products related to Neutrogena. To continue exploring the website, I will scroll down to view more products or information about Neutrogena.\",\n \"action\": \"ScrollDown(1000, 540)\"\n}", + "action_string": "ScrollDown(1000, 540)", + "intent_string": "The current page is showing products related to Neutrogena. To continue exploring the website, I will scroll down to view more products or information about Neutrogena.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 6, + 59, + 132, + 96 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 136, + 143 + ] + }, + { + "text": " a", + "boxes": [ + 130, + 103, + 224, + 155 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 298, + 115, + 410, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1452, + 115, + 1476, + 137 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1580, + 147 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "day's Deals", + "boxes": [ + 106, + 169, + 172, + 187 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 634, + 225, + 1276, + 377 + ] + }, + { + "text": "BEAUTYTOA SCIENCE", + "boxes": [ + 746, + 427, + 1162, + 463 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 338, + 512, + 402, + 532 + ] + }, + { + "text": "Follow", + "boxes": [ + 358, + 548, + 404, + 574 + ] + }, + { + "text": "凸", + "boxes": [ + 450, + 550, + 474, + 574 + ] + }, + { + "text": "HOME", + "boxes": [ + 492, + 552, + 544, + 578 + ] + }, + { + "text": "Q Search all Neutrogena", + "boxes": [ + 1406, + 544, + 1606, + 580 + ] + }, + { + "text": "FESTIVAL", + "boxes": [ + 568, + 556, + 638, + 574 + ] + }, + { + "text": "COLLAGEN", + "boxes": [ + 930, + 556, + 1012, + 574 + ] + }, + { + "text": "MORE", + "boxes": [ + 1278, + 556, + 1324, + 574 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1004, + 655, + 1372, + 697 + ] + }, + { + "text": "Neutrogena:", + "boxes": [ + 647, + 670, + 881, + 729 + ] + }, + { + "text": "Sheer Berry", + "boxes": [ + 999, + 696, + 1195, + 747 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 1407, + 698, + 1531, + 737 + ] + }, + { + "text": "BEAUTYTOASCIENCE", + "boxes": [ + 694, + 741, + 834, + 758 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and ", + "boxes": [ + 1002, + 752, + 1366, + 782 + ] + }, + { + "text": "DRO", + "boxes": [ + 1426, + 744, + 1460, + 764 + ] + }, + { + "text": " shea butter to instantly nourish dry ips ", + "boxes": [ + 998, + 772, + 1336, + 812 + ] + }, + { + "text": "$970 ($13.86/ounce) ", + "boxes": [ + 999, + 813, + 1135, + 857 + ] + }, + { + "text": "Pre-order. This item will be released on May 20,", + "boxes": [ + 1004, + 850, + 1332, + 874 + ] + }, + { + "text": "Nourishing", + "boxes": [ + 617, + 863, + 909, + 942 + ] + }, + { + "text": "2025.", + "boxes": [ + 1006, + 866, + 1046, + 891 + ] + }, + { + "text": "hydration", + "boxes": [ + 644, + 935, + 886, + 991 + ] + }, + { + "text": "for iuicv lips", + "boxes": [ + 604, + 982, + 924, + 1055 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1590, + 68, + 1660, + 89 + ] + }, + { + "text": "BEAUTY TO A SCIENCE", + "boxes": [ + 742, + 135, + 1162, + 177 + ] + }, + { + "text": "Follow", + "boxes": [ + 360, + 264, + 404, + 290 + ] + }, + { + "text": "凸", + "boxes": [ + 450, + 264, + 474, + 288 + ] + }, + { + "text": "HOME", + "boxes": [ + 496, + 270, + 542, + 288 + ] + }, + { + "text": "α", + "boxes": [ + 1412, + 262, + 1446, + 290 + ] + }, + { + "text": " Search all Neutrogena", + "boxes": [ + 1438, + 262, + 1604, + 292 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1004, + 371, + 1372, + 413 + ] + }, + { + "text": "Neutrogena:", + "boxes": [ + 649, + 382, + 881, + 447 + ] + }, + { + "text": "Sheer Berry", + "boxes": [ + 1001, + 412, + 1193, + 461 + ] + }, + { + "text": "Neutrogenar", + "boxes": [ + 1405, + 410, + 1529, + 455 + ] + }, + { + "text": "BEAUTY TO A SCIENCE", + "boxes": [ + 690, + 451, + 836, + 480 + ] + }, + { + "text": "Hyaluronic acid ip balm with vitamin E and", + "boxes": [ + 1002, + 465, + 1368, + 500 + ] + }, + { + "text": "DRO", + "boxes": [ + 1424, + 463, + 1458, + 476 + ] + }, + { + "text": " shea butter to instantly nourish dry lips", + "boxes": [ + 1000, + 486, + 1336, + 526 + ] + }, + { + "text": "970 ($13.86/ouncel ", + "boxes": [ + 995, + 528, + 1133, + 570 + ] + }, + { + "text": "Nourishing", + "boxes": [ + 621, + 581, + 907, + 654 + ] + }, + { + "text": " Quick look", + "boxes": [ + 1016, + 602, + 1104, + 619 + ] + }, + { + "text": "hydration", + "boxes": [ + 638, + 645, + 890, + 711 + ] + }, + { + "text": "for juicy lips", + "boxes": [ + 603, + 696, + 925, + 777 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1276, + 739, + 1640, + 780 + ] + }, + { + "text": "Neutrogenar", + "boxes": [ + 1024, + 782, + 1148, + 820 + ] + }, + { + "text": "Sheer lce", + "boxes": [ + 1492, + 782, + 1642, + 826 + ] + }, + { + "text": "HYDRO", + "boxes": [ + 1032, + 828, + 1082, + 846 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and ", + "boxes": [ + 1278, + 832, + 1642, + 866 + ] + }, + { + "text": "B00ST", + "boxes": [ + 1032, + 844, + 1080, + 864 + ] + }, + { + "text": "shea butter to instantly nourish dry lips", + "boxes": [ + 1312, + 862, + 1640, + 891 + ] + }, + { + "text": "From the #1 Hyaluronic Acid", + "boxes": [ + 618, + 901, + 908, + 931 + ] + }, + { + "text": "$955 ($13.64/ounce)", + "boxes": [ + 1505, + 897, + 1637, + 938 + ] + }, + { + "text": "facial care line in the US*", + "boxes": [ + 634, + 929, + 888, + 959 + ] + }, + { + "text": "Pre-order. This item willbe released on May 20.", + "boxes": [ + 1312, + 929, + 1638, + 959 + ] + }, + { + "text": "2025.", + "boxes": [ + 1596, + 945, + 1638, + 973 + ] + }, + { + "text": "TateMcRae", + "boxes": [ + 264, + 983, + 372, + 1007 + ] + } + ] + }, + { + "frame_id": "120_7_7", + "episode_id": 120, + "env_id": 7, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.698098182678223, + "advantages": -0.47340819239616394, + "generated_text": "{\n \"intent\": \"To continue exploring the file system by navigating through the files and folders. The 'Desktop' folder is currently selected, and the goal is to open one of the available files. Clicking on a file will allow us to view its contents or perform further actions.\",\n \"action\": \"DoubleClick(732, 197)\"\n}", + "action_string": "DoubleClick(732, 197)", + "intent_string": "To continue exploring the file system by navigating through the files and folders. The 'Desktop' folder is currently selected, and the goal is to open one of the available files. Clicking on a file will allow us to view its contents or perform further actions.", + "before_screen_ocr_results": [ + { + "text": "D", + "boxes": [ + 0, + 0, + 30, + 28 + ] + }, + { + "text": "Untitled 1- Mousepad", + "boxes": [ + 872, + 4, + 1048, + 22 + ] + }, + { + "text": "@", + "boxes": [ + 1860, + 6, + 1914, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 30, + 34, + 44 + ] + }, + { + "text": " Edit ", + "boxes": [ + 40, + 28, + 84, + 46 + ] + }, + { + "text": "View", + "boxes": [ + 134, + 30, + 184, + 48 + ] + }, + { + "text": "Open File", + "boxes": [ + 920, + 103, + 1004, + 127 + ] + }, + { + "text": " Recent", + "boxes": [ + 418, + 131, + 494, + 157 + ] + }, + { + "text": "仑root", + "boxes": [ + 566, + 131, + 652, + 163 + ] + }, + { + "text": " Desktop", + "boxes": [ + 654, + 131, + 748, + 163 + ] + }, + { + "text": " Home", + "boxes": [ + 418, + 165, + 486, + 191 + ] + }, + { + "text": "Name", + "boxes": [ + 568, + 169, + 612, + 189 + ] + }, + { + "text": "Size", + "boxes": [ + 1268, + 173, + 1302, + 187 + ] + }, + { + "text": "Type", + "boxes": [ + 1356, + 173, + 1386, + 187 + ] + }, + { + "text": "[ Modified", + "boxes": [ + 1410, + 167, + 1486, + 191 + ] + }, + { + "text": "96 bytes", + "boxes": [ + 1274, + 193, + 1330, + 210 + ] + }, + { + "text": "7 May", + "boxes": [ + 1420, + 195, + 1460, + 210 + ] + }, + { + "text": " galculator.desktop", + "boxes": [ + 582, + 208, + 718, + 240 + ] + }, + { + "text": "105 bytes", + "boxes": [ + 1270, + 210, + 1344, + 242 + ] + }, + { + "text": "Text", + "boxes": [ + 1346, + 216, + 1382, + 236 + ] + }, + { + "text": "7 May", + "boxes": [ + 1413, + 209, + 1463, + 239 + ] + }, + { + "text": "e-calc.desktop", + "boxes": [ + 652, + 236, + 746, + 258 + ] + }, + { + "text": "112 bytes", + "boxes": [ + 1270, + 232, + 1344, + 264 + ] + }, + { + "text": "Text", + "boxes": [ + 1347, + 235, + 1383, + 262 + ] + }, + { + "text": "7 May", + "boxes": [ + 1417, + 233, + 1463, + 261 + ] + }, + { + "text": "draw.desktop", + "boxes": [ + 668, + 262, + 750, + 280 + ] + }, + { + "text": "11lbytes", + "boxes": [ + 1270, + 256, + 1344, + 288 + ] + }, + { + "text": "Text", + "boxes": [ + 1348, + 256, + 1386, + 286 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 258, + 1464, + 286 + ] + }, + { + "text": "132 bytes", + "boxes": [ + 1270, + 278, + 1344, + 310 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 280, + 1386, + 310 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 280, + 1464, + 308 + ] + }, + { + "text": "117 bytes", + "boxes": [ + 1270, + 302, + 1344, + 334 + ] + }, + { + "text": "Text", + "boxes": [ + 1354, + 306, + 1382, + 330 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 304, + 1464, + 332 + ] + }, + { + "text": " Ixterminal.desktop", + "boxes": [ + 572, + 328, + 720, + 351 + ] + }, + { + "text": "103 bytes", + "boxes": [ + 1268, + 322, + 1346, + 359 + ] + }, + { + "text": "Text", + "boxes": [ + 1344, + 319, + 1390, + 360 + ] + }, + { + "text": "7 May", + "boxes": [ + 1416, + 324, + 1468, + 357 + ] + }, + { + "text": "114 bytes", + "boxes": [ + 1270, + 347, + 1344, + 379 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 349, + 1384, + 377 + ] + }, + { + "text": "19 Feb 2", + "boxes": [ + 1418, + 349, + 1472, + 379 + ] + }, + { + "text": "2024", + "boxes": [ + 1462, + 353, + 1502, + 373 + ] + }, + { + "text": " pcmanfm.desktop", + "boxes": [ + 568, + 371, + 716, + 401 + ] + }, + { + "text": "106 bytes", + "boxes": [ + 1270, + 371, + 1344, + 403 + ] + }, + { + "text": "Text", + "boxes": [ + 1350, + 373, + 1386, + 401 + ] + }, + { + "text": "7 May", + "boxes": [ + 1418, + 373, + 1462, + 401 + ] + }, + { + "text": "OpenFile", + "boxes": [ + 370, + 1052, + 428, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 6, + 26, + 36, + 46 + ] + }, + { + "text": "Search View", + "boxes": [ + 90, + 24, + 186, + 48 + ] + }, + { + "text": "Help", + "boxes": [ + 270, + 21, + 310, + 49 + ] + }, + { + "text": "URL=/usr/share/applications/firefox.desktop", + "boxes": [ + 0, + 139, + 352, + 169 + ] + }, + { + "text": "/root/Desktop", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "120_0_8", + "episode_id": 120, + "env_id": 0, + "step": 8, + "format_rewards": 1.0, + "rewards": 8.896360397338867, + "advantages": -1.6088733673095703, + "generated_text": "{\n \"intent\": \"The goal is to explore the environment by interacting with the website and the downloaded page. Clicking 'Close' would allow us to exit the current file explorer window and continue exploring other parts of the desktop.\",\n \"action\": \"DoubleClick(1042, 879)\"\n}", + "action_string": "DoubleClick(1042, 879)", + "intent_string": "The goal is to explore the environment by interacting with the website and the downloaded page. Clicking 'Close' would allow us to exit the current file explorer window and continue exploring other parts of the desktop.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口", + "boxes": [ + 1118, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Whichis: TAR Archive (69.8 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now", + "boxes": [ + 764, + 594, + 1102, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system", + "boxes": [ + 702, + 887, + 1200, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 30, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 64, + 1662, + 89 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1876, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 353, + 1088, + 383 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 383, + 922, + 411 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "which is: TAR Archive (69.8 MB)", + "boxes": [ + 772, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from nov", + "boxes": [ + 764, + 594, + 1096, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 655, + 1060, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distrbuton?", + "boxes": [ + 732, + 681, + 1172, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 698, + 881, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1016, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "120_1_8", + "episode_id": 120, + "env_id": 1, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.84304428100586, + "advantages": -0.26812756061553955, + "generated_text": "{\n \"intent\": \"I have been exploring the Amazon website by clicking on various icons on the desktop. To expand my exploration further, I will continue scrolling down the page to view more product listings related to jeans.\",\n \"action\": \"ScrollDown(768, 546)\"\n}", + "action_string": "ScrollDown(768, 546)", + "intent_string": "I have been exploring the Amazon website by clicking on various icons on the desktop. To expand my exploration further, I will continue scrolling down the page to view more product listings related to jeans.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 251, + 61, + 311, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "、", + "boxes": [ + 276, + 121, + 298, + 135 + ] + }, + { + "text": "jeans", + "boxes": [ + 294, + 115, + 348, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 189 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 174, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Thin", + "boxes": [ + 14, + 284, + 44, + 306 + ] + }, + { + "text": "High Waist", + "boxes": [ + 12, + 302, + 86, + 328 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1876, + 322 + ] + }, + { + "text": "Straight ", + "boxes": [ + 10, + 320, + 74, + 351 + ] + }, + { + "text": "Bootcut", + "boxes": [ + 14, + 343, + 68, + 363 + ] + }, + { + "text": "Gender", + "boxes": [ + 10, + 409, + 64, + 435 + ] + }, + { + "text": "Men ", + "boxes": [ + 9, + 426, + 71, + 462 + ] + }, + { + "text": "women", + "boxes": [ + 10, + 453, + 86, + 478 + ] + }, + { + "text": " Boys", + "boxes": [ + 9, + 467, + 71, + 502 + ] + }, + { + "text": " Girls", + "boxes": [ + 12, + 494, + 68, + 514 + ] + }, + { + "text": "Babies", + "boxes": [ + 34, + 514, + 76, + 534 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "25", + "boxes": [ + 62, + 600, + 98, + 613 + ] + }, + { + "text": "26", + "boxes": [ + 108, + 600, + 138, + 613 + ] + }, + { + "text": "Men's Classic Relaxed Fit Jean ", + "boxes": [ + 608, + 613, + 830, + 643 + ] + }, + { + "text": "Wrangler Authentics Men's Regular", + "boxes": [ + 900, + 613, + 1160, + 643 + ] + }, + { + "text": "Gloria Vanderbilt Women's Amanda", + "boxes": [ + 1194, + 611, + 1460, + 641 + ] + }, + { + "text": " Trendy Queen Womens Baggy Jeans", + "boxes": [ + 1486, + 613, + 1758, + 641 + ] + }, + { + "text": "32", + "boxes": [ + 108, + 627, + 138, + 641 + ] + }, + { + "text": "33", + "boxes": [ + 156, + 627, + 186, + 641 + ] + }, + { + "text": "Amazon Essentials Men's Slim-Fit ", + "boxes": [ + 314, + 641, + 562, + 665 + ] + }, + { + "text": "★★★★☆28.090", + "boxes": [ + 610, + 639, + 758, + 667 + ] + }, + { + "text": "Fit Comfort Flex Waist Jean", + "boxes": [ + 900, + 635, + 1108, + 667 + ] + }, + { + "text": "Classic High Rise Tapered Jean", + "boxes": [ + 1196, + 635, + 1422, + 667 + ] + }, + { + "text": "Wide Leg Denim Pants Boyfriend", + "boxes": [ + 1486, + 635, + 1734, + 665 + ] + }, + { + "text": "Jeans", + "boxes": [ + 314, + 665, + 360, + 691 + ] + }, + { + "text": "★★★★★ 83.863", + "boxes": [ + 904, + 663, + 1056, + 693 + ] + }, + { + "text": "★★★★☆75.800", + "boxes": [ + 1198, + 667, + 1346, + 691 + ] + }, + { + "text": "Loose Streetwear Trousers Fall", + "boxes": [ + 1486, + 661, + 1720, + 691 + ] + }, + { + "text": "$1398 List: $4.99", + "boxes": [ + 603, + 688, + 749, + 735 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 902, + 679, + 1080, + 717 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 1194, + 679, + 1376, + 717 + ] + }, + { + "text": "★★★★☆42.605", + "boxes": [ + 318, + 691, + 458, + 715 + ] + }, + { + "text": "Fashion y2k Outfits 2025", + "boxes": [ + 1486, + 685, + 1680, + 715 + ] + }, + { + "text": "$3155", + "boxes": [ + 898, + 713, + 970, + 752 + ] + }, + { + "text": "$29 st: 4-00", + "boxes": [ + 1187, + 708, + 1339, + 759 + ] + }, + { + "text": "★★★★☆91", + "boxes": [ + 1490, + 715, + 1612, + 739 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1810, + 707, + 1862, + 733 + ] + }, + { + "text": "$2660", + "boxes": [ + 312, + 719, + 378, + 758 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 608, + 728, + 748, + 757 + ] + }, + { + "text": "100+ bought in past month", + "boxes": [ + 1486, + 728, + 1670, + 765 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 314, + 760, + 444, + 784 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 900, + 752, + 1048, + 782 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 1194, + 752, + 1342, + 782 + ] + }, + { + "text": "$3299", + "boxes": [ + 1488, + 764, + 1552, + 798 + ] + }, + { + "text": "1-2", + "boxes": [ + 114, + 798, + 134, + 812 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 1486, + 800, + 1636, + 832 + ] + }, + { + "text": "11-12", + "boxes": [ + 28, + 860, + 68, + 876 + ] + }, + { + "text": "13-14", + "boxes": [ + 101, + 853, + 149, + 876 + ] + }, + { + "text": "isored ", + "boxes": [ + 1220, + 883, + 1270, + 903 + ] + }, + { + "text": " Lysse Women's Denim Straight Leg", + "boxes": [ + 1190, + 899, + 1484, + 933 + ] + }, + { + "text": "★★★★☆324", + "boxes": [ + 1196, + 927, + 1326, + 957 + ] + }, + { + "text": "14", + "boxes": [ + 24, + 959, + 42, + 973 + ] + }, + { + "text": "22八", + "boxes": [ + 194, + 957, + 248, + 975 + ] + }, + { + "text": "24", + "boxes": [ + 244, + 959, + 270, + 973 + ] + }, + { + "text": "$8800", + "boxes": [ + 1189, + 958, + 1257, + 997 + ] + }, + { + "text": "Delivery May 27 - 31", + "boxes": [ + 1196, + 999, + 1330, + 1022 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1654, + 85 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1874, + 91 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1816, + 109, + 1870, + 129 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1880, + 149 + ] + }, + { + "text": "Girl", + "boxes": [ + 38, + 149, + 58, + 165 + ] + }, + { + "text": "Men's Classic Relaxed Fit Jean", + "boxes": [ + 608, + 266, + 830, + 296 + ] + }, + { + "text": "Wrangler Authentics Men's Regular", + "boxes": [ + 900, + 268, + 1158, + 298 + ] + }, + { + "text": "Gloria Vanderbilt Women's Amanda", + "boxes": [ + 1196, + 270, + 1456, + 294 + ] + }, + { + "text": "Trendy Queen Womens Baggy Jeans ", + "boxes": [ + 1490, + 268, + 1760, + 298 + ] + }, + { + "text": "Amazon Essentials Men's Slim-Fit", + "boxes": [ + 316, + 294, + 564, + 324 + ] + }, + { + "text": "★★★★★28.090", + "boxes": [ + 610, + 294, + 760, + 324 + ] + }, + { + "text": " Fit Comfort Flex Waist Jean", + "boxes": [ + 898, + 290, + 1106, + 322 + ] + }, + { + "text": "Classic High Rise Tapered Jean", + "boxes": [ + 1196, + 292, + 1418, + 322 + ] + }, + { + "text": "Wide Leg Denim Pants Boyfriend", + "boxes": [ + 1488, + 292, + 1736, + 322 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 294, + 1880, + 326 + ] + }, + { + "text": "ean:", + "boxes": [ + 322, + 326, + 352, + 339 + ] + }, + { + "text": "ought in past month", + "boxes": [ + 654, + 316, + 784, + 342 + ] + }, + { + "text": "★★★★★83.863", + "boxes": [ + 904, + 320, + 1054, + 349 + ] + }, + { + "text": "★★★★☆75.800", + "boxes": [ + 1196, + 320, + 1346, + 349 + ] + }, + { + "text": "Loose Streetwear Trousers Fall", + "boxes": [ + 1490, + 320, + 1718, + 343 + ] + }, + { + "text": "$1398 List: $4.99", + "boxes": [ + 603, + 343, + 747, + 390 + ] + }, + { + "text": "00+", + "boxes": [ + 914, + 345, + 938, + 359 + ] + }, + { + "text": "00+", + "boxes": [ + 1204, + 345, + 1234, + 359 + ] + }, + { + "text": "Fashion y2k Outfits 2025", + "boxes": [ + 1488, + 339, + 1678, + 369 + ] + }, + { + "text": "$3155", + "boxes": [ + 900, + 371, + 966, + 405 + ] + }, + { + "text": "$229 ist: $40:09 ", + "boxes": [ + 1189, + 367, + 1337, + 412 + ] + }, + { + "text": "★★★★☆91", + "boxes": [ + 1490, + 367, + 1608, + 397 + ] + }, + { + "text": "$2660", + "boxes": [ + 314, + 377, + 376, + 411 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 383, + 750, + 413 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 316, + 415, + 446, + 439 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 900, + 407, + 1046, + 435 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 1196, + 407, + 1338, + 435 + ] + }, + { + "text": "ric Clothing Size", + "boxes": [ + 116, + 421, + 222, + 445 + ] + }, + { + "text": "$3299", + "boxes": [ + 1488, + 421, + 1552, + 455 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 316, + 435, + 406, + 459 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 906, + 435, + 990, + 453 + ] + }, + { + "text": "1-2", + "boxes": [ + 106, + 451, + 140, + 471 + ] + }, + { + "text": "3-4", + "boxes": [ + 180, + 451, + 212, + 471 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 1486, + 455, + 1634, + 484 + ] + }, + { + "text": "5-6", + "boxes": [ + 34, + 480, + 64, + 500 + ] + }, + { + "text": "7-8", + "boxes": [ + 106, + 478, + 140, + 500 + ] + }, + { + "text": "9-10", + "boxes": [ + 176, + 476, + 218, + 502 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1492, + 482, + 1576, + 500 + ] + }, + { + "text": "11-12", + "boxes": [ + 24, + 508, + 76, + 534 + ] + }, + { + "text": "13-14", + "boxes": [ + 98, + 508, + 150, + 534 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "Lysse Women's Denim Straight Leg", + "boxes": [ + 1192, + 552, + 1484, + 590 + ] + }, + { + "text": "#", + "boxes": [ + 1792, + 548, + 1822, + 578 + ] + }, + { + "text": "en's Plus N", + "boxes": [ + 48, + 584, + 116, + 602 + ] + }, + { + "text": "icClot", + "boxes": [ + 156, + 586, + 196, + 602 + ] + }, + { + "text": "★★★★☆324", + "boxes": [ + 1198, + 586, + 1322, + 609 + ] + }, + { + "text": "18", + "boxes": [ + 104, + 611, + 140, + 631 + ] + }, + { + "text": "2022[24", + "boxes": [ + 150, + 609, + 278, + 633 + ] + }, + { + "text": "$8800", + "boxes": [ + 1194, + 617, + 1256, + 649 + ] + }, + { + "text": "26", + "boxes": [ + 20, + 643, + 50, + 657 + ] + }, + { + "text": "28", + "boxes": [ + 66, + 639, + 102, + 659 + ] + }, + { + "text": "30", + "boxes": [ + 106, + 639, + 140, + 661 + ] + }, + { + "text": "Delivery May 27 - 31", + "boxes": [ + 1198, + 653, + 1334, + 677 + ] + }, + { + "text": "Special Clothing Size", + "boxes": [ + 10, + 679, + 152, + 705 + ] + }, + { + "text": "Or fastest delivery May 26 - 30", + "boxes": [ + 1198, + 675, + 1392, + 699 + ] + }, + { + "text": " Big & Tall", + "boxes": [ + 9, + 700, + 99, + 731 + ] + }, + { + "text": " Husky", + "boxes": [ + 12, + 723, + 78, + 748 + ] + }, + { + "text": " Petite", + "boxes": [ + 12, + 764, + 72, + 784 + ] + }, + { + "text": " plus Size", + "boxes": [ + 7, + 779, + 93, + 811 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 816, + 136, + 840 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 14, + 840, + 144, + 870 + ] + }, + { + "text": "Brands", + "boxes": [ + 9, + 873, + 63, + 904 + ] + }, + { + "text": "More results", + "boxes": [ + 304, + 866, + 426, + 895 + ] + }, + { + "text": " sidefeel", + "boxes": [ + 9, + 916, + 91, + 948 + ] + }, + { + "text": "Levi's", + "boxes": [ + 12, + 941, + 72, + 967 + ] + }, + { + "text": "Lee", + "boxes": [ + 14, + 963, + 58, + 981 + ] + } + ] + }, + { + "frame_id": "120_2_8", + "episode_id": 120, + "env_id": 2, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.082091331481934, + "advantages": -1.3458309173583984, + "generated_text": "{\n \"intent\": \"I need to download the Firefox software by clicking the 'OK' button in the file dialog. This will initiate the download process, which is part of exploring the installation or downloading area.\",\n \"action\": \"DoubleClick(1119, 672)\"\n}", + "action_string": "DoubleClick(1119, 672)", + "intent_string": "I need to download the Firefox software by clicking the 'OK' button in the file dialog. This will initiate the download process, which is part of exploring the installation or downloading area.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三四?", + "boxes": [ + 1788, + 66, + 1882, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TARArchive(69.8 MB)", + "boxes": [ + 772, + 433, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1036, + 532 + ] + }, + { + "text": " Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "OSave File", + "boxes": [ + 764, + 558, + 846, + 584 + ] + }, + { + "text": "+", + "boxes": [ + 902, + 580, + 914, + 592 + ] + }, + { + "text": " Do this automatically for files like this from now on.", + "boxes": [ + 762, + 594, + 1124, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 732, + 681, + 1176, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1122, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "firet", + "boxes": [ + 422, + 1054, + 456, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1574, + 68, + 1628, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "firefox-138.0.4.tar.z", + "boxes": [ + 1238, + 105, + 1384, + 135 + ] + }, + { + "text": "Completed — 69.8 MB", + "boxes": [ + 1246, + 133, + 1378, + 151 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1118, + 731 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_3_8", + "episode_id": 120, + "env_id": 3, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.610115051269531, + "advantages": -0.5980147123336792, + "generated_text": "{\n \"intent\": \"The current page shows a product listing for Lee Women's Plus Size Relaxed Fit Straight Leg Jeans. The 'Add to Cart' button is visible, indicating that clicking this button will add the item to the shopping cart. This action furthers the exploration by adding an item to the cart, which could be useful for purchasing or reviewing later.\",\n \"action\": \"DoubleClick(1512, 324)\"\n}", + "action_string": "DoubleClick(1512, 324)", + "intent_string": "The current page shows a product listing for Lee Women's Plus Size Relaxed Fit Straight Leg Jeans. The 'Add to Cart' button is visible, indicating that clicking this button will add the item to the shopping cart. This action furthers the exploration by adding an item to the cart, which could be useful for purchasing or reviewing later.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1457, + 309, + 1537, + 346 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": " Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spandex", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise,straight leg, Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "Add t Rart", + "boxes": [ + 1455, + 305, + 1539, + 348 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": " Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spande", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise straight leg,Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ] + }, + { + "frame_id": "120_4_8", + "episode_id": 120, + "env_id": 4, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.375378608703613, + "advantages": -0.9304614067077637, + "generated_text": "{\n \"intent\": \"Since this appears to be an application interface, clicking on the 'File' menu will allow me to open files, save documents, or exit the application. This action should be useful for exploring functionalities related to file management.\",\n \"action\": \"Click(15, 36)\"\n}", + "action_string": "Click(15, 36)", + "intent_string": "Since this appears to be an application interface, clicking on the 'File' menu will allow me to open files, save documents, or exit the application. This action should be useful for exploring functionalities related to file management.", + "before_screen_ocr_results": [ + { + "text": "---@", + "boxes": [ + 4, + 47, + 270, + 88 + ] + }, + { + "text": "X电旨", + "boxes": [ + 258, + 60, + 348, + 77 + ] + }, + { + "text": "園口", + "boxes": [ + 996, + 56, + 1146, + 79 + ] + }, + { + "text": "C", + "boxes": [ + 184, + 101, + 206, + 115 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 240 + ] + }, + { + "text": "", + "boxes": [ + 1886, + 234, + 1912, + 304 + ] + }, + { + "text": "O", + "boxes": [ + 1884, + 308, + 1914, + 342 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New", + "boxes": [ + 26, + 54, + 58, + 73 + ] + }, + { + "text": "T曼·Ω", + "boxes": [ + 726, + 54, + 888, + 83 + ] + }, + { + "text": "園一→田", + "boxes": [ + 992, + 54, + 1252, + 83 + ] + }, + { + "text": "Open..", + "boxes": [ + 24, + 77, + 72, + 103 + ] + }, + { + "text": "Ctrl+(", + "boxes": [ + 254, + 81, + 288, + 95 + ] + }, + { + "text": "a", + "boxes": [ + 520, + 101, + 538, + 115 + ] + }, + { + "text": "Recent Docu", + "boxes": [ + 30, + 135, + 106, + 153 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 24, + 191, + 86, + 216 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 238 + ] + }, + { + "text": "Template", + "boxes": [ + 30, + 222, + 90, + 240 + ] + }, + { + "text": "Reload", + "boxes": [ + 30, + 252, + 74, + 272 + ] + }, + { + "text": "Save", + "boxes": [ + 26, + 312, + 60, + 332 + ] + }, + { + "text": "Ctrl+s", + "boxes": [ + 250, + 312, + 294, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1874, + 303, + 1920, + 350 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 298, + 359 + ] + }, + { + "text": " Save Remote", + "boxes": [ + 24, + 365, + 116, + 389 + ] + }, + { + "text": "Save a Copy..", + "boxes": [ + 26, + 389, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 22, + 447, + 84, + 478 + ] + }, + { + "text": "Export As", + "boxes": [ + 24, + 476, + 94, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 68, + 528 + ] + }, + { + "text": "Preview in Web Bro", + "boxes": [ + 26, + 534, + 156, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 298, + 586 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 248, + 588, + 296, + 613 + ] + }, + { + "text": " Printer Settings.", + "boxes": [ + 24, + 617, + 134, + 641 + ] + }, + { + "text": " Properties..", + "boxes": [ + 24, + 649, + 102, + 673 + ] + }, + { + "text": " Exit LibreOffice", + "boxes": [ + 22, + 705, + 134, + 735 + ] + }, + { + "text": "Ctrl+Q", + "boxes": [ + 250, + 709, + 294, + 729 + ] + } + ] + }, + { + "frame_id": "120_5_8", + "episode_id": 120, + "env_id": 5, + "step": 8, + "format_rewards": 1.0, + "rewards": 8.794734001159668, + "advantages": -1.7528022527694702, + "generated_text": "{\n \"intent\": \"To start exploring the desktop environment, I should look for an icon or file to open. The most logical place to do this is on the desktop area where files and folders are typically stored. Clicking on one will allow us to see its contents.\",\n \"action\": \"DoubleClick(5, 67)\"\n}", + "action_string": "DoubleClick(5, 67)", + "intent_string": "To start exploring the desktop environment, I should look for an icon or file to open. The most logical place to do this is on the desktop area where files and folders are typically stored. Clicking on one will allow us to see its contents.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "FileEditView", + "boxes": [ + 6, + 24, + 124, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "画·口·目·目@%血-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "G-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "、", + "boxes": [ + 130, + 129, + 160, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 258, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "--- @-", + "boxes": [ + 0, + 48, + 372, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "勺-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "v田·I·子I", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberation San ", + "boxes": [ + 20, + 95, + 118, + 119 + ] + }, + { + "text": "、", + "boxes": [ + 124, + 101, + 146, + 115 + ] + }, + { + "text": "10", + "boxes": [ + 154, + 97, + 210, + 117 + ] + }, + { + "text": "、", + "boxes": [ + 202, + 97, + 228, + 117 + ] + }, + { + "text": "Jaaa.", + "boxes": [ + 222, + 93, + 350, + 123 + ] + }, + { + "text": "要日日%00三图目", + "boxes": [ + 336, + 93, + 1208, + 123 + ] + }, + { + "text": "?", + "boxes": [ + 130, + 129, + 156, + 149 + ] + }, + { + "text": "、=", + "boxes": [ + 148, + 125, + 256, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 234, + 154, + 256, + 176 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "120_6_8", + "episode_id": 120, + "env_id": 6, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.845749855041504, + "advantages": -0.264295756816864, + "generated_text": "{\"intent\": \"The user has arrived at an Amazon page for Neutrogena products. To explore more about the brand or related products, clicking on the menu button (represented by the icon next to 'Neutrogena') might provide additional navigation options.\", \"action\": \"Click(461, 274)\"}", + "action_string": "Click(461, 274)", + "intent_string": "The user has arrived at an Amazon page for Neutrogena products. To explore more about the brand or related products, clicking on the menu button (represented by the icon next to 'Neutrogena') might provide additional navigation options.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1590, + 68, + 1660, + 89 + ] + }, + { + "text": "BEAUTY TO A SCIENCE", + "boxes": [ + 742, + 135, + 1162, + 177 + ] + }, + { + "text": "Follow", + "boxes": [ + 360, + 264, + 404, + 290 + ] + }, + { + "text": "凸", + "boxes": [ + 450, + 264, + 474, + 288 + ] + }, + { + "text": "HOME", + "boxes": [ + 496, + 270, + 542, + 288 + ] + }, + { + "text": "α", + "boxes": [ + 1412, + 262, + 1446, + 290 + ] + }, + { + "text": " Search all Neutrogena", + "boxes": [ + 1438, + 262, + 1604, + 292 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1004, + 371, + 1372, + 413 + ] + }, + { + "text": "Neutrogena:", + "boxes": [ + 649, + 382, + 881, + 447 + ] + }, + { + "text": "Sheer Berry", + "boxes": [ + 1001, + 412, + 1193, + 461 + ] + }, + { + "text": "Neutrogenar", + "boxes": [ + 1405, + 410, + 1529, + 455 + ] + }, + { + "text": "BEAUTY TO A SCIENCE", + "boxes": [ + 690, + 451, + 836, + 480 + ] + }, + { + "text": "Hyaluronic acid ip balm with vitamin E and", + "boxes": [ + 1002, + 465, + 1368, + 500 + ] + }, + { + "text": "DRO", + "boxes": [ + 1424, + 463, + 1458, + 476 + ] + }, + { + "text": " shea butter to instantly nourish dry lips", + "boxes": [ + 1000, + 486, + 1336, + 526 + ] + }, + { + "text": "970 ($13.86/ouncel ", + "boxes": [ + 995, + 528, + 1133, + 570 + ] + }, + { + "text": "Nourishing", + "boxes": [ + 621, + 581, + 907, + 654 + ] + }, + { + "text": " Quick look", + "boxes": [ + 1016, + 602, + 1104, + 619 + ] + }, + { + "text": "hydration", + "boxes": [ + 638, + 645, + 890, + 711 + ] + }, + { + "text": "for juicy lips", + "boxes": [ + 603, + 696, + 925, + 777 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1276, + 739, + 1640, + 780 + ] + }, + { + "text": "Neutrogenar", + "boxes": [ + 1024, + 782, + 1148, + 820 + ] + }, + { + "text": "Sheer lce", + "boxes": [ + 1492, + 782, + 1642, + 826 + ] + }, + { + "text": "HYDRO", + "boxes": [ + 1032, + 828, + 1082, + 846 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and ", + "boxes": [ + 1278, + 832, + 1642, + 866 + ] + }, + { + "text": "B00ST", + "boxes": [ + 1032, + 844, + 1080, + 864 + ] + }, + { + "text": "shea butter to instantly nourish dry lips", + "boxes": [ + 1312, + 862, + 1640, + 891 + ] + }, + { + "text": "From the #1 Hyaluronic Acid", + "boxes": [ + 618, + 901, + 908, + 931 + ] + }, + { + "text": "$955 ($13.64/ounce)", + "boxes": [ + 1505, + 897, + 1637, + 938 + ] + }, + { + "text": "facial care line in the US*", + "boxes": [ + 634, + 929, + 888, + 959 + ] + }, + { + "text": "Pre-order. This item willbe released on May 20.", + "boxes": [ + 1312, + 929, + 1638, + 959 + ] + }, + { + "text": "2025.", + "boxes": [ + 1596, + 945, + 1638, + 973 + ] + }, + { + "text": "TateMcRae", + "boxes": [ + 264, + 983, + 372, + 1007 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "BEAUTY TOA SCIENCE", + "boxes": [ + 746, + 139, + 1160, + 175 + ] + }, + { + "text": "Q", + "boxes": [ + 1414, + 264, + 1442, + 286 + ] + }, + { + "text": " Search all Neutrogen", + "boxes": [ + 1440, + 264, + 1600, + 288 + ] + }, + { + "text": " Email", + "boxes": [ + 344, + 308, + 436, + 339 + ] + }, + { + "text": " Pinterest ", + "boxes": [ + 344, + 345, + 466, + 383 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1004, + 371, + 1372, + 413 + ] + }, + { + "text": " Facebook", + "boxes": [ + 348, + 389, + 468, + 421 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 647, + 382, + 881, + 447 + ] + }, + { + "text": "Sheer Berry", + "boxes": [ + 1001, + 410, + 1193, + 461 + ] + }, + { + "text": "Neutrogenar", + "boxes": [ + 1405, + 410, + 1531, + 455 + ] + }, + { + "text": "×", + "boxes": [ + 344, + 425, + 402, + 467 + ] + }, + { + "text": "BEAUTY TO A SCIENCE", + "boxes": [ + 686, + 451, + 836, + 480 + ] + }, + { + "text": "β Copy Link ", + "boxes": [ + 348, + 470, + 472, + 507 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and ", + "boxes": [ + 1002, + 467, + 1366, + 496 + ] + }, + { + "text": " shea butter to instantly nourish dry lips ", + "boxes": [ + 1000, + 486, + 1336, + 526 + ] + }, + { + "text": "$970 ($13.86/ounce) ", + "boxes": [ + 1001, + 529, + 1133, + 568 + ] + }, + { + "text": "Pre-order. This item willbe released on May 20.", + "boxes": [ + 1004, + 562, + 1332, + 592 + ] + }, + { + "text": "Nourishing", + "boxes": [ + 621, + 581, + 905, + 654 + ] + }, + { + "text": "2025.", + "boxes": [ + 1006, + 580, + 1046, + 606 + ] + }, + { + "text": "hydration", + "boxes": [ + 642, + 651, + 884, + 707 + ] + }, + { + "text": "for juicy lips", + "boxes": [ + 607, + 696, + 919, + 779 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1276, + 739, + 1640, + 780 + ] + }, + { + "text": "Neutrogena*", + "boxes": [ + 1026, + 782, + 1150, + 820 + ] + }, + { + "text": "Sheer lce", + "boxes": [ + 1496, + 786, + 1642, + 824 + ] + }, + { + "text": "YDRO", + "boxes": [ + 1036, + 828, + 1082, + 846 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and", + "boxes": [ + 1278, + 834, + 1638, + 864 + ] + }, + { + "text": "B0OST", + "boxes": [ + 1032, + 844, + 1078, + 862 + ] + }, + { + "text": "shea butter to instantly nourish dry lips", + "boxes": [ + 1312, + 862, + 1640, + 891 + ] + }, + { + "text": "From the #1 Hyaluronic Acid", + "boxes": [ + 616, + 897, + 910, + 933 + ] + }, + { + "text": "$sg55 ($13.64/ounce)", + "boxes": [ + 1505, + 895, + 1637, + 938 + ] + }, + { + "text": "facial care line in the US*", + "boxes": [ + 636, + 929, + 888, + 959 + ] + }, + { + "text": "Pre-order. This item will be released on May 20.", + "boxes": [ + 1316, + 929, + 1638, + 959 + ] + }, + { + "text": "2025.", + "boxes": [ + 1596, + 945, + 1638, + 973 + ] + }, + { + "text": "Tate McRae", + "boxes": [ + 262, + 981, + 378, + 1011 + ] + } + ] + }, + { + "frame_id": "120_7_8", + "episode_id": 120, + "env_id": 7, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.846027374267578, + "advantages": -0.2639027237892151, + "generated_text": "{\n \"intent\": \"Clicking the 'File' menu item will allow me to explore file-related options such as opening files, saving, or exiting the application. This action will help me understand the environment by navigating through different types of resources, hence expanding my knowledge about the interface.\",\n \"action\": \"Click(20, 37)\"\n}", + "action_string": "Click(20, 37)", + "intent_string": "Clicking the 'File' menu item will allow me to explore file-related options such as opening files, saving, or exiting the application. This action will help me understand the environment by navigating through different types of resources, hence expanding my knowledge about the interface.", + "before_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 6, + 26, + 36, + 46 + ] + }, + { + "text": "Search View", + "boxes": [ + 90, + 24, + 186, + 48 + ] + }, + { + "text": "Help", + "boxes": [ + 270, + 21, + 310, + 49 + ] + }, + { + "text": "URL=/usr/share/applications/firefox.desktop", + "boxes": [ + 0, + 139, + 352, + 169 + ] + }, + { + "text": "/root/Desktop", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 10, + 30, + 36, + 52 + ] + }, + { + "text": "Edit", + "boxes": [ + 42, + 26, + 76, + 46 + ] + }, + { + "text": "Nev", + "boxes": [ + 10, + 54, + 50, + 73 + ] + }, + { + "text": "Ctrl+N", + "boxes": [ + 194, + 54, + 242, + 73 + ] + }, + { + "text": " New Window", + "boxes": [ + 18, + 79, + 112, + 103 + ] + }, + { + "text": "Shift+Ctrl+N ", + "boxes": [ + 152, + 77, + 244, + 103 + ] + }, + { + "text": "1 open.", + "boxes": [ + 1, + 106, + 67, + 138 + ] + }, + { + "text": "Ctrl+o", + "boxes": [ + 192, + 109, + 244, + 135 + ] + }, + { + "text": "-efox.desktop", + "boxes": [ + 224, + 137, + 350, + 167 + ] + }, + { + "text": " save", + "boxes": [ + 0, + 165, + 64, + 197 + ] + }, + { + "text": "Ctrl+s", + "boxes": [ + 194, + 167, + 248, + 197 + ] + }, + { + "text": " Save As.", + "boxes": [ + 0, + 195, + 82, + 220 + ] + }, + { + "text": "shift+Ctrl+S ", + "boxes": [ + 156, + 195, + 246, + 220 + ] + }, + { + "text": "Save All ", + "boxes": [ + 22, + 222, + 84, + 248 + ] + }, + { + "text": " Revert", + "boxes": [ + 2, + 250, + 72, + 276 + ] + }, + { + "text": " Print.", + "boxes": [ + 2, + 282, + 62, + 308 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 194, + 282, + 246, + 308 + ] + }, + { + "text": " Detach Tab", + "boxes": [ + 22, + 314, + 100, + 339 + ] + }, + { + "text": "Ctrl+D", + "boxes": [ + 192, + 312, + 244, + 338 + ] + }, + { + "text": " Close Tab", + "boxes": [ + 4, + 345, + 94, + 369 + ] + }, + { + "text": "Ctrl+w", + "boxes": [ + 188, + 341, + 246, + 373 + ] + }, + { + "text": " close Window", + "boxes": [ + 2, + 371, + 120, + 401 + ] + }, + { + "text": "Ctrl+Q", + "boxes": [ + 192, + 373, + 246, + 399 + ] + }, + { + "text": "root/Desl", + "boxes": [ + 218, + 1050, + 282, + 1068 + ] + } + ] + }, + { + "frame_id": "120_0_9", + "episode_id": 120, + "env_id": 0, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.009786605834961, + "advantages": -1.448232889175415, + "generated_text": "{\n \"intent\": \"The goal is to continue exploring the desktop environment and specifically to download Firefox for desktop. The current window is open to the Firefox download page, which suggests an intention to initiate the download process. Clicking 'OK' on the dialog box will proceed with the opening of the file, aligning with the exploration goal.\",\n \"action\": \"DoubleClick(1178, 673)\"\n}", + "action_string": "DoubleClick(1178, 673)", + "intent_string": "The goal is to continue exploring the desktop environment and specifically to download Firefox for desktop. The current window is open to the Firefox download page, which suggests an intention to initiate the download process. Clicking 'OK' on the dialog box will proceed with the opening of the file, aligning with the exploration goal.", + "before_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 30, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 64, + 1662, + 89 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1876, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 353, + 1088, + 383 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 383, + 922, + 411 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "which is: TAR Archive (69.8 MB)", + "boxes": [ + 772, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from nov", + "boxes": [ + 764, + 594, + 1096, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 655, + 1060, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distrbuton?", + "boxes": [ + 732, + 681, + 1172, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 698, + 881, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1016, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口", + "boxes": [ + 1118, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Whichis: TAR Archive (69.8 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now on.", + "boxes": [ + 764, + 594, + 1124, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distrbuton?", + "boxes": [ + 732, + 681, + 1172, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 702, + 883, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_1_9", + "episode_id": 120, + "env_id": 1, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.967039108276367, + "advantages": -0.09251925349235535, + "generated_text": "{\n \"intent\": \"To continue exploring the options available on Amazon for purchasing jeans, I will double-click on one of the listed items.\",\n \"action\": \"DoubleClick(446, 323)\"\n}", + "action_string": "DoubleClick(446, 323)", + "intent_string": "To continue exploring the options available on Amazon for purchasing jeans, I will double-click on one of the listed items.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1654, + 85 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1874, + 91 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1814, + 105, + 1872, + 131 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1806, + 125, + 1880, + 149 + ] + }, + { + "text": "Men's Classic Relaxed Fit Jean", + "boxes": [ + 608, + 266, + 830, + 296 + ] + }, + { + "text": "Wrangler Authentics Men's Regular", + "boxes": [ + 900, + 268, + 1158, + 298 + ] + }, + { + "text": "Gloria Vanderbilt Women's Amanda", + "boxes": [ + 1194, + 268, + 1458, + 298 + ] + }, + { + "text": "Trendy Queen Womens Baggy Jeans ", + "boxes": [ + 1490, + 268, + 1760, + 298 + ] + }, + { + "text": "Amazon Essentials Men's Slim-Fit ", + "boxes": [ + 314, + 292, + 562, + 322 + ] + }, + { + "text": "★★★★★28.090", + "boxes": [ + 610, + 294, + 760, + 324 + ] + }, + { + "text": "Fit Comfort Flex Waist Jean", + "boxes": [ + 898, + 290, + 1106, + 322 + ] + }, + { + "text": "Classic High Rise Tapered Jean", + "boxes": [ + 1196, + 292, + 1418, + 322 + ] + }, + { + "text": "Wide Leg Denim Pants Boyfriend", + "boxes": [ + 1488, + 292, + 1736, + 322 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 294, + 1880, + 326 + ] + }, + { + "text": "ought in past month", + "boxes": [ + 652, + 316, + 784, + 342 + ] + }, + { + "text": "★★★★★83,863", + "boxes": [ + 902, + 315, + 1054, + 350 + ] + }, + { + "text": "★★★★75,800", + "boxes": [ + 1196, + 315, + 1346, + 350 + ] + }, + { + "text": "Loose Streetwear Trousers Fall", + "boxes": [ + 1486, + 314, + 1718, + 346 + ] + }, + { + "text": "★★★★☆42,605", + "boxes": [ + 318, + 345, + 460, + 369 + ] + }, + { + "text": "$1398 List: $4.99", + "boxes": [ + 603, + 343, + 747, + 390 + ] + }, + { + "text": "00+", + "boxes": [ + 912, + 345, + 940, + 359 + ] + }, + { + "text": "00+", + "boxes": [ + 1204, + 345, + 1234, + 359 + ] + }, + { + "text": " bought in past i", + "boxes": [ + 1228, + 341, + 1334, + 365 + ] + }, + { + "text": "Fashion y2k Outfits 2025", + "boxes": [ + 1488, + 339, + 1678, + 369 + ] + }, + { + "text": "$3155", + "boxes": [ + 900, + 371, + 966, + 405 + ] + }, + { + "text": "$229 ist: $40:09 ", + "boxes": [ + 1189, + 367, + 1337, + 412 + ] + }, + { + "text": "★★★★☆91", + "boxes": [ + 1490, + 367, + 1608, + 397 + ] + }, + { + "text": "$2660", + "boxes": [ + 314, + 377, + 376, + 411 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 608, + 383, + 750, + 413 + ] + }, + { + "text": "100+ bought in past n", + "boxes": [ + 1490, + 387, + 1630, + 415 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 314, + 413, + 448, + 443 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 902, + 407, + 1048, + 437 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 1196, + 407, + 1338, + 435 + ] + }, + { + "text": "$3299", + "boxes": [ + 1488, + 421, + 1552, + 455 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 316, + 435, + 408, + 459 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 906, + 435, + 988, + 453 + ] + }, + { + "text": "1-2", + "boxes": [ + 110, + 453, + 138, + 467 + ] + }, + { + "text": "3-4", + "boxes": [ + 180, + 451, + 212, + 471 + ] + }, + { + "text": " 1 sustainabiltyfeature ", + "boxes": [ + 316, + 455, + 502, + 484 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 1486, + 455, + 1634, + 484 + ] + }, + { + "text": "5-6", + "boxes": [ + 34, + 480, + 64, + 500 + ] + }, + { + "text": "7-8", + "boxes": [ + 108, + 480, + 140, + 500 + ] + }, + { + "text": "9-10", + "boxes": [ + 178, + 478, + 220, + 504 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1492, + 482, + 1576, + 500 + ] + }, + { + "text": "11-12", + "boxes": [ + 24, + 508, + 76, + 534 + ] + }, + { + "text": "13-14", + "boxes": [ + 100, + 510, + 156, + 530 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1866, + 526 + ] + }, + { + "text": "Lysse Women's Denim Straight Leg", + "boxes": [ + 1190, + 552, + 1484, + 588 + ] + }, + { + "text": "ric Clothing Size", + "boxes": [ + 150, + 580, + 256, + 604 + ] + }, + { + "text": "★★★★☆324", + "boxes": [ + 1198, + 586, + 1322, + 609 + ] + }, + { + "text": "2022[24", + "boxes": [ + 150, + 609, + 280, + 633 + ] + }, + { + "text": "$8800", + "boxes": [ + 1194, + 617, + 1256, + 649 + ] + }, + { + "text": "28", + "boxes": [ + 66, + 639, + 108, + 659 + ] + }, + { + "text": "30", + "boxes": [ + 104, + 637, + 144, + 663 + ] + }, + { + "text": "Delivery May 27 - 31", + "boxes": [ + 1200, + 653, + 1334, + 677 + ] + }, + { + "text": "Special Clothing Size", + "boxes": [ + 12, + 681, + 152, + 705 + ] + }, + { + "text": "r fastest delivery May 26 - 30", + "boxes": [ + 1204, + 675, + 1392, + 699 + ] + }, + { + "text": " Big & Tall", + "boxes": [ + 9, + 700, + 99, + 731 + ] + }, + { + "text": " Husky", + "boxes": [ + 12, + 723, + 78, + 748 + ] + }, + { + "text": " Petite", + "boxes": [ + 12, + 764, + 74, + 784 + ] + }, + { + "text": " plus Size", + "boxes": [ + 7, + 779, + 95, + 811 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 816, + 136, + 840 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 14, + 840, + 144, + 870 + ] + }, + { + "text": "More results", + "boxes": [ + 304, + 866, + 426, + 895 + ] + }, + { + "text": "Brands", + "boxes": [ + 12, + 878, + 62, + 903 + ] + }, + { + "text": " sidefeel", + "boxes": [ + 9, + 916, + 91, + 948 + ] + }, + { + "text": "Levi's", + "boxes": [ + 12, + 941, + 72, + 967 + ] + }, + { + "text": "Lee", + "boxes": [ + 14, + 963, + 58, + 981 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Canara", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "jeans", + "boxes": [ + 294, + 115, + 348, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1646, + 139 + ] + }, + { + "text": "Prime Video", + "boxes": [ + 260, + 167, + 342, + 191 + ] + }, + { + "text": " Gift Cards ", + "boxes": [ + 352, + 167, + 430, + 191 + ] + }, + { + "text": "Visit the Ar", + "boxes": [ + 658, + 264, + 724, + 284 + ] + }, + { + "text": "entialsStore", + "boxes": [ + 788, + 264, + 868, + 282 + ] + }, + { + "text": "Amazon Essentials Men's Slim-Fit Jeans", + "boxes": [ + 652, + 286, + 1088, + 316 + ] + }, + { + "text": "$2660", + "boxes": [ + 1394, + 280, + 1456, + 314 + ] + }, + { + "text": "4.2★★★★☆ 42.605 ratings| Search this page", + "boxes": [ + 654, + 312, + 1022, + 341 + ] + }, + { + "text": "No Import Charges & $7.30", + "boxes": [ + 1390, + 319, + 1576, + 352 + ] + }, + { + "text": " 1 sustainability feature ", + "boxes": [ + 658, + 338, + 850, + 361 + ] + }, + { + "text": "Shipping to Canada Details ", + "boxes": [ + 1392, + 341, + 1590, + 371 + ] + }, + { + "text": "Delivery Friday, May 23. Order", + "boxes": [ + 1394, + 365, + 1592, + 389 + ] + }, + { + "text": "$260", + "boxes": [ + 652, + 381, + 720, + 421 + ] + }, + { + "text": "within 23 hrs 13 mins", + "boxes": [ + 1392, + 381, + 1536, + 411 + ] + }, + { + "text": " Deliver", + "boxes": [ + 1392, + 411, + 1462, + 443 + ] + }, + { + "text": "to Canada", + "boxes": [ + 1416, + 435, + 1472, + 455 + ] + }, + { + "text": "36W x 31L v", + "boxes": [ + 664, + 478, + 756, + 502 + ] + }, + { + "text": "In Stock", + "boxes": [ + 1392, + 469, + 1468, + 500 + ] + }, + { + "text": "Quantity: 1", + "boxes": [ + 1404, + 512, + 1484, + 536 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1460, + 558, + 1534, + 576 + ] + }, + { + "text": " Buy Now", + "boxes": [ + 1466, + 594, + 1528, + 625 + ] + }, + { + "text": "Ships from", + "boxes": [ + 1404, + 639, + 1460, + 659 + ] + }, + { + "text": "Sold by", + "boxes": [ + 1401, + 656, + 1445, + 682 + ] + }, + { + "text": "30-day", + "boxes": [ + 1468, + 677, + 1508, + 703 + ] + }, + { + "text": "refund/r", + "boxes": [ + 1470, + 693, + 1516, + 711 + ] + }, + { + "text": " Product details", + "boxes": [ + 648, + 711, + 806, + 746 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 744, + 737, + 781 + ] + }, + { + "text": "99% Cotton, 1% Elastane", + "boxes": [ + 824, + 750, + 994, + 774 + ] + }, + { + "text": "口", + "boxes": [ + 1396, + 766, + 1420, + 788 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 778, + 762, + 806 + ] + }, + { + "text": "Machine Wash Warm, Tumble Dry Lo", + "boxes": [ + 824, + 776, + 1058, + 806 + ] + }, + { + "text": "Add agift receipt for easy", + "boxes": [ + 1416, + 784, + 1586, + 814 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 806, + 702, + 834 + ] + }, + { + "text": "Imported", + "boxes": [ + 827, + 807, + 891, + 830 + ] + }, + { + "text": "returns", + "boxes": [ + 1416, + 806, + 1470, + 832 + ] + }, + { + "text": "Closure type", + "boxes": [ + 651, + 831, + 743, + 864 + ] + }, + { + "text": "Button and Zippe", + "boxes": [ + 826, + 836, + 938, + 858 + ] + }, + { + "text": "Add to List", + "boxes": [ + 1397, + 851, + 1481, + 888 + ] + }, + { + "text": "About this item", + "boxes": [ + 650, + 893, + 810, + 929 + ] + }, + { + "text": "Shopping from Canada?", + "boxes": [ + 1392, + 921, + 1588, + 951 + ] + }, + { + "text": "· SLIM FIT DESIGN: These mens jeans slim fit are designed with a tapered", + "boxes": [ + 662, + 935, + 1126, + 959 + ] + }, + { + "text": "Saveonshipping andgetfaster", + "boxes": [ + 1394, + 939, + 1598, + 971 + ] + }, + { + "text": "delivery on eligible products", + "boxes": [ + 1394, + 961, + 1580, + 991 + ] + }, + { + "text": "ALL-DAY COMFORT: Crafted from mid-weight cotton", + "boxes": [ + 666, + 973, + 1014, + 997 + ] + } + ] + }, + { + "frame_id": "120_2_9", + "episode_id": 120, + "env_id": 2, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.592742919921875, + "advantages": -0.6226180791854858, + "generated_text": "{\n \"intent\": \"The user is likely interested in downloading Firefox for desktop. The first step would be to click on one of the download options to proceed towards selecting the appropriate version for their operating system.\",\n \"action\": \"DoubleClick(957, 656)\"\n}", + "action_string": "DoubleClick(957, 656)", + "intent_string": "The user is likely interested in downloading Firefox for desktop. The first step would be to click on one of the download options to proceed towards selecting the appropriate version for their operating system.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1574, + 68, + 1628, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "firefox-138.0.4.tar.z", + "boxes": [ + 1238, + 105, + 1384, + 135 + ] + }, + { + "text": "Completed — 69.8 MB", + "boxes": [ + 1246, + 133, + 1378, + 151 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1118, + 731 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "。☆", + "boxes": [ + 1561, + 61, + 1631, + 92 + ] + }, + { + "text": "→三目", + "boxes": [ + 1756, + 66, + 1874, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (69.8 MB)", + "boxes": [ + 772, + 433, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/download-intaller.cdn.moilenet ", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1036, + 532 + ] + }, + { + "text": " Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "OSave File", + "boxes": [ + 764, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for files like this from now on.", + "boxes": [ + 762, + 594, + 1124, + 617 + ] + }, + { + "text": "?", + "boxes": [ + 951, + 654, + 974, + 676 + ] + }, + { + "text": "Cancel", + "boxes": [ + 1010, + 653, + 1066, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 784, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "120_3_9", + "episode_id": 120, + "env_id": 3, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.457015037536621, + "advantages": -0.8148434162139893, + "generated_text": "{\n \"intent\": \"The goal is to explore the product details and ensure the jeans meet the user's requirements. The current step involves clicking 'Select Size' to choose a suitable size for purchase.\",\n \"action\": \"Click(703, 547)\"\n}", + "action_string": "Click(703, 547)", + "intent_string": "The goal is to explore the product details and ensure the jeans meet the user's requirements. The current step involves clicking 'Select Size' to choose a suitable size for purchase.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→白", + "boxes": [ + 36, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jea", + "boxes": [ + 648, + 282, + 1212, + 320 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "Select Size from the left", + "boxes": [ + 1214, + 300, + 1374, + 324 + ] + }, + { + "text": "Add t Rart", + "boxes": [ + 1455, + 305, + 1539, + 348 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings | Search this page", + "boxes": [ + 792, + 314, + 1020, + 338 + ] + }, + { + "text": "to add to Shopping Cart", + "boxes": [ + 1215, + 315, + 1373, + 352 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "Size:", + "boxes": [ + 658, + 506, + 688, + 526 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Select ", + "boxes": [ + 662, + 534, + 730, + 560 + ] + }, + { + "text": "Size Chart ", + "boxes": [ + 656, + 574, + 742, + 598 + ] + }, + { + "text": " Product details", + "boxes": [ + 650, + 602, + 804, + 631 + ] + }, + { + "text": "Fabric type", + "boxes": [ + 647, + 632, + 737, + 670 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spande", + "boxes": [ + 824, + 637, + 1086, + 661 + ] + }, + { + "text": "Care instructions", + "boxes": [ + 654, + 667, + 764, + 691 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "Origin", + "boxes": [ + 652, + 695, + 702, + 721 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "Closure type", + "boxes": [ + 649, + 718, + 745, + 755 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": " About this item", + "boxes": [ + 650, + 782, + 810, + 818 + ] + }, + { + "text": "· Mid rise straight leg,Weave Type: Woven", + "boxes": [ + 654, + 820, + 938, + 850 + ] + }, + { + "text": "· Relaxed fit through the thigh", + "boxes": [ + 660, + 842, + 862, + 866 + ] + }, + { + "text": "· Embroidered back pockets, and flexible waistband for comfort", + "boxes": [ + 658, + 860, + 1076, + 889 + ] + }, + { + "text": "·Rise:12inches, ening17inchs,neam:etit29inche,meum31,lng:33inh", + "boxes": [ + 660, + 879, + 1270, + 909 + ] + }, + { + "text": "oy size", + "boxes": [ + 676, + 929, + 716, + 945 + ] + }, + { + "text": "Shop the newAmazon Essentials x Sofia Grainge collection", + "boxes": [ + 748, + 985, + 1142, + 1009 + ] + }, + { + "text": "Effortless cuteness for little ones Shop now", + "boxes": [ + 750, + 1005, + 1028, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 64, + 310, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1648, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "Jeans", + "boxes": [ + 296, + 115, + 348, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1448, + 147 + ] + }, + { + "text": "$16,091.98", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 426, + 187 + ] + }, + { + "text": "Visit the LeeStore", + "boxes": [ + 656, + 262, + 770, + 286 + ] + }, + { + "text": "Lee Women's Plus Size Relaxed Fit Straight Leg Jean", + "boxes": [ + 652, + 286, + 1224, + 316 + ] + }, + { + "text": "To buy, select Size", + "boxes": [ + 1440, + 278, + 1560, + 302 + ] + }, + { + "text": "$1,244.78", + "boxes": [ + 1800, + 296, + 1878, + 322 + ] + }, + { + "text": "4.3★★★★☆", + "boxes": [ + 656, + 316, + 784, + 339 + ] + }, + { + "text": "10,791 ratings I Search this page", + "boxes": [ + 792, + 316, + 1020, + 339 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1462, + 312, + 1536, + 338 + ] + }, + { + "text": "Price:", + "boxes": [ + 656, + 351, + 698, + 371 + ] + }, + { + "text": "$28.00", + "boxes": [ + 688, + 349, + 758, + 375 + ] + }, + { + "text": " Add to List ", + "boxes": [ + 1404, + 367, + 1478, + 393 + ] + }, + { + "text": "人", + "boxes": [ + 174, + 423, + 188, + 441 + ] + }, + { + "text": "$144.95", + "boxes": [ + 1810, + 506, + 1868, + 526 + ] + }, + { + "text": "Seleca", + "boxes": [ + 656, + 538, + 718, + 564 + ] + }, + { + "text": "立", + "boxes": [ + 1791, + 551, + 1822, + 578 + ] + }, + { + "text": "14 Plus", + "boxes": [ + 666, + 564, + 722, + 590 + ] + }, + { + "text": "14 Plus", + "boxes": [ + 672, + 588, + 720, + 615 + ] + }, + { + "text": "Petite", + "boxes": [ + 670, + 604, + 716, + 637 + ] + }, + { + "text": "lils", + "boxes": [ + 768, + 606, + 802, + 627 + ] + }, + { + "text": "14 Plus", + "boxes": [ + 670, + 629, + 722, + 663 + ] + }, + { + "text": "75% Cotton, 24% Polyester, 1% Spandex", + "boxes": [ + 826, + 639, + 1088, + 663 + ] + }, + { + "text": "Long", + "boxes": [ + 670, + 653, + 710, + 681 + ] + }, + { + "text": "Machine Wash", + "boxes": [ + 824, + 667, + 922, + 693 + ] + }, + { + "text": "16", + "boxes": [ + 674, + 683, + 690, + 701 + ] + }, + { + "text": "Imported", + "boxes": [ + 824, + 695, + 892, + 721 + ] + }, + { + "text": "16Petite", + "boxes": [ + 674, + 705, + 728, + 725 + ] + }, + { + "text": "$16.99", + "boxes": [ + 1814, + 711, + 1860, + 731 + ] + }, + { + "text": "16 Plus", + "boxes": [ + 672, + 727, + 726, + 756 + ] + }, + { + "text": "Zipper", + "boxes": [ + 824, + 723, + 870, + 750 + ] + }, + { + "text": "16 Plus", + "boxes": [ + 672, + 750, + 722, + 778 + ] + }, + { + "text": "Petite", + "boxes": [ + 672, + 772, + 712, + 800 + ] + }, + { + "text": "em", + "boxes": [ + 766, + 786, + 806, + 816 + ] + }, + { + "text": "16 Plus", + "boxes": [ + 672, + 798, + 720, + 824 + ] + }, + { + "text": "Long", + "boxes": [ + 670, + 818, + 710, + 846 + ] + }, + { + "text": "t leg, Weave Type: Woven", + "boxes": [ + 768, + 820, + 942, + 850 + ] + }, + { + "text": "18", + "boxes": [ + 674, + 846, + 690, + 866 + ] + }, + { + "text": "ugh the thigh", + "boxes": [ + 770, + 842, + 862, + 866 + ] + }, + { + "text": "18 Petite", + "boxes": [ + 672, + 868, + 732, + 893 + ] + }, + { + "text": "ick pockets, and flexible waistband for comfort", + "boxes": [ + 768, + 860, + 1076, + 889 + ] + }, + { + "text": "18 Long", + "boxes": [ + 669, + 887, + 729, + 924 + ] + }, + { + "text": "leg opening: 17 inches, Inseam: petite: 29 inches, medium: 31, long: 3 inches ", + "boxes": [ + 768, + 879, + 1270, + 909 + ] + }, + { + "text": "ements were taken using size 2ow medium. Please note that Measurements may vary", + "boxes": [ + 774, + 903, + 1316, + 927 + ] + }, + { + "text": "18 Plus", + "boxes": [ + 672, + 917, + 726, + 947 + ] + }, + { + "text": "18 Plus", + "boxes": [ + 672, + 941, + 720, + 967 + ] + }, + { + "text": "Petite", + "boxes": [ + 674, + 965, + 708, + 985 + ] + }, + { + "text": "op the newAmazon EssentialsxSofia Grainge collection", + "boxes": [ + 768, + 985, + 1142, + 1009 + ] + }, + { + "text": "Etfortless cuteness for little ones Shop now", + "boxes": [ + 752, + 1005, + 1028, + 1028 + ] + } + ] + }, + { + "frame_id": "120_4_9", + "episode_id": 120, + "env_id": 4, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.384319305419922, + "advantages": 0.4984559118747711, + "generated_text": "{\n \"intent\": \"The goal is to explore the desktop environment and open an application or file. Double-clicking 'File' in the LibreOffice Writer will likely display more options such as 'Open', which could lead to interacting with files or applications.\",\n \"action\": \"DoubleClick(56, 97)\"\n}", + "action_string": "DoubleClick(56, 97)", + "intent_string": "The goal is to explore the desktop environment and open an application or file. Double-clicking 'File' in the LibreOffice Writer will likely display more options such as 'Open', which could lead to interacting with files or applications.", + "before_screen_ocr_results": [ + { + "text": "New", + "boxes": [ + 26, + 54, + 58, + 73 + ] + }, + { + "text": "T曼·Ω", + "boxes": [ + 724, + 54, + 886, + 83 + ] + }, + { + "text": "園→-", + "boxes": [ + 994, + 54, + 1254, + 83 + ] + }, + { + "text": "Open..", + "boxes": [ + 24, + 77, + 72, + 103 + ] + }, + { + "text": "Ctrl+(", + "boxes": [ + 254, + 81, + 288, + 95 + ] + }, + { + "text": "a", + "boxes": [ + 520, + 101, + 538, + 115 + ] + }, + { + "text": "Recent Docu", + "boxes": [ + 30, + 135, + 106, + 153 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 24, + 191, + 86, + 216 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 238 + ] + }, + { + "text": "Template", + "boxes": [ + 30, + 222, + 90, + 240 + ] + }, + { + "text": "Reload", + "boxes": [ + 28, + 252, + 74, + 272 + ] + }, + { + "text": "Save", + "boxes": [ + 26, + 312, + 60, + 332 + ] + }, + { + "text": "Ctrl+s", + "boxes": [ + 250, + 312, + 294, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1874, + 303, + 1920, + 350 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 298, + 359 + ] + }, + { + "text": " Save Remote", + "boxes": [ + 24, + 365, + 116, + 389 + ] + }, + { + "text": "Save a Copy..", + "boxes": [ + 26, + 389, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 22, + 447, + 84, + 478 + ] + }, + { + "text": "Export As", + "boxes": [ + 24, + 476, + 94, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 68, + 528 + ] + }, + { + "text": "Preview in Web Bro", + "boxes": [ + 26, + 534, + 156, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 298, + 586 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 248, + 588, + 296, + 613 + ] + }, + { + "text": " Printer Settings.", + "boxes": [ + 24, + 617, + 134, + 641 + ] + }, + { + "text": " Properties..", + "boxes": [ + 24, + 649, + 102, + 673 + ] + }, + { + "text": " Exit LibreOffice", + "boxes": [ + 22, + 705, + 134, + 735 + ] + }, + { + "text": "Ctrl+Q", + "boxes": [ + 250, + 709, + 294, + 729 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "国", + "boxes": [ + 6, + 2, + 28, + 26 + ] + }, + { + "text": "Untitled 1 - LibreOffice Writer", + "boxes": [ + 840, + 2, + 1078, + 26 + ] + }, + { + "text": "File", + "boxes": [ + 8, + 26, + 36, + 46 + ] + }, + { + "text": "Table ", + "boxes": [ + 308, + 30, + 348, + 46 + ] + }, + { + "text": "·-目·@x自· ", + "boxes": [ + 10, + 52, + 378, + 87 + ] + }, + { + "text": "二", + "boxes": [ + 380, + 58, + 408, + 79 + ] + }, + { + "text": "G--", + "boxes": [ + 408, + 56, + 510, + 81 + ] + }, + { + "text": "IE-T→", + "boxes": [ + 510, + 52, + 1256, + 87 + ] + }, + { + "text": "QG", + "boxes": [ + 148, + 95, + 214, + 121 + ] + }, + { + "text": "12", + "boxes": [ + 370, + 97, + 396, + 117 + ] + }, + { + "text": " Reo", + "boxes": [ + 402, + 127, + 456, + 153 + ] + }, + { + "text": "[ Modified", + "boxes": [ + 1432, + 161, + 1510, + 185 + ] + }, + { + "text": " Desktop", + "boxes": [ + 402, + 189, + 490, + 220 + ] + }, + { + "text": "日", + "boxes": [ + 558, + 189, + 578, + 204 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 206, + 654, + 230 + ] + }, + { + "text": "7 May", + "boxes": [ + 1446, + 208, + 1484, + 228 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 202, + 1914, + 240 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 222, + 470, + 248 + ] + }, + { + "text": " clipboard app", + "boxes": [ + 552, + 224, + 674, + 261 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 230, + 1518, + 254 + ] + }, + { + "text": "", + "boxes": [ + 1886, + 234, + 1912, + 306 + ] + }, + { + "text": "Other Locations", + "boxes": [ + 418, + 266, + 534, + 290 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 306, + 1914, + 343 + ] + }, + { + "text": " Allfles", + "boxes": [ + 1160, + 895, + 1220, + 927 + ] + }, + { + "text": "1open", + "boxes": [ + 1440, + 961, + 1494, + 987 + ] + }, + { + "text": "Default Style", + "boxes": [ + 660, + 1018, + 738, + 1042 + ] + }, + { + "text": "English (USA)", + "boxes": [ + 1070, + 1020, + 1148, + 1038 + ] + }, + { + "text": "ntitled 1 - L", + "boxes": [ + 378, + 1050, + 452, + 1068 + ] + } + ] + }, + { + "frame_id": "120_5_9", + "episode_id": 120, + "env_id": 5, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.091499328613281, + "advantages": -1.3325068950653076, + "generated_text": "{\n \"intent\": \"The current environment is an open spreadsheet application on the desktop. To continue exploring, interacting with the file menu option would be useful. This could lead to further exploration of the application or accessing different menu options for additional functionality.\",\n \"action\": \"DoubleClick(37, 37)\"\n}", + "action_string": "DoubleClick(37, 37)", + "intent_string": "The current environment is an open spreadsheet application on the desktop. To continue exploring, interacting with the file menu option would be useful. This could lead to further exploration of the application or accessing different menu options for additional functionality.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "--- @-", + "boxes": [ + 0, + 48, + 372, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "勺-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "v田·I·子I", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberation San ", + "boxes": [ + 20, + 95, + 118, + 119 + ] + }, + { + "text": "、", + "boxes": [ + 124, + 101, + 146, + 115 + ] + }, + { + "text": "10", + "boxes": [ + 152, + 97, + 210, + 117 + ] + }, + { + "text": "、", + "boxes": [ + 202, + 97, + 228, + 117 + ] + }, + { + "text": "Jaaa.", + "boxes": [ + 222, + 93, + 350, + 123 + ] + }, + { + "text": "要日日%00三图目", + "boxes": [ + 336, + 93, + 1208, + 123 + ] + }, + { + "text": "?", + "boxes": [ + 130, + 129, + 156, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 256, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 234, + 154, + 256, + 176 + ] + }, + { + "text": "SS", + "boxes": [ + 1890, + 169, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "于", + "boxes": [ + 30, + 32, + 42, + 44 + ] + }, + { + "text": "dit", + "boxes": [ + 52, + 30, + 74, + 44 + ] + }, + { + "text": "Data", + "boxes": [ + 366, + 30, + 388, + 44 + ] + }, + { + "text": "New", + "boxes": [ + 30, + 56, + 56, + 69 + ] + }, + { + "text": "自、", + "boxes": [ + 294, + 54, + 370, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 378, + 54, + 444, + 85 + ] + }, + { + "text": "G-.", + "boxes": [ + 452, + 56, + 550, + 81 + ] + }, + { + "text": "v田·I·子I", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Open..", + "boxes": [ + 26, + 81, + 70, + 101 + ] + }, + { + "text": "Ctrl+C", + "boxes": [ + 252, + 79, + 290, + 99 + ] + }, + { + "text": "a", + "boxes": [ + 312, + 99, + 332, + 119 + ] + }, + { + "text": "a要日日%00三田二口", + "boxes": [ + 342, + 93, + 1204, + 123 + ] + }, + { + "text": "Close", + "boxes": [ + 24, + 159, + 68, + 185 + ] + }, + { + "text": "Wizards", + "boxes": [ + 26, + 191, + 84, + 216 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 270 + ] + }, + { + "text": "Reload", + "boxes": [ + 30, + 252, + 74, + 270 + ] + }, + { + "text": "Save", + "boxes": [ + 26, + 312, + 60, + 332 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "Save As..", + "boxes": [ + 24, + 336, + 92, + 361 + ] + }, + { + "text": "Shift+Ctrl+S", + "boxes": [ + 210, + 336, + 296, + 359 + ] + }, + { + "text": "Save Remote", + "boxes": [ + 28, + 367, + 114, + 385 + ] + }, + { + "text": "Save a Copy...", + "boxes": [ + 24, + 387, + 120, + 415 + ] + }, + { + "text": " Save All", + "boxes": [ + 24, + 417, + 86, + 443 + ] + }, + { + "text": "Export..", + "boxes": [ + 24, + 449, + 82, + 474 + ] + }, + { + "text": "Export as PDF", + "boxes": [ + 26, + 476, + 118, + 502 + ] + }, + { + "text": "Send ", + "boxes": [ + 24, + 500, + 68, + 528 + ] + }, + { + "text": "Preview in WebBrows", + "boxes": [ + 26, + 534, + 172, + 552 + ] + }, + { + "text": " Print Preview", + "boxes": [ + 8, + 557, + 116, + 588 + ] + }, + { + "text": "Shift+Ctrl+0", + "boxes": [ + 208, + 560, + 296, + 584 + ] + }, + { + "text": " Prin...", + "boxes": [ + 24, + 588, + 72, + 613 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 250, + 590, + 292, + 609 + ] + }, + { + "text": "Printer Settings.", + "boxes": [ + 26, + 619, + 132, + 637 + ] + }, + { + "text": " Properties.", + "boxes": [ + 24, + 649, + 100, + 673 + ] + }, + { + "text": "Digital :", + "boxes": [ + 26, + 679, + 78, + 697 + ] + }, + { + "text": " Exit Libreoffice", + "boxes": [ + 24, + 707, + 126, + 731 + ] + }, + { + "text": "Ctrl+c", + "boxes": [ + 254, + 711, + 290, + 725 + ] + }, + { + "text": "35", + "boxes": [ + 12, + 750, + 32, + 766 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "一→", + "boxes": [ + 20, + 997, + 92, + 1014 + ] + } + ] + }, + { + "frame_id": "120_6_9", + "episode_id": 120, + "env_id": 6, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.11340618133545, + "advantages": 0.11477384716272354, + "generated_text": "{\n \"intent\": \"The goal here is to further explore the Neutrogena section on Amazon. Clicking the 'Email' option might allow sharing or saving information about the products displayed, which could be useful for future reference. This action aims to explore the webpage's features related to sharing content.\",\n \"action\": \"Click(423, 274)\"\n}", + "action_string": "Click(423, 274)", + "intent_string": "The goal here is to further explore the Neutrogena section on Amazon. Clicking the 'Email' option might allow sharing or saving information about the products displayed, which could be useful for future reference. This action aims to explore the webpage's features related to sharing content.", + "before_screen_ocr_results": [ + { + "text": "BEAUTY TOA SCIENCE", + "boxes": [ + 746, + 139, + 1160, + 175 + ] + }, + { + "text": "Q", + "boxes": [ + 1414, + 264, + 1442, + 286 + ] + }, + { + "text": " Search all Neutrogen", + "boxes": [ + 1440, + 264, + 1600, + 288 + ] + }, + { + "text": " Email", + "boxes": [ + 344, + 308, + 436, + 339 + ] + }, + { + "text": " Pinterest ", + "boxes": [ + 344, + 345, + 466, + 383 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1004, + 371, + 1372, + 413 + ] + }, + { + "text": " Facebook", + "boxes": [ + 348, + 389, + 468, + 421 + ] + }, + { + "text": "Neutrogena", + "boxes": [ + 647, + 382, + 881, + 447 + ] + }, + { + "text": "Sheer Berry", + "boxes": [ + 1001, + 410, + 1193, + 461 + ] + }, + { + "text": "Neutrogenar", + "boxes": [ + 1405, + 410, + 1531, + 455 + ] + }, + { + "text": "×", + "boxes": [ + 344, + 425, + 402, + 467 + ] + }, + { + "text": "BEAUTY TO A SCIENCE", + "boxes": [ + 686, + 451, + 836, + 480 + ] + }, + { + "text": "β Copy Link ", + "boxes": [ + 348, + 470, + 472, + 507 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and ", + "boxes": [ + 1002, + 467, + 1366, + 496 + ] + }, + { + "text": " shea butter to instantly nourish dry lips ", + "boxes": [ + 1000, + 486, + 1336, + 526 + ] + }, + { + "text": "$970 ($13.86/ounce) ", + "boxes": [ + 1001, + 529, + 1133, + 568 + ] + }, + { + "text": "Pre-order. This item willbe released on May 20,", + "boxes": [ + 1004, + 562, + 1332, + 592 + ] + }, + { + "text": "Nourishing", + "boxes": [ + 621, + 581, + 905, + 654 + ] + }, + { + "text": "2025.", + "boxes": [ + 1006, + 580, + 1046, + 606 + ] + }, + { + "text": "hydration", + "boxes": [ + 642, + 651, + 884, + 707 + ] + }, + { + "text": "for juicy lips", + "boxes": [ + 607, + 696, + 919, + 779 + ] + }, + { + "text": "Hydro Boost Lip Balm -", + "boxes": [ + 1276, + 739, + 1640, + 780 + ] + }, + { + "text": "Neutrogena*", + "boxes": [ + 1026, + 782, + 1150, + 820 + ] + }, + { + "text": "Sheer lce", + "boxes": [ + 1496, + 786, + 1642, + 824 + ] + }, + { + "text": "YDRO", + "boxes": [ + 1036, + 828, + 1082, + 846 + ] + }, + { + "text": "Hyaluronic acid lip balm with vitamin E and", + "boxes": [ + 1278, + 834, + 1638, + 864 + ] + }, + { + "text": "B0OST", + "boxes": [ + 1032, + 844, + 1078, + 862 + ] + }, + { + "text": "shea butter to instantly nourish dry lips", + "boxes": [ + 1312, + 862, + 1640, + 891 + ] + }, + { + "text": "From the #1 Hyaluronic Acid", + "boxes": [ + 616, + 897, + 910, + 933 + ] + }, + { + "text": "facial care line in the US*", + "boxes": [ + 636, + 929, + 888, + 959 + ] + }, + { + "text": "Pre-order. This item will be released on May 20.", + "boxes": [ + 1316, + 929, + 1638, + 959 + ] + }, + { + "text": "2025.", + "boxes": [ + 1596, + 945, + 1638, + 973 + ] + }, + { + "text": "Tate McRae", + "boxes": [ + 262, + 981, + 378, + 1011 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "→C", + "boxes": [ + 36, + 66, + 126, + 89 + ] + }, + { + "text": "?", + "boxes": [ + 262, + 73, + 274, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1576, + 66, + 1662, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": "of Use and Privacy Notice.", + "boxes": [ + 810, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice ", + "boxes": [ + 936, + 582, + 1062, + 606 + ] + }, + { + "text": "Help", + "boxes": [ + 1056, + 588, + 1086, + 602 + ] + } + ] + }, + { + "frame_id": "120_7_9", + "episode_id": 120, + "env_id": 7, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.809463500976562, + "advantages": -0.31568649411201477, + "generated_text": "{\n \"intent\": \"Clicking 'New' will allow us to create a new document in Mousepad, which could give us an opportunity to explore the desktop more by saving the document or viewing its contents.\",\n \"action\": \"DoubleClick(56, 88)\"\n}", + "action_string": "DoubleClick(56, 88)", + "intent_string": "Clicking 'New' will allow us to create a new document in Mousepad, which could give us an opportunity to explore the desktop more by saving the document or viewing its contents.", + "before_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 10, + 30, + 36, + 52 + ] + }, + { + "text": "Edit", + "boxes": [ + 42, + 26, + 76, + 46 + ] + }, + { + "text": "Nev", + "boxes": [ + 10, + 54, + 50, + 73 + ] + }, + { + "text": "Ctrl+N", + "boxes": [ + 194, + 54, + 242, + 73 + ] + }, + { + "text": " New Window", + "boxes": [ + 18, + 79, + 112, + 103 + ] + }, + { + "text": "Shift+Ctrl+N ", + "boxes": [ + 152, + 77, + 244, + 103 + ] + }, + { + "text": " open.", + "boxes": [ + 6, + 113, + 64, + 133 + ] + }, + { + "text": "Ctrl+o", + "boxes": [ + 192, + 109, + 244, + 135 + ] + }, + { + "text": "-efox.desktop", + "boxes": [ + 224, + 137, + 350, + 167 + ] + }, + { + "text": " save", + "boxes": [ + 0, + 165, + 64, + 197 + ] + }, + { + "text": "Ctrl+s", + "boxes": [ + 194, + 167, + 248, + 197 + ] + }, + { + "text": " Save As.", + "boxes": [ + 0, + 195, + 82, + 220 + ] + }, + { + "text": "shift+Ctrl+S ", + "boxes": [ + 156, + 195, + 246, + 220 + ] + }, + { + "text": "Save All ", + "boxes": [ + 22, + 222, + 84, + 248 + ] + }, + { + "text": " Revert", + "boxes": [ + 2, + 250, + 72, + 276 + ] + }, + { + "text": " Print.", + "boxes": [ + 2, + 282, + 62, + 308 + ] + }, + { + "text": "Ctrl+P", + "boxes": [ + 194, + 282, + 246, + 308 + ] + }, + { + "text": " Detach Tab", + "boxes": [ + 22, + 314, + 100, + 339 + ] + }, + { + "text": "Ctrl+D", + "boxes": [ + 192, + 312, + 244, + 338 + ] + }, + { + "text": " Close Tab", + "boxes": [ + 4, + 345, + 94, + 369 + ] + }, + { + "text": "Ctrl+w", + "boxes": [ + 188, + 341, + 246, + 373 + ] + }, + { + "text": " close Window", + "boxes": [ + 2, + 371, + 120, + 401 + ] + }, + { + "text": "Ctrl+Q", + "boxes": [ + 192, + 373, + 246, + 399 + ] + }, + { + "text": "root/Desl", + "boxes": [ + 218, + 1050, + 282, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit Search View Document Help", + "boxes": [ + 4, + 22, + 312, + 52 + ] + }, + { + "text": " Untitled 2 - M..", + "boxes": [ + 358, + 1046, + 470, + 1076 + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_120/scores_v3.json b/ScreenExplorer-7B-E1/episode_120/scores_v3.json new file mode 100644 index 0000000000000000000000000000000000000000..d4763ec2f22b285a907322e5b569f67553edee00 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_120/scores_v3.json @@ -0,0 +1,19 @@ +{ + "directory_path": "logs/ScreenExplorer_2025-05-17_22-11-27ab2072/episode_120", + "single_trajectory_image_diversity": 0.2820312576368451, + "single_trajectory_text_diversity": 0.31795383617281914, + "single_trajectory_image_diversity_max": 0.72998046875, + "single_trajectory_image_diversity_min": 0.00048828125, + "single_trajectory_image_diversity_std": 0.24444580078125, + "single_trajectory_text_diversity_max": 0.544921875, + "single_trajectory_text_diversity_min": 0.009033203125, + "single_trajectory_text_diversity_std": 0.15448760986328125, + "group_trajectories_image_diversity": 0.3834565281867981, + "group_trajectories_text_diversity": 0.4840033948421478, + "group_trajectories_image_diversity_max": 1.15625, + "group_trajectories_image_diversity_min": 0.0, + "group_trajectories_image_diversity_std": 0.2451171875, + "group_trajectories_text_diversity_max": 0.724609375, + "group_trajectories_text_diversity_min": -0.0009765625, + "group_trajectories_text_diversity_std": 0.1436767578125 +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_130/actor_model_130/adapter_config.json b/ScreenExplorer-7B-E1/episode_130/actor_model_130/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c138106acf79edf195f0e28248c173bb4c71f7 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_130/actor_model_130/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen2.5-VL-7B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "k_proj", + "v_proj", + "o_proj", + "q_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_130/recording_env_rank_0_19076.h5 b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_0_19076.h5 new file mode 100644 index 0000000000000000000000000000000000000000..2c6f7a45ed69ec1a28c37139c9c2a9059f9b7f01 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_0_19076.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a90df9bbe436005b5463a6eb82beedef30d7aba12713767d06d9ca0744b380ea +size 15370052 diff --git a/ScreenExplorer-7B-E1/episode_130/recording_env_rank_1_40d7e.h5 b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_1_40d7e.h5 new file mode 100644 index 0000000000000000000000000000000000000000..25fccab2e10228f4b8d51c8b7df01396d78bc3c7 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_1_40d7e.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:918320435fc8b6ef0328c3f665741de46565df3affb2027152d630f9ed7dd044 +size 21689757 diff --git a/ScreenExplorer-7B-E1/episode_130/recording_env_rank_4_68675.h5 b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_4_68675.h5 new file mode 100644 index 0000000000000000000000000000000000000000..2e3310a9722ef88b0e12de9575d0325bcdbed84c --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_4_68675.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77cbb7d37d58e8144d2f11c9eb38dad45c7a428f43a5cc000b44f9fd3c9b70b9 +size 12531761 diff --git a/ScreenExplorer-7B-E1/episode_130/recording_env_rank_5_d5675.h5 b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_5_d5675.h5 new file mode 100644 index 0000000000000000000000000000000000000000..2755ddae3dcea4dc0de9d1949eba7166bfd40eb1 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_5_d5675.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2565a5eac33ab0701a994e39e49874abe4b2c7a65cf69c3d7712c8c2c88f1c1e +size 10180117 diff --git a/ScreenExplorer-7B-E1/episode_130/recording_env_rank_7_a4666.h5 b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_7_a4666.h5 new file mode 100644 index 0000000000000000000000000000000000000000..38b5524d0f7ce62f5325525f121563f235856e4b --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_130/recording_env_rank_7_a4666.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:892ccee4493ace4316ff24cc218ef9bc33340ce9228862b828ef84a347b3576e +size 13349677 diff --git a/ScreenExplorer-7B-E1/episode_140/actor_model_140/adapter_config.json b/ScreenExplorer-7B-E1/episode_140/actor_model_140/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c138106acf79edf195f0e28248c173bb4c71f7 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_140/actor_model_140/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen2.5-VL-7B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "k_proj", + "v_proj", + "o_proj", + "q_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_140/recording_env_rank_3_f3894.h5 b/ScreenExplorer-7B-E1/episode_140/recording_env_rank_3_f3894.h5 new file mode 100644 index 0000000000000000000000000000000000000000..195fa93c59a41d2cfe117884053e3c0d579565e4 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_140/recording_env_rank_3_f3894.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37557cce798526dc09db59cbee8c428d1b7ba10348bb6998272e8feb1952347f +size 13490837 diff --git a/ScreenExplorer-7B-E1/episode_140/recording_env_rank_7_c31e1.h5 b/ScreenExplorer-7B-E1/episode_140/recording_env_rank_7_c31e1.h5 new file mode 100644 index 0000000000000000000000000000000000000000..56541d1d10043297e3b53d4ce84e428a0ec48cc3 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_140/recording_env_rank_7_c31e1.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a8ac691abb9d0952d3e98c80f9ca8283715a7521e72d24b275dde4beaf67a3 +size 16913105 diff --git a/ScreenExplorer-7B-E1/episode_150/recording_env_rank_0_171c2.h5 b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_0_171c2.h5 new file mode 100644 index 0000000000000000000000000000000000000000..5518a8862c5400f4a1e511d3b316d9e775987b5f --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_0_171c2.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e13aba4cea1eb4a2a12d960ac4a958e946e38409a41e76ef250948895b7fdc4 +size 9027362 diff --git a/ScreenExplorer-7B-E1/episode_150/recording_env_rank_1_586f8.h5 b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_1_586f8.h5 new file mode 100644 index 0000000000000000000000000000000000000000..74a9a69d1175bddf42af61025c749635b85b2a18 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_1_586f8.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33197c7fad4be1540c6ebcef4fdf62a9f3552c6031b5281e91159743c0887b26 +size 12809467 diff --git a/ScreenExplorer-7B-E1/episode_150/recording_env_rank_2_69c15.h5 b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_2_69c15.h5 new file mode 100644 index 0000000000000000000000000000000000000000..53c99d69aa5cab599e1dc1c527de21d7e9da266c --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_2_69c15.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1902278f7a1fa760529a551e83fde644bf155f66533c4e03f2ad4ed7b6c3fce5 +size 18158441 diff --git a/ScreenExplorer-7B-E1/episode_150/recording_env_rank_3_2f211.h5 b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_3_2f211.h5 new file mode 100644 index 0000000000000000000000000000000000000000..1b68c738360c13d524c9de8b46333e03ab72364c --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_3_2f211.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a105f9b1bcd028d6505ad633ce8b6b09c522e9b476912d87029b702eeafc322 +size 7671961 diff --git a/ScreenExplorer-7B-E1/episode_150/recording_env_rank_4_7679c.h5 b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_4_7679c.h5 new file mode 100644 index 0000000000000000000000000000000000000000..545cd7f70d63d03c66706273e0db1c8363ca10b1 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_4_7679c.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95c52427acebe6b1c348daf60f9e1bcf16f5af76a0f19291683d7ab363371bdc +size 12377229 diff --git a/ScreenExplorer-7B-E1/episode_150/recording_env_rank_5_c6e0d.h5 b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_5_c6e0d.h5 new file mode 100644 index 0000000000000000000000000000000000000000..f94e68f3099319def53b6ec428f570519d56fb1f --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_5_c6e0d.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c14502f669cd39ea8f2ba7023da079b3766274a0900ee54b49554a74fb0517a3 +size 14291206 diff --git a/ScreenExplorer-7B-E1/episode_150/recording_env_rank_7_49bdd.h5 b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_7_49bdd.h5 new file mode 100644 index 0000000000000000000000000000000000000000..567059e5293cd36e9aa8e3826aaad6cdffe9f29f --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_150/recording_env_rank_7_49bdd.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a98750beb324fec161bd55538df5dc2df912078e67601b14fb8595301024011 +size 14207966 diff --git a/ScreenExplorer-7B-E1/episode_160/actor_model_160/adapter_config.json b/ScreenExplorer-7B-E1/episode_160/actor_model_160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c138106acf79edf195f0e28248c173bb4c71f7 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/actor_model_160/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen2.5-VL-7B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "k_proj", + "v_proj", + "o_proj", + "q_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_160/recording_env_rank_0_ebea0.h5 b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_0_ebea0.h5 new file mode 100644 index 0000000000000000000000000000000000000000..c50c387c0662a1ab09867cf13870501890cfe705 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_0_ebea0.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8155cc1f9642c5a269c2260fed3415a9cca0d010eb68478f4b76c00d8787a369 +size 10537870 diff --git a/ScreenExplorer-7B-E1/episode_160/recording_env_rank_1_7070d.h5 b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_1_7070d.h5 new file mode 100644 index 0000000000000000000000000000000000000000..e3f113711c28621721ae4fe23069bcdee665835a --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_1_7070d.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab7e5155ec8bb7b1097c43c3ff1576010bfb5216d330fe53c8a31fd52bb336bf +size 18030918 diff --git a/ScreenExplorer-7B-E1/episode_160/recording_env_rank_2_350c5.h5 b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_2_350c5.h5 new file mode 100644 index 0000000000000000000000000000000000000000..e9e15e9a36cb7b9b0a8272386dbc5d5ec86b6932 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_2_350c5.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e00a9fc9e203381fd83036243e71ccba5c68b66e5c8f4e64ba17d4d240f55b3e +size 13698816 diff --git a/ScreenExplorer-7B-E1/episode_160/recording_env_rank_3_1afcc.h5 b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_3_1afcc.h5 new file mode 100644 index 0000000000000000000000000000000000000000..b0693bbc4003552bf4349dafae48a97b2f552e24 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_3_1afcc.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dde820edba8d1f89a4b528c3895ad4b6ecac2236eb3e8e5ac2c060307b00f329 +size 4854760 diff --git a/ScreenExplorer-7B-E1/episode_160/recording_env_rank_4_d3f87.h5 b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_4_d3f87.h5 new file mode 100644 index 0000000000000000000000000000000000000000..fc8bf95c26724ba7d13395fd502f6f6817007ea4 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_4_d3f87.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:566199bc50151b3e492fe8a44d37103be7a2ea7189a45cc808013262bf5be49b +size 12094136 diff --git a/ScreenExplorer-7B-E1/episode_160/recording_env_rank_5_4ed32.h5 b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_5_4ed32.h5 new file mode 100644 index 0000000000000000000000000000000000000000..0f9a9334192752f1c37102df2989082d80b9287e --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_5_4ed32.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48ff49726de01a6879c32a72bb2850b7e254c9c0a1ff2be75b47b3706d857863 +size 6686696 diff --git a/ScreenExplorer-7B-E1/episode_160/recording_env_rank_6_17f73.h5 b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_6_17f73.h5 new file mode 100644 index 0000000000000000000000000000000000000000..c703ad670dd92340f5d1c17576984c0ba0257868 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_6_17f73.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff66fafb630a5011fa64747fd4a18434dc196d361a80321788f196d59371f056 +size 16120684 diff --git a/ScreenExplorer-7B-E1/episode_160/recording_env_rank_7_bb950.h5 b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_7_bb950.h5 new file mode 100644 index 0000000000000000000000000000000000000000..18d06f8e81eaec3eead72d630161e00d2138cd1f --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_160/recording_env_rank_7_bb950.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb7667d1813f14f86d981e62e79e9c8771e9d3e9d1ae1df002640118932108a2 +size 12552523 diff --git a/ScreenExplorer-7B-E1/episode_180/game_record_20250518_205519.json b/ScreenExplorer-7B-E1/episode_180/game_record_20250518_205519.json new file mode 100644 index 0000000000000000000000000000000000000000..057900445595767559af7839b5453bcf0e2c7b9d --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_180/game_record_20250518_205519.json @@ -0,0 +1,38457 @@ +{ + "start_time": "2025-05-18T20:55:19.070907", + "end_time": "2025-05-18T21:02:09.284776", + "buffer_size": 80, + "all_records": [ + { + "frame_id": "180_0_0", + "episode_id": 180, + "env_id": 0, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.514909744262695, + "advantages": 1.601567029953003, + "generated_text": "{\n \"intent\": \"To start interacting by clicking an application from the desktop.\",\n \"action\": \"DoubleClick(68, 78)\"\n}", + "action_string": "DoubleClick(68, 78)", + "intent_string": "To start interacting by clicking an application from the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 48, + 68, + 128, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "bookmarks..", + "boxes": [ + 82, + 101, + 162, + 119 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_1_0", + "episode_id": 180, + "env_id": 1, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.474891662597656, + "advantages": 1.54634428024292, + "generated_text": "{\n \"intent\": \"To possibly start interacting with applications by clicking \",\n \"action\": \"DoubleClick(45, 75)\"\n}", + "action_string": "DoubleClick(45, 75)", + "intent_string": "To possibly start interacting with applications by clicking ", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "√C白", + "boxes": [ + 34, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_2_0", + "episode_id": 180, + "env_id": 2, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.599042892456055, + "advantages": 1.7176660299301147, + "generated_text": "{\n \"intent\": \"To start by interacting with an application available from the desktop environment.\",\n \"action\": \"DoubleClick(67, 75)\"\n}", + "action_string": "DoubleClick(67, 75)", + "intent_string": "To start by interacting with an application available from the desktop environment.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 48, + 68, + 128, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 516, + 479, + 567, + 535 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 988, + 671, + 1172, + 719 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_3_0", + "episode_id": 180, + "env_id": 3, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.686721801757812, + "advantages": 1.838658094406128, + "generated_text": "{\n \"intent\": \"To begin the exploration by interacting with applications available on the desktop.\",\n \"action\": \"DoubleClick(70, 72)\"\n}", + "action_string": "DoubleClick(70, 72)", + "intent_string": "To begin the exploration by interacting with applications available on the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→白", + "boxes": [ + 44, + 66, + 128, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_4_0", + "episode_id": 180, + "env_id": 4, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.804319381713867, + "advantages": 2.000936269760132, + "generated_text": "{\n \"intent\": \"Opening an application like web browser will provide access to web resources which could be explored.\",\n \"action\": \"DoubleClick(72, 87)\"\n}", + "action_string": "DoubleClick(72, 87)", + "intent_string": "Opening an application like web browser will provide access to web resources which could be explored.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "ookmarks..", + "boxes": [ + 92, + 101, + 162, + 119 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_5_0", + "episode_id": 180, + "env_id": 5, + "step": 0, + "format_rewards": 1.0, + "rewards": 9.451339721679688, + "advantages": -1.2460448741912842, + "generated_text": "{\n \"intent\": \"To start by clicking an application icon like Firefox which is often used for web browsing or general internet tasks.\",\n \"action\": \"DoubleClick(14, 79)\"\n}", + "action_string": "DoubleClick(14, 79)", + "intent_string": "To start by clicking an application icon like Firefox which is often used for web browsing or general internet tasks.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox", + "boxes": [ + 1, + 59, + 101, + 102 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "180_6_0", + "episode_id": 180, + "env_id": 6, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.626077651977539, + "advantages": 1.7549724578857422, + "generated_text": "{\n \"intent\": \"To start my exploration by engaging with an application.\",\n \"action\": \"DoubleClick(67, 74)\"\n}", + "action_string": "DoubleClick(67, 74)", + "intent_string": "To start my exploration by engaging with an application.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 46, + 68, + 128, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 524, + 488, + 562, + 528 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 988, + 671, + 1172, + 719 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_7_0", + "episode_id": 180, + "env_id": 7, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.762371063232422, + "advantages": 1.9430497884750366, + "generated_text": "{\n \"intent\": \"To start by exploring applications available through an action like launching Firefox browser.\",\n \"action\": \"DoubleClick(72, 72)\"\n}", + "action_string": "DoubleClick(72, 72)", + "intent_string": "To start by exploring applications available through an action like launching Firefox browser.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_0_1", + "episode_id": 180, + "env_id": 0, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.855120658874512, + "advantages": 0.6910947561264038, + "generated_text": "{\n \"intent\": \"Clicking the Bing icon will help in accessing search functionality or browsing web content.\",\n \"action\": \"DoubleClick(860, 661)\"\n}", + "action_string": "DoubleClick(860, 661)", + "intent_string": "Clicking the Bing icon will help in accessing search functionality or browsing web content.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→,", + "boxes": [ + 44, + 68, + 130, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1354, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "语", + "boxes": [ + 1632, + 381, + 1656, + 403 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 796, + 453, + 970, + 476 + ] + }, + { + "text": "）图卢兹的暮", + "boxes": [ + 1534, + 681, + 1620, + 699 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 729, + 268, + 746 + ] + }, + { + "text": "经济观察报·31分钟", + "boxes": [ + 828, + 919, + 976, + 949 + ] + }, + { + "text": "游民星空·1天", + "boxes": [ + 1146, + 921, + 1254, + 945 + ] + }, + { + "text": "汽车“奇\"设计：在安全与创", + "boxes": [ + 822, + 937, + 1106, + 979 + ] + }, + { + "text": "意式浪漫暴击！全新手工超跑配自吸V8+", + "boxes": [ + 202, + 949, + 748, + 985 + ] + }, + { + "text": "《街霸6》制作人：格斗游戏", + "boxes": [ + 1144, + 941, + 1402, + 977 + ] + }, + { + "text": "新之间游走", + "boxes": [ + 826, + 969, + 936, + 1001 + ] + }, + { + "text": "必须要“变简单”！", + "boxes": [ + 1136, + 967, + 1308, + 1003 + ] + }, + { + "text": "手动挡，全球仅30台！", + "boxes": [ + 202, + 981, + 524, + 1022 + ] + }, + { + "text": "3", + "boxes": [ + 188, + 1052, + 202, + 1068 + ] + } + ] + }, + { + "frame_id": "180_1_1", + "episode_id": 180, + "env_id": 1, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.341184616088867, + "advantages": -0.01810840144753456, + "generated_text": "{\n \"intent\": \"To start exploring by selecting an appealing website from the top sites.\",\n \"action\": \"DoubleClick(759, 727)\"\n}", + "action_string": "DoubleClick(759, 727)", + "intent_string": "To start exploring by selecting an appealing website from the top sites.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "√C白", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 10, + 66, + 36, + 89 + ] + }, + { + "text": "→白", + "boxes": [ + 32, + 64, + 132, + 95 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 18, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 492, + 248, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 566, + 250, + 636, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 470, + 295, + 612, + 332 + ] + }, + { + "text": "O", + "boxes": [ + 1342, + 302, + 1356, + 324 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "", + "boxes": [ + 1544, + 363, + 1568, + 385 + ] + }, + { + "text": "中虹中", + "boxes": [ + 1318, + 373, + 1350, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "京", + "boxes": [ + 1636, + 387, + 1654, + 403 + ] + }, + { + "text": "8", + "boxes": [ + 1879, + 374, + 1896, + 391 + ] + }, + { + "text": "福", + "boxes": [ + 976, + 402, + 999, + 424 + ] + }, + { + "text": "1 雷军回应SU7事故", + "boxes": [ + 482, + 413, + 638, + 443 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 413, + 916, + 443 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 446, + 968, + 477 + ] + }, + { + "text": "6朱雀玄武救令", + "boxes": [ + 1108, + 453, + 1226, + 471 + ] + }, + { + "text": "）图卢兹的暮", + "boxes": [ + 1532, + 679, + 1620, + 697 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 270, + 748 + ] + } + ] + }, + { + "frame_id": "180_2_1", + "episode_id": 180, + "env_id": 2, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.070722579956055, + "advantages": -0.3913309574127197, + "generated_text": "{\n \"intent\": \"To start by clicking an icon is beneficial to get immediate interactions which could potentially lead to new exploration opportunities or actions.\",\n \"action\": \"DoubleClick(1077, 728)\"\n}", + "action_string": "DoubleClick(1077, 728)", + "intent_string": "To start by clicking an icon is beneficial to get immediate interactions which could potentially lead to new exploration opportunities or actions.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 48, + 68, + 128, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→×白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 139, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 121, + 1474, + 135 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "94", + "boxes": [ + 1823, + 112, + 1861, + 138 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 660, + 294, + 942, + 349 + ] + }, + { + "text": "You are on amazon.com.", + "boxes": [ + 510, + 461, + 662, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1336, + 486, + 1416, + 510 + ] + }, + { + "text": "", + "boxes": [ + 1442, + 486, + 1544, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 982, + 530, + 1288, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1534, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 354, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1462, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1553, + 996, + 1647, + 1025 + ] + }, + { + "text": "e-65df-49f5-af6f-19", + "boxes": [ + 1520, + 1020, + 1644, + 1038 + ] + } + ] + }, + { + "frame_id": "180_3_1", + "episode_id": 180, + "env_id": 3, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.64904499053955, + "advantages": 0.4067218005657196, + "generated_text": "{\n \"intent\": \"To start interacting with available resources or explore more actively by clicking relevant sites like Bing or Amazon.\",\n \"action\": \"DoubleClick(674, 480)\"\n}", + "action_string": "DoubleClick(674, 480)", + "intent_string": "To start interacting with available resources or explore more actively by clicking relevant sites like Bing or Amazon.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 46, + 66, + 128, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "earch - Microsoft Bing", + "boxes": [ + 44, + 32, + 188, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 68, + 1658, + 87 + ] + }, + { + "text": "口", + "boxes": [ + 1788, + 68, + 1858, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 406, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1354, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 448, + 968, + 477 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1522, + 679, + 1624, + 703 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 729, + 268, + 746 + ] + }, + { + "text": "4经济观察报·31分钟", + "boxes": [ + 826, + 919, + 976, + 949 + ] + }, + { + "text": "游民星空·2天", + "boxes": [ + 1146, + 921, + 1254, + 945 + ] + }, + { + "text": "汽车“奇\"设计：在安全与创", + "boxes": [ + 822, + 937, + 1106, + 979 + ] + }, + { + "text": "意式浪漫暴击！全新手工超跑配自吸V8+", + "boxes": [ + 202, + 949, + 750, + 985 + ] + }, + { + "text": "《暗黑4》偷偷增添功能：节", + "boxes": [ + 1146, + 941, + 1402, + 977 + ] + }, + { + "text": "新之间游走", + "boxes": [ + 826, + 969, + 936, + 1001 + ] + }, + { + "text": "省大量跑路时间！", + "boxes": [ + 1138, + 967, + 1306, + 1003 + ] + }, + { + "text": "手动挡，全球仅30台！", + "boxes": [ + 202, + 981, + 524, + 1022 + ] + } + ] + }, + { + "frame_id": "180_4_1", + "episode_id": 180, + "env_id": 4, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.45695686340332, + "advantages": 0.14165085554122925, + "generated_text": "{\n \"intent\": \"To start by clicking an interesting website or resource from the 'Highlights' section, which could lead to engaging content exploration.\",\n \"action\": \"DoubleClick(1075, 706)\"\n}", + "action_string": "DoubleClick(1075, 706)", + "intent_string": "To start by clicking an interesting website or resource from the 'Highlights' section, which could lead to engaging content exploration.", + "before_screen_ocr_results": [ + { + "text": "Q日", + "boxes": [ + 72, + 66, + 128, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": "ookmarks..", + "boxes": [ + 88, + 101, + 162, + 119 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "94", + "boxes": [ + 1823, + 110, + 1862, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 121, + 1474, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 918, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "You are on ama", + "boxes": [ + 510, + 461, + 606, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 240, + 526, + 464, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 916, + 560 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 982, + 530, + 1178, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See all deals", + "boxes": [ + 240, + 889, + 318, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See more", + "boxes": [ + 982, + 891, + 1042, + 911 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1462, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1555, + 996, + 1647, + 1025 + ] + } + ] + }, + { + "frame_id": "180_5_1", + "episode_id": 180, + "env_id": 5, + "step": 1, + "format_rewards": 1.0, + "rewards": 11.386568069458008, + "advantages": 1.4244625568389893, + "generated_text": "{\n \"intent\": \"To start by interacting with applications available from this desktop.\",\n \"action\": \"DoubleClick(47, 70)\"\n}", + "action_string": "DoubleClick(47, 70)", + "intent_string": "To start by interacting with applications available from this desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox", + "boxes": [ + 1, + 59, + 101, + 102 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 130, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_6_1", + "episode_id": 180, + "env_id": 6, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.584675788879395, + "advantages": 0.3178958594799042, + "generated_text": "{\n \"intent\": \"I will click the 'Getting Started' button to explore new resources or guides provided by Mozilla Firefox.\",\n \"action\": \"DoubleClick(257, 109)\"\n}", + "action_string": "DoubleClick(257, 109)", + "intent_string": "I will click the 'Getting Started' button to explore new resources or guides provided by Mozilla Firefox.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 46, + 68, + 128, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozilla I", + "boxes": [ + 251, + 103, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "180_7_1", + "episode_id": 180, + "env_id": 7, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.66054916381836, + "advantages": 0.42259690165519714, + "generated_text": "{\n \"intent\": \"To gather information or start interacting online by selecting an icon such as 'bing' from the top sites.\",\n \"action\": \"DoubleClick(675, 487)\"\n}", + "action_string": "DoubleClick(675, 487)", + "intent_string": "To gather information or start interacting online by selecting an icon such as 'bing' from the top sites.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 10, + 66, + 36, + 89 + ] + }, + { + "text": "→白", + "boxes": [ + 32, + 64, + 132, + 95 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 18, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 492, + 248, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 566, + 250, + 636, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 470, + 295, + 612, + 332 + ] + }, + { + "text": "O", + "boxes": [ + 1342, + 302, + 1356, + 324 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "", + "boxes": [ + 1544, + 363, + 1568, + 385 + ] + }, + { + "text": "中虹中", + "boxes": [ + 1318, + 373, + 1350, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "京", + "boxes": [ + 1636, + 387, + 1654, + 403 + ] + }, + { + "text": "8", + "boxes": [ + 1879, + 374, + 1896, + 391 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 482, + 411, + 664, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 976, + 402, + 999, + 424 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 413, + 916, + 443 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生蛆", + "boxes": [ + 482, + 451, + 636, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 448, + 968, + 477 + ] + }, + { + "text": "6朱雀玄武救令", + "boxes": [ + 1108, + 453, + 1226, + 471 + ] + }, + { + "text": "）图卢兹的暮", + "boxes": [ + 1532, + 679, + 1620, + 697 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 270, + 748 + ] + } + ] + }, + { + "frame_id": "180_0_2", + "episode_id": 180, + "env_id": 0, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.73666763305664, + "advantages": 0.5276361703872681, + "generated_text": "{\n \"intent\": \"Click the 'Images' tab in Microsoft Bing to start exploring visual content.\",\n \"action\": \"DoubleClick(278, 130)\"\n}", + "action_string": "DoubleClick(278, 130)", + "intent_string": "Click the 'Images' tab in Microsoft Bing to start exploring visual content.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 796, + 453, + 970, + 476 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1460, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1462, + 808, + 1502, + 838 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "商时中国A50", + "boxes": [ + 1451, + 889, + 1549, + 940 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "经济观察报·31分钟", + "boxes": [ + 824, + 919, + 976, + 949 + ] + }, + { + "text": "游民星空·1天", + "boxes": [ + 1136, + 921, + 1252, + 945 + ] + }, + { + "text": "中国建成全球最大信息通信网5G－A与6G", + "boxes": [ + 200, + 945, + 766, + 987 + ] + }, + { + "text": "汽车“奇\"设计：在安全与创", + "boxes": [ + 822, + 937, + 1106, + 979 + ] + }, + { + "text": "《街霸6》制作人：格斗游戏", + "boxes": [ + 1144, + 941, + 1404, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "觀点", + "boxes": [ + 428, + 961, + 568, + 1038 + ] + }, + { + "text": "新之间游走", + "boxes": [ + 826, + 969, + 936, + 1001 + ] + }, + { + "text": "必须要“变简单”！", + "boxes": [ + 1138, + 967, + 1310, + 1003 + ] + }, + { + "text": "", + "boxes": [ + 1581, + 931, + 1641, + 990 + ] + }, + { + "text": "研发加速推进", + "boxes": [ + 202, + 985, + 384, + 1022 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国版", + "boxes": [ + 260, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ] + }, + { + "frame_id": "180_1_2", + "episode_id": 180, + "env_id": 1, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.208686828613281, + "advantages": -0.20094797015190125, + "generated_text": "{\n \"intent\": \"The current environment is a web browser on Bing's homepage. To continue exploring, I will click the search bar to view more content related to '雷军回应SU7事故'. This action helps in broadening the exploration by access potentially related or interesting information.\",\n \"action\": \"DoubleClick(524, 315)\"\n}", + "action_string": "DoubleClick(524, 315)", + "intent_string": "The current environment is a web browser on Bing's homepage. To continue exploring, I will click the search bar to view more content related to '雷军回应SU7事故'. This action helps in broadening the exploration by access potentially related or interesting information.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": " Trending on Bing", + "boxes": [ + 480, + 369, + 646, + 405 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故", + "boxes": [ + 484, + 411, + 638, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 488, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1466, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1462, + 808, + 1502, + 838 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": ",61023", + "boxes": [ + 1630, + 850, + 1698, + 891 + ] + }, + { + "text": "钢筋带错子弹赚百万", + "boxes": [ + 332, + 876, + 658, + 925 + ] + }, + { + "text": "商时中国A50", + "boxes": [ + 1451, + 889, + 1549, + 940 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1698, + 941 + ] + }, + { + "text": "经济观察报·31分钟", + "boxes": [ + 824, + 919, + 976, + 949 + ] + }, + { + "text": "游民星空·1天", + "boxes": [ + 1136, + 921, + 1252, + 945 + ] + }, + { + "text": "进入游戏天塌了7套猛攻竟然忘带子弹！和", + "boxes": [ + 198, + 943, + 764, + 987 + ] + }, + { + "text": "汽车“奇”设计：在安全与创", + "boxes": [ + 822, + 937, + 1106, + 979 + ] + }, + { + "text": "《街霸6》制作人：格斗游戏", + "boxes": [ + 1144, + 941, + 1404, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.2879", + "boxes": [ + 1626, + 939, + 1698, + 989 + ] + }, + { + "text": "L.", + "boxes": [ + 1582, + 936, + 1636, + 985 + ] + }, + { + "text": "新之间游走", + "boxes": [ + 826, + 969, + 936, + 1001 + ] + }, + { + "text": "必须要“变简单”！", + "boxes": [ + 1138, + 967, + 1310, + 1003 + ] + }, + { + "text": "平精英地铁逃生地铁逃生赚百万 和平精英..", + "boxes": [ + 202, + 981, + 764, + 1022 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Images", + "boxes": [ + 252, + 123, + 292, + 143 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 490, + 250, + 638, + 282 + ] + }, + { + "text": "雷军应SU7事故", + "boxes": [ + 478, + 300, + 614, + 330 + ] + }, + { + "text": "", + "boxes": [ + 1332, + 296, + 1364, + 328 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 371, + 1422, + 405 + ] + }, + { + "text": "", + "boxes": [ + 1546, + 367, + 1566, + 383 + ] + }, + { + "text": "电啤肉", + "boxes": [ + 1136, + 377, + 1176, + 414 + ] + }, + { + "text": "中享中", + "boxes": [ + 1320, + 377, + 1348, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故", + "boxes": [ + 484, + 411, + 640, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 918, + 441 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生蛆", + "boxes": [ + 486, + 451, + 636, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 790, + 449, + 968, + 478 + ] + }, + { + "text": "6“朱雀玄武救令“改名", + "boxes": [ + 1100, + 451, + 1270, + 474 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "行堂娜归", + "boxes": [ + 669, + 772, + 791, + 829 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1458, + 766, + 1550, + 792 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 840 + ] + }, + { + "text": "+2.09 ", + "boxes": [ + 1626, + 798, + 1700, + 846 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 848, + 1514, + 889 + ] + }, + { + "text": ",114023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,.87%6", + "boxes": [ + 1626, + 891, + 1698, + 941 + ] + }, + { + "text": "经济观察报·31分钟", + "boxes": [ + 826, + 917, + 976, + 947 + ] + }, + { + "text": "游民星空·1天", + "boxes": [ + 1138, + 921, + 1254, + 945 + ] + }, + { + "text": "汽车“奇葩”设计：在安全与创", + "boxes": [ + 824, + 937, + 1104, + 977 + ] + }, + { + "text": "《街霸6》制作人：格斗游戏", + "boxes": [ + 1142, + 941, + 1404, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "r", + "boxes": [ + 1592, + 941, + 1632, + 979 + ] + }, + { + "text": "382", + "boxes": [ + 1620, + 935, + 1702, + 989 + ] + }, + { + "text": "斤选择比亚迪海狮07DM-i石家庄上市", + "boxes": [ + 282, + 977, + 776, + 1023 + ] + }, + { + "text": "新之间游走", + "boxes": [ + 828, + 969, + 936, + 1001 + ] + }, + { + "text": "必须要“变简单”！", + "boxes": [ + 1140, + 969, + 1308, + 999 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "WLAM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "凸2", + "boxes": [ + 1136, + 1028, + 1170, + 1042 + ] + }, + { + "text": "态丢白选股建过", + "boxes": [ + 1630, + 1028, + 1716, + 1046 + ] + } + ] + }, + { + "frame_id": "180_2_2", + "episode_id": 180, + "env_id": 2, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.25223159790039, + "advantages": -0.14085862040519714, + "generated_text": "{\n \"intent\": \"To continue exploring the environment, clicking the 'Account & Lists' drop-down to see if there are any actions or features available under the account. Clicking the 'Sign in' button will further allow user interaction if the goal is exploration, which can be useful for accessing account details later.\",\n \"action\": \"Click(1670, 122)\"\n}", + "action_string": "Click(1670, 122)", + "intent_string": "To continue exploring the environment, clicking the 'Account & Lists' drop-down to see if there are any actions or features available under the account. Clicking the 'Sign in' button will further allow user interaction if the goal is exploration, which can be useful for accessing account details later.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Returnder", + "boxes": [ + 1732, + 109, + 1788, + 151 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "94", + "boxes": [ + 1823, + 110, + 1862, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": " Signin", + "boxes": [ + 1622, + 175, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1226, + 488, + 1276, + 508 + ] + }, + { + "text": "1", + "boxes": [ + 1332, + 488, + 1364, + 508 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 608, + 526, + 834, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 528, + 1288, + 558 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": " See all deals", + "boxes": [ + 610, + 889, + 690, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 967, + 1546, + 1003 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 354, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1358, + 1001, + 1462, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1555, + 996, + 1647, + 1025 + ] + }, + { + "text": "=UTF8&k=", + "boxes": [ + 264, + 1020, + 334, + 1042 + ] + }, + { + "text": "e-65df-49f5-af6f-19", + "boxes": [ + 1518, + 1020, + 1646, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1601, + 65, + 1659, + 88 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "180_3_2", + "episode_id": 180, + "env_id": 3, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.517945289611816, + "advantages": 0.22581149637699127, + "generated_text": "{\n \"intent\": \"Clicking the search icon to start clicking around the top right corner will help me explore the website more effectively by interacting with other features like images, videos, translate, maps, etc.\",\n \"action\": \"Click(279, 129)\"\n}", + "action_string": "Click(279, 129)", + "intent_string": "Clicking the search icon to start clicking around the top right corner will help me explore the website more effectively by interacting with other features like images, videos, translate, maps, etc.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 486, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 796, + 453, + 970, + 476 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 673, + 1630, + 703 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 272, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "DANIE", + "boxes": [ + 205, + 776, + 355, + 870 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "Ww", + "boxes": [ + 1594, + 810, + 1628, + 830 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1632, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1456, + 850, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13, 287", + "boxes": [ + 1626, + 893, + 1700, + 941 + ] + }, + { + "text": "经济观察报·31分钟", + "boxes": [ + 824, + 913, + 976, + 949 + ] + }, + { + "text": "游民星空·2天", + "boxes": [ + 1140, + 921, + 1254, + 945 + ] + }, + { + "text": "汽车“奇\"设计：在安全与创", + "boxes": [ + 822, + 937, + 1106, + 979 + ] + }, + { + "text": "《暗黑4》偷偷增添功能：节", + "boxes": [ + 1144, + 941, + 1402, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "小黑盒游戏·10小时", + "boxes": [ + 200, + 953, + 352, + 983 + ] + }, + { + "text": "新之间游走", + "boxes": [ + 826, + 969, + 936, + 1001 + ] + }, + { + "text": "省大量跑路时间！", + "boxes": [ + 1138, + 967, + 1306, + 1003 + ] + }, + { + "text": "《死亡岛2》新手排雷心得", + "boxes": [ + 214, + 981, + 548, + 1022 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "Lr~", + "boxes": [ + 1592, + 997, + 1630, + 1026 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 70, + 69, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国版", + "boxes": [ + 258, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ] + }, + { + "frame_id": "180_4_2", + "episode_id": 180, + "env_id": 4, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.095727920532227, + "advantages": -0.35682499408721924, + "generated_text": "{\n \"intent\": \"Click the Account & Lists button to see if the goal is to manage accounts or orders.\",\n \"action\": \"Click(1672, 122)\"\n}", + "action_string": "Click(1672, 122)", + "intent_string": "Click the Account & Lists button to see if the goal is to manage accounts or orders.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Returnder", + "boxes": [ + 1732, + 109, + 1788, + 151 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "94", + "boxes": [ + 1826, + 115, + 1860, + 137 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": " Signin", + "boxes": [ + 1622, + 175, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 240, + 526, + 466, + 562 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 980, + 530, + 1178, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": " See all deals", + "boxes": [ + 240, + 889, + 318, + 913 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See more", + "boxes": [ + 982, + 891, + 1042, + 911 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 967, + 1546, + 1003 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1462, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1555, + 996, + 1647, + 1025 + ] + }, + { + "text": "2-4b87-4618-96b2-19", + "boxes": [ + 1508, + 1020, + 1652, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1599, + 65, + 1659, + 88 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "180_5_2", + "episode_id": 180, + "env_id": 5, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.754315376281738, + "advantages": 0.5519890785217285, + "generated_text": "{\n \"intent\": \"To start by exploring the web, selecting a search engine such as Bing would be beneficial.\",\n \"action\": \"DoubleClick(669, 516)\"\n}", + "action_string": "DoubleClick(669, 516)", + "intent_string": "To start by exploring the web, selecting a search engine such as Bing would be beneficial.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→C白", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com ", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 449, + 972, + 478 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 729, + 268, + 746 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 889 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "网上车市·2小时", + "boxes": [ + 826, + 921, + 956, + 945 + ] + }, + { + "text": "小黑盒游戏·10小时", + "boxes": [ + 1142, + 921, + 1286, + 945 + ] + }, + { + "text": "仅靠一只狗就能通关！《博德3》玩家用宠物", + "boxes": [ + 200, + 945, + 784, + 987 + ] + }, + { + "text": "捷豹路虎产品线将迎大调整，", + "boxes": [ + 824, + 943, + 1086, + 973 + ] + }, + { + "text": "《死亡岛2》新手排雷心得", + "boxes": [ + 1148, + 941, + 1382, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "这与退出中国市场有啥区别？", + "boxes": [ + 826, + 969, + 1092, + 999 + ] + }, + { + "text": "", + "boxes": [ + 1581, + 931, + 1641, + 990 + ] + }, + { + "text": "咬死BOSS ", + "boxes": [ + 204, + 985, + 358, + 1018 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + } + ] + }, + { + "frame_id": "180_6_2", + "episode_id": 180, + "env_id": 6, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.562280654907227, + "advantages": 0.28699183464050293, + "generated_text": "{\n \"intent\": \"To click the Download Firefox button to continue browser interaction or to continue exploring different functionalities available on the page.\",\n \"action\": \"DoubleClick(447, 594)\"\n}", + "action_string": "DoubleClick(447, 594)", + "intent_string": "To click the Download Firefox button to continue browser interaction or to continue exploring different functionalities available on the page.", + "before_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozilla I", + "boxes": [ + 251, + 103, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "180_7_2", + "episode_id": 180, + "env_id": 7, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.509119033813477, + "advantages": 0.2136317491531372, + "generated_text": "{\n \"intent\": \"The current action is important because by clicking the 'Images' tab, the user can explore visual content or search more about the topic by image. This could lead to discovering new resources or perspectives related to the interest discussed in the search query.\",\n \"action\": \"DoubleClick(274, 129)\"\n}", + "action_string": "DoubleClick(274, 129)", + "intent_string": "The current action is important because by clicking the 'Images' tab, the user can explore visual content or search more about the topic by image. This could lead to discovering new resources or perspectives related to the interest discussed in the search query.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 32, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 558, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1342, + 302, + 1356, + 324 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "化", + "boxes": [ + 1546, + 367, + 1562, + 383 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "", + "boxes": [ + 1135, + 375, + 1180, + 420 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 729, + 268, + 746 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "小尖是妙手", + "boxes": [ + 1206, + 806, + 1366, + 844 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1632, + 802, + 1698, + 842 + ] + }, + { + "text": "：点据了棋形的要点", + "boxes": [ + 1166, + 838, + 1408, + 879 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13, 87%", + "boxes": [ + 1626, + 891, + 1700, + 941 + ] + }, + { + "text": "经济观察报·31分钟", + "boxes": [ + 824, + 913, + 976, + 949 + ] + }, + { + "text": "1", + "boxes": [ + 1144, + 925, + 1164, + 941 + ] + }, + { + "text": "一点资讯·22小时", + "boxes": [ + 1158, + 921, + 1274, + 945 + ] + }, + { + "text": "汽车“奇\"设计：在安全与创", + "boxes": [ + 822, + 937, + 1106, + 979 + ] + }, + { + "text": "黑棋小尖是妙手，占据棋形要", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "中国车企“造船远航”：自主可控将运力握在", + "boxes": [ + 198, + 945, + 784, + 987 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "+3.287", + "boxes": [ + 1626, + 939, + 1700, + 987 + ] + }, + { + "text": "自己手中！八人", + "boxes": [ + 199, + 976, + 509, + 1031 + ] + }, + { + "text": "新之间游走", + "boxes": [ + 828, + 969, + 936, + 1001 + ] + }, + { + "text": "点，使黑棋先手收官破掉白棋", + "boxes": [ + 1134, + 967, + 1408, + 1003 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1452, + 981, + 1524, + 1038 + ] + }, + { + "text": "#建食教具刻复具", + "boxes": [ + 505, + 992, + 605, + 1031 + ] + }, + { + "text": "-11420 ", + "boxes": [ + 1634, + 983, + 1700, + 1030 + ] + }, + { + "text": "最大的空", + "boxes": [ + 1134, + 993, + 1230, + 1030 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "2", + "boxes": [ + 1140, + 1028, + 1172, + 1042 + ] + }, + { + "text": "R", + "boxes": [ + 1180, + 1028, + 1198, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国际版", + "boxes": [ + 260, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ] + }, + { + "frame_id": "180_0_3", + "episode_id": 180, + "env_id": 0, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.755438804626465, + "advantages": 0.5535393357276917, + "generated_text": "{\n \"intent\": \"Choosing an action here will help me consume a step and scroll through different tourist attraction images to gather more information or choices.\",\n \"action\": \"ScrollDown(739, 319)\"\n}", + "action_string": "ScrollDown(739, 319)", + "intent_string": "Choosing an action here will help me consume a step and scroll through different tourist attraction images to gather more information or choices.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国版", + "boxes": [ + 260, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "fii886ocr", + "boxes": [ + 1424, + 365, + 1514, + 389 + ] + }, + { + "text": "Most complained about tourist attractions in the", + "boxes": [ + 1294, + 397, + 1604, + 415 + ] + }, + { + "text": "10 of the Most Popular Tourist", + "boxes": [ + 1638, + 393, + 1832, + 417 + ] + }, + { + "text": "Wor", + "boxes": [ + 22, + 417, + 44, + 431 + ] + }, + { + "text": "vorld - Travel News", + "boxes": [ + 1296, + 415, + 1420, + 433 + ] + }, + { + "text": "Attractions in Paris", + "boxes": [ + 1642, + 415, + 1758, + 433 + ] + }, + { + "text": "Why I choose to do some of the", + "boxes": [ + 302, + 637, + 506, + 655 + ] + }, + { + "text": "MostBea", + "boxes": [ + 550, + 637, + 604, + 655 + ] + }, + { + "text": "Top10", + "boxes": [ + 1120, + 637, + 1164, + 655 + ] + }, + { + "text": "lost Popular Tourist Attractions Ii", + "boxes": [ + 1172, + 637, + 1380, + 655 + ] + }, + { + "text": "World Touri", + "boxes": [ + 1454, + 637, + 1524, + 655 + ] + }, + { + "text": "The 25 best-valu", + "boxes": [ + 1584, + 637, + 1692, + 655 + ] + }, + { + "text": "2023I", + "boxes": [ + 20, + 655, + 60, + 671 + ] + }, + { + "text": "touristy things and see the", + "boxes": [ + 302, + 651, + 478, + 675 + ] + }, + { + "text": " Tanya Tany", + "boxes": [ + 548, + 651, + 626, + 675 + ] + }, + { + "text": "England Th - vrogue.co", + "boxes": [ + 1116, + 649, + 1266, + 675 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 653, + 1516, + 671 + ] + }, + { + "text": "Tourist Attractions", + "boxes": [ + 936, + 868, + 1046, + 891 + ] + }, + { + "text": "Travel | uS Ne", + "boxes": [ + 22, + 891, + 116, + 909 + ] + }, + { + "text": "Collage·", + "boxes": [ + 936, + 891, + 988, + 909 + ] + }, + { + "text": "Bucket List", + "boxes": [ + 986, + 885, + 1068, + 911 + ] + }, + { + "text": "Attractions of Duba", + "boxes": [ + 1114, + 885, + 1238, + 911 + ] + } + ] + }, + { + "frame_id": "180_1_3", + "episode_id": 180, + "env_id": 1, + "step": 3, + "format_rewards": 1.0, + "rewards": 11.40539836883545, + "advantages": 1.4504473209381104, + "generated_text": "{\n \"intent\": \"The user has opened Microsoft Bing, which allows them to search and browse various topics online. Clicking the 'Back' button will allow the user to go back to the previous page, continuing their browsing session.\",\n \"action\": \"Click(24, 73)\"\n}", + "action_string": "Click(24, 73)", + "intent_string": "The user has opened Microsoft Bing, which allows them to search and browse various topics online. Clicking the 'Back' button will allow the user to go back to the previous page, continuing their browsing session.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 40, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 28, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1607, + 65, + 1659, + 90 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Images", + "boxes": [ + 252, + 123, + 292, + 143 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 490, + 250, + 636, + 282 + ] + }, + { + "text": "雷军应SU7事故", + "boxes": [ + 470, + 295, + 614, + 332 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "啤", + "boxes": [ + 1136, + 376, + 1176, + 415 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 371, + 1422, + 405 + ] + }, + { + "text": "A人", + "boxes": [ + 1542, + 364, + 1565, + 385 + ] + }, + { + "text": "1 雷军回应SU7事故", + "boxes": [ + 484, + 411, + 640, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 918, + 441 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生蛆", + "boxes": [ + 486, + 451, + 636, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 790, + 449, + 968, + 478 + ] + }, + { + "text": "6“朱雀玄武救令“改名", + "boxes": [ + 1100, + 451, + 1270, + 474 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1458, + 766, + 1550, + 792 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 840 + ] + }, + { + "text": "+2.09 ", + "boxes": [ + 1626, + 798, + 1700, + 846 + ] + }, + { + "text": "MWY", + "boxes": [ + 1591, + 803, + 1631, + 831 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 848, + 1514, + 889 + ] + }, + { + "text": "-61013", + "boxes": [ + 1630, + 850, + 1698, + 891 + ] + }, + { + "text": "13.27", + "boxes": [ + 1628, + 891, + 1698, + 941 + ] + }, + { + "text": "娱乐扒客", + "boxes": [ + 204, + 923, + 276, + 941 + ] + }, + { + "text": "经济观察报·31分钟", + "boxes": [ + 824, + 915, + 976, + 949 + ] + }, + { + "text": "游民星空·1天", + "boxes": [ + 1138, + 921, + 1254, + 945 + ] + }, + { + "text": "汽车“奇葩”设计：在安全与创", + "boxes": [ + 822, + 937, + 1104, + 977 + ] + }, + { + "text": "《街霸6》制作人：格斗游戏", + "boxes": [ + 1142, + 941, + 1404, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "382", + "boxes": [ + 1620, + 935, + 1702, + 989 + ] + }, + { + "text": "马筱梅婚纱照曝光，汪小菲眼神坚定像入党", + "boxes": [ + 202, + 947, + 762, + 983 + ] + }, + { + "text": "新之间游走", + "boxes": [ + 826, + 969, + 936, + 1001 + ] + }, + { + "text": "必须要“变简单”！", + "boxes": [ + 1140, + 969, + 1308, + 999 + ] + }, + { + "text": "娘家陪嫁数目惊人", + "boxes": [ + 202, + 985, + 448, + 1020 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "-11620 ", + "boxes": [ + 1634, + 983, + 1702, + 1030 + ] + }, + { + "text": "WLAM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "凸2", + "boxes": [ + 1138, + 1028, + 1170, + 1042 + ] + }, + { + "text": "李丢白选股建过", + "boxes": [ + 1628, + 1028, + 1716, + 1046 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "→白", + "boxes": [ + 10, + 64, + 132, + 93 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 10, + 97, + 166, + 125 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_2_3", + "episode_id": 180, + "env_id": 2, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.894134521484375, + "advantages": 0.744931697845459, + "generated_text": "{\n \"intent\": \"To continue exploring the environment, I need to close the Amazon sign-in page to resume the exploration. The most valuable action is to click the close button to close the Amazon page.\",\n \"action\": \"Click(207, 37)\"\n}", + "action_string": "Click(207, 37)", + "intent_string": "To continue exploring the environment, I need to close the Amazon sign-in page to resume the exploration. The most valuable action is to click the close button to close the Amazon page.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1599, + 65, + 1659, + 88 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 22, + 332, + 86, + 381 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "180_3_3", + "episode_id": 180, + "env_id": 3, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.525853157043457, + "advantages": 0.2367239147424698, + "generated_text": "{\n \"intent\": \"Clicking an image to view more information about the tourist attraction will help me explore the details of different locations.\",\n \"action\": \"DoubleClick(841, 345)\"\n}", + "action_string": "DoubleClick(841, 345)", + "intent_string": "Clicking an image to view more information about the tourist attraction will help me explore the details of different locations.", + "before_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 70, + 69, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国版", + "boxes": [ + 258, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 262, + 117, + 306, + 143 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 171, + 234, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "Tourist Places Wallpapers - Wallpaper Cave", + "boxes": [ + 1320, + 240, + 1650, + 270 + ] + }, + { + "text": "", + "boxes": [ + 1452, + 294, + 1518, + 320 + ] + }, + { + "text": "", + "boxes": [ + 1532, + 290, + 1564, + 304 + ] + }, + { + "text": "ave.com I 2500 x 1666ipeg I 4 yrs ago", + "boxes": [ + 1410, + 337, + 1666, + 369 + ] + }, + { + "text": "wal", + "boxes": [ + 1330, + 347, + 1362, + 361 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1330, + 375, + 1374, + 395 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "180_4_3", + "episode_id": 180, + "env_id": 4, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.817205429077148, + "advantages": -0.7411705255508423, + "generated_text": "{\n \"intent\": \"The goal of exploring the computer desktop environment has been achieved. The user has successfully navigated to the Amazon sign-in page, which is a meaningful action in terms of interaction with the web browser and the site's functionality. Click(241, 36) action is chosen to click on the home icon to close the current tab and continue to the home screen, which provides a fresh start for exploring other applications or websites.\",\n \"action\": \"RightClick(115, 72)\"\n}", + "action_string": "RightClick(115, 72)", + "intent_string": "The goal of exploring the computer desktop environment has been achieved. The user has successfully navigated to the Amazon sign-in page, which is a meaningful action in terms of interaction with the web browser and the site's functionality. Click(241, 36) action is chosen to click on the home icon to close the current tab and continue to the home screen, which provides a fresh start for exploring other applications or websites.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1599, + 65, + 1659, + 88 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 806, + 434, + 930, + 471 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 902, + 4, + 1020, + 22 + ] + }, + { + "text": "C", + "boxes": [ + 56, + 68, + 100, + 87 + ] + }, + { + "text": "低", + "boxes": [ + 94, + 64, + 128, + 91 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1392, + 66, + 1582, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "Pin to Overflow Menu", + "boxes": [ + 142, + 81, + 284, + 99 + ] + }, + { + "text": "Remove from Toolbar", + "boxes": [ + 146, + 107, + 284, + 125 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Menu Bar", + "boxes": [ + 138, + 135, + 208, + 157 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": "Enter mobile number or email", + "boxes": [ + 806, + 216, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice ", + "boxes": [ + 936, + 582, + 1062, + 606 + ] + }, + { + "text": "Help", + "boxes": [ + 1054, + 588, + 1086, + 602 + ] + } + ] + }, + { + "frame_id": "180_5_3", + "episode_id": 180, + "env_id": 5, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.257428169250488, + "advantages": -1.51363205909729, + "generated_text": "{\n \"intent\": \"Opening Bing homepage will allow me to explore various features or search functionalities that might be interesting or useful during exploration.\",\n \"action\": \"DoubleClick(0, 37)\"\n}", + "action_string": "DoubleClick(0, 37)", + "intent_string": "Opening Bing homepage will allow me to explore various features or search functionalities that might be interesting or useful during exploration.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "中", + "boxes": [ + 956, + 435, + 968, + 447 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 448, + 970, + 477 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 270, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "点资讯·49分钟", + "boxes": [ + 240, + 921, + 334, + 945 + ] + }, + { + "text": "网上车市·2小时", + "boxes": [ + 826, + 921, + 956, + 945 + ] + }, + { + "text": "小黑盒游戏·10小时", + "boxes": [ + 1142, + 921, + 1286, + 945 + ] + }, + { + "text": "进化5亿年肠道竟成“第二大脑”！专家：我们", + "boxes": [ + 198, + 941, + 794, + 989 + ] + }, + { + "text": "捷豹路虎产品线将迎大调整，", + "boxes": [ + 826, + 943, + 1086, + 973 + ] + }, + { + "text": "《死亡岛2》新手排雷心得", + "boxes": [ + 1148, + 941, + 1382, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "这与退出中国市场有啥区别？", + "boxes": [ + 824, + 969, + 1090, + 999 + ] + }, + { + "text": "", + "boxes": [ + 1581, + 931, + 1641, + 990 + ] + }, + { + "text": "的喜怒哀乐被它操控", + "boxes": [ + 200, + 981, + 478, + 1022 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 558, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 488, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 274, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "全新2025款星纪元ES纯电 上市价格", + "boxes": [ + 202, + 794, + 604, + 824 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 840 + ] + }, + { + "text": "+2.0996 ", + "boxes": [ + 1636, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "Max", + "boxes": [ + 339, + 878, + 383, + 911 + ] + }, + { + "text": "Ultra四驱旗舰", + "boxes": [ + 474, + 879, + 594, + 909 + ] + }, + { + "text": "13, 287", + "boxes": [ + 1626, + 893, + 1700, + 941 + ] + }, + { + "text": "威区辅助驾驶", + "boxes": [ + 484, + 903, + 536, + 921 + ] + }, + { + "text": "网上车市·2小时", + "boxes": [ + 826, + 921, + 954, + 945 + ] + }, + { + "text": "小黑盒游戏·10小时", + "boxes": [ + 1142, + 921, + 1286, + 945 + ] + }, + { + "text": "26全新2025款星纪元", + "boxes": [ + 498, + 945, + 786, + 987 + ] + }, + { + "text": "捷豹路虎产品线将迎大调整，", + "boxes": [ + 824, + 943, + 1086, + 973 + ] + }, + { + "text": "《死亡岛2》新手排雷心得", + "boxes": [ + 1148, + 941, + 1382, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "[~.", + "boxes": [ + 1592, + 945, + 1628, + 977 + ] + }, + { + "text": "+3.287", + "boxes": [ + 1626, + 939, + 1700, + 987 + ] + }, + { + "text": "ES纯电售18.98万起", + "boxes": [ + 204, + 976, + 482, + 1021 + ] + }, + { + "text": "这与退出中国市场有啥区别？", + "boxes": [ + 828, + 973, + 1090, + 997 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "-11420 ", + "boxes": [ + 1634, + 983, + 1700, + 1030 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + } + ] + }, + { + "frame_id": "180_6_3", + "episode_id": 180, + "env_id": 6, + "step": 3, + "format_rewards": 1.0, + "rewards": 8.887311935424805, + "advantages": -2.024371862411499, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment, the next logical step is to close the current tab and return to the previous page to continue exploring the desktop. The objective is to continue interacting with the desktop resources, which may include files, applications, or system settings.\",\n \"action\": \"Click(124, 37)\"\n}", + "action_string": "Click(124, 37)", + "intent_string": "To continue exploring the desktop environment, the next logical step is to close the current tab and return to the previous page to continue exploring the desktop. The objective is to continue interacting with the desktop resources, which may include files, applications, or system settings.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 471 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "180_7_3", + "episode_id": 180, + "env_id": 7, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.533994674682617, + "advantages": 0.24795874953269958, + "generated_text": "{\n \"intent\": \"To continue exploring and potentially clicking on an interesting image or link about tourist attractions.\",\n \"action\": \"DoubleClick(872, 310)\"\n}", + "action_string": "DoubleClick(872, 310)", + "intent_string": "To continue exploring and potentially clicking on an interesting image or link about tourist attractions.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国际版", + "boxes": [ + 260, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→×", + "boxes": [ + 28, + 66, + 128, + 91 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 171, + 234, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "Tourist Places Wallpapers - Wallpaper Cave", + "boxes": [ + 1320, + 240, + 1650, + 270 + ] + }, + { + "text": "", + "boxes": [ + 1452, + 294, + 1518, + 320 + ] + }, + { + "text": "", + "boxes": [ + 1532, + 290, + 1564, + 304 + ] + }, + { + "text": "ave.com I 2500 x 1666ipeg I 4 yrs ago", + "boxes": [ + 1410, + 337, + 1666, + 369 + ] + }, + { + "text": "wal", + "boxes": [ + 1334, + 347, + 1362, + 361 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1330, + 375, + 1374, + 395 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 84, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 308, + 1072 + ] + } + ] + }, + { + "frame_id": "180_0_4", + "episode_id": 180, + "env_id": 0, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.696183204650879, + "advantages": -0.908174455165863, + "generated_text": "{\n \"intent\": \"Clicking on one of these images will allow us to see more details about the world tourist attraction and expand our understanding of the topic.\",\n \"action\": \"DoubleClick(32, 77)\"\n}", + "action_string": "DoubleClick(32, 77)", + "intent_string": "Clicking on one of these images will allow us to see more details about the world tourist attraction and expand our understanding of the topic.", + "before_screen_ocr_results": [ + { + "text": "fii886ocr", + "boxes": [ + 1424, + 365, + 1514, + 389 + ] + }, + { + "text": "Most complained about tourist attractions in the", + "boxes": [ + 1294, + 397, + 1604, + 415 + ] + }, + { + "text": "10 of the Most Popular Tourist", + "boxes": [ + 1638, + 393, + 1832, + 417 + ] + }, + { + "text": "Wor", + "boxes": [ + 22, + 417, + 44, + 431 + ] + }, + { + "text": "vorld - Travel News", + "boxes": [ + 1296, + 415, + 1420, + 433 + ] + }, + { + "text": "Attractions in Paris", + "boxes": [ + 1642, + 415, + 1758, + 433 + ] + }, + { + "text": "Why I choose to do some of the", + "boxes": [ + 302, + 637, + 506, + 655 + ] + }, + { + "text": "MostBea", + "boxes": [ + 550, + 637, + 604, + 655 + ] + }, + { + "text": "Top10", + "boxes": [ + 1120, + 637, + 1164, + 655 + ] + }, + { + "text": "lost Popular Tourist Attractions Ii", + "boxes": [ + 1172, + 637, + 1380, + 655 + ] + }, + { + "text": "World Touri", + "boxes": [ + 1454, + 637, + 1524, + 655 + ] + }, + { + "text": "The 25 best-valu", + "boxes": [ + 1584, + 637, + 1692, + 655 + ] + }, + { + "text": "2023I", + "boxes": [ + 20, + 655, + 60, + 671 + ] + }, + { + "text": "touristy things and see the", + "boxes": [ + 302, + 651, + 478, + 675 + ] + }, + { + "text": " Tanya Tany", + "boxes": [ + 548, + 651, + 626, + 675 + ] + }, + { + "text": "England Th - vrogue.co", + "boxes": [ + 1116, + 649, + 1266, + 675 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 653, + 1516, + 671 + ] + }, + { + "text": "Tourist Attractions", + "boxes": [ + 936, + 868, + 1046, + 891 + ] + }, + { + "text": "Travel | uS Ne", + "boxes": [ + 22, + 891, + 116, + 909 + ] + }, + { + "text": "Collage·", + "boxes": [ + 936, + 891, + 988, + 909 + ] + }, + { + "text": "Bucket List", + "boxes": [ + 986, + 885, + 1068, + 911 + ] + }, + { + "text": "Attractions of Duba", + "boxes": [ + 1114, + 885, + 1238, + 911 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 66, + 69, + 124, + 87 + ] + }, + { + "text": "ir", + "boxes": [ + 1108, + 256, + 1180, + 274 + ] + }, + { + "text": "fii886ocr", + "boxes": [ + 1424, + 365, + 1514, + 389 + ] + }, + { + "text": "Most complained about tourist attractions in the", + "boxes": [ + 1290, + 393, + 1606, + 419 + ] + }, + { + "text": "10 of the Most Popular Tourist", + "boxes": [ + 1638, + 393, + 1832, + 417 + ] + }, + { + "text": "Wor", + "boxes": [ + 22, + 417, + 44, + 431 + ] + }, + { + "text": "vorld - Travel News", + "boxes": [ + 1296, + 415, + 1420, + 433 + ] + }, + { + "text": "Attractions in Paris", + "boxes": [ + 1642, + 415, + 1758, + 433 + ] + }, + { + "text": "Why I choose to do some of the", + "boxes": [ + 302, + 637, + 506, + 655 + ] + }, + { + "text": "MostBea", + "boxes": [ + 550, + 637, + 604, + 655 + ] + }, + { + "text": "Top10", + "boxes": [ + 1120, + 637, + 1164, + 655 + ] + }, + { + "text": "World Tourism", + "boxes": [ + 1454, + 637, + 1540, + 655 + ] + }, + { + "text": "The 25 best-value", + "boxes": [ + 1584, + 637, + 1700, + 655 + ] + }, + { + "text": "2023 r", + "boxes": [ + 18, + 653, + 62, + 671 + ] + }, + { + "text": "touristy things and see the", + "boxes": [ + 302, + 651, + 478, + 675 + ] + }, + { + "text": " Tanya Tanya", + "boxes": [ + 548, + 651, + 634, + 675 + ] + }, + { + "text": "England Th - vrogue.co", + "boxes": [ + 1114, + 649, + 1270, + 675 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 653, + 1516, + 671 + ] + }, + { + "text": "ones you can ski", + "boxes": [ + 1584, + 653, + 1692, + 671 + ] + }, + { + "text": "Tourist Attractions", + "boxes": [ + 932, + 868, + 1046, + 891 + ] + }, + { + "text": "Travel | uS New", + "boxes": [ + 20, + 891, + 128, + 909 + ] + }, + { + "text": "Collage-", + "boxes": [ + 933, + 887, + 989, + 912 + ] + }, + { + "text": "- Bucket List", + "boxes": [ + 982, + 885, + 1068, + 911 + ] + }, + { + "text": "Attractions of Dubai", + "boxes": [ + 1114, + 887, + 1240, + 911 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "180_1_4", + "episode_id": 180, + "env_id": 1, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.573336601257324, + "advantages": 0.30224841833114624, + "generated_text": "{\n \"intent\": \"To continue exploring by clicking an interesting or frequently used site like Bing.\",\n \"action\": \"DoubleClick(650, 492)\"\n}", + "action_string": "DoubleClick(650, 492)", + "intent_string": "To continue exploring by clicking an interesting or frequently used site like Bing.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "→白", + "boxes": [ + 10, + 64, + 132, + 93 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 10, + 97, + 166, + 125 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "earch - Microsoft Bing", + "boxes": [ + 44, + 32, + 188, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 68, + 1658, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 406, + 121, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "中", + "boxes": [ + 956, + 435, + 968, + 447 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 272, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "", + "boxes": [ + 1594, + 810, + 1628, + 832 + ] + }, + { + "text": "+2.09%60", + "boxes": [ + 1632, + 802, + 1698, + 844 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "", + "boxes": [ + 946, + 895, + 996, + 913 + ] + }, + { + "text": "-027", + "boxes": [ + 1621, + 886, + 1703, + 945 + ] + }, + { + "text": "点资讯·50分钟", + "boxes": [ + 240, + 921, + 334, + 945 + ] + }, + { + "text": "科普世界·25分钟", + "boxes": [ + 830, + 921, + 962, + 945 + ] + }, + { + "text": "游民星空·2天", + "boxes": [ + 1136, + 919, + 1256, + 949 + ] + }, + { + "text": "进化5亿年肠道竟成“第二大脑”！专家：我们", + "boxes": [ + 198, + 941, + 794, + 989 + ] + }, + { + "text": "海拔高达4500米！世界最高", + "boxes": [ + 826, + 941, + 1094, + 977 + ] + }, + { + "text": "《暗黑4》偷偷增添功能：节", + "boxes": [ + 1144, + 941, + 1402, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "*329 ", + "boxes": [ + 1624, + 935, + 1702, + 991 + ] + }, + { + "text": "海拔光热电站西藏投产", + "boxes": [ + 826, + 967, + 1034, + 1003 + ] + }, + { + "text": "省大量跑路时间！", + "boxes": [ + 1138, + 967, + 1306, + 1003 + ] + }, + { + "text": "的喜怒哀乐被它操控", + "boxes": [ + 202, + 981, + 478, + 1022 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "Lr~", + "boxes": [ + 1592, + 997, + 1630, + 1026 + ] + } + ] + }, + { + "frame_id": "180_2_4", + "episode_id": 180, + "env_id": 2, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.999152183532715, + "advantages": 0.8898502588272095, + "generated_text": "{\n \"intent\": \"Opening Firefox is a good starting point because locating installed applications helps understand the desktop and its functionalities.\",\n \"action\": \"DoubleClick(76, 40)\"\n}", + "action_string": "DoubleClick(76, 40)", + "intent_string": "Opening Firefox is a good starting point because locating installed applications helps understand the desktop and its functionalities.", + "before_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 22, + 332, + 86, + 381 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 86, + 969 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C", + "boxes": [ + 72, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1040, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 488, + 615, + 584, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 508, + 671, + 692, + 721 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1226, + 677, + 1414, + 733 + ] + }, + { + "text": "", + "boxes": [ + 668, + 744, + 692, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 912, + 750, + 928, + 766 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 746, + 768, + 786, + 794 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 981, + 765, + 1035, + 797 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1228, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 746, + 790, + 790, + 818 + ] + }, + { + "text": "Search -Microsoft Bing", + "boxes": [ + 982, + 788, + 1144, + 816 + ] + } + ] + }, + { + "frame_id": "180_3_4", + "episode_id": 180, + "env_id": 3, + "step": 4, + "format_rewards": 1.0, + "rewards": 11.545488357543945, + "advantages": 1.643763780593872, + "generated_text": "{\n \"intent\": \"To continue exploring World tourist attraction images on Bing search results. The goal is to gather more information about different tourist attractions worldwide.\",\n \"action\": \"Click(126, 78)\"\n}", + "action_string": "Click(126, 78)", + "intent_string": "To continue exploring World tourist attraction images on Bing search results. The goal is to gather more information about different tourist attractions worldwide.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 262, + 117, + 306, + 143 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 171, + 234, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "Tourist Places Wallpapers - Wallpaper Cave", + "boxes": [ + 1320, + 240, + 1650, + 270 + ] + }, + { + "text": "", + "boxes": [ + 1452, + 294, + 1518, + 320 + ] + }, + { + "text": "", + "boxes": [ + 1532, + 290, + 1564, + 304 + ] + }, + { + "text": "ave.com I 2500 x 1666ipeg I 4 yrs ago", + "boxes": [ + 1410, + 337, + 1666, + 369 + ] + }, + { + "text": "wal", + "boxes": [ + 1330, + 347, + 1362, + 361 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1330, + 375, + 1374, + 395 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "- Microsoft Bing ", + "boxes": [ + 80, + 30, + 190, + 54 + ] + }, + { + "text": "×", + "boxes": [ + 194, + 36, + 218, + 50 + ] + }, + { + "text": "New Tab", + "boxes": [ + 248, + 32, + 314, + 52 + ] + }, + { + "text": "①→Q白", + "boxes": [ + 6, + 57, + 138, + 100 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_4_4", + "episode_id": 180, + "env_id": 4, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.432319641113281, + "advantages": -1.2722915410995483, + "generated_text": "{\n \"intent\": \"To start the exploration by clicking on the 'X' button to close the Amazon sign-in page, allowing the browser to continue exploring other available options. This action is significant because it removes the obstacle of the Facebook sign-in page and allows the focus to be re-directed back to the desktop.\", \n \"action\": \"Click(119, 38)\"\n}", + "action_string": "Click(119, 38)", + "intent_string": "To start the exploration by clicking on the 'X' button to close the Amazon sign-in page, allowing the browser to continue exploring other available options. This action is significant because it removes the obstacle of the Facebook sign-in page and allows the focus to be re-directed back to the desktop.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 902, + 4, + 1020, + 22 + ] + }, + { + "text": "C", + "boxes": [ + 56, + 68, + 100, + 87 + ] + }, + { + "text": "低", + "boxes": [ + 94, + 64, + 128, + 91 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1392, + 66, + 1582, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "Pin to Overflow Menu", + "boxes": [ + 142, + 81, + 284, + 99 + ] + }, + { + "text": "Remove from Toolbar", + "boxes": [ + 146, + 107, + 284, + 125 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Menu Bar", + "boxes": [ + 138, + 135, + 208, + 157 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": "Enter mobile number or email", + "boxes": [ + 806, + 216, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice ", + "boxes": [ + 936, + 582, + 1062, + 606 + ] + }, + { + "text": "Help", + "boxes": [ + 1054, + 588, + 1086, + 602 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "nap/sigi", + "boxes": [ + 108, + 29, + 165, + 57 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1384, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "180_5_4", + "episode_id": 180, + "env_id": 5, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.099649429321289, + "advantages": -1.7313579320907593, + "generated_text": "{\n \"intent\": \"To continue exploring various content like images, videos, or translate by clicking on trending topics or articles.\",\n \"action\": \"DoubleClick(297, 132)\"\n}", + "action_string": "DoubleClick(297, 132)", + "intent_string": "To continue exploring various content like images, videos, or translate by clicking on trending topics or articles.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 62, + 130, + 93 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 558, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1520, + 675, + 1630, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 272, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 840 + ] + }, + { + "text": "+2.09.9%6 ", + "boxes": [ + 1636, + 802, + 1696, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 889 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,2876", + "boxes": [ + 1626, + 893, + 1698, + 941 + ] + }, + { + "text": "网上车市·2小时", + "boxes": [ + 826, + 921, + 956, + 945 + ] + }, + { + "text": "小黑盒游戏·10小时", + "boxes": [ + 1142, + 921, + 1286, + 945 + ] + }, + { + "text": "寻找你的躲藏地点“躲避”未来巨人！植物大", + "boxes": [ + 200, + 943, + 782, + 987 + ] + }, + { + "text": "捷豹路虎产品线将迎大调整，", + "boxes": [ + 824, + 943, + 1086, + 973 + ] + }, + { + "text": "《死亡岛2》新手排雷心得", + "boxes": [ + 1148, + 941, + 1382, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.2879", + "boxes": [ + 1626, + 939, + 1698, + 987 + ] + }, + { + "text": "这与退出中国市场有啥区别？", + "boxes": [ + 826, + 969, + 1092, + 999 + ] + }, + { + "text": "战僵尸 单机游戏 寻找你的躲藏地点 躲猫猫", + "boxes": [ + 198, + 983, + 764, + 1024 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "4770", + "boxes": [ + 204, + 1014, + 306, + 1045 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 792, + 449, + 972, + 478 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 729, + 268, + 746 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 889 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "网上车市·2小时", + "boxes": [ + 826, + 921, + 956, + 945 + ] + }, + { + "text": "小黑盒游戏·10小时", + "boxes": [ + 1142, + 921, + 1286, + 945 + ] + }, + { + "text": "仅靠一只狗就能通关！《博德3》玩家用宠物", + "boxes": [ + 200, + 945, + 784, + 987 + ] + }, + { + "text": "捷豹路虎产品线将迎大调整，", + "boxes": [ + 824, + 943, + 1086, + 973 + ] + }, + { + "text": "《死亡岛2》新手排雷心得", + "boxes": [ + 1148, + 941, + 1382, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "这与退出中国市场有啥区别？", + "boxes": [ + 826, + 969, + 1092, + 999 + ] + }, + { + "text": "", + "boxes": [ + 1581, + 931, + 1641, + 990 + ] + }, + { + "text": "咬死BOSS ", + "boxes": [ + 204, + 985, + 358, + 1018 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + } + ] + }, + { + "frame_id": "180_6_4", + "episode_id": 180, + "env_id": 6, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.64095687866211, + "advantages": -0.9843837022781372, + "generated_text": "{\n \"intent\": \"Click on the 'Firefox browsers' link to explore further desktop environments. This helps to understand different web browser options available.\",\n \"action\": \"Click(844, 129)\"\n}", + "action_string": "Click(844, 129)", + "intent_string": "Click on the 'Firefox browsers' link to explore further desktop environments. This helps to understand different web browser options available.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 471 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "①", + "boxes": [ + 252, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1874, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1048, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": " Firefox for Desktop", + "boxes": [ + 636, + 191, + 836, + 220 + ] + }, + { + "text": " Firefox for ios", + "boxes": [ + 638, + 228, + 792, + 258 + ] + }, + { + "text": " Firefox for Android", + "boxes": [ + 636, + 266, + 832, + 296 + ] + }, + { + "text": " Firefox Focus", + "boxes": [ + 634, + 299, + 786, + 334 + ] + }, + { + "text": " Firefox blog", + "boxes": [ + 638, + 337, + 772, + 374 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 415, + 1238, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": " Download for Linux 64-bit", + "boxes": [ + 846, + 637, + 1062, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "Get Firefox fo.", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "180_7_4", + "episode_id": 180, + "env_id": 7, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.220327377319336, + "advantages": -1.5648289918899536, + "generated_text": "{\n \"intent\": \"To continue exploring world tourist attractions by selecting and clicking the image of the Taj Mahal to potentially view additional images or details about this popular landmark.\",\n \"action\": \"Click(1317, 335)\"\n}", + "action_string": "Click(1317, 335)", + "intent_string": "To continue exploring world tourist attractions by selecting and clicking the image of the Taj Mahal to potentially view additional images or details about this popular landmark.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 66, + 128, + 91 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 171, + 234, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "Tourist Places Wallpapers - Wallpaper Cave", + "boxes": [ + 1320, + 240, + 1650, + 270 + ] + }, + { + "text": "", + "boxes": [ + 1452, + 294, + 1518, + 320 + ] + }, + { + "text": "", + "boxes": [ + 1532, + 290, + 1564, + 304 + ] + }, + { + "text": "ave.com I 2500 x 1666ipeg I 4 yrs ago", + "boxes": [ + 1410, + 337, + 1666, + 369 + ] + }, + { + "text": "wal", + "boxes": [ + 1330, + 347, + 1362, + 361 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1330, + 375, + 1374, + 395 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 308, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 171, + 234, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "Tourist Places Wallpapers - Wallpaper Cave", + "boxes": [ + 1320, + 240, + 1650, + 270 + ] + }, + { + "text": "", + "boxes": [ + 1454, + 294, + 1518, + 320 + ] + }, + { + "text": "", + "boxes": [ + 1532, + 290, + 1562, + 304 + ] + }, + { + "text": "ve.com I 2500 x 1666 ipeg I 4 yrs ago", + "boxes": [ + 1418, + 339, + 1666, + 367 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "180_0_5", + "episode_id": 180, + "env_id": 0, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.395764350891113, + "advantages": 0.0572085976600647, + "generated_text": "{\n \"intent\": \"Clicking on an image will provide more information about a specific world tourist attraction, which contributes to my exploration by obtaining detailed information and images.\",\n \"action\": \"DoubleClick(175, 288)\"\n}", + "action_string": "DoubleClick(175, 288)", + "intent_string": "Clicking on an image will provide more information about a specific world tourist attraction, which contributes to my exploration by obtaining detailed information and images.", + "before_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 66, + 69, + 124, + 87 + ] + }, + { + "text": "ir", + "boxes": [ + 1108, + 256, + 1180, + 274 + ] + }, + { + "text": "fii886ocr", + "boxes": [ + 1424, + 365, + 1514, + 389 + ] + }, + { + "text": "Most complained about tourist attractions in the", + "boxes": [ + 1290, + 393, + 1606, + 419 + ] + }, + { + "text": "10 of the Most Popular Tourist", + "boxes": [ + 1638, + 393, + 1832, + 417 + ] + }, + { + "text": "Wor", + "boxes": [ + 22, + 417, + 44, + 431 + ] + }, + { + "text": "vorld - Travel News", + "boxes": [ + 1296, + 415, + 1420, + 433 + ] + }, + { + "text": "Attractions in Paris", + "boxes": [ + 1642, + 415, + 1758, + 433 + ] + }, + { + "text": "Why I choose to do some of the", + "boxes": [ + 302, + 637, + 506, + 655 + ] + }, + { + "text": "MostBea", + "boxes": [ + 550, + 637, + 604, + 655 + ] + }, + { + "text": "Top10", + "boxes": [ + 1120, + 637, + 1164, + 655 + ] + }, + { + "text": "World Tourism", + "boxes": [ + 1454, + 637, + 1540, + 655 + ] + }, + { + "text": "The 25 best-value", + "boxes": [ + 1584, + 637, + 1700, + 655 + ] + }, + { + "text": "2023 r", + "boxes": [ + 18, + 653, + 62, + 671 + ] + }, + { + "text": "touristy things and see the", + "boxes": [ + 302, + 651, + 478, + 675 + ] + }, + { + "text": " Tanya Tanya", + "boxes": [ + 548, + 651, + 634, + 675 + ] + }, + { + "text": "England Th - vrogue.co", + "boxes": [ + 1114, + 649, + 1270, + 675 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 653, + 1516, + 671 + ] + }, + { + "text": "ones you can ski", + "boxes": [ + 1584, + 653, + 1692, + 671 + ] + }, + { + "text": "Tourist Attractions", + "boxes": [ + 932, + 868, + 1046, + 891 + ] + }, + { + "text": "Travel | uS New", + "boxes": [ + 20, + 891, + 128, + 909 + ] + }, + { + "text": "Collage-", + "boxes": [ + 933, + 887, + 989, + 912 + ] + }, + { + "text": "- Bucket List", + "boxes": [ + 982, + 885, + 1068, + 911 + ] + }, + { + "text": "Attractions of Dubai", + "boxes": [ + 1114, + 887, + 1240, + 911 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 308, + 87 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 28, + 121, + 190, + 145 + ] + }, + { + "text": "Q", + "boxes": [ + 282, + 123, + 298, + 141 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 262, + 169, + 432, + 199 + ] + }, + { + "text": "Top 50 Most Visited Tourist Attractions in the World", + "boxes": [ + 1218, + 242, + 1608, + 266 + ] + }, + { + "text": "e.com I 840×560 png |1yrago", + "boxes": [ + 1326, + 335, + 1542, + 370 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1228, + 375, + 1272, + 395 + ] + }, + { + "text": "Top 50 Most", + "boxes": [ + 13, + 386, + 95, + 420 + ] + }, + { + "text": "Worl", + "boxes": [ + 22, + 417, + 48, + 431 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 653, + 76, + 671 + ] + }, + { + "text": "lorld tourist", + "boxes": [ + 224, + 1048, + 302, + 1072 + ] + } + ] + }, + { + "frame_id": "180_1_5", + "episode_id": 180, + "env_id": 1, + "step": 5, + "format_rewards": 1.0, + "rewards": 11.341047286987305, + "advantages": 1.3616464138031006, + "generated_text": "{\n \"intent\": \"The current page is Microsoft Bing search engine. By clicking the 'Translation' option, we aim to further understand the documents or links related to translation services that might be available. This will help in getting more information about potential tools or resources related to the desktop environment exploration.\",\n \"action\": \"DoubleClick(432, 130)\"\n}", + "action_string": "DoubleClick(432, 130)", + "intent_string": "The current page is Microsoft Bing search engine. By clicking the 'Translation' option, we aim to further understand the documents or links related to translation services that might be available. This will help in getting more information about potential tools or resources related to the desktop environment exploration.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 488, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 448, + 970, + 477 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 673, + 1630, + 703 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 187, + 720, + 271, + 751 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "Ww", + "boxes": [ + 1594, + 810, + 1628, + 830 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1632, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1456, + 850, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13, 287", + "boxes": [ + 1626, + 893, + 1700, + 941 + ] + }, + { + "text": "科普世界·25分钟", + "boxes": [ + 830, + 921, + 962, + 945 + ] + }, + { + "text": "游民星空·2天", + "boxes": [ + 1138, + 921, + 1254, + 945 + ] + }, + { + "text": "海拔高达4500米！世界最高", + "boxes": [ + 826, + 941, + 1094, + 977 + ] + }, + { + "text": "《暗黑4》偷偷增添功能：节", + "boxes": [ + 1144, + 941, + 1402, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "游民星空·1天", + "boxes": [ + 202, + 955, + 316, + 979 + ] + }, + { + "text": "海拔光热电站西藏投产", + "boxes": [ + 824, + 967, + 1034, + 1003 + ] + }, + { + "text": "省大量跑路时间！", + "boxes": [ + 1138, + 967, + 1306, + 1003 + ] + }, + { + "text": "恐怖3A大作新史低！将近半折入库", + "boxes": [ + 192, + 977, + 656, + 1028 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "-47%", + "boxes": [ + 742, + 1003, + 790, + 1028 + ] + }, + { + "text": "LrM", + "boxes": [ + 1592, + 999, + 1630, + 1026 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "- Microsoft Bing ", + "boxes": [ + 80, + 28, + 190, + 52 + ] + }, + { + "text": "×", + "boxes": [ + 194, + 36, + 222, + 50 + ] + }, + { + "text": "Q必应翻译-Search", + "boxes": [ + 224, + 28, + 370, + 52 + ] + }, + { + "text": "Q必应翻译·", + "boxes": [ + 450, + 32, + 540, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 62, + 132, + 93 + ] + }, + { + "text": "国内版", + "boxes": [ + 184, + 113, + 268, + 145 + ] + }, + { + "text": "国际版", + "boxes": [ + 258, + 115, + 312, + 145 + ] + }, + { + "text": "人", + "boxes": [ + 428, + 133, + 442, + 149 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 48, + 159, + 146, + 183 + ] + }, + { + "text": "必应翻译", + "boxes": [ + 176, + 155, + 250, + 187 + ] + }, + { + "text": "WEB", + "boxes": [ + 178, + 212, + 206, + 232 + ] + }, + { + "text": "MAGE", + "boxes": [ + 248, + 214, + 282, + 228 + ] + }, + { + "text": "/IDEOS", + "boxes": [ + 332, + 214, + 368, + 228 + ] + }, + { + "text": "ACADEMI", + "boxes": [ + 410, + 212, + 462, + 230 + ] + }, + { + "text": "About 1,070,000 results", + "boxes": [ + 176, + 260, + 328, + 284 + ] + }, + { + "text": "?", + "boxes": [ + 184, + 306, + 214, + 328 + ] + }, + { + "text": "豆包", + "boxes": [ + 212, + 298, + 248, + 318 + ] + }, + { + "text": "翻译难题找豆包！－需要翻译？豆包来助力！", + "boxes": [ + 176, + 341, + 660, + 377 + ] + }, + { + "text": "无论是商务文件还是日常交流内容，都能准确翻译，为你打破语言障碍，迅速处理文本，在不同语", + "boxes": [ + 176, + 375, + 778, + 409 + ] + }, + { + "text": "言间搭建桥梁，使你在语言转换中毫无压力，轻松理解异国语言内容 Ad", + "boxes": [ + 176, + 399, + 618, + 431 + ] + }, + { + "text": "图像生成", + "boxes": [ + 192, + 441, + 280, + 472 + ] + }, + { + "text": "写作助手", + "boxes": [ + 510, + 437, + 600, + 476 + ] + }, + { + "text": "自定风格", + "boxes": [ + 194, + 471, + 256, + 496 + ] + }, + { + "text": "多种体裁", + "boxes": [ + 512, + 467, + 576, + 500 + ] + }, + { + "text": "搜索灵感，复制同款", + "boxes": [ + 192, + 490, + 328, + 520 + ] + }, + { + "text": "润色校对，一键成文", + "boxes": [ + 512, + 490, + 646, + 520 + ] + }, + { + "text": "数据分析", + "boxes": [ + 190, + 534, + 282, + 572 + ] + }, + { + "text": "一键完成周报范文生成", + "boxes": [ + 524, + 538, + 720, + 568 + ] + }, + { + "text": "处理数据", + "boxes": [ + 194, + 566, + 258, + 592 + ] + }, + { + "text": "支持自由拓展，为你节省时间", + "boxes": [ + 512, + 564, + 690, + 594 + ] + }, + { + "text": "精确分析，绘制图表", + "boxes": [ + 192, + 588, + 326, + 617 + ] + }, + { + "text": "提升工作效率，立即体验！", + "boxes": [ + 514, + 590, + 672, + 613 + ] + }, + { + "text": "让AI成为你的得力助手", + "boxes": [ + 194, + 633, + 398, + 663 + ] + }, + { + "text": "助你更好地管理生活，提高工作效率", + "boxes": [ + 196, + 663, + 410, + 687 + ] + }, + { + "text": "享受科技带来的便捷与乐趣.", + "boxes": [ + 196, + 687, + 368, + 711 + ] + }, + { + "text": " See more at doubao.com", + "boxes": [ + 192, + 723, + 358, + 752 + ] + }, + { + "text": "星火大模型-深度思考·精准推理", + "boxes": [ + 214, + 780, + 412, + 804 + ] + }, + { + "text": "https://xinghuo.xfyun.cn 讯飞星火>更懂你的Ai助手", + "boxes": [ + 214, + 798, + 544, + 828 + ] + }, + { + "text": "立即开始", + "boxes": [ + 710, + 790, + 778, + 816 + ] + }, + { + "text": "『讯飞星火』全能AI助手－『讯飞星火』O元解锁全部AI功能", + "boxes": [ + 186, + 826, + 710, + 862 + ] + }, + { + "text": "拥有跨领域的知识和理解能力，能够基于自然对话方式理解与执行任务.免费注册体验》，必应。『讯飞", + "boxes": [ + 176, + 854, + 784, + 887 + ] + }, + { + "text": "星火Spark人工智能Al助手，解决写作/绘图/问答/搜索/翻译/解题等多样需求Ad", + "boxes": [ + 180, + 881, + 660, + 905 + ] + }, + { + "text": "Bing", + "boxes": [ + 214, + 935, + 248, + 955 + ] + }, + { + "text": "https://ww.bing.com translator Translate his result", + "boxes": [ + 206, + 947, + 544, + 979 + ] + }, + { + "text": "Microsoft Translator-从英语翻译", + "boxes": [ + 174, + 973, + 482, + 1009 + ] + }, + { + "text": "15 hours ago·免费翻译英语，并提供准确结果。每天有数百万人使用必应翻译100多种语言", + "boxes": [ + 180, + 1005, + 766, + 1028 + ] + } + ] + }, + { + "frame_id": "180_2_5", + "episode_id": 180, + "env_id": 2, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.381027221679688, + "advantages": 0.03687217831611633, + "generated_text": "{\n \"intent\": \"To begin web browsing for more information regarding the icons available or various websites.\",\n \"action\": \"DoubleClick(756, 716)\"\n}", + "action_string": "DoubleClick(756, 716)", + "intent_string": "To begin web browsing for more information regarding the icons available or various websites.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1040, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 488, + 615, + 584, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 508, + 671, + 692, + 721 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1226, + 677, + 1414, + 733 + ] + }, + { + "text": "", + "boxes": [ + 668, + 744, + 692, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 912, + 750, + 928, + 766 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 746, + 768, + 786, + 794 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 981, + 765, + 1035, + 797 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1228, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 746, + 790, + 790, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 982, + 788, + 1144, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 189, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 822, + 318, + 932, + 361 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "180_3_5", + "episode_id": 180, + "env_id": 3, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.59914493560791, + "advantages": 0.337862491607666, + "generated_text": "{\n \"intent\": \"Clicking on 'Getting Started' is a good initial step towards learning about the browser's features or starting content exploration.\",\n \"action\": \"DoubleClick(251, 108)\"\n}", + "action_string": "DoubleClick(251, 108)", + "intent_string": "Clicking on 'Getting Started' is a good initial step towards learning about the browser's features or starting content exploration.", + "before_screen_ocr_results": [ + { + "text": "- Microsoft Bing ", + "boxes": [ + 80, + 30, + 190, + 54 + ] + }, + { + "text": "×", + "boxes": [ + 194, + 36, + 218, + 50 + ] + }, + { + "text": "New Tab", + "boxes": [ + 248, + 32, + 314, + 52 + ] + }, + { + "text": "①→Q白", + "boxes": [ + 6, + 57, + 138, + 100 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 253, + 104, + 373, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 111, + 1048, + 143 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 242, + 171, + 326, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "180_4_5", + "episode_id": 180, + "env_id": 4, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.85096549987793, + "advantages": -0.694583535194397, + "generated_text": "{\n \"intent\": \"Click on the home icon to return to the main internet browser page and continue the exploration of the desktop environment.\",\n \"action\": \"Click(113, 71)\"\n}", + "action_string": "Click(113, 71)", + "intent_string": "Click on the home icon to return to the main internet browser page and continue the exploration of the desktop environment.", + "before_screen_ocr_results": [ + { + "text": "miap/sigr", + "boxes": [ + 107, + 29, + 165, + 55 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 68, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "1%2F%3Fref_%3DnaV_ya_signi", + "boxes": [ + 1168, + 69, + 1378, + 87 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1384, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing, you agree to Amazon's Conditions", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 14, + 32, + 86, + 50 + ] + }, + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 132, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_5_5", + "episode_id": 180, + "env_id": 5, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.350818634033203, + "advantages": -0.004813991952687502, + "generated_text": "{\n \"intent\": \"To continue exploring various web pages and eventually open new meaningful content.\",\n \"action\": \"DoubleClick(208, 37)\"\n}", + "action_string": "DoubleClick(208, 37)", + "intent_string": "To continue exploring various web pages and eventually open new meaningful content.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "中", + "boxes": [ + 956, + 435, + 968, + 447 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 446, + 970, + 477 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 270, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "点资讯·49分钟", + "boxes": [ + 240, + 921, + 334, + 945 + ] + }, + { + "text": "网上车市·2小时", + "boxes": [ + 826, + 921, + 956, + 945 + ] + }, + { + "text": "小黑盒游戏·10小时", + "boxes": [ + 1142, + 921, + 1286, + 945 + ] + }, + { + "text": "进化5亿年肠道竟成“第二大脑”！专家：我们", + "boxes": [ + 198, + 941, + 794, + 989 + ] + }, + { + "text": "捷豹路虎产品线将迎大调整，", + "boxes": [ + 826, + 943, + 1086, + 973 + ] + }, + { + "text": "《死亡岛2》新手排雷心得", + "boxes": [ + 1148, + 941, + 1382, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "这与退出中国市场有啥区别？", + "boxes": [ + 824, + 969, + 1090, + 999 + ] + }, + { + "text": "", + "boxes": [ + 1581, + 931, + 1641, + 990 + ] + }, + { + "text": "的喜怒哀乐被它操控", + "boxes": [ + 200, + 981, + 478, + 1022 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 22, + 332, + 86, + 381 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 86, + 969 + ] + } + ] + }, + { + "frame_id": "180_6_5", + "episode_id": 180, + "env_id": 6, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.39649486541748, + "advantages": -1.3217277526855469, + "generated_text": "{\n \"intent\": \"Click the 'Will use it on my Windows' option to continue exploring the browser downsloading page.\",\n \"action\": \"Click(1096, 124)\"\n}", + "action_string": "Click(1096, 124)", + "intent_string": "Click the 'Will use it on my Windows' option to continue exploring the browser downsloading page.", + "before_screen_ocr_results": [ + { + "text": "①", + "boxes": [ + 252, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1874, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1048, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": " Firefox for Desktop", + "boxes": [ + 636, + 191, + 836, + 220 + ] + }, + { + "text": " Firefox for ios", + "boxes": [ + 638, + 228, + 792, + 258 + ] + }, + { + "text": " Firefox for Android", + "boxes": [ + 636, + 266, + 832, + 296 + ] + }, + { + "text": " Firefox Focus", + "boxes": [ + 634, + 299, + 786, + 334 + ] + }, + { + "text": " Firefox blog", + "boxes": [ + 638, + 337, + 772, + 374 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 415, + 1238, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": " Download for Linux 64-bit", + "boxes": [ + 846, + 637, + 1062, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "Get Firefox fo.", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→", + "boxes": [ + 12, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 64, + 1662, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 109, + 1048, + 141 + ] + }, + { + "text": "Ahout us", + "boxes": [ + 1086, + 111, + 1174, + 143 + ] + }, + { + "text": "Our Mission", + "boxes": [ + 638, + 185, + 738, + 210 + ] + }, + { + "text": " Our Work", + "boxes": [ + 856, + 181, + 942, + 212 + ] + }, + { + "text": "About Mozilla", + "boxes": [ + 634, + 224, + 764, + 254 + ] + }, + { + "text": "Mozilla Foundation", + "boxes": [ + 858, + 224, + 1032, + 254 + ] + }, + { + "text": "The MozillaManifesto", + "boxes": [ + 636, + 260, + 836, + 290 + ] + }, + { + "text": " Mozilla.ai", + "boxes": [ + 856, + 260, + 948, + 292 + ] + }, + { + "text": "Get Involved", + "boxes": [ + 632, + 294, + 752, + 324 + ] + }, + { + "text": "Mozilla Ventures", + "boxes": [ + 858, + 296, + 1008, + 328 + ] + }, + { + "text": "Blog", + "boxes": [ + 632, + 332, + 686, + 365 + ] + }, + { + "text": " Mozilla Advertising ", + "boxes": [ + 856, + 330, + 1036, + 365 + ] + }, + { + "text": "Mozilla Builders", + "boxes": [ + 856, + 367, + 1004, + 397 + ] + }, + { + "text": "Mozilla New Products", + "boxes": [ + 856, + 401, + 1058, + 437 + ] + }, + { + "text": "pEIUW", + "boxes": [ + 886, + 484, + 1020, + 516 + ] + }, + { + "text": "Download for Linux 32-bit", + "boxes": [ + 848, + 592, + 1054, + 615 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 850, + 641, + 1062, + 665 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 788, + 707, + 1118, + 731 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompany", + "boxes": [ + 738, + 1030, + 792, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "Get Firefox fo.", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "180_7_5", + "episode_id": 180, + "env_id": 7, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.705329895019531, + "advantages": -0.8955525159835815, + "generated_text": "{\n \"intent\": \"To continue exploring world tourist attractions online by clicking on an image to see more details or related websites.\",\n \"action\": \"DoubleClick(165, 329)\"\n}", + "action_string": "DoubleClick(165, 329)", + "intent_string": "To continue exploring world tourist attractions online by clicking on an image to see more details or related websites.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 171, + 234, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "Tourist Places Wallpapers - Wallpaper Cave", + "boxes": [ + 1320, + 240, + 1650, + 270 + ] + }, + { + "text": "", + "boxes": [ + 1454, + 294, + 1518, + 320 + ] + }, + { + "text": "", + "boxes": [ + 1532, + 290, + 1562, + 304 + ] + }, + { + "text": "ve.com I 2500 x 1666 ipeg I 4 yrs ago", + "boxes": [ + 1418, + 339, + 1666, + 367 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 310, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 141 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 173, + 236, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "These Are The 10 Most Popular Tourist Attractions in the", + "boxes": [ + 1322, + 242, + 1750, + 266 + ] + }, + { + "text": "World -Journeyz", + "boxes": [ + 1322, + 262, + 1454, + 286 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 84, + 465 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 943, + 76, + 963 + ] + } + ] + }, + { + "frame_id": "180_0_6", + "episode_id": 180, + "env_id": 0, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.180397987365723, + "advantages": -0.23998500406742096, + "generated_text": "{\n \"intent\": \"Click the close button to close the image view window and continue exploring the environment outside of the image view.\",\n \"action\": \"Click(1787, 173)\"\n}", + "action_string": "Click(1787, 173)", + "intent_string": "Click the close button to close the image view window and continue exploring the environment outside of the image view.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 308, + 87 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 28, + 121, + 190, + 145 + ] + }, + { + "text": "Q", + "boxes": [ + 282, + 123, + 298, + 141 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 262, + 169, + 432, + 199 + ] + }, + { + "text": "Top 50 Most Visited Tourist Attractions in the World", + "boxes": [ + 1218, + 242, + 1608, + 266 + ] + }, + { + "text": "e.com I 840×560 png |1yrago", + "boxes": [ + 1326, + 335, + 1542, + 370 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1228, + 375, + 1272, + 395 + ] + }, + { + "text": "Top 50 Most", + "boxes": [ + 13, + 386, + 95, + 420 + ] + }, + { + "text": "Worl", + "boxes": [ + 22, + 417, + 48, + 431 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 653, + 76, + 671 + ] + }, + { + "text": "lorld tourist", + "boxes": [ + 224, + 1048, + 302, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 44, + 69, + 124, + 87 + ] + }, + { + "text": "fii886ocr", + "boxes": [ + 1424, + 365, + 1514, + 389 + ] + }, + { + "text": "Most complained about tourist attractions in the", + "boxes": [ + 1294, + 397, + 1604, + 415 + ] + }, + { + "text": "10 of the Most Popular Tourist", + "boxes": [ + 1638, + 393, + 1832, + 417 + ] + }, + { + "text": "Wor", + "boxes": [ + 22, + 417, + 44, + 431 + ] + }, + { + "text": "lorld - Travel News", + "boxes": [ + 1298, + 415, + 1420, + 433 + ] + }, + { + "text": "Attractions in Paris", + "boxes": [ + 1642, + 415, + 1758, + 433 + ] + }, + { + "text": "Why I choose to do some of the", + "boxes": [ + 302, + 637, + 506, + 655 + ] + }, + { + "text": "MostBea", + "boxes": [ + 552, + 637, + 604, + 655 + ] + }, + { + "text": "Top10", + "boxes": [ + 1120, + 637, + 1164, + 655 + ] + }, + { + "text": "lost Popular Tourist Attractions Ii", + "boxes": [ + 1172, + 637, + 1380, + 655 + ] + }, + { + "text": "World Touri", + "boxes": [ + 1454, + 637, + 1524, + 655 + ] + }, + { + "text": "The 25 best-valu", + "boxes": [ + 1584, + 637, + 1692, + 655 + ] + }, + { + "text": "2023I", + "boxes": [ + 20, + 655, + 60, + 671 + ] + }, + { + "text": "touristy things and see the", + "boxes": [ + 302, + 651, + 478, + 675 + ] + }, + { + "text": " Tanya Tany", + "boxes": [ + 548, + 651, + 626, + 675 + ] + }, + { + "text": "England Th - vrogue.co", + "boxes": [ + 1116, + 649, + 1266, + 675 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 653, + 1516, + 671 + ] + }, + { + "text": "Tourist Attractions", + "boxes": [ + 936, + 868, + 1046, + 891 + ] + }, + { + "text": "Travel | uS Ne", + "boxes": [ + 20, + 891, + 116, + 909 + ] + }, + { + "text": "Collage·", + "boxes": [ + 936, + 891, + 988, + 909 + ] + }, + { + "text": "Bucket List", + "boxes": [ + 986, + 885, + 1068, + 911 + ] + }, + { + "text": "Attractions of Duba", + "boxes": [ + 1114, + 885, + 1238, + 911 + ] + } + ] + }, + { + "frame_id": "180_1_6", + "episode_id": 180, + "env_id": 1, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.505661010742188, + "advantages": 0.2088598757982254, + "generated_text": "{\n \"intent\": \"The search result page related to '必应翻译' suggests interest in exploring potential resources about industry-specific tools like Microsoft Translator. Selecting the first search result could be an action that helps in finding detailed information about available translation services.\",\n \"action\": \"DoubleClick(286, 999)\"\n}", + "action_string": "DoubleClick(286, 999)", + "intent_string": "The search result page related to '必应翻译' suggests interest in exploring potential resources about industry-specific tools like Microsoft Translator. Selecting the first search result could be an action that helps in finding detailed information about available translation services.", + "before_screen_ocr_results": [ + { + "text": "- Microsoft Bing ", + "boxes": [ + 80, + 28, + 190, + 52 + ] + }, + { + "text": "×", + "boxes": [ + 194, + 36, + 222, + 50 + ] + }, + { + "text": "Q必应翻译-Search", + "boxes": [ + 224, + 28, + 370, + 52 + ] + }, + { + "text": "Q必应翻译·", + "boxes": [ + 450, + 32, + 540, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 62, + 132, + 93 + ] + }, + { + "text": "国内版", + "boxes": [ + 184, + 113, + 268, + 145 + ] + }, + { + "text": "国际版", + "boxes": [ + 258, + 115, + 312, + 145 + ] + }, + { + "text": "人", + "boxes": [ + 428, + 133, + 442, + 149 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 48, + 159, + 146, + 183 + ] + }, + { + "text": "必应翻译", + "boxes": [ + 176, + 155, + 250, + 187 + ] + }, + { + "text": "WEB", + "boxes": [ + 178, + 212, + 206, + 232 + ] + }, + { + "text": "MAGE", + "boxes": [ + 248, + 214, + 282, + 228 + ] + }, + { + "text": "/IDEOS", + "boxes": [ + 332, + 214, + 368, + 228 + ] + }, + { + "text": "ACADEMI", + "boxes": [ + 410, + 212, + 462, + 230 + ] + }, + { + "text": "About 1,070,000 results", + "boxes": [ + 176, + 260, + 328, + 284 + ] + }, + { + "text": "?", + "boxes": [ + 184, + 306, + 214, + 328 + ] + }, + { + "text": "豆包", + "boxes": [ + 212, + 298, + 248, + 318 + ] + }, + { + "text": "翻译难题找豆包！－需要翻译？豆包来助力！", + "boxes": [ + 176, + 341, + 660, + 377 + ] + }, + { + "text": "无论是商务文件还是日常交流内容，都能准确翻译，为你打破语言障碍，迅速处理文本，在不同语", + "boxes": [ + 176, + 375, + 778, + 409 + ] + }, + { + "text": "言间搭建桥梁，使你在语言转换中毫无压力，轻松理解异国语言内容 Ad", + "boxes": [ + 176, + 399, + 618, + 431 + ] + }, + { + "text": "图像生成", + "boxes": [ + 192, + 441, + 280, + 472 + ] + }, + { + "text": "写作助手", + "boxes": [ + 510, + 437, + 600, + 476 + ] + }, + { + "text": "自定风格", + "boxes": [ + 194, + 471, + 256, + 496 + ] + }, + { + "text": "多种体裁", + "boxes": [ + 512, + 467, + 576, + 500 + ] + }, + { + "text": "搜索灵感，复制同款", + "boxes": [ + 192, + 490, + 328, + 520 + ] + }, + { + "text": "润色校对，一键成文", + "boxes": [ + 512, + 490, + 646, + 520 + ] + }, + { + "text": "数据分析", + "boxes": [ + 190, + 534, + 282, + 572 + ] + }, + { + "text": "一键完成周报范文生成", + "boxes": [ + 524, + 538, + 720, + 568 + ] + }, + { + "text": "处理数据", + "boxes": [ + 194, + 566, + 258, + 592 + ] + }, + { + "text": "支持自由拓展，为你节省时间", + "boxes": [ + 512, + 564, + 690, + 594 + ] + }, + { + "text": "精确分析，绘制图表", + "boxes": [ + 192, + 588, + 326, + 617 + ] + }, + { + "text": "提升工作效率，立即体验！", + "boxes": [ + 514, + 590, + 672, + 613 + ] + }, + { + "text": "让AI成为你的得力助手", + "boxes": [ + 194, + 633, + 398, + 663 + ] + }, + { + "text": "助你更好地管理生活，提高工作效率", + "boxes": [ + 196, + 663, + 410, + 687 + ] + }, + { + "text": "享受科技带来的便捷与乐趣.", + "boxes": [ + 196, + 687, + 368, + 711 + ] + }, + { + "text": " See more at doubao.com", + "boxes": [ + 192, + 723, + 358, + 752 + ] + }, + { + "text": "星火大模型-深度思考·精准推理", + "boxes": [ + 214, + 780, + 412, + 804 + ] + }, + { + "text": "https://xinghuo.xfyun.cn 讯飞星火>更懂你的Ai助手", + "boxes": [ + 214, + 798, + 544, + 828 + ] + }, + { + "text": "立即开始", + "boxes": [ + 710, + 790, + 778, + 816 + ] + }, + { + "text": "『讯飞星火』全能AI助手－『讯飞星火』O元解锁全部AI功能", + "boxes": [ + 186, + 826, + 710, + 862 + ] + }, + { + "text": "拥有跨领域的知识和理解能力，能够基于自然对话方式理解与执行任务.免费注册体验》，必应。『讯飞", + "boxes": [ + 176, + 854, + 784, + 887 + ] + }, + { + "text": "星火Spark人工智能Al助手，解决写作/绘图/问答/搜索/翻译/解题等多样需求Ad", + "boxes": [ + 180, + 881, + 660, + 905 + ] + }, + { + "text": "Bing", + "boxes": [ + 214, + 935, + 248, + 955 + ] + }, + { + "text": "https://ww.bing.com translator Translate his result", + "boxes": [ + 206, + 947, + 544, + 979 + ] + }, + { + "text": "Microsoft Translator-从英语翻译", + "boxes": [ + 174, + 973, + 482, + 1009 + ] + }, + { + "text": "15 hours ago·免费翻译英语，并提供准确结果。每天有数百万人使用必应翻译100多种语言", + "boxes": [ + 180, + 1005, + 766, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": "一", + "boxes": [ + 24, + 133, + 52, + 155 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 40, + 125, + 184, + 165 + ] + }, + { + "text": "在 Web上搜索", + "boxes": [ + 642, + 129, + 758, + 161 + ] + }, + { + "text": "文本", + "boxes": [ + 770, + 204, + 804, + 224 + ] + }, + { + "text": "翻译工具", + "boxes": [ + 826, + 202, + 892, + 228 + ] + }, + { + "text": "对话", + "boxes": [ + 910, + 195, + 959, + 230 + ] + }, + { + "text": "应用", + "boxes": [ + 978, + 201, + 1018, + 228 + ] + }, + { + "text": "商务用", + "boxes": [ + 1040, + 202, + 1094, + 228 + ] + }, + { + "text": "帮助", + "boxes": [ + 1122, + 202, + 1152, + 224 + ] + }, + { + "text": "英语", + "boxes": [ + 286, + 298, + 308, + 312 + ] + }, + { + "text": "1", + "boxes": [ + 952, + 298, + 970, + 314 + ] + }, + { + "text": "中文 (简体)", + "boxes": [ + 1008, + 294, + 1082, + 318 + ] + }, + { + "text": "输入文本", + "boxes": [ + 256, + 341, + 462, + 405 + ] + }, + { + "text": "←国", + "boxes": [ + 820, + 455, + 948, + 498 + ] + }, + { + "text": "语气", + "boxes": [ + 1010, + 469, + 1042, + 490 + ] + }, + { + "text": "广泛使用的短语", + "boxes": [ + 258, + 522, + 390, + 554 + ] + }, + { + "text": "Most popular", + "boxes": [ + 262, + 562, + 354, + 594 + ] + }, + { + "text": ">", + "boxes": [ + 1636, + 570, + 1652, + 588 + ] + }, + { + "text": "Thank you", + "boxes": [ + 989, + 609, + 1059, + 642 + ] + }, + { + "text": "Good morni", + "boxes": [ + 264, + 657, + 334, + 675 + ] + }, + { + "text": "Nice to meet you", + "boxes": [ + 260, + 697, + 368, + 721 + ] + }, + { + "text": "You are welcome", + "boxes": [ + 990, + 697, + 1100, + 721 + ] + }, + { + "text": "Good bye", + "boxes": [ + 255, + 732, + 327, + 763 + ] + }, + { + "text": " Good night ", + "boxes": [ + 985, + 728, + 1067, + 767 + ] + }, + { + "text": "隐私声明和Cookie", + "boxes": [ + 18, + 1005, + 132, + 1028 + ] + }, + { + "text": " 2025 Mic", + "boxes": [ + 1780, + 1009, + 1844, + 1026 + ] + }, + { + "text": "icrosoft Tra", + "boxes": [ + 226, + 1050, + 300, + 1068 + ] + } + ] + }, + { + "frame_id": "180_2_6", + "episode_id": 180, + "env_id": 2, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.19927978515625, + "advantages": -0.2139291763305664, + "generated_text": "{\n \"intent\": \"To continue exploring the page and cover different sections of the webpage.\",\n \"action\": \"DoubleClick(712, 119)\"\n}", + "action_string": "DoubleClick(712, 119)", + "intent_string": "To continue exploring the page and cover different sections of the webpage.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 822, + 318, + 932, + 361 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "iPhone - Apple", + "boxes": [ + 26, + 28, + 126, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "?", + "boxes": [ + 262, + 73, + 274, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1608, + 69, + 1658, + 87 + ] + }, + { + "text": "Watch", + "boxes": [ + 768, + 113, + 798, + 127 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 986, + 109, + 1046, + 129 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1082, + 111, + 1158, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1368, + 108, + 1391, + 128 + ] + }, + { + "text": "B", + "boxes": [ + 454, + 157, + 522, + 218 + ] + }, + { + "text": "B", + "boxes": [ + 562, + 157, + 630, + 224 + ] + }, + { + "text": "C", + "boxes": [ + 754, + 159, + 820, + 218 + ] + }, + { + "text": "18", + "boxes": [ + 1106, + 155, + 1170, + 222 + ] + }, + { + "text": "D", + "boxes": [ + 1380, + 153, + 1464, + 226 + ] + }, + { + "text": "0", + "boxes": [ + 670, + 163, + 718, + 214 + ] + }, + { + "text": "00", + "boxes": [ + 846, + 165, + 922, + 222 + ] + }, + { + "text": "口", + "boxes": [ + 1210, + 173, + 1276, + 220 + ] + }, + { + "text": "OS", + "boxes": [ + 1306, + 171, + 1354, + 218 + ] + }, + { + "text": "9?", + "boxes": [ + 944, + 181, + 992, + 218 + ] + }, + { + "text": "·", + "boxes": [ + 1019, + 172, + 1072, + 223 + ] + }, + { + "text": "ipPhone 16 Pro", + "boxes": [ + 448, + 218, + 530, + 244 + ] + }, + { + "text": "iPhone 16", + "boxes": [ + 564, + 218, + 626, + 244 + ] + }, + { + "text": "iphone 16e", + "boxes": [ + 660, + 218, + 726, + 244 + ] + }, + { + "text": "iPhone 15", + "boxes": [ + 758, + 218, + 820, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 850, + 216, + 912, + 248 + ] + }, + { + "text": "AirPods", + "boxes": [ + 944, + 218, + 994, + 246 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1026, + 218, + 1066, + 246 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1102, + 218, + 1172, + 244 + ] + }, + { + "text": "Apple Card", + "boxes": [ + 1210, + 218, + 1274, + 244 + ] + }, + { + "text": "ios 18", + "boxes": [ + 1308, + 218, + 1350, + 244 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 1386, + 218, + 1458, + 244 + ] + }, + { + "text": "iPhone", + "boxes": [ + 99, + 384, + 369, + 488 + ] + }, + { + "text": "Designed to be loved.", + "boxes": [ + 1520, + 433, + 1800, + 474 + ] + }, + { + "text": "9:41", + "boxes": [ + 836, + 566, + 1044, + 693 + ] + } + ] + }, + { + "frame_id": "180_3_6", + "episode_id": 180, + "env_id": 3, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.677574157714844, + "advantages": 0.4460904598236084, + "generated_text": "{\n \"intent\": \"Click the Download Firefox button to explore the Firefox browser page and prepare the environment to start browsing other web sites.\",\n \"action\": \"Click(441, 595)\"\n}", + "action_string": "Click(441, 595)", + "intent_string": "Click the Download Firefox button to explore the Firefox browser page and prepare the environment to start browsing other web sites.", + "before_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 253, + 104, + 373, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 111, + 1048, + 143 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 242, + 171, + 326, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "180_4_6", + "episode_id": 180, + "env_id": 4, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.169958114624023, + "advantages": -0.25439146161079407, + "generated_text": "{\n \"intent\": \"To begin by exploring various available applications or browsing pages by selecting an interesting choice.\",\n \"action\": \"DoubleClick(769, 740)\"\n}", + "action_string": "DoubleClick(769, 740)", + "intent_string": "To begin by exploring various available applications or browsing pages by selecting an interesting choice.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 14, + 32, + 86, + 50 + ] + }, + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 132, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 634, + 474 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 673, + 1630, + 703 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 729, + 266, + 746 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "+2,49 26 ", + "boxes": [ + 1634, + 802, + 1698, + 844 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1456, + 850, + 1514, + 891 + ] + }, + { + "text": "261029 ", + "boxes": [ + 1632, + 846, + 1700, + 893 + ] + }, + { + "text": "13,67", + "boxes": [ + 1628, + 891, + 1700, + 941 + ] + }, + { + "text": "LOOKING FLOAT", + "boxes": [ + 267, + 913, + 389, + 952 + ] + }, + { + "text": "南方+客户端·14小时", + "boxes": [ + 826, + 921, + 984, + 945 + ] + }, + { + "text": "新京报·10小时", + "boxes": [ + 1160, + 921, + 1262, + 945 + ] + }, + { + "text": "AI还原！中山市博物馆这些干", + "boxes": [ + 826, + 941, + 1100, + 977 + ] + }, + { + "text": "幼儿园厨师餐具池里洗墩布被", + "boxes": [ + 1134, + 937, + 1408, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+32829 ", + "boxes": [ + 1630, + 939, + 1700, + 989 + ] + }, + { + "text": "4小时", + "boxes": [ + 284, + 961, + 324, + 977 + ] + }, + { + "text": "年“神器”，实用", + "boxes": [ + 826, + 967, + 982, + 1003 + ] + }, + { + "text": "辞退，起诉幼儿园要求赔偿被", + "boxes": [ + 1138, + 969, + 1408, + 999 + ] + }, + { + "text": "极氪回归吉利，是妙手也是必然", + "boxes": [ + 196, + 979, + 620, + 1026 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "驳回", + "boxes": [ + 1134, + 995, + 1190, + 1028 + ] + }, + { + "text": "2", + "boxes": [ + 1140, + 1028, + 1170, + 1042 + ] + } + ] + }, + { + "frame_id": "180_5_6", + "episode_id": 180, + "env_id": 5, + "step": 6, + "format_rewards": 1.0, + "rewards": 11.526768684387207, + "advantages": 1.617931604385376, + "generated_text": "{\n \"intent\": \"To start by clicking an application icon like Firefox from the available tools.\",\n \"action\": \"DoubleClick(73, 73)\"\n}", + "action_string": "DoubleClick(73, 73)", + "intent_string": "To start by clicking an application icon like Firefox from the available tools.", + "before_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 22, + 332, + 86, + 381 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 86, + 969 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 522, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "IPnon", + "boxes": [ + 564, + 453, + 586, + 467 + ] + }, + { + "text": "T", + "boxes": [ + 858, + 453, + 938, + 508 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing ", + "boxes": [ + 630, + 530, + 680, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": " Highlights", + "boxes": [ + 488, + 615, + 588, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 988, + 671, + 1172, + 719 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 663, + 739, + 697, + 772 + ] + }, + { + "text": "?", + "boxes": [ + 912, + 750, + 928, + 766 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1410, + 766 + ] + }, + { + "text": "cn.bing ", + "boxes": [ + 506, + 772, + 554, + 790 + ] + }, + { + "text": "apple", + "boxes": [ + 744, + 768, + 788, + 796 + ] + }, + { + "text": "amazo", + "boxes": [ + 992, + 774, + 1028, + 788 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": " Search - Microsoft Bing", + "boxes": [ + 502, + 788, + 664, + 816 + ] + }, + { + "text": "Apple", + "boxes": [ + 744, + 790, + 792, + 818 + ] + } + ] + }, + { + "frame_id": "180_6_6", + "episode_id": 180, + "env_id": 6, + "step": 6, + "format_rewards": 1.0, + "rewards": 8.854043960571289, + "advantages": -2.070279836654663, + "generated_text": "{\n \"intent\": \"Click on the close button at the top right corner to close this window and continue exploring the desktop.\",\n \"action\": \"Click(1270, 178)\"\n}", + "action_string": "Click(1270, 178)", + "intent_string": "Click on the close button at the top right corner to close this window and continue exploring the desktop.", + "before_screen_ocr_results": [ + { + "text": "←→", + "boxes": [ + 12, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 64, + 1662, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 109, + 1048, + 141 + ] + }, + { + "text": "Ahout us", + "boxes": [ + 1086, + 111, + 1174, + 143 + ] + }, + { + "text": "Our Mission", + "boxes": [ + 638, + 185, + 738, + 210 + ] + }, + { + "text": " Our Work", + "boxes": [ + 856, + 181, + 942, + 212 + ] + }, + { + "text": "About Mozilla", + "boxes": [ + 634, + 224, + 764, + 254 + ] + }, + { + "text": "Mozilla Foundation", + "boxes": [ + 858, + 224, + 1032, + 254 + ] + }, + { + "text": "The MozillaManifesto", + "boxes": [ + 636, + 260, + 836, + 290 + ] + }, + { + "text": " Mozilla.ai", + "boxes": [ + 856, + 260, + 948, + 292 + ] + }, + { + "text": "Get Involved", + "boxes": [ + 632, + 294, + 752, + 324 + ] + }, + { + "text": "Mozilla Ventures", + "boxes": [ + 858, + 296, + 1008, + 328 + ] + }, + { + "text": "Blog", + "boxes": [ + 632, + 332, + 686, + 365 + ] + }, + { + "text": " Mozilla Advertising ", + "boxes": [ + 856, + 330, + 1036, + 365 + ] + }, + { + "text": "Mozilla Builders", + "boxes": [ + 856, + 367, + 1004, + 397 + ] + }, + { + "text": "Mozilla New Products", + "boxes": [ + 856, + 401, + 1058, + 437 + ] + }, + { + "text": "pEIUW", + "boxes": [ + 886, + 484, + 1020, + 516 + ] + }, + { + "text": "Download for Linux 32-bit", + "boxes": [ + 848, + 592, + 1054, + 615 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 850, + 641, + 1062, + 665 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 788, + 707, + 1118, + 731 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompany", + "boxes": [ + 738, + 1030, + 792, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "Get Firefox fo.", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1608, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 471 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "180_7_6", + "episode_id": 180, + "env_id": 7, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.276773452758789, + "advantages": -1.4869365692138672, + "generated_text": "{\n \"intent\": \"To continue exploring various world tourist attractions by clicking on different images and learning more about them.\",\n \"action\": \"DoubleClick(650, 975)\"\n}", + "action_string": "DoubleClick(650, 975)", + "intent_string": "To continue exploring various world tourist attractions by clicking on different images and learning more about them.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 310, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 141 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 173, + 236, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "These Are The 10 Most Popular Tourist Attractions in the", + "boxes": [ + 1322, + 242, + 1750, + 266 + ] + }, + { + "text": "World -Journeyz", + "boxes": [ + 1322, + 262, + 1454, + 286 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 84, + 465 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 943, + 76, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 310, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 141 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 173, + 236, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "These Are The 10 Most Popular Tourist Attractions in the", + "boxes": [ + 1322, + 242, + 1750, + 266 + ] + }, + { + "text": "World -Journeyz", + "boxes": [ + 1322, + 262, + 1454, + 286 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 943, + 76, + 963 + ] + } + ] + }, + { + "frame_id": "180_0_7", + "episode_id": 180, + "env_id": 0, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.33525562286377, + "advantages": -0.026290081441402435, + "generated_text": "{\n \"intent\": \"To continue exploring various world tourist attractions by selecting an interesting link from the search results.\",\n \"action\": \"DoubleClick(137, 275)\"\n}", + "action_string": "DoubleClick(137, 275)", + "intent_string": "To continue exploring various world tourist attractions by selecting an interesting link from the search results.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 44, + 69, + 124, + 87 + ] + }, + { + "text": "fii886ocr", + "boxes": [ + 1424, + 365, + 1514, + 389 + ] + }, + { + "text": "Most complained about tourist attractions in the", + "boxes": [ + 1294, + 397, + 1604, + 415 + ] + }, + { + "text": "10 of the Most Popular Tourist", + "boxes": [ + 1638, + 393, + 1832, + 417 + ] + }, + { + "text": "Wor", + "boxes": [ + 22, + 417, + 44, + 431 + ] + }, + { + "text": "lorld - Travel News", + "boxes": [ + 1298, + 415, + 1420, + 433 + ] + }, + { + "text": "Attractions in Paris", + "boxes": [ + 1642, + 415, + 1758, + 433 + ] + }, + { + "text": "Why I choose to do some of the", + "boxes": [ + 302, + 637, + 506, + 655 + ] + }, + { + "text": "MostBea", + "boxes": [ + 552, + 637, + 604, + 655 + ] + }, + { + "text": "Top10", + "boxes": [ + 1120, + 637, + 1164, + 655 + ] + }, + { + "text": "lost Popular Tourist Attractions Ii", + "boxes": [ + 1172, + 637, + 1380, + 655 + ] + }, + { + "text": "World Touri", + "boxes": [ + 1454, + 637, + 1524, + 655 + ] + }, + { + "text": "The 25 best-valu", + "boxes": [ + 1584, + 637, + 1692, + 655 + ] + }, + { + "text": "2023I", + "boxes": [ + 20, + 655, + 60, + 671 + ] + }, + { + "text": "touristy things and see the", + "boxes": [ + 302, + 651, + 478, + 675 + ] + }, + { + "text": " Tanya Tany", + "boxes": [ + 548, + 651, + 626, + 675 + ] + }, + { + "text": "England Th - vrogue.co", + "boxes": [ + 1116, + 649, + 1266, + 675 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 653, + 1516, + 671 + ] + }, + { + "text": "Tourist Attractions", + "boxes": [ + 936, + 868, + 1046, + 891 + ] + }, + { + "text": "Travel | uS Ne", + "boxes": [ + 20, + 891, + 116, + 909 + ] + }, + { + "text": "Collage·", + "boxes": [ + 936, + 891, + 988, + 909 + ] + }, + { + "text": "Bucket List", + "boxes": [ + 986, + 885, + 1068, + 911 + ] + }, + { + "text": "Attractions of Duba", + "boxes": [ + 1114, + 885, + 1238, + 911 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 308, + 87 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 26, + 119, + 190, + 147 + ] + }, + { + "text": "Q", + "boxes": [ + 282, + 123, + 298, + 141 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 262, + 169, + 432, + 199 + ] + }, + { + "text": "Top 50 Most Visited Tourist Attractions in the World", + "boxes": [ + 1218, + 242, + 1608, + 266 + ] + }, + { + "text": "e.com I 840×560 png |1yrago", + "boxes": [ + 1326, + 335, + 1542, + 370 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1228, + 375, + 1272, + 395 + ] + }, + { + "text": "Top 50 Most", + "boxes": [ + 13, + 386, + 95, + 420 + ] + }, + { + "text": "Worl", + "boxes": [ + 22, + 417, + 48, + 431 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 653, + 76, + 671 + ] + }, + { + "text": "lorld tourist", + "boxes": [ + 224, + 1048, + 302, + 1072 + ] + } + ] + }, + { + "frame_id": "180_1_7", + "episode_id": 180, + "env_id": 1, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.955045700073242, + "advantages": -0.5509586334228516, + "generated_text": "{\n \"intent\": \"My goal is to explore different functionalities and tools available on this computer desktop environment. Currently, I'm focusing on selecting an action that will allow me to explore Bing search and translation features effectively.\",\n \"action\": \"DoubleClick(750, 145)\"\n}", + "action_string": "DoubleClick(750, 145)", + "intent_string": "My goal is to explore different functionalities and tools available on this computer desktop environment. Currently, I'm focusing on selecting an action that will allow me to explore Bing search and translation features effectively.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": "一", + "boxes": [ + 24, + 133, + 52, + 155 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 40, + 125, + 184, + 165 + ] + }, + { + "text": "在 Web上搜索", + "boxes": [ + 642, + 129, + 758, + 161 + ] + }, + { + "text": "文本", + "boxes": [ + 770, + 204, + 804, + 224 + ] + }, + { + "text": "翻译工具", + "boxes": [ + 826, + 202, + 892, + 228 + ] + }, + { + "text": "对话", + "boxes": [ + 910, + 195, + 959, + 230 + ] + }, + { + "text": "应用", + "boxes": [ + 978, + 201, + 1018, + 228 + ] + }, + { + "text": "商务用", + "boxes": [ + 1040, + 202, + 1094, + 228 + ] + }, + { + "text": "帮助", + "boxes": [ + 1122, + 202, + 1152, + 224 + ] + }, + { + "text": "英语", + "boxes": [ + 286, + 298, + 308, + 312 + ] + }, + { + "text": "1", + "boxes": [ + 952, + 298, + 970, + 314 + ] + }, + { + "text": "中文 (简体)", + "boxes": [ + 1008, + 294, + 1082, + 318 + ] + }, + { + "text": "输入文本", + "boxes": [ + 256, + 341, + 462, + 405 + ] + }, + { + "text": "←国", + "boxes": [ + 820, + 455, + 948, + 498 + ] + }, + { + "text": "语气", + "boxes": [ + 1010, + 469, + 1042, + 490 + ] + }, + { + "text": "广泛使用的短语", + "boxes": [ + 258, + 522, + 390, + 554 + ] + }, + { + "text": "Most popular", + "boxes": [ + 262, + 562, + 354, + 594 + ] + }, + { + "text": ">", + "boxes": [ + 1636, + 570, + 1652, + 588 + ] + }, + { + "text": "Thank you", + "boxes": [ + 989, + 609, + 1059, + 642 + ] + }, + { + "text": "Good morni", + "boxes": [ + 262, + 657, + 334, + 675 + ] + }, + { + "text": "Nice to meet you", + "boxes": [ + 260, + 697, + 368, + 721 + ] + }, + { + "text": "You are welcome", + "boxes": [ + 990, + 697, + 1100, + 721 + ] + }, + { + "text": "Good bye", + "boxes": [ + 255, + 732, + 327, + 763 + ] + }, + { + "text": " Good night ", + "boxes": [ + 985, + 728, + 1067, + 767 + ] + }, + { + "text": "隐私声明和Cookie", + "boxes": [ + 18, + 1005, + 132, + 1028 + ] + }, + { + "text": "@ 2025 Mic", + "boxes": [ + 1778, + 1009, + 1844, + 1026 + ] + }, + { + "text": "icrosoft Tra", + "boxes": [ + 226, + 1050, + 300, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": "一", + "boxes": [ + 24, + 133, + 52, + 153 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 40, + 125, + 184, + 165 + ] + }, + { + "text": "在 Web上搜索", + "boxes": [ + 640, + 127, + 762, + 165 + ] + }, + { + "text": "文本", + "boxes": [ + 770, + 204, + 804, + 224 + ] + }, + { + "text": "翻译工具", + "boxes": [ + 826, + 202, + 892, + 228 + ] + }, + { + "text": "对话", + "boxes": [ + 911, + 197, + 957, + 230 + ] + }, + { + "text": "应用", + "boxes": [ + 978, + 201, + 1018, + 228 + ] + }, + { + "text": "商务用", + "boxes": [ + 1040, + 202, + 1094, + 228 + ] + }, + { + "text": "帮助", + "boxes": [ + 1122, + 202, + 1152, + 224 + ] + }, + { + "text": "英语", + "boxes": [ + 286, + 298, + 308, + 312 + ] + }, + { + "text": "1", + "boxes": [ + 952, + 298, + 970, + 314 + ] + }, + { + "text": "中文 (简体)", + "boxes": [ + 1008, + 294, + 1082, + 318 + ] + }, + { + "text": "输入文本", + "boxes": [ + 256, + 341, + 462, + 405 + ] + }, + { + "text": "←国", + "boxes": [ + 820, + 455, + 948, + 498 + ] + }, + { + "text": "语气", + "boxes": [ + 1010, + 469, + 1042, + 490 + ] + }, + { + "text": "广泛使用的短语", + "boxes": [ + 258, + 522, + 390, + 554 + ] + }, + { + "text": "Most popular", + "boxes": [ + 262, + 562, + 354, + 594 + ] + }, + { + "text": ">", + "boxes": [ + 1636, + 570, + 1652, + 588 + ] + }, + { + "text": "Thank you", + "boxes": [ + 989, + 609, + 1059, + 642 + ] + }, + { + "text": "Good morning", + "boxes": [ + 262, + 657, + 346, + 675 + ] + }, + { + "text": "Nice to meet you", + "boxes": [ + 258, + 697, + 368, + 721 + ] + }, + { + "text": "You are welcome", + "boxes": [ + 990, + 697, + 1100, + 721 + ] + }, + { + "text": "Good bye", + "boxes": [ + 255, + 732, + 327, + 763 + ] + }, + { + "text": " Good night ", + "boxes": [ + 985, + 728, + 1067, + 767 + ] + }, + { + "text": "隐私声明和Cookie", + "boxes": [ + 18, + 1005, + 130, + 1028 + ] + }, + { + "text": "法律声明广告帮助", + "boxes": [ + 144, + 1003, + 320, + 1032 + ] + }, + { + "text": "反馈", + "boxes": [ + 318, + 1007, + 358, + 1032 + ] + }, + { + "text": " 2025 Mic", + "boxes": [ + 1780, + 1009, + 1844, + 1026 + ] + } + ] + }, + { + "frame_id": "180_2_7", + "episode_id": 180, + "env_id": 2, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.188720703125, + "advantages": -0.2285001128911972, + "generated_text": "{\n \"intent\": \"Click the search icon to start exploring more about iPhones or other Apple products available.\",\n \"action\": \"Click(777, 120)\"\n}", + "action_string": "Click(777, 120)", + "intent_string": "Click the search icon to start exploring more about iPhones or other Apple products available.", + "before_screen_ocr_results": [ + { + "text": "iPhone - Apple", + "boxes": [ + 30, + 32, + 126, + 50 + ] + }, + { + "text": "?", + "boxes": [ + 262, + 73, + 274, + 85 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 986, + 109, + 1046, + 129 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1082, + 111, + 1158, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "O", + "boxes": [ + 1366, + 105, + 1394, + 133 + ] + }, + { + "text": "B", + "boxes": [ + 456, + 157, + 522, + 218 + ] + }, + { + "text": "C0", + "boxes": [ + 560, + 155, + 630, + 226 + ] + }, + { + "text": "0", + "boxes": [ + 668, + 161, + 718, + 220 + ] + }, + { + "text": "D", + "boxes": [ + 754, + 159, + 818, + 222 + ] + }, + { + "text": "18", + "boxes": [ + 1106, + 155, + 1170, + 222 + ] + }, + { + "text": "00", + "boxes": [ + 846, + 165, + 920, + 222 + ] + }, + { + "text": "?", + "boxes": [ + 942, + 179, + 994, + 222 + ] + }, + { + "text": "·", + "boxes": [ + 1026, + 179, + 1066, + 218 + ] + }, + { + "text": "口", + "boxes": [ + 1210, + 173, + 1276, + 220 + ] + }, + { + "text": "OS", + "boxes": [ + 1306, + 171, + 1354, + 218 + ] + }, + { + "text": "iphone 16 Pro", + "boxes": [ + 448, + 218, + 530, + 244 + ] + }, + { + "text": "ipPhone 16 ", + "boxes": [ + 564, + 218, + 626, + 244 + ] + }, + { + "text": "iphone 16e", + "boxes": [ + 660, + 218, + 726, + 244 + ] + }, + { + "text": "iPhone 15", + "boxes": [ + 758, + 218, + 820, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 852, + 218, + 908, + 244 + ] + }, + { + "text": "AirPods", + "boxes": [ + 944, + 218, + 994, + 246 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1026, + 218, + 1066, + 246 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1102, + 218, + 1172, + 244 + ] + }, + { + "text": "Apple Card", + "boxes": [ + 1210, + 218, + 1274, + 244 + ] + }, + { + "text": "ios 18", + "boxes": [ + 1308, + 218, + 1350, + 244 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 1386, + 218, + 1458, + 244 + ] + }, + { + "text": "iPhone", + "boxes": [ + 101, + 386, + 367, + 483 + ] + }, + { + "text": "Designed to be loved.", + "boxes": [ + 1520, + 433, + 1800, + 474 + ] + }, + { + "text": "MondaySeptember9", + "boxes": [ + 832, + 554, + 1050, + 584 + ] + }, + { + "text": "9:41", + "boxes": [ + 834, + 576, + 1052, + 695 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "", + "boxes": [ + 370, + 159, + 414, + 212 + ] + }, + { + "text": "0", + "boxes": [ + 468, + 155, + 520, + 214 + ] + }, + { + "text": "0", + "boxes": [ + 574, + 163, + 618, + 206 + ] + }, + { + "text": "", + "boxes": [ + 676, + 163, + 716, + 204 + ] + }, + { + "text": "6", + "boxes": [ + 770, + 161, + 828, + 214 + ] + }, + { + "text": "0", + "boxes": [ + 864, + 159, + 940, + 214 + ] + }, + { + "text": "00", + "boxes": [ + 964, + 163, + 1034, + 216 + ] + }, + { + "text": "", + "boxes": [ + 1050, + 167, + 1078, + 210 + ] + }, + { + "text": "?", + "boxes": [ + 1208, + 169, + 1246, + 210 + ] + }, + { + "text": "00", + "boxes": [ + 1476, + 161, + 1562, + 214 + ] + }, + { + "text": "9?", + "boxes": [ + 1118, + 177, + 1166, + 214 + ] + }, + { + "text": "?", + "boxes": [ + 1302, + 171, + 1358, + 214 + ] + }, + { + "text": "", + "boxes": [ + 1400, + 171, + 1454, + 212 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 458, + 214, + 532, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 556, + 214, + 636, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 660, + 214, + 738, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 764, + 214, + 838, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 864, + 212, + 940, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 964, + 214, + 1022, + 240 + ] + }, + { + "text": "Bands", + "boxes": [ + 1048, + 216, + 1084, + 236 + ] + }, + { + "text": "AirPods", + "boxes": [ + 1122, + 218, + 1162, + 234 + ] + }, + { + "text": "atchos11", + "boxes": [ + 1408, + 216, + 1460, + 234 + ] + }, + { + "text": "Ultra 2", + "boxes": [ + 468, + 232, + 516, + 252 + ] + }, + { + "text": "Nike", + "boxes": [ + 680, + 232, + 710, + 252 + ] + }, + { + "text": "Hermes", + "boxes": [ + 774, + 232, + 824, + 250 + ] + }, + { + "text": "Studio", + "boxes": [ + 878, + 232, + 924, + 252 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 112, + 387, + 574, + 488 + ] + }, + { + "text": "The ultimate device", + "boxes": [ + 1536, + 393, + 1796, + 439 + ] + }, + { + "text": "for a healthy life.", + "boxes": [ + 1538, + 433, + 1752, + 469 + ] + }, + { + "text": "ple Watch", + "boxes": [ + 234, + 1050, + 300, + 1070 + ] + } + ] + }, + { + "frame_id": "180_3_7", + "episode_id": 180, + "env_id": 3, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.38378620147705, + "advantages": -1.339264988899231, + "generated_text": "{\n \"intent\": \"Clicking the 'Get Firefox for desktop' link will take the user to the Firefox download page where they can download the browser. This is a relevant action to continue exploring the environment by selecting and clicking an icon on a web page.\",\n \"action\": \"Click(347, 36)\"\n}", + "action_string": "Click(347, 36)", + "intent_string": "Clicking the 'Get Firefox for desktop' link will take the user to the Firefox download page where they can download the browser. This is a relevant action to continue exploring the environment by selecting and clicking an icon on a web page.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "↑→白", + "boxes": [ + 10, + 64, + 130, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 471 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": "Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "180_4_7", + "episode_id": 180, + "env_id": 4, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.610065460205078, + "advantages": 0.3529322147369385, + "generated_text": "{\n \"intent\": \"To continue exploring outside of the Bing search page, I will click the Home icon to return to the main desktop environment.\",\n \"action\": \"Click(113, 72)\"\n}", + "action_string": "Click(113, 72)", + "intent_string": "To continue exploring outside of the Bing search page, I will click the Home icon to return to the main desktop environment.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 614, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭中", + "boxes": [ + 1318, + 375, + 1358, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "京", + "boxes": [ + 1636, + 385, + 1654, + 401 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 792, + 449, + 972, + 478 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 673, + 1630, + 703 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1632, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1456, + 850, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13, 287", + "boxes": [ + 1626, + 893, + 1700, + 941 + ] + }, + { + "text": "南方+客户端·14小时", + "boxes": [ + 826, + 921, + 984, + 945 + ] + }, + { + "text": "新京报·10小时", + "boxes": [ + 1134, + 917, + 1264, + 947 + ] + }, + { + "text": "《双影奇境》电影改编权的竞拍报价已达数百", + "boxes": [ + 212, + 941, + 792, + 991 + ] + }, + { + "text": "AI还原！中山市博物馆这些干", + "boxes": [ + 826, + 941, + 1100, + 977 + ] + }, + { + "text": "幼儿园厨师餐具池里洗墩布被", + "boxes": [ + 1134, + 937, + 1408, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "r'.", + "boxes": [ + 1584, + 937, + 1636, + 983 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "年“神器”，实用", + "boxes": [ + 826, + 967, + 982, + 1003 + ] + }, + { + "text": "辞退，起诉幼儿园要求赔偿被", + "boxes": [ + 1132, + 961, + 1412, + 1007 + ] + }, + { + "text": "万美元级别", + "boxes": [ + 202, + 985, + 358, + 1022 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "驳回", + "boxes": [ + 1134, + 995, + 1190, + 1028 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "2", + "boxes": [ + 1140, + 1028, + 1172, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 11, + 25, + 87, + 54 + ] + }, + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "↑→", + "boxes": [ + 8, + 64, + 134, + 93 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_5_7", + "episode_id": 180, + "env_id": 5, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.391125679016113, + "advantages": 0.05080748721957207, + "generated_text": "{\n \"intent\": \"Clicking the Bing logo will help me access popular search engines or related content which could provide insights into different web pages.\",\n \"action\": \"DoubleClick(663, 507)\"\n}", + "action_string": "DoubleClick(663, 507)", + "intent_string": "Clicking the Bing logo will help me access popular search engines or related content which could provide insights into different web pages.", + "before_screen_ocr_results": [ + { + "text": "百日", + "boxes": [ + 61, + 63, + 129, + 94 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlights", + "boxes": [ + 492, + 619, + 580, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 988, + 671, + 1174, + 721 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "?", + "boxes": [ + 916, + 752, + 928, + 764 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 510, + 774, + 546, + 788 + ] + }, + { + "text": "apple", + "boxes": [ + 744, + 768, + 786, + 796 + ] + }, + { + "text": "amazo", + "boxes": [ + 992, + 774, + 1028, + 788 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1230, + 772, + 1268, + 792 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 504, + 788, + 664, + 816 + ] + }, + { + "text": "Apple", + "boxes": [ + 744, + 790, + 790, + 818 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 10, + 66, + 36, + 89 + ] + }, + { + "text": "→白", + "boxes": [ + 32, + 64, + 130, + 95 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 18, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 492, + 248, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 566, + 250, + 636, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 470, + 295, + 612, + 332 + ] + }, + { + "text": "O", + "boxes": [ + 1342, + 302, + 1356, + 324 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "", + "boxes": [ + 1544, + 363, + 1568, + 385 + ] + }, + { + "text": "中虹中", + "boxes": [ + 1318, + 373, + 1350, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "京", + "boxes": [ + 1636, + 387, + 1654, + 403 + ] + }, + { + "text": "8", + "boxes": [ + 1879, + 374, + 1896, + 391 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 482, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 413, + 916, + 443 + ] + }, + { + "text": "囍", + "boxes": [ + 976, + 404, + 997, + 424 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "欢甲", + "boxes": [ + 950, + 427, + 970, + 448 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 448, + 968, + 477 + ] + }, + { + "text": "6朱雀玄武救令", + "boxes": [ + 1108, + 453, + 1226, + 471 + ] + }, + { + "text": "）图卢兹的暮", + "boxes": [ + 1532, + 679, + 1620, + 697 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 270, + 748 + ] + } + ] + }, + { + "frame_id": "180_6_7", + "episode_id": 180, + "env_id": 6, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.88693618774414, + "advantages": -0.6449459791183472, + "generated_text": "{\n \"intent\": \"Click on Download for Linux 32-bit to select a download option and continue exploring the environment.\",\n \"action\": \"DoubleClick(960, 606)\"\n}", + "action_string": "DoubleClick(960, 606)", + "intent_string": "Click on Download for Linux 32-bit to select a download option and continue exploring the environment.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1608, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 471 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三四?", + "boxes": [ + 1788, + 66, + 1880, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "pn", + "boxes": [ + 1184, + 427, + 1238, + 463 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "180_7_7", + "episode_id": 180, + "env_id": 7, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.93855094909668, + "advantages": 0.8062239289283752, + "generated_text": "{\n \"intent\": \"To continue exploring the available information about world tourist attractions and potentially collect additional images, I will click on the first image preview to view the page.\",\n \"action\": \"DoubleClick(678, 42)\"\n}", + "action_string": "DoubleClick(678, 42)", + "intent_string": "To continue exploring the available information about world tourist attractions and potentially collect additional images, I will click on the first image preview to view the page.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 310, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 141 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 173, + 236, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "These Are The 10 Most Popular Tourist Attractions in the", + "boxes": [ + 1322, + 242, + 1750, + 266 + ] + }, + { + "text": "World -Journeyz", + "boxes": [ + 1322, + 262, + 1454, + 286 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 943, + 76, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "lozilla Fi", + "boxes": [ + 908, + 4, + 980, + 22 + ] + }, + { + "text": "→白", + "boxes": [ + 38, + 66, + 130, + 91 + ] + }, + { + "text": "Qearch with Bing or enter address", + "boxes": [ + 244, + 57, + 524, + 97 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 164, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 996, + 534, + 1036, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "180_0_8", + "episode_id": 180, + "env_id": 0, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.046843528747559, + "advantages": -0.4242827296257019, + "generated_text": "{\n \"intent\": \"Click the close button of the Bing search window to continue exploring the environment outside of the current search result.\",\n \"action\": \"Click(1787, 182)\"\n}", + "action_string": "Click(1787, 182)", + "intent_string": "Click the close button of the Bing search window to continue exploring the environment outside of the current search result.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 308, + 87 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 26, + 119, + 190, + 147 + ] + }, + { + "text": "Q", + "boxes": [ + 282, + 123, + 298, + 141 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 262, + 169, + 432, + 199 + ] + }, + { + "text": "Top 50 Most Visited Tourist Attractions in the World", + "boxes": [ + 1218, + 242, + 1608, + 266 + ] + }, + { + "text": "e.com I 840×560 png |1yrago", + "boxes": [ + 1326, + 335, + 1542, + 370 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1228, + 375, + 1272, + 395 + ] + }, + { + "text": "Top 50 Most", + "boxes": [ + 13, + 386, + 95, + 420 + ] + }, + { + "text": "Worl", + "boxes": [ + 22, + 417, + 48, + 431 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 653, + 76, + 671 + ] + }, + { + "text": "lorld tourist", + "boxes": [ + 224, + 1048, + 302, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 44, + 69, + 124, + 87 + ] + }, + { + "text": "fii886ocr", + "boxes": [ + 1424, + 365, + 1514, + 389 + ] + }, + { + "text": "Most complained about tourist attractions in the", + "boxes": [ + 1294, + 397, + 1604, + 415 + ] + }, + { + "text": "10 of the Most Popular Tourist", + "boxes": [ + 1638, + 393, + 1832, + 417 + ] + }, + { + "text": "Wor", + "boxes": [ + 22, + 417, + 44, + 431 + ] + }, + { + "text": "lorld - Travel News", + "boxes": [ + 1298, + 415, + 1420, + 433 + ] + }, + { + "text": "Attractions in Paris", + "boxes": [ + 1642, + 415, + 1758, + 433 + ] + }, + { + "text": "Why I choose to do some of the", + "boxes": [ + 302, + 637, + 506, + 655 + ] + }, + { + "text": "MostBea", + "boxes": [ + 552, + 637, + 604, + 655 + ] + }, + { + "text": "Top10", + "boxes": [ + 1120, + 637, + 1164, + 655 + ] + }, + { + "text": "lost Popular Tourist Attractions Ii", + "boxes": [ + 1172, + 637, + 1380, + 655 + ] + }, + { + "text": "World Touri", + "boxes": [ + 1454, + 637, + 1524, + 655 + ] + }, + { + "text": "The 25 best-valu", + "boxes": [ + 1584, + 637, + 1692, + 655 + ] + }, + { + "text": "2023I", + "boxes": [ + 20, + 655, + 60, + 671 + ] + }, + { + "text": "touristy things and see the", + "boxes": [ + 302, + 651, + 478, + 675 + ] + }, + { + "text": " Tanya Tany", + "boxes": [ + 548, + 651, + 626, + 675 + ] + }, + { + "text": "England Th - vrogue.co", + "boxes": [ + 1116, + 649, + 1266, + 675 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 653, + 1516, + 671 + ] + }, + { + "text": "Tourist Attractions", + "boxes": [ + 936, + 868, + 1046, + 891 + ] + }, + { + "text": "Travel | uS Ne", + "boxes": [ + 20, + 891, + 116, + 909 + ] + }, + { + "text": "Collage·", + "boxes": [ + 936, + 891, + 988, + 909 + ] + }, + { + "text": "Bucket List", + "boxes": [ + 986, + 885, + 1068, + 911 + ] + }, + { + "text": "Attractions of Duba", + "boxes": [ + 1114, + 885, + 1238, + 911 + ] + } + ] + }, + { + "frame_id": "180_1_8", + "episode_id": 180, + "env_id": 1, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.984996795654297, + "advantages": -0.5096277594566345, + "generated_text": "{\n \"intent\": \"The current screen is showing the Bing Translator page, which can be used for learning languages or translating texts. Clicking the search bar would allow me to explore more by searching for related topics or resources.\",\n \"action\": \"Click(750, 141)\"\n}", + "action_string": "Click(750, 141)", + "intent_string": "The current screen is showing the Bing Translator page, which can be used for learning languages or translating texts. Clicking the search bar would allow me to explore more by searching for related topics or resources.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": "一", + "boxes": [ + 24, + 133, + 52, + 153 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 40, + 125, + 184, + 165 + ] + }, + { + "text": "在 Web上搜索", + "boxes": [ + 640, + 127, + 762, + 165 + ] + }, + { + "text": "文本", + "boxes": [ + 770, + 204, + 804, + 224 + ] + }, + { + "text": "翻译工具", + "boxes": [ + 826, + 202, + 892, + 228 + ] + }, + { + "text": "对话", + "boxes": [ + 911, + 197, + 957, + 230 + ] + }, + { + "text": "应用", + "boxes": [ + 978, + 201, + 1018, + 228 + ] + }, + { + "text": "商务用", + "boxes": [ + 1040, + 202, + 1094, + 228 + ] + }, + { + "text": "帮助", + "boxes": [ + 1122, + 202, + 1152, + 224 + ] + }, + { + "text": "英语", + "boxes": [ + 286, + 298, + 308, + 312 + ] + }, + { + "text": "1", + "boxes": [ + 952, + 298, + 970, + 314 + ] + }, + { + "text": "中文 (简体)", + "boxes": [ + 1008, + 294, + 1082, + 318 + ] + }, + { + "text": "输入文本", + "boxes": [ + 256, + 341, + 462, + 405 + ] + }, + { + "text": "←国", + "boxes": [ + 820, + 455, + 948, + 498 + ] + }, + { + "text": "语气", + "boxes": [ + 1010, + 469, + 1042, + 490 + ] + }, + { + "text": "广泛使用的短语", + "boxes": [ + 258, + 522, + 390, + 554 + ] + }, + { + "text": "Most popular", + "boxes": [ + 262, + 562, + 354, + 594 + ] + }, + { + "text": ">", + "boxes": [ + 1636, + 570, + 1652, + 588 + ] + }, + { + "text": "Thank you", + "boxes": [ + 989, + 609, + 1059, + 642 + ] + }, + { + "text": "Good morning", + "boxes": [ + 262, + 657, + 346, + 675 + ] + }, + { + "text": "Nice to meet you", + "boxes": [ + 258, + 697, + 368, + 721 + ] + }, + { + "text": "You are welcome", + "boxes": [ + 990, + 697, + 1100, + 721 + ] + }, + { + "text": "Good bye", + "boxes": [ + 255, + 732, + 327, + 763 + ] + }, + { + "text": " Good night ", + "boxes": [ + 985, + 728, + 1067, + 767 + ] + }, + { + "text": "隐私声明和Cookie", + "boxes": [ + 18, + 1005, + 130, + 1028 + ] + }, + { + "text": "法律声明广告帮助", + "boxes": [ + 144, + 1003, + 320, + 1032 + ] + }, + { + "text": "反馈", + "boxes": [ + 318, + 1007, + 358, + 1032 + ] + }, + { + "text": " 2025 Mic", + "boxes": [ + 1780, + 1009, + 1844, + 1026 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": "一", + "boxes": [ + 24, + 133, + 52, + 153 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 40, + 125, + 184, + 165 + ] + }, + { + "text": "在 Web上搜", + "boxes": [ + 642, + 127, + 760, + 159 + ] + }, + { + "text": "文本", + "boxes": [ + 770, + 204, + 804, + 224 + ] + }, + { + "text": "翻译工具", + "boxes": [ + 826, + 202, + 892, + 228 + ] + }, + { + "text": "对话", + "boxes": [ + 914, + 201, + 958, + 228 + ] + }, + { + "text": "应用", + "boxes": [ + 978, + 201, + 1018, + 228 + ] + }, + { + "text": "商务用", + "boxes": [ + 1040, + 202, + 1094, + 228 + ] + }, + { + "text": "帮助", + "boxes": [ + 1122, + 202, + 1152, + 224 + ] + }, + { + "text": "英语", + "boxes": [ + 286, + 298, + 308, + 312 + ] + }, + { + "text": "1", + "boxes": [ + 952, + 298, + 970, + 314 + ] + }, + { + "text": "中文 (简体)", + "boxes": [ + 1008, + 294, + 1082, + 318 + ] + }, + { + "text": "输入文本", + "boxes": [ + 256, + 341, + 462, + 405 + ] + }, + { + "text": "←国", + "boxes": [ + 820, + 455, + 948, + 498 + ] + }, + { + "text": "语气", + "boxes": [ + 1010, + 469, + 1042, + 490 + ] + }, + { + "text": "广泛使用的短语", + "boxes": [ + 258, + 522, + 390, + 554 + ] + }, + { + "text": "Most popular", + "boxes": [ + 262, + 562, + 354, + 594 + ] + }, + { + "text": ">", + "boxes": [ + 1636, + 570, + 1652, + 588 + ] + }, + { + "text": "Thank you", + "boxes": [ + 989, + 609, + 1059, + 642 + ] + }, + { + "text": "Good morning", + "boxes": [ + 262, + 657, + 346, + 675 + ] + }, + { + "text": "How are you?", + "boxes": [ + 996, + 657, + 1076, + 675 + ] + }, + { + "text": "Nice to meet you", + "boxes": [ + 258, + 697, + 368, + 721 + ] + }, + { + "text": "You are welcome", + "boxes": [ + 990, + 697, + 1100, + 721 + ] + }, + { + "text": "Good bye", + "boxes": [ + 255, + 732, + 327, + 763 + ] + }, + { + "text": " Good night ", + "boxes": [ + 985, + 728, + 1067, + 767 + ] + }, + { + "text": "隐私声明和Cookie", + "boxes": [ + 18, + 1005, + 130, + 1028 + ] + }, + { + "text": "法律声明广告帮助", + "boxes": [ + 144, + 1003, + 320, + 1032 + ] + }, + { + "text": "反馈", + "boxes": [ + 318, + 1007, + 358, + 1032 + ] + }, + { + "text": " 2025 Mic", + "boxes": [ + 1780, + 1009, + 1844, + 1026 + ] + } + ] + }, + { + "frame_id": "180_2_8", + "episode_id": 180, + "env_id": 2, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.579495429992676, + "advantages": 0.31074726581573486, + "generated_text": "{\n \"intent\": \"The webpage is about Apple Watch products, and exploring the link 'Shop Apple Watch' will likely provide product information and allows you to engage further with the webpage by clicking on specific products or the element 'Watch Studio', which could have more detailed information. This action aims to get more information about Apple Watch products to achieve the goal of exploring this environment.\",\n \"action\": \"DoubleClick(1139, 291)\"\n}", + "action_string": "DoubleClick(1139, 291)", + "intent_string": "The webpage is about Apple Watch products, and exploring the link 'Shop Apple Watch' will likely provide product information and allows you to engage further with the webpage by clicking on specific products or the element 'Watch Studio', which could have more detailed information. This action aims to get more information about Apple Watch products to achieve the goal of exploring this environment.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 204, + 34, + 244, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 68, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 66, + 128, + 91 + ] + }, + { + "text": "", + "boxes": [ + 370, + 159, + 414, + 212 + ] + }, + { + "text": "0", + "boxes": [ + 468, + 155, + 520, + 214 + ] + }, + { + "text": "0", + "boxes": [ + 574, + 163, + 618, + 206 + ] + }, + { + "text": "O", + "boxes": [ + 674, + 161, + 718, + 210 + ] + }, + { + "text": "6", + "boxes": [ + 770, + 161, + 828, + 214 + ] + }, + { + "text": "0", + "boxes": [ + 864, + 159, + 940, + 214 + ] + }, + { + "text": "00", + "boxes": [ + 964, + 163, + 1034, + 216 + ] + }, + { + "text": "", + "boxes": [ + 1050, + 167, + 1078, + 210 + ] + }, + { + "text": "?", + "boxes": [ + 1208, + 169, + 1246, + 210 + ] + }, + { + "text": "00", + "boxes": [ + 1476, + 161, + 1562, + 214 + ] + }, + { + "text": "9?", + "boxes": [ + 1118, + 177, + 1166, + 214 + ] + }, + { + "text": "?", + "boxes": [ + 1302, + 171, + 1358, + 214 + ] + }, + { + "text": "O", + "boxes": [ + 1398, + 171, + 1454, + 212 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 458, + 214, + 532, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 556, + 214, + 636, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 660, + 214, + 738, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 764, + 214, + 838, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 864, + 212, + 940, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 964, + 214, + 1022, + 240 + ] + }, + { + "text": "Bands", + "boxes": [ + 1048, + 216, + 1084, + 236 + ] + }, + { + "text": "AirPods", + "boxes": [ + 1122, + 218, + 1162, + 234 + ] + }, + { + "text": "atchos 11", + "boxes": [ + 1406, + 216, + 1462, + 234 + ] + }, + { + "text": "Shop Watch", + "boxes": [ + 1490, + 216, + 1554, + 234 + ] + }, + { + "text": "Ultra 2", + "boxes": [ + 468, + 232, + 516, + 252 + ] + }, + { + "text": "Nike", + "boxes": [ + 680, + 232, + 710, + 252 + ] + }, + { + "text": "Hermes", + "boxes": [ + 774, + 232, + 824, + 250 + ] + }, + { + "text": "Studio", + "boxes": [ + 878, + 232, + 924, + 252 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 108, + 381, + 574, + 489 + ] + }, + { + "text": "The ultimate device", + "boxes": [ + 1536, + 393, + 1796, + 439 + ] + }, + { + "text": "for a healthy life.", + "boxes": [ + 1538, + 433, + 1752, + 469 + ] + }, + { + "text": "e Watch", + "boxes": [ + 250, + 1050, + 302, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Apple x", + "boxes": [ + 158, + 32, + 220, + 50 + ] + }, + { + "text": "目", + "boxes": [ + 1544, + 69, + 1562, + 85 + ] + }, + { + "text": "AirPod", + "boxes": [ + 912, + 113, + 946, + 127 + ] + }, + { + "text": "TV & Hom", + "boxes": [ + 986, + 111, + 1042, + 129 + ] + }, + { + "text": "Shop Apple Watch", + "boxes": [ + 138, + 222, + 532, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1562, + 230, + 1706, + 254 + ] + }, + { + "text": "Watch Specialist ", + "boxes": [ + 1650, + 254, + 1764, + 272 + ] + }, + { + "text": "Visit an Apple Store", + "boxes": [ + 1556, + 284, + 1694, + 312 + ] + }, + { + "text": "?", + "boxes": [ + 1524, + 294, + 1550, + 318 + ] + }, + { + "text": "All Models ", + "boxes": [ + 144, + 389, + 230, + 413 + ] + }, + { + "text": " Shopping guides ", + "boxes": [ + 260, + 387, + 400, + 417 + ] + }, + { + "text": "Ways to Save", + "boxes": [ + 430, + 387, + 542, + 417 + ] + }, + { + "text": "Apple Watch Bands", + "boxes": [ + 574, + 387, + 730, + 417 + ] + }, + { + "text": "Setup and Support", + "boxes": [ + 764, + 389, + 914, + 413 + ] + }, + { + "text": "The Apple Watch Experience", + "boxes": [ + 946, + 387, + 1176, + 417 + ] + }, + { + "text": "Special Stores", + "boxes": [ + 1210, + 389, + 1322, + 413 + ] + }, + { + "text": "All models. Take your pick.", + "boxes": [ + 138, + 472, + 464, + 508 + ] + }, + { + "text": "Apple Watch Series 10", + "boxes": [ + 168, + 584, + 456, + 619 + ] + }, + { + "text": "Apple Watch Ultra 2", + "boxes": [ + 584, + 573, + 846, + 624 + ] + }, + { + "text": "Apple Watch SE", + "boxes": [ + 1004, + 575, + 1216, + 624 + ] + }, + { + "text": "Apple Watch Hermes", + "boxes": [ + 1424, + 575, + 1702, + 624 + ] + }, + { + "text": "Appl", + "boxes": [ + 1846, + 582, + 1914, + 621 + ] + }, + { + "text": "Series 10", + "boxes": [ + 1426, + 615, + 1546, + 647 + ] + }, + { + "text": "Uitra", + "boxes": [ + 1845, + 611, + 1915, + 654 + ] + }, + { + "text": "?", + "boxes": [ + 658, + 705, + 742, + 744 + ] + }, + { + "text": "10:09", + "boxes": [ + 664, + 737, + 738, + 768 + ] + }, + { + "text": "", + "boxes": [ + 668, + 802, + 716, + 830 + ] + }, + { + "text": "$1399 or", + "boxes": [ + 1848, + 977, + 1906, + 1003 + ] + } + ] + }, + { + "frame_id": "180_3_8", + "episode_id": 180, + "env_id": 3, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.07690715789795, + "advantages": -0.38279658555984497, + "generated_text": "{\n \"intent\": \"Click the 'Firefox browsers' button to explore the download options and potentially select a download option for Linux.\",\n \"action\": \"Click(838, 124)\"\n}", + "action_string": "Click(838, 124)", + "intent_string": "Click the 'Firefox browsers' button to explore the download options and potentially select a download option for Linux.", + "before_screen_ocr_results": [ + { + "text": "↑→白", + "boxes": [ + 10, + 64, + 130, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 471 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": "Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "↑→白", + "boxes": [ + 10, + 64, + 130, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1874, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": "Firefox bsowsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 109, + 1050, + 141 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": " Firefox for Desktop", + "boxes": [ + 636, + 191, + 836, + 220 + ] + }, + { + "text": " Firefox for ios", + "boxes": [ + 638, + 228, + 792, + 258 + ] + }, + { + "text": " Firefox for Android", + "boxes": [ + 636, + 266, + 832, + 296 + ] + }, + { + "text": " Firefox Focus", + "boxes": [ + 634, + 299, + 786, + 334 + ] + }, + { + "text": " Firefox blog", + "boxes": [ + 638, + 337, + 772, + 374 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 415, + 1238, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": "Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "Get Firefox fo.", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "180_4_8", + "episode_id": 180, + "env_id": 4, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.199018478393555, + "advantages": -0.21428976953029633, + "generated_text": "{\n \"intent\": \"To open an interesting or useful website by clicking on one of the highlighted sites.\",\n \"action\": \"DoubleClick(1087, 705)\"\n}", + "action_string": "DoubleClick(1087, 705)", + "intent_string": "To open an interesting or useful website by clicking on one of the highlighted sites.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 14, + 32, + 86, + 50 + ] + }, + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "↑→", + "boxes": [ + 8, + 64, + 134, + 93 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "94", + "boxes": [ + 1823, + 110, + 1862, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 121, + 1474, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 918, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 916, + 560 + ] + }, + { + "text": "Shop deals in Fashion ", + "boxes": [ + 980, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 354, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1460, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1555, + 996, + 1647, + 1025 + ] + } + ] + }, + { + "frame_id": "180_5_8", + "episode_id": 180, + "env_id": 5, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.335901260375977, + "advantages": -1.4053435325622559, + "generated_text": "{\n \"intent\": \"Click(277, 150) to check images, videos or maps section for potential information and further exploration.\",\n \"action\": \"Click(277, 150)\"\n}", + "action_string": "Click(277, 150)", + "intent_string": "Click(277, 150) to check images, videos or maps section for potential information and further exploration.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 10, + 66, + 36, + 89 + ] + }, + { + "text": "→白", + "boxes": [ + 32, + 64, + 130, + 95 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 18, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 492, + 248, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 566, + 250, + 636, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1342, + 302, + 1356, + 324 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "", + "boxes": [ + 1544, + 363, + 1568, + 385 + ] + }, + { + "text": "中虹中", + "boxes": [ + 1318, + 373, + 1350, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "京", + "boxes": [ + 1636, + 387, + 1654, + 403 + ] + }, + { + "text": "8", + "boxes": [ + 1879, + 374, + 1896, + 391 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 482, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 413, + 916, + 443 + ] + }, + { + "text": "囍", + "boxes": [ + 976, + 404, + 997, + 424 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "欢甲", + "boxes": [ + 950, + 427, + 970, + 448 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 448, + 968, + 477 + ] + }, + { + "text": "6朱雀玄武救令", + "boxes": [ + 1108, + 453, + 1226, + 471 + ] + }, + { + "text": "）图卢兹的暮", + "boxes": [ + 1532, + 679, + 1620, + 697 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 270, + 748 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "arch - Microsoft Bing", + "boxes": [ + 46, + 32, + 188, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 406, + 121, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 614, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1354, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "中", + "boxes": [ + 956, + 435, + 968, + 447 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 449, + 972, + 478 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 673, + 1630, + 703 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "Ww", + "boxes": [ + 1594, + 810, + 1628, + 830 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1632, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1456, + 850, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "高附时中国A50", + "boxes": [ + 1452, + 889, + 1550, + 941 + ] + }, + { + "text": "13, 287", + "boxes": [ + 1626, + 893, + 1700, + 941 + ] + }, + { + "text": "游侠网·5天", + "boxes": [ + 828, + 921, + 932, + 945 + ] + }, + { + "text": "1", + "boxes": [ + 1142, + 925, + 1162, + 941 + ] + }, + { + "text": "一点资讯·51分钟", + "boxes": [ + 1158, + 919, + 1274, + 949 + ] + }, + { + "text": "仅靠一只狗就能通关！《博德", + "boxes": [ + 824, + 941, + 1098, + 977 + ] + }, + { + "text": "进化5亿年肠道竟成“第二大", + "boxes": [ + 1136, + 941, + 1394, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "AI还原！中山市博物馆这些", + "boxes": [ + 206, + 951, + 552, + 981 + ] + }, + { + "text": "，实", + "boxes": [ + 718, + 953, + 766, + 979 + ] + }, + { + "text": "3》玩家用宠物咬死BOSS", + "boxes": [ + 828, + 971, + 1068, + 1001 + ] + }, + { + "text": "脑”！专家：我们的喜怒哀乐", + "boxes": [ + 1132, + 963, + 1402, + 1005 + ] + }, + { + "text": "用", + "boxes": [ + 202, + 983, + 240, + 1022 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "被它操控", + "boxes": [ + 1134, + 993, + 1230, + 1030 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + } + ] + }, + { + "frame_id": "180_6_8", + "episode_id": 180, + "env_id": 6, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.546915054321289, + "advantages": -1.1141562461853027, + "generated_text": "{\n \"intent\": \"Clicking the 'OK' button will open the Firefox installer and permit further exploration of the internet or the desktop environment after the installation is complete.\",\n \"action\": \"DoubleClick(1133, 670)\"\n}", + "action_string": "DoubleClick(1133, 670)", + "intent_string": "Clicking the 'OK' button will open the Firefox installer and permit further exploration of the internet or the desktop environment after the installation is complete.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三四?", + "boxes": [ + 1788, + 66, + 1880, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "pn", + "boxes": [ + 1184, + 427, + 1238, + 463 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop - Mozilla (Us) ", + "boxes": [ + 738, + 2, + 1048, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "。☆", + "boxes": [ + 1561, + 61, + 1631, + 92 + ] + }, + { + "text": "三四◎", + "boxes": [ + 1752, + 64, + 1876, + 93 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "firefox-138.0.4.tar.xz - xarchiver 0.5.4.14", + "boxes": [ + 776, + 322, + 1138, + 345 + ] + }, + { + "text": "Archive Action Help", + "boxes": [ + 664, + 343, + 828, + 373 + ] + }, + { + "text": "口", + "boxes": [ + 678, + 379, + 762, + 405 + ] + }, + { + "text": "&", + "boxes": [ + 752, + 373, + 1034, + 409 + ] + }, + { + "text": "Location:", + "boxes": [ + 668, + 419, + 730, + 445 + ] + }, + { + "text": "Archive tree", + "boxes": [ + 668, + 453, + 762, + 478 + ] + }, + { + "text": "Download in another language or for another operating system", + "boxes": [ + 704, + 889, + 1198, + 913 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "180_7_8", + "episode_id": 180, + "env_id": 7, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.571642875671387, + "advantages": 0.29991117119789124, + "generated_text": "{\n \"intent\": \"Clicking the Bing icon will allow me to search online and gather more information about world tourist attractions.\",\n \"action\": \"DoubleClick(664, 484)\"\n}", + "action_string": "DoubleClick(664, 484)", + "intent_string": "Clicking the Bing icon will allow me to search online and gather more information about world tourist attractions.", + "before_screen_ocr_results": [ + { + "text": "lozilla Fi", + "boxes": [ + 908, + 4, + 980, + 22 + ] + }, + { + "text": "→白", + "boxes": [ + 38, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 522, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 164, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 996, + 534, + 1036, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 8, + 64, + 42, + 91 + ] + }, + { + "text": "→", + "boxes": [ + 28, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 486, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 976, + 402, + 999, + 425 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "爆料正新鸡排生姐", + "boxes": [ + 492, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 272, + 750 + ] + }, + { + "text": "TRIBE NINE：战极死游", + "boxes": [ + 212, + 758, + 386, + 794 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "福", + "boxes": [ + 310, + 802, + 470, + 965 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 850, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1698, + 941 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 925, + 850, + 941 + ] + }, + { + "text": "一点资讯·51分钟", + "boxes": [ + 844, + 915, + 964, + 949 + ] + }, + { + "text": "EVCar·5天", + "boxes": [ + 1136, + 917, + 1250, + 947 + ] + }, + { + "text": "进化5亿年肠道竟成“第二大", + "boxes": [ + 822, + 937, + 1084, + 977 + ] + }, + { + "text": "全铝车身+性能升级！1953", + "boxes": [ + 1138, + 943, + 1414, + 973 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1630, + 981 + ] + }, + { + "text": "+3.2879", + "boxes": [ + 1626, + 939, + 1698, + 989 + ] + }, + { + "text": "一点资讯·22小时", + "boxes": [ + 234, + 953, + 336, + 983 + ] + }, + { + "text": "脑”！专家：我们的喜怒哀乐", + "boxes": [ + 822, + 963, + 1092, + 1005 + ] + }, + { + "text": "小高和刚原案游戏光速去世", + "boxes": [ + 196, + 973, + 560, + 1026 + ] + }, + { + "text": "款捷豹XK120经典重现，致", + "boxes": [ + 1134, + 967, + 1416, + 1003 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "敬赛道传奇", + "boxes": [ + 1135, + 992, + 1249, + 1031 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "被它操控", + "boxes": [ + 822, + 993, + 918, + 1030 + ] + }, + { + "text": "WrM", + "boxes": [ + 1594, + 1003, + 1628, + 1024 + ] + }, + { + "text": "8", + "boxes": [ + 236, + 1022, + 270, + 1036 + ] + }, + { + "text": "S", + "boxes": [ + 852, + 1028, + 874, + 1042 + ] + }, + { + "text": "24", + "boxes": [ + 1140, + 1026, + 1180, + 1046 + ] + } + ] + }, + { + "frame_id": "180_0_9", + "episode_id": 180, + "env_id": 0, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.28335952758789, + "advantages": -0.09790380299091339, + "generated_text": "{\n \"intent\": \"Selecting a hyperlink related to tourist attractions reduces the number of steps by narrowing the focus and fulfills the goal of exploring information about world tourist attractions.\",\n \"action\": \"DoubleClick(197, 855)\"\n}", + "action_string": "DoubleClick(197, 855)", + "intent_string": "Selecting a hyperlink related to tourist attractions reduces the number of steps by narrowing the focus and fulfills the goal of exploring information about world tourist attractions.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 44, + 69, + 124, + 87 + ] + }, + { + "text": "fii886ocr", + "boxes": [ + 1424, + 365, + 1514, + 389 + ] + }, + { + "text": "Most complained about tourist attractions in the", + "boxes": [ + 1294, + 397, + 1604, + 415 + ] + }, + { + "text": "10 of the Most Popular Tourist", + "boxes": [ + 1638, + 393, + 1832, + 417 + ] + }, + { + "text": "Wor", + "boxes": [ + 22, + 417, + 44, + 431 + ] + }, + { + "text": "lorld - Travel News", + "boxes": [ + 1298, + 415, + 1420, + 433 + ] + }, + { + "text": "Attractions in Paris", + "boxes": [ + 1642, + 415, + 1758, + 433 + ] + }, + { + "text": "Why I choose to do some of the", + "boxes": [ + 302, + 637, + 506, + 655 + ] + }, + { + "text": "MostBea", + "boxes": [ + 552, + 637, + 604, + 655 + ] + }, + { + "text": "Top10", + "boxes": [ + 1120, + 637, + 1164, + 655 + ] + }, + { + "text": "lost Popular Tourist Attractions Ii", + "boxes": [ + 1172, + 637, + 1380, + 655 + ] + }, + { + "text": "World Touri", + "boxes": [ + 1454, + 637, + 1524, + 655 + ] + }, + { + "text": "The 25 best-valu", + "boxes": [ + 1584, + 637, + 1692, + 655 + ] + }, + { + "text": "2023I", + "boxes": [ + 20, + 655, + 60, + 671 + ] + }, + { + "text": "touristy things and see the", + "boxes": [ + 302, + 651, + 478, + 675 + ] + }, + { + "text": " Tanya Tany", + "boxes": [ + 548, + 651, + 626, + 675 + ] + }, + { + "text": "England Th - vrogue.co", + "boxes": [ + 1116, + 649, + 1266, + 675 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 653, + 1516, + 671 + ] + }, + { + "text": "Tourist Attractions", + "boxes": [ + 936, + 868, + 1046, + 891 + ] + }, + { + "text": "Travel | uS Ne", + "boxes": [ + 20, + 891, + 116, + 909 + ] + }, + { + "text": "Collage·", + "boxes": [ + 936, + 891, + 988, + 909 + ] + }, + { + "text": "Bucket List", + "boxes": [ + 986, + 885, + 1068, + 911 + ] + }, + { + "text": "Attractions of Duba", + "boxes": [ + 1114, + 885, + 1238, + 911 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→×", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 69, + 1656, + 87 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 30, + 121, + 190, + 145 + ] + }, + { + "text": "Q", + "boxes": [ + 282, + 123, + 298, + 141 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 264, + 171, + 430, + 195 + ] + }, + { + "text": "The World's 50 Best Tourist Attractions I Travel | US News", + "boxes": [ + 1240, + 238, + 1684, + 268 + ] + }, + { + "text": " U.S. News Travel I 970 x 647 ipeg I 6 yrs ago", + "boxes": [ + 1246, + 262, + 1560, + 288 + ] + }, + { + "text": "Vist sit", + "boxes": [ + 1244, + 292, + 1306, + 324 + ] + }, + { + "text": "Top 50 Most", + "boxes": [ + 16, + 393, + 94, + 417 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 653, + 76, + 671 + ] + } + ] + }, + { + "frame_id": "180_1_9", + "episode_id": 180, + "env_id": 1, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.47168254852295, + "advantages": -1.2179728746414185, + "generated_text": "{\n \"intent\": \"Clicking the 'Hello' translation to see more content or understand how the translation works.\",\n \"action\": \"DoubleClick(275, 627)\"\n}", + "action_string": "DoubleClick(275, 627)", + "intent_string": "Clicking the 'Hello' translation to see more content or understand how the translation works.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": "一", + "boxes": [ + 24, + 133, + 52, + 153 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 40, + 125, + 184, + 165 + ] + }, + { + "text": "在 Web上搜累", + "boxes": [ + 636, + 125, + 758, + 157 + ] + }, + { + "text": "文本", + "boxes": [ + 770, + 204, + 804, + 224 + ] + }, + { + "text": "翻译工具", + "boxes": [ + 826, + 202, + 892, + 228 + ] + }, + { + "text": "对话", + "boxes": [ + 914, + 201, + 956, + 228 + ] + }, + { + "text": "应用", + "boxes": [ + 978, + 201, + 1018, + 228 + ] + }, + { + "text": "商务用", + "boxes": [ + 1040, + 202, + 1094, + 228 + ] + }, + { + "text": "帮助", + "boxes": [ + 1122, + 202, + 1152, + 224 + ] + }, + { + "text": "英语", + "boxes": [ + 286, + 298, + 308, + 312 + ] + }, + { + "text": "1", + "boxes": [ + 952, + 298, + 970, + 314 + ] + }, + { + "text": "中文 (简体)", + "boxes": [ + 1008, + 294, + 1082, + 318 + ] + }, + { + "text": "输入文本", + "boxes": [ + 256, + 341, + 462, + 405 + ] + }, + { + "text": "←国", + "boxes": [ + 820, + 455, + 948, + 498 + ] + }, + { + "text": "语气", + "boxes": [ + 1010, + 469, + 1042, + 490 + ] + }, + { + "text": "广泛使用的短语", + "boxes": [ + 258, + 522, + 390, + 554 + ] + }, + { + "text": "Most popular", + "boxes": [ + 262, + 562, + 354, + 594 + ] + }, + { + "text": ">", + "boxes": [ + 1636, + 570, + 1652, + 588 + ] + }, + { + "text": "Thank you", + "boxes": [ + 989, + 609, + 1059, + 642 + ] + }, + { + "text": "Good morning", + "boxes": [ + 262, + 657, + 346, + 675 + ] + }, + { + "text": "How are you?", + "boxes": [ + 996, + 657, + 1076, + 675 + ] + }, + { + "text": "Nice to meet you", + "boxes": [ + 258, + 697, + 368, + 721 + ] + }, + { + "text": "You are welcome", + "boxes": [ + 990, + 697, + 1100, + 721 + ] + }, + { + "text": "Good bye", + "boxes": [ + 255, + 732, + 327, + 763 + ] + }, + { + "text": " Good night ", + "boxes": [ + 985, + 728, + 1067, + 767 + ] + }, + { + "text": "隐私声明和Cookie", + "boxes": [ + 18, + 1005, + 130, + 1028 + ] + }, + { + "text": "法律声明广告帮助", + "boxes": [ + 144, + 1003, + 320, + 1032 + ] + }, + { + "text": "反馈", + "boxes": [ + 318, + 1007, + 358, + 1032 + ] + }, + { + "text": " 2025 Mic", + "boxes": [ + 1780, + 1009, + 1844, + 1026 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 50, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1878, + 91 + ] + }, + { + "text": "一", + "boxes": [ + 26, + 133, + 52, + 153 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 40, + 125, + 184, + 165 + ] + }, + { + "text": "在 Web 上搜索", + "boxes": [ + 634, + 129, + 752, + 161 + ] + }, + { + "text": "文本", + "boxes": [ + 764, + 204, + 796, + 226 + ] + }, + { + "text": "翻译工具", + "boxes": [ + 820, + 202, + 886, + 228 + ] + }, + { + "text": "对话", + "boxes": [ + 910, + 201, + 950, + 226 + ] + }, + { + "text": "应用", + "boxes": [ + 972, + 201, + 1014, + 226 + ] + }, + { + "text": "商务用", + "boxes": [ + 1036, + 202, + 1086, + 228 + ] + }, + { + "text": "帮助", + "boxes": [ + 1114, + 202, + 1144, + 224 + ] + }, + { + "text": "下", + "boxes": [ + 950, + 300, + 962, + 312 + ] + }, + { + "text": "中文（简体", + "boxes": [ + 1002, + 296, + 1070, + 314 + ] + }, + { + "text": "Hello", + "boxes": [ + 252, + 341, + 376, + 399 + ] + }, + { + "text": "十", + "boxes": [ + 893, + 332, + 925, + 363 + ] + }, + { + "text": ">", + "boxes": [ + 1622, + 343, + 1634, + 355 + ] + }, + { + "text": "一一国", + "boxes": [ + 774, + 457, + 938, + 494 + ] + }, + { + "text": "语气", + "boxes": [ + 1002, + 467, + 1046, + 492 + ] + }, + { + "text": "其他表达方式 hello", + "boxes": [ + 254, + 520, + 422, + 556 + ] + }, + { + "text": "名词", + "boxes": [ + 256, + 556, + 294, + 586 + ] + }, + { + "text": "》你好(nihao)", + "boxes": [ + 260, + 588, + 368, + 615 + ] + }, + { + "text": "您好 (ninhao)", + "boxes": [ + 960, + 588, + 1054, + 611 + ] + }, + { + "text": "hello, hi, hey. howdy, greeting", + "boxes": [ + 284, + 605, + 472, + 634 + ] + }, + { + "text": "》哈罗(haluo)", + "boxes": [ + 258, + 647, + 372, + 671 + ] + }, + { + "text": "hello, harrow, aloha, hale", + "boxes": [ + 284, + 669, + 442, + 693 + ] + }, + { + "text": "广泛使用的短语", + "boxes": [ + 258, + 737, + 390, + 768 + ] + }, + { + "text": "Hellco", + "boxes": [ + 260, + 828, + 292, + 850 + ] + }, + { + "text": "Thank you", + "boxes": [ + 984, + 828, + 1054, + 854 + ] + }, + { + "text": "How are you?", + "boxes": [ + 988, + 869, + 1068, + 892 + ] + }, + { + "text": "Good bye", + "boxes": [ + 254, + 949, + 326, + 981 + ] + }, + { + "text": " Good night", + "boxes": [ + 981, + 948, + 1059, + 979 + ] + }, + { + "text": "隐私声明和Cookie", + "boxes": [ + 18, + 1005, + 128, + 1028 + ] + }, + { + "text": "法律声明", + "boxes": [ + 146, + 1009, + 212, + 1026 + ] + }, + { + "text": "反馈", + "boxes": [ + 320, + 1007, + 358, + 1034 + ] + }, + { + "text": "@ 2025 Micr", + "boxes": [ + 1766, + 1009, + 1836, + 1026 + ] + } + ] + }, + { + "frame_id": "180_2_9", + "episode_id": 180, + "env_id": 2, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.398188591003418, + "advantages": 0.060553912073373795, + "generated_text": "{\n \"intent\": \"To continue exploring the environment, clicking on a watch model will allow me to view more details about that product or other relevant products on the webpage.\",\n \"action\": \"Click(318, 734)\"\n}", + "action_string": "Click(318, 734)", + "intent_string": "To continue exploring the environment, clicking on a watch model will allow me to view more details about that product or other relevant products on the webpage.", + "before_screen_ocr_results": [ + { + "text": "Apple x", + "boxes": [ + 158, + 32, + 220, + 50 + ] + }, + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1599, + 63, + 1657, + 88 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 944, + 127 + ] + }, + { + "text": "TV & H", + "boxes": [ + 988, + 113, + 1024, + 127 + ] + }, + { + "text": "Shop Apple Watch", + "boxes": [ + 138, + 222, + 532, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1562, + 230, + 1706, + 254 + ] + }, + { + "text": "Watch Specialist ", + "boxes": [ + 1650, + 254, + 1764, + 272 + ] + }, + { + "text": "Visit an Apple Store", + "boxes": [ + 1556, + 284, + 1694, + 312 + ] + }, + { + "text": "?", + "boxes": [ + 1524, + 294, + 1550, + 318 + ] + }, + { + "text": "All Models ", + "boxes": [ + 144, + 389, + 230, + 413 + ] + }, + { + "text": " Shopping guides ", + "boxes": [ + 260, + 387, + 400, + 417 + ] + }, + { + "text": "Ways to Save", + "boxes": [ + 430, + 387, + 542, + 417 + ] + }, + { + "text": "Apple Watch Bands", + "boxes": [ + 574, + 387, + 730, + 417 + ] + }, + { + "text": "Setup and Support", + "boxes": [ + 764, + 389, + 914, + 413 + ] + }, + { + "text": "The Apple Watch Experience", + "boxes": [ + 946, + 387, + 1176, + 417 + ] + }, + { + "text": "Special Stores", + "boxes": [ + 1210, + 389, + 1322, + 413 + ] + }, + { + "text": "All models. Take your pick.", + "boxes": [ + 138, + 472, + 464, + 508 + ] + }, + { + "text": "Apple Watch Series 10", + "boxes": [ + 166, + 582, + 454, + 617 + ] + }, + { + "text": "Apple Watch Ultra 2", + "boxes": [ + 584, + 573, + 846, + 624 + ] + }, + { + "text": "Apple Watch SE", + "boxes": [ + 1004, + 575, + 1216, + 624 + ] + }, + { + "text": "Apple Watch Hermes ", + "boxes": [ + 1424, + 575, + 1702, + 624 + ] + }, + { + "text": "Appl", + "boxes": [ + 1846, + 582, + 1914, + 621 + ] + }, + { + "text": "Series 10", + "boxes": [ + 1426, + 615, + 1546, + 647 + ] + }, + { + "text": "Uitra", + "boxes": [ + 1845, + 611, + 1915, + 654 + ] + }, + { + "text": "?", + "boxes": [ + 658, + 705, + 742, + 744 + ] + }, + { + "text": "10:09", + "boxes": [ + 664, + 737, + 738, + 768 + ] + }, + { + "text": "", + "boxes": [ + 668, + 802, + 716, + 830 + ] + }, + { + "text": "$1399 or", + "boxes": [ + 1848, + 977, + 1906, + 1003 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Buy Apple Watch - Apple ×+", + "boxes": [ + 22, + 24, + 246, + 54 + ] + }, + { + "text": "←→", + "boxes": [ + 8, + 62, + 130, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "I四@三", + "boxes": [ + 1784, + 62, + 1912, + 93 + ] + }, + { + "text": "Shop Apple Wa", + "boxes": [ + 134, + 216, + 472, + 282 + ] + }, + { + "text": "Apple Watch Series 10", + "boxes": [ + 482, + 222, + 640, + 252 + ] + }, + { + "text": "Apple Watch Ultra 2", + "boxes": [ + 678, + 222, + 818, + 252 + ] + }, + { + "text": "Apple Watch SE", + "boxes": [ + 852, + 222, + 968, + 252 + ] + }, + { + "text": "Apple Watch Hermes Series 10", + "boxes": [ + 1004, + 226, + 1214, + 250 + ] + }, + { + "text": "Apple Watch Herme", + "boxes": [ + 1252, + 222, + 1388, + 252 + ] + }, + { + "text": "+", + "boxes": [ + 1540, + 294, + 1558, + 312 + ] + }, + { + "text": "Apple Watch Series 10", + "boxes": [ + 954, + 343, + 1278, + 379 + ] + }, + { + "text": "All Models", + "boxes": [ + 144, + 387, + 226, + 413 + ] + }, + { + "text": " Shopping guides", + "boxes": [ + 260, + 389, + 388, + 413 + ] + }, + { + "text": "From $399 or $33.25/mo.for 12 mo.t", + "boxes": [ + 950, + 381, + 1196, + 413 + ] + }, + { + "text": "Thethinest pleWathever, wit", + "boxes": [ + 1010, + 445, + 1338, + 480 + ] + }, + { + "text": "All models. Take your pick", + "boxes": [ + 138, + 470, + 458, + 508 + ] + }, + { + "text": "α", + "boxes": [ + 960, + 469, + 994, + 498 + ] + }, + { + "text": "biggest display.Featuring a Wide-Angle", + "boxes": [ + 1010, + 471, + 1336, + 500 + ] + }, + { + "text": "Always-On OLED Retina display", + "boxes": [ + 1012, + 488, + 1276, + 522 + ] + }, + { + "text": "Get health insights day and night. Now with", + "boxes": [ + 1008, + 538, + 1366, + 576 + ] + }, + { + "text": "O", + "boxes": [ + 958, + 552, + 992, + 582 + ] + }, + { + "text": "sleep apnea notificationss", + "boxes": [ + 1010, + 562, + 1226, + 598 + ] + }, + { + "text": "Apple Watch Series 10", + "boxes": [ + 168, + 584, + 456, + 619 + ] + }, + { + "text": "pple Watch Hermes", + "boxes": [ + 1444, + 578, + 1700, + 619 + ] + }, + { + "text": "Appl", + "boxes": [ + 1848, + 586, + 1908, + 619 + ] + }, + { + "text": "Track all your workouts and activity. Depth and", + "boxes": [ + 1012, + 615, + 1392, + 645 + ] + }, + { + "text": "eries 10", + "boxes": [ + 1448, + 617, + 1548, + 649 + ] + }, + { + "text": "UItra", + "boxes": [ + 1848, + 615, + 1910, + 649 + ] + }, + { + "text": "罗", + "boxes": [ + 964, + 637, + 988, + 661 + ] + }, + { + "text": "water temperature sensors enhance every", + "boxes": [ + 1010, + 637, + 1354, + 667 + ] + }, + { + "text": "swim", + "boxes": [ + 1014, + 659, + 1060, + 687 + ] + }, + { + "text": "2", + "boxes": [ + 955, + 715, + 993, + 752 + ] + }, + { + "text": "Faster charging gives you up to 80 percent", + "boxes": [ + 1010, + 711, + 1362, + 741 + ] + }, + { + "text": "battery in 30 minutes4", + "boxes": [ + 1014, + 731, + 1194, + 760 + ] + }, + { + "text": "Get help in an emergency or if you fll or are in", + "boxes": [ + 1008, + 778, + 1388, + 816 + ] + }, + { + "text": "(SOS)", + "boxes": [ + 962, + 794, + 990, + 822 + ] + }, + { + "text": "a car crash*", + "boxes": [ + 1012, + 808, + 1110, + 832 + ] + }, + { + "text": "Explore Apple Watch Series 10 further >", + "boxes": [ + 950, + 846, + 1260, + 876 + ] + }, + { + "text": "Apple Card Monthly Installments", + "boxes": [ + 504, + 927, + 760, + 957 + ] + }, + { + "text": " Apple Trade Inih", + "boxes": [ + 818, + 927, + 958, + 959 + ] + }, + { + "text": " Carbon Neutral", + "boxes": [ + 1130, + 925, + 1272, + 957 + ] + }, + { + "text": "Pay over time,interest-free when you", + "boxes": [ + 540, + 947, + 744, + 965 + ] + }, + { + "text": "Tradeinyoureliibledevice for crecit", + "boxes": [ + 852, + 943, + 1064, + 973 + ] + }, + { + "text": "choosetcheckoutwithAppleCard", + "boxes": [ + 532, + 953, + 748, + 989 + ] + }, + { + "text": "Look for this logo tofind carbon neutral", + "boxes": [ + 1164, + 945, + 1384, + 969 + ] + }, + { + "text": "combinations. Learn", + "boxes": [ + 1166, + 961, + 1278, + 985 + ] + }, + { + "text": "1 more about our plan", + "boxes": [ + 1272, + 961, + 1392, + 985 + ] + }, + { + "text": "Monthly Installments.\"", + "boxes": [ + 538, + 977, + 662, + 1001 + ] + }, + { + "text": "at apple.com/2030", + "boxes": [ + 1166, + 979, + 1270, + 1005 + ] + } + ] + }, + { + "frame_id": "180_3_9", + "episode_id": 180, + "env_id": 3, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.468088150024414, + "advantages": 0.15701141953468323, + "generated_text": "{\n \"intent\": \"Clicking the 'Firefox for Desktop' link allows us to continue exploring the browser's download page and eventually close the current window. This is the most valuable action because clicking this link will help us continue browsing the desired information about the Firefox browser. Click(750, 204)\",\n \"action\": \"Click(750, 204)\"\n}", + "action_string": "Click(750, 204)", + "intent_string": "Clicking the 'Firefox for Desktop' link allows us to continue exploring the browser's download page and eventually close the current window. This is the most valuable action because clicking this link will help us continue browsing the desired information about the Firefox browser. Click(750, 204)", + "before_screen_ocr_results": [ + { + "text": "↑→白", + "boxes": [ + 10, + 64, + 130, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1874, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": "Firefox bsowsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 109, + 1050, + 141 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": " Firefox for Desktop", + "boxes": [ + 636, + 191, + 836, + 220 + ] + }, + { + "text": " Firefox for ios", + "boxes": [ + 638, + 228, + 792, + 258 + ] + }, + { + "text": " Firefox for Android", + "boxes": [ + 636, + 266, + 832, + 296 + ] + }, + { + "text": " Firefox Focus", + "boxes": [ + 634, + 299, + 786, + 334 + ] + }, + { + "text": " Firefox blog", + "boxes": [ + 638, + 337, + 772, + 374 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 415, + 1238, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": "Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "Get Firefox fo.", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 111, + 1048, + 143 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "180_4_9", + "episode_id": 180, + "env_id": 4, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.842620849609375, + "advantages": -0.7060986757278442, + "generated_text": "{\n \"intent\": \"Click on 'Sign in' to continue exploring account-related options or returning to previous tasks on Amazon.\",\n \"action\": \"Click(1650, 124)\"\n}", + "action_string": "Click(1650, 124)", + "intent_string": "Click on 'Sign in' to continue exploring account-related options or returning to previous tasks on Amazon.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Returnder", + "boxes": [ + 1732, + 109, + 1788, + 151 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "94", + "boxes": [ + 1826, + 115, + 1860, + 137 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": " Signin", + "boxes": [ + 1622, + 175, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 614, + 891, + 748, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 967, + 1546, + 1003 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 354, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1358, + 1001, + 1462, + 1024 + ] + }, + { + "text": "U", + "boxes": [ + 1555, + 994, + 1647, + 1025 + ] + }, + { + "text": "=UTF8&k=", + "boxes": [ + 264, + 1020, + 334, + 1042 + ] + }, + { + "text": "d-17e7-4ceb-9773-90", + "boxes": [ + 1504, + 1020, + 1644, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1600, + 68, + 1658, + 87 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1872, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 806, + 434, + 930, + 471 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "180_5_9", + "episode_id": 180, + "env_id": 5, + "step": 9, + "format_rewards": 1.0, + "rewards": 11.024020195007324, + "advantages": 0.924166738986969, + "generated_text": "{\n \"intent\": \"To continue exploring the environment and gather more information or content related to the trend on Bing.\",\n \"action\": \"DoubleClick(590, 427)\"\n}", + "action_string": "DoubleClick(590, 427)", + "intent_string": "To continue exploring the environment and gather more information or content related to the trend on Bing.", + "before_screen_ocr_results": [ + { + "text": "earch - Microsoft Bing", + "boxes": [ + 44, + 32, + 188, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 406, + 121, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 614, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭中", + "boxes": [ + 1318, + 375, + 1358, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 792, + 449, + 972, + 478 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 274, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "TAL", + "boxes": [ + 218, + 778, + 344, + 838 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "Ww", + "boxes": [ + 1594, + 810, + 1628, + 830 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1632, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1456, + 850, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "15087%", + "boxes": [ + 1623, + 888, + 1702, + 944 + ] + }, + { + "text": "游戏天地·1天", + "boxes": [ + 226, + 925, + 308, + 943 + ] + }, + { + "text": "游侠网·5天", + "boxes": [ + 828, + 921, + 932, + 945 + ] + }, + { + "text": "D", + "boxes": [ + 1140, + 923, + 1166, + 943 + ] + }, + { + "text": "一点资讯·51分钟", + "boxes": [ + 1158, + 919, + 1274, + 949 + ] + }, + { + "text": "仅靠一只狗就能通关！《博德", + "boxes": [ + 824, + 941, + 1098, + 977 + ] + }, + { + "text": "进化5亿年肠道竟成“第二大", + "boxes": [ + 1136, + 941, + 1394, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "娱美德3A大作TAL ：The Arcane Land首", + "boxes": [ + 200, + 949, + 772, + 985 + ] + }, + { + "text": "3》玩家用宠物咬死BOSS", + "boxes": [ + 824, + 967, + 1068, + 1001 + ] + }, + { + "text": "脑”！专家：我们的喜怒哀乐", + "boxes": [ + 1132, + 963, + 1402, + 1005 + ] + }, + { + "text": "度曝光", + "boxes": [ + 195, + 976, + 305, + 1025 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "被它操控", + "boxes": [ + 1134, + 993, + 1230, + 1030 + ] + }, + { + "text": "Lr~", + "boxes": [ + 1592, + 997, + 1630, + 1026 + ] + }, + { + "text": "O", + "boxes": [ + 218, + 1024, + 286, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 10, + 62, + 132, + 93 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 184, + 113, + 310, + 145 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 48, + 159, + 146, + 185 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 176, + 157, + 312, + 187 + ] + }, + { + "text": "WEB", + "boxes": [ + 178, + 212, + 208, + 232 + ] + }, + { + "text": "IMAGES", + "boxes": [ + 246, + 212, + 288, + 232 + ] + }, + { + "text": "ACADEMI", + "boxes": [ + 410, + 212, + 460, + 230 + ] + }, + { + "text": " About 776,000 results", + "boxes": [ + 174, + 256, + 318, + 288 + ] + }, + { + "text": "腾讯网", + "boxes": [ + 212, + 294, + 262, + 322 + ] + }, + { + "text": "雷军最新演讲内容已回应小米SU7事故，相关视频时长.", + "boxes": [ + 176, + 336, + 674, + 371 + ] + }, + { + "text": "2 days ago·雷军面向小米员工发布的最新演讲被业内外关注，视频内容流出了一部分；流出部", + "boxes": [ + 178, + 363, + 776, + 393 + ] + }, + { + "text": "分主要是雷军对3月底小米SU7交通事故的回应，但就演讲内容来看", + "boxes": [ + 178, + 389, + 598, + 413 + ] + }, + { + "text": "澎湃新闻", + "boxes": [ + 208, + 433, + 280, + 465 + ] + }, + { + "text": "", + "boxes": [ + 586, + 425, + 602, + 445 + ] + }, + { + "text": "澎", + "boxes": [ + 182, + 447, + 212, + 469 + ] + }, + { + "text": "https://wwv", + "boxes": [ + 214, + 455, + 284, + 472 + ] + }, + { + "text": "w.thepaper.cn Translae this reslt ", + "boxes": [ + 272, + 448, + 492, + 483 + ] + }, + { + "text": "雷军回应SU7车祸致三人死亡：很多问题此刻还没办法回答.", + "boxes": [ + 176, + 476, + 728, + 512 + ] + }, + { + "text": "4月1日晚间，小米CEO雷军回应广受关注的\"小米SU7高速车祸\"事件。雷军表示：29日晚上的这起", + "boxes": [ + 176, + 504, + 790, + 534 + ] + }, + { + "text": "事故，我的心情非常沉重。三位年轻女孩不幸离世，这样的消息，对她们的家人、朋友，乃至，", + "boxes": [ + 178, + 530, + 766, + 554 + ] + }, + { + "text": "雷军线下回应小米SU7事故：有错就改，持续改进", + "boxes": [ + 176, + 613, + 624, + 649 + ] + }, + { + "text": "2 days ago·大象新闻记者赵朝阳5月15日晚，小米集团创始人雷军在小米价值观大赛后发表了演", + "boxes": [ + 176, + 645, + 792, + 675 + ] + }, + { + "text": "讲，回应了备受关注的小米SU7事故，雷军说：“受到了狂风暴雨般的质疑、批评和指责，我和", + "boxes": [ + 178, + 667, + 762, + 691 + ] + }, + { + "text": "中国新闻网", + "boxes": [ + 214, + 715, + 292, + 741 + ] + }, + { + "text": "PH", + "boxes": [ + 188, + 729, + 202, + 744 + ] + }, + { + "text": "https://www.china", + "boxes": [ + 216, + 733, + 326, + 756 + ] + }, + { + "text": "雷军回应小米SU7爆燃事故－中国新闻网", + "boxes": [ + 176, + 756, + 542, + 792 + ] + }, + { + "text": "Apr1,2025．小米创办人雷军表示，他的心情沉重，代表小米向事故家属致慰问，承诺配合警方", + "boxes": [ + 178, + 784, + 776, + 814 + ] + }, + { + "text": "调查，尽力回应社会关心。小米汽车官方微博也发文，回答了网友关于事故原因、车速、车门.", + "boxes": [ + 176, + 808, + 766, + 838 + ] + }, + { + "text": "哗哩哗哩", + "boxes": [ + 210, + 854, + 278, + 879 + ] + }, + { + "text": "百", + "boxes": [ + 182, + 868, + 208, + 887 + ] + }, + { + "text": "https://www.bibil.com video Translatethis esult", + "boxes": [ + 208, + 872, + 528, + 901 + ] + }, + { + "text": "雷军首度线下回应小米SU7燃爆事故全程紧张看稿 曾对质量，", + "boxes": [ + 178, + 899, + 718, + 929 + ] + }, + { + "text": "2 days ago·近日，网上一段视频显示，雷军首度线下公开回应小米SU7燃爆事故，在台上的发言", + "boxes": [ + 176, + 923, + 786, + 953 + ] + }, + { + "text": "中，雷军全程在紧张看稿，并声称，突如其来的交通事故让小米受到了狂风暴雨般的质疑、批，", + "boxes": [ + 176, + 947, + 762, + 977 + ] + }, + { + "text": "搜狐", + "boxes": [ + 208, + 993, + 250, + 1020 + ] + }, + { + "text": "#", + "boxes": [ + 182, + 1005, + 212, + 1026 + ] + }, + { + "text": "https:/ww.sohu.com Translaethisrsut", + "boxes": [ + 210, + 1006, + 480, + 1045 + ] + } + ] + }, + { + "frame_id": "180_6_9", + "episode_id": 180, + "env_id": 6, + "step": 9, + "format_rewards": 1.0, + "rewards": 8.935874938964844, + "advantages": -1.9573575258255005, + "generated_text": "{\n \"intent\": \"The goal is to continue exploring the computer desktop environment. Clicking the close button on the Firefox browser window is a good action to select because it will allow the user to return to the desktop and continue exploring other applications or files.\",\n \"action\": \"Click(189, 37)\"\n}", + "action_string": "Click(189, 37)", + "intent_string": "The goal is to continue exploring the computer desktop environment. Clicking the close button on the Firefox browser window is a good action to select because it will allow the user to return to the desktop and continue exploring other applications or files.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop - Mozilla (Us) -", + "boxes": [ + 738, + 2, + 1050, + 26 + ] + }, + { + "text": "-Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 89 + ] + }, + { + "text": "① h", + "boxes": [ + 250, + 62, + 316, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1559, + 61, + 1631, + 92 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "firefox-138.0.4.tar.xz - xarchiver 0.5.4.14", + "boxes": [ + 774, + 322, + 1140, + 345 + ] + }, + { + "text": "Archive Action Help", + "boxes": [ + 666, + 343, + 826, + 371 + ] + }, + { + "text": "PIO*", + "boxes": [ + 674, + 373, + 1034, + 409 + ] + }, + { + "text": "ocation: firefox/", + "boxes": [ + 670, + 419, + 788, + 443 + ] + }, + { + "text": "Archive tree", + "boxes": [ + 666, + 453, + 766, + 478 + ] + }, + { + "text": "Filename", + "boxes": [ + 866, + 453, + 942, + 478 + ] + }, + { + "text": " Points to Original Size | Date", + "boxes": [ + 1014, + 451, + 1252, + 480 + ] + }, + { + "text": "firefox", + "boxes": [ + 672, + 478, + 738, + 498 + ] + }, + { + "text": "libplds4.s0", + "boxes": [ + 882, + 472, + 966, + 504 + ] + }, + { + "text": "11164", + "boxes": [ + 1098, + 476, + 1142, + 502 + ] + }, + { + "text": "2025-0", + "boxes": [ + 1210, + 478, + 1254, + 498 + ] + }, + { + "text": "』 browser", + "boxes": [ + 678, + 494, + 776, + 526 + ] + }, + { + "text": " precomplete", + "boxes": [ + 882, + 498, + 974, + 524 + ] + }, + { + "text": "1478", + "boxes": [ + 1094, + 496, + 1138, + 524 + ] + }, + { + "text": "2025-05", + "boxes": [ + 1204, + 494, + 1266, + 526 + ] + }, + { + "text": " defaults", + "boxes": [ + 684, + 520, + 774, + 546 + ] + }, + { + "text": " defaults", + "boxes": [ + 864, + 518, + 950, + 550 + ] + }, + { + "text": "I fonts", + "boxes": [ + 694, + 544, + 758, + 570 + ] + }, + { + "text": "libmozgtk.so", + "boxes": [ + 884, + 546, + 980, + 572 + ] + }, + { + "text": "3316", + "boxes": [ + 1094, + 544, + 1136, + 572 + ] + }, + { + "text": "2025-09", + "boxes": [ + 1204, + 542, + 1264, + 574 + ] + }, + { + "text": " gmp-clearkey", + "boxes": [ + 682, + 566, + 812, + 596 + ] + }, + { + "text": "removed-files", + "boxes": [ + 888, + 568, + 982, + 592 + ] + }, + { + "text": "2025-0S", + "boxes": [ + 1204, + 564, + 1262, + 598 + ] + }, + { + "text": "libfreeblpiv3.so ", + "boxes": [ + 886, + 590, + 1000, + 619 + ] + }, + { + "text": "1084392", + "boxes": [ + 1096, + 592, + 1158, + 617 + ] + }, + { + "text": "2025-05", + "boxes": [ + 1204, + 588, + 1264, + 619 + ] + }, + { + "text": "libmozavutil.so", + "boxes": [ + 888, + 615, + 994, + 639 + ] + }, + { + "text": "474320", + "boxes": [ + 1098, + 613, + 1150, + 639 + ] + }, + { + "text": "2025-0", + "boxes": [ + 1208, + 613, + 1256, + 641 + ] + }, + { + "text": "liblgpliso ", + "boxes": [ + 886, + 633, + 978, + 665 + ] + }, + { + "text": "268384", + "boxes": [ + 1098, + 637, + 1150, + 663 + ] + }, + { + "text": "2025-05", + "boxes": [ + 1208, + 635, + 1262, + 665 + ] + }, + { + "text": "libss3.so0 ", + "boxes": [ + 886, + 657, + 960, + 689 + ] + }, + { + "text": "461076", + "boxes": [ + 1091, + 652, + 1155, + 698 + ] + }, + { + "text": "2025-05", + "boxes": [ + 1208, + 659, + 1262, + 689 + ] + }, + { + "text": "libnssutil3.so", + "boxes": [ + 886, + 683, + 978, + 709 + ] + }, + { + "text": "178640", + "boxes": [ + 1096, + 683, + 1150, + 709 + ] + }, + { + "text": "2025-05", + "boxes": [ + 1206, + 683, + 1260, + 709 + ] + }, + { + "text": "libnss3.50", + "boxes": [ + 888, + 709, + 954, + 729 + ] + }, + { + "text": "792072", + "boxes": [ + 1096, + 707, + 1150, + 733 + ] + }, + { + "text": "2025-(", + "boxes": [ + 1206, + 707, + 1250, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "efox-138.0", + "boxes": [ + 380, + 1048, + 454, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1572, + 68, + 1626, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1062, + 667 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "firefox-138.0..", + "boxes": [ + 366, + 1048, + 462, + 1072 + ] + } + ] + }, + { + "frame_id": "180_7_9", + "episode_id": 180, + "env_id": 7, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.632855415344238, + "advantages": 0.38438108563423157, + "generated_text": "{\n \"intent\": \"Click on 'Images' to see if there's any interesting content about world tourist attractions.\",\n \"action\": \"Click(274, 131)\"\n}", + "action_string": "Click(274, 131)", + "intent_string": "Click on 'Images' to see if there's any interesting content about world tourist attractions.", + "before_screen_ocr_results": [ + { + "text": "- Microsoft Bing ", + "boxes": [ + 76, + 32, + 194, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 66, + 40, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 274, + 750 + ] + }, + { + "text": "各种各样的", + "boxes": [ + 224, + 772, + 370, + 810 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "·下列城市新增了树之乌鸦：", + "boxes": [ + 230, + 840, + 414, + 870 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "。塞里加尔", + "boxes": [ + 256, + 866, + 340, + 891 + ] + }, + { + "text": "盖亚库尔", + "boxes": [ + 254, + 887, + 344, + 919 + ] + }, + { + "text": "15087%", + "boxes": [ + 1623, + 888, + 1702, + 944 + ] + }, + { + "text": "游民量空尔社", + "boxes": [ + 217, + 909, + 357, + 954 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 925, + 850, + 941 + ] + }, + { + "text": "一点资讯·51分钟", + "boxes": [ + 844, + 915, + 964, + 949 + ] + }, + { + "text": "EVCar·5天", + "boxes": [ + 1136, + 917, + 1250, + 947 + ] + }, + { + "text": "进化5亿年肠道竟成“第二大", + "boxes": [ + 822, + 937, + 1084, + 977 + ] + }, + { + "text": "全铝车身+性能升级！1953", + "boxes": [ + 1138, + 943, + 1414, + 973 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "《暗黑4》偷偷增添功能：节省大量跑路时", + "boxes": [ + 216, + 945, + 752, + 987 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1630, + 981 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "脑”！专家：我们的喜怒哀乐", + "boxes": [ + 822, + 963, + 1092, + 1005 + ] + }, + { + "text": "款捷豹XK120经典重现，致", + "boxes": [ + 1134, + 967, + 1416, + 1003 + ] + }, + { + "text": "间！。扎宾泽特", + "boxes": [ + 204, + 987, + 338, + 1018 + ] + }, + { + "text": "敬赛道传奇", + "boxes": [ + 1135, + 990, + 1249, + 1031 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "被它操控", + "boxes": [ + 824, + 993, + 918, + 1030 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "S", + "boxes": [ + 852, + 1028, + 874, + 1042 + ] + }, + { + "text": "24", + "boxes": [ + 1140, + 1026, + 1180, + 1046 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国际版", + "boxes": [ + 262, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_180/scores_v3.json b/ScreenExplorer-7B-E1/episode_180/scores_v3.json new file mode 100644 index 0000000000000000000000000000000000000000..92edde37eedef4c436181882ebef24cb3b6aab9a --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_180/scores_v3.json @@ -0,0 +1,19 @@ +{ + "directory_path": "logs/ScreenExplorer_2025-05-17_22-11-27ab2072/episode_180", + "single_trajectory_image_diversity": 0.5342122567817569, + "single_trajectory_text_diversity": 0.330386022105813, + "single_trajectory_image_diversity_max": 1.02490234375, + "single_trajectory_image_diversity_min": 0.00390625, + "single_trajectory_image_diversity_std": 0.32073974609375, + "single_trajectory_text_diversity_max": 0.54986572265625, + "single_trajectory_text_diversity_min": 0.0050048828125, + "single_trajectory_text_diversity_std": 0.144073486328125, + "group_trajectories_image_diversity": 0.6488689184188843, + "group_trajectories_text_diversity": 0.42015519738197327, + "group_trajectories_image_diversity_max": 1.2890625, + "group_trajectories_image_diversity_min": -0.0078125, + "group_trajectories_image_diversity_std": 0.353515625, + "group_trajectories_text_diversity_max": 0.6865234375, + "group_trajectories_text_diversity_min": -0.0009765625, + "group_trajectories_text_diversity_std": 0.1346435546875 +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_20/actor_model_20/adapter_config.json b/ScreenExplorer-7B-E1/episode_20/actor_model_20/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c138106acf79edf195f0e28248c173bb4c71f7 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_20/actor_model_20/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen2.5-VL-7B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "k_proj", + "v_proj", + "o_proj", + "q_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_20/recording_env_rank_1_a743e.h5 b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_1_a743e.h5 new file mode 100644 index 0000000000000000000000000000000000000000..df56370daac694ac9f177a9f572906b2983e3eab --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_1_a743e.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4062daeada459ebdced581cb893a257a919bdd4f43165239b34af5bbdfb3267f +size 10917844 diff --git a/ScreenExplorer-7B-E1/episode_20/recording_env_rank_2_0d74f.h5 b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_2_0d74f.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8d19e314683f7ea9d5b445f24181dfe668ce5e6b --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_2_0d74f.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e18c8cf1b05c72f40c892f5a540cf9bd4f41da50d53cf996d02cd2e5ce136091 +size 11972112 diff --git a/ScreenExplorer-7B-E1/episode_20/recording_env_rank_3_c51d4.h5 b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_3_c51d4.h5 new file mode 100644 index 0000000000000000000000000000000000000000..295a948c77352498d6ac432825ccfe0f149d7b83 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_3_c51d4.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:674943ca6ef4654a7668e47656b5f67cf83d8ed6cead57557ba54c76bd7e5eff +size 11685571 diff --git a/ScreenExplorer-7B-E1/episode_20/recording_env_rank_4_cc89e.h5 b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_4_cc89e.h5 new file mode 100644 index 0000000000000000000000000000000000000000..15a3ffb9b1fa9bab083fe2e2b40d72e3c4af7a30 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_4_cc89e.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f51ffcbeb80f054e1b29b98814c4f1ecec9d8b00e56ba2d6b5c9b0831f6fe4 +size 7198950 diff --git a/ScreenExplorer-7B-E1/episode_20/recording_env_rank_5_5a8b9.h5 b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_5_5a8b9.h5 new file mode 100644 index 0000000000000000000000000000000000000000..5d452bb6c67b0b7fd9a8bdf572a57e76ecf0fbe2 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_5_5a8b9.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85582969e2675a5f49a83186ddababaaae0cdba82608fb291f8c6de002b86434 +size 13403081 diff --git a/ScreenExplorer-7B-E1/episode_20/recording_env_rank_6_7789f.h5 b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_6_7789f.h5 new file mode 100644 index 0000000000000000000000000000000000000000..4a6cd73689f66d2d6b8b0091a3c4a0413bfd28da --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_6_7789f.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:344e4c8a61b52e200ccac21e8020da943fdd631ee5731f1a312c4c0eb577d2ac +size 25239719 diff --git a/ScreenExplorer-7B-E1/episode_20/recording_env_rank_7_cdfbd.h5 b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_7_cdfbd.h5 new file mode 100644 index 0000000000000000000000000000000000000000..20c7af79f04c2d7e70793a87a3427c4a168c8345 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_20/recording_env_rank_7_cdfbd.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60cb0fc96d39569f7397b62b07db196e31e50a0c474e190d41827616f7110f9b +size 11885193 diff --git a/ScreenExplorer-7B-E1/episode_40/actor_model_40/README.md b/ScreenExplorer-7B-E1/episode_40/actor_model_40/README.md new file mode 100644 index 0000000000000000000000000000000000000000..589ec54ac19affe115ba89fb3473bc212c5146a5 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_40/actor_model_40/README.md @@ -0,0 +1,202 @@ +--- +base_model: Qwen/Qwen2.5-VL-7B-Instruct +library_name: peft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.15.2 \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_40/actor_model_40/adapter_config.json b/ScreenExplorer-7B-E1/episode_40/actor_model_40/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c138106acf79edf195f0e28248c173bb4c71f7 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_40/actor_model_40/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen2.5-VL-7B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "k_proj", + "v_proj", + "o_proj", + "q_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_40/recording_env_rank_0_c2724.h5 b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_0_c2724.h5 new file mode 100644 index 0000000000000000000000000000000000000000..ac506ba44898efbbe79918a18565f4da7d2d1af4 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_0_c2724.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd5996ec2d4a9bcb19c8b518989d35e58406c50a0291807b838b8cf6bfdf209a +size 11790784 diff --git a/ScreenExplorer-7B-E1/episode_40/recording_env_rank_1_7c052.h5 b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_1_7c052.h5 new file mode 100644 index 0000000000000000000000000000000000000000..415d619954803aa067a026df3d69f5de266a51e3 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_1_7c052.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69e5f310bb23cfdbd4a8f58c0a3a9ebc1d176966cf2e91c623766fd6792bbf37 +size 9071295 diff --git a/ScreenExplorer-7B-E1/episode_40/recording_env_rank_2_02db7.h5 b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_2_02db7.h5 new file mode 100644 index 0000000000000000000000000000000000000000..c133c28d6e3320ef1ff0f6c34fd011b1000d4527 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_2_02db7.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fc29be9136de13b0025f64c35ec2825b51b127331db9b3cdd77f94671e1b4a2 +size 15023348 diff --git a/ScreenExplorer-7B-E1/episode_40/recording_env_rank_4_c697d.h5 b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_4_c697d.h5 new file mode 100644 index 0000000000000000000000000000000000000000..faa2d97acfb1d322d6310a629fc7db0150382854 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_4_c697d.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4c169e14a241e9ef251b2f6555219c58e6d3344fb8b7f1f710c28759b4128a5 +size 14028879 diff --git a/ScreenExplorer-7B-E1/episode_40/recording_env_rank_5_eac4f.h5 b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_5_eac4f.h5 new file mode 100644 index 0000000000000000000000000000000000000000..661704675db727512edfd2a33805ee779c825e8c --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_5_eac4f.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e3c655e98ef426e1ea1707d02f324cec40e038435ba78b459044dfd72e6873 +size 17448817 diff --git a/ScreenExplorer-7B-E1/episode_40/recording_env_rank_6_6dad2.h5 b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_6_6dad2.h5 new file mode 100644 index 0000000000000000000000000000000000000000..4a4cb2d9071e08c0d1d308fe476edf4b206587a6 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_40/recording_env_rank_6_6dad2.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f5c2a4757c9067048972bf3e7216d98457f4bd5a1389ce62fd2692e368a9659 +size 10414745 diff --git a/ScreenExplorer-7B-E1/episode_50/game_record_20250518_034407.json b/ScreenExplorer-7B-E1/episode_50/game_record_20250518_034407.json new file mode 100644 index 0000000000000000000000000000000000000000..011259497d0f7d24da8ac7d67c21ed7e45bcce8b --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_50/game_record_20250518_034407.json @@ -0,0 +1,39897 @@ +{ + "start_time": "2025-05-18T03:44:07.171219", + "end_time": "2025-05-18T03:50:52.422090", + "buffer_size": 80, + "all_records": [ + { + "frame_id": "50_0_0", + "episode_id": 50, + "env_id": 0, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.447541236877441, + "advantages": 0.975701093673706, + "generated_text": "{\n \"intent\": \"To explore further, I will start by clicking on one of the application icons like Firefox to open it.\",\n \"action\": \"DoubleClick(45, 39)\"\n}", + "action_string": "DoubleClick(45, 39)", + "intent_string": "To explore further, I will start by clicking on one of the application icons like Firefox to open it.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C", + "boxes": [ + 72, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_1_0", + "episode_id": 50, + "env_id": 1, + "step": 0, + "format_rewards": 1.0, + "rewards": 10.110225677490234, + "advantages": 0.3172726035118103, + "generated_text": "{\n \"intent\": \"To explore the environment by interacting with an icon file or organize files on the desktop.\",\n \"action\": \"DoubleClick(76, 483)\"\n}", + "action_string": "DoubleClick(76, 483)", + "intent_string": "To explore the environment by interacting with an icon file or organize files on the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 26, + 66, + 60, + 93 + ] + }, + { + "text": " open File", + "boxes": [ + 60, + 62, + 208, + 105 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 92, + 115, + 236, + 149 + ] + }, + { + "text": "Mous", + "boxes": [ + 10, + 171, + 56, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 66, + 173, + 104, + 210 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 102, + 181, + 226, + 212 + ] + }, + { + "text": " Templates ", + "boxes": [ + 92, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 63, + 279, + 141, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 102, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 403 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "", + "boxes": [ + 55, + 405, + 109, + 459 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 419, + 314, + 449 + ] + }, + { + "text": "D", + "boxes": [ + 66, + 459, + 102, + 500 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 102, + 462, + 252, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "圆", + "boxes": [ + 64, + 508, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 96, + 508, + 244, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 26, + 606, + 59, + 635 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1374, + 699 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "Extensions", + "boxes": [ + 220, + 981, + 294, + 1005 + ] + }, + { + "text": "breoffice", + "boxes": [ + 226, + 1050, + 288, + 1068 + ] + } + ] + }, + { + "frame_id": "50_2_0", + "episode_id": 50, + "env_id": 2, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.466675758361816, + "advantages": 0.9851219654083252, + "generated_text": "{\n \"intent\": \"Beginning by clicking on an icon on the desktop will allow us to quickly access applications such as Firefox, Mousepad, Terminal, Calculator, and others for further exploration.\",\n \"action\": \"DoubleClick(57, 49)\"\n}", + "action_string": "DoubleClick(57, 49)", + "intent_string": "Beginning by clicking on an icon on the desktop will allow us to quickly access applications such as Firefox, Mousepad, Terminal, Calculator, and others for further exploration.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Qbearch with Bing orenteraddress", + "boxes": [ + 244, + 57, + 522, + 100 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_3_0", + "episode_id": 50, + "env_id": 3, + "step": 0, + "format_rewards": 1.0, + "rewards": 9.135489463806152, + "advantages": -0.16263961791992188, + "generated_text": "{\n \"intent\": \"To start interacting with the environment by exploring various applications.\",\n \"action\": \"Move(60, 45)\"\n}", + "action_string": "Move(60, 45)", + "intent_string": "To start interacting with the environment by exploring various applications.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "工", + "boxes": [ + 52, + 28, + 68, + 44 + ] + }, + { + "text": "Fref Firefox", + "boxes": [ + 13, + 44, + 139, + 105 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 6, + 385, + 118, + 423 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "50_4_0", + "episode_id": 50, + "env_id": 4, + "step": 0, + "format_rewards": 1.0, + "rewards": 8.971527099609375, + "advantages": -0.24336662888526917, + "generated_text": "{\n \"intent\": \"To continue exploring, I will first open one of the applications to gain more insight into its features.\",\n \"action\": \"DoubleClick(67, 277)\"\n}", + "action_string": "DoubleClick(67, 277)", + "intent_string": "To continue exploring, I will first open one of the applications to gain more insight into its features.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 38, + 129, + 78, + 159 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 6, + 169, + 114, + 201 + ] + }, + { + "text": "Termmal", + "boxes": [ + 10, + 272, + 110, + 316 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 32, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "oot@02b525", + "boxes": [ + 224, + 1048, + 312, + 1072 + ] + } + ] + }, + { + "frame_id": "50_5_0", + "episode_id": 50, + "env_id": 5, + "step": 0, + "format_rewards": 1.0, + "rewards": 9.8319673538208, + "advantages": 0.18027187883853912, + "generated_text": "{\n \"intent\": \"To execute the first action systematically by starting with clicking on an icon, which likely leads to navigating into another application or utility.\",\n \"action\": \"DoubleClick(31, 166)\"\n}", + "action_string": "DoubleClick(31, 166)", + "intent_string": "To execute the first action systematically by starting with clicking on an icon, which likely leads to navigating into another application or utility.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 42, + 131, + 76, + 153 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 3, + 160, + 117, + 205 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "50_6_0", + "episode_id": 50, + "env_id": 6, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.923095703125, + "advantages": 1.2098407745361328, + "generated_text": "{\n \"intent\": \"To start exploring applications available on the desktop by opening one of them'\",\n \"action\": \"DoubleClick(48, 735)\"\n}", + "action_string": "DoubleClick(48, 735)", + "intent_string": "To start exploring applications available on the desktop by opening one of them'", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "··目、自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "勺·、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 16, + 943, + 28, + 955 + ] + }, + { + "text": "K一→一", + "boxes": [ + 14, + 997, + 104, + 1014 + ] + } + ] + }, + { + "frame_id": "50_7_0", + "episode_id": 50, + "env_id": 7, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.523933410644531, + "advantages": 1.013312816619873, + "generated_text": "{\n \"intent\": \"To start by interacting with an application to explore its interface and functionalities.\",\n \"action\": \"DoubleClick(35, 68)\"\n}", + "action_string": "DoubleClick(35, 68)", + "intent_string": "To start by interacting with an application to explore its interface and functionalities.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C", + "boxes": [ + 24, + 66, + 128, + 89 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_0_1", + "episode_id": 50, + "env_id": 0, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.722882270812988, + "advantages": 0.6189146041870117, + "generated_text": "{\n \"intent\": \"To begin exploring the web by navigating to a popular or interesting link.\",\n \"action\": \"DoubleClick(751, 552)\"\n}", + "action_string": "DoubleClick(751, 552)", + "intent_string": "To begin exploring the web by navigating to a popular or interesting link.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C", + "boxes": [ + 72, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 139, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1446, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your.home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1352, + 969, + 1508, + 999 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1028 + ] + } + ] + }, + { + "frame_id": "50_1_1", + "episode_id": 50, + "env_id": 1, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.944239616394043, + "advantages": 0.7279000878334045, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment by investigating available applications such as LibreOffice.\",\n \"action\": \"DoubleClick(130, 87)\"\n}", + "action_string": "DoubleClick(130, 87)", + "intent_string": "To continue exploring the desktop environment by investigating available applications such as LibreOffice.", + "before_screen_ocr_results": [ + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 26, + 66, + 60, + 93 + ] + }, + { + "text": " open File", + "boxes": [ + 60, + 62, + 208, + 105 + ] + }, + { + "text": "一", + "boxes": [ + 72, + 121, + 98, + 135 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 92, + 115, + 236, + 149 + ] + }, + { + "text": "Mous", + "boxes": [ + 10, + 171, + 56, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 66, + 173, + 104, + 210 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 102, + 181, + 226, + 212 + ] + }, + { + "text": "T", + "boxes": [ + 74, + 230, + 100, + 254 + ] + }, + { + "text": " Templates ", + "boxes": [ + 92, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 63, + 279, + 141, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 102, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 403 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 419, + 314, + 449 + ] + }, + { + "text": "D", + "boxes": [ + 66, + 459, + 102, + 500 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 102, + 462, + 252, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "圆", + "boxes": [ + 64, + 508, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 96, + 508, + 244, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 26, + 606, + 59, + 635 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1374, + 699 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "Extensions", + "boxes": [ + 220, + 981, + 294, + 1005 + ] + }, + { + "text": "breoffice", + "boxes": [ + 226, + 1050, + 286, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": " onen ile ", + "boxes": [ + 57, + 59, + 209, + 112 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "』 clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates ", + "boxes": [ + 100, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_1", + "episode_id": 50, + "env_id": 2, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.351095199584961, + "advantages": -0.05648594722151756, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment, clicking the 'New Tab' button will allow us to test different browsing functionalities or search options.\",\n \"action\": \"Click(34, 72)\"\n}", + "action_string": "Click(34, 72)", + "intent_string": "To continue exploring the desktop environment, clicking the 'New Tab' button will allow us to test different browsing functionalities or search options.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→→", + "boxes": [ + 24, + 66, + 132, + 93 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_3_1", + "episode_id": 50, + "env_id": 3, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.508493423461914, + "advantages": 0.021009203046560287, + "generated_text": "{\n \"intent\": \"To start by exploring the functionality of each application on the desktop.\",\n \"action\": \"DoubleClick(105, 42)\"\n}", + "action_string": "DoubleClick(105, 42)", + "intent_string": "To start by exploring the functionality of each application on the desktop.", + "before_screen_ocr_results": [ + { + "text": "工", + "boxes": [ + 52, + 28, + 68, + 44 + ] + }, + { + "text": "Fref Firefox", + "boxes": [ + 13, + 44, + 139, + 105 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 6, + 385, + 118, + 423 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 102, + 99 + ] + }, + { + "text": "1", + "boxes": [ + 34, + 119, + 82, + 165 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 38, + 234, + 76, + 264 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "50_4_1", + "episode_id": 50, + "env_id": 4, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.13178825378418, + "advantages": -0.1644619107246399, + "generated_text": "{\n \"intent\": \"Clicking the Terminal icon will allow me to explore commands or applications installed on the system, which will help me understand more about the system's capabilities.\",\n \"action\": \"Click(59, 297)\"\n}", + "action_string": "Click(59, 297)", + "intent_string": "Clicking the Terminal icon will allow me to explore commands or applications installed on the system, which will help me understand more about the system's capabilities.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 38, + 129, + 78, + 159 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 6, + 169, + 114, + 201 + ] + }, + { + "text": "Termmal", + "boxes": [ + 10, + 272, + 110, + 316 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 32, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "oot@02b525", + "boxes": [ + 224, + 1048, + 312, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 38, + 129, + 78, + 155 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 112, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 9, + 271, + 111, + 316 + ] + }, + { + "text": "Terminal", + "boxes": [ + 64, + 310, + 144, + 336 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 16, + 498, + 96, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 32, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + } + ] + }, + { + "frame_id": "50_5_1", + "episode_id": 50, + "env_id": 5, + "step": 1, + "format_rewards": 1.0, + "rewards": 11.423075675964355, + "advantages": 0.963655412197113, + "generated_text": "{\n \"intent\": \"To open Firefox and browse the web, which provides new information and explores internet resources.\",\n \"action\": \"DoubleClick(55, 34)\"\n}", + "action_string": "DoubleClick(55, 34)", + "intent_string": "To open Firefox and browse the web, which provides new information and explores internet resources.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 42, + 131, + 76, + 153 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 3, + 160, + 117, + 205 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→e", + "boxes": [ + 46, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_6_1", + "episode_id": 50, + "env_id": 6, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.798051834106445, + "advantages": 0.16357353329658508, + "generated_text": "{\n \"intent\": \"Double-clicking might help me explore or work within the spreadsheet application more effectively.\",\n \"action\": \"DoubleClick(50, 70)\"\n}", + "action_string": "DoubleClick(50, 70)", + "intent_string": "Double-clicking might help me explore or work within the spreadsheet application more effectively.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "·口·目·目@自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "白·>、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "=[", + "boxes": [ + 148, + 125, + 260, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "46", + "boxes": [ + 16, + 943, + 28, + 955 + ] + }, + { + "text": "K一", + "boxes": [ + 14, + 997, + 104, + 1014 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "File", + "boxes": [ + 8, + 26, + 36, + 46 + ] + }, + { + "text": "Edit View", + "boxes": [ + 42, + 26, + 118, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 524, + 30, + 550, + 44 + ] + }, + { + "text": "画口·目·目@%血、", + "boxes": [ + 12, + 54, + 378, + 83 + ] + }, + { + "text": "上&", + "boxes": [ + 369, + 51, + 445, + 86 + ] + }, + { + "text": "-、", + "boxes": [ + 450, + 56, + 554, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "New (Ctrl+N)", + "boxes": [ + 12, + 89, + 102, + 113 + ] + }, + { + "text": "1aαaa-=要日日-%00+三至图-目", + "boxes": [ + 126, + 93, + 1208, + 123 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "KA→→", + "boxes": [ + 14, + 997, + 104, + 1014 + ] + } + ] + }, + { + "frame_id": "50_7_1", + "episode_id": 50, + "env_id": 7, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.37833309173584, + "advantages": -0.04307534918189049, + "generated_text": "{\n \"intent\": \"To continue exploring the computer desktop environment, I will scroll down slightly to view more content.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring the computer desktop environment, I will scroll down slightly to view more content.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C", + "boxes": [ + 24, + 66, + 128, + 89 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "→C", + "boxes": [ + 42, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "6", + "boxes": [ + 1233, + 479, + 1271, + 506 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_0_2", + "episode_id": 50, + "env_id": 0, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.176481246948242, + "advantages": 0.34989356994628906, + "generated_text": "{\n \"intent\": \"To continue exploring the Amazon website by interacting further with the navigation options or content, I will scroll down slightly to see more content below.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring the Amazon website by interacting further with the navigation options or content, I will scroll down slightly to see more content below.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 249, + 63, + 313, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop for yourhome essentials", + "boxes": [ + 608, + 528, + 916, + 558 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1352, + 969, + 1508, + 999 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1660, + 87 + ] + }, + { + "text": "lions of products withfast localdelivery Click heretogotoa", + "boxes": [ + 952, + 109, + 1344, + 139 + ] + }, + { + "text": "", + "boxes": [ + 1444, + 143, + 1502, + 161 + ] + }, + { + "text": "Gamingaccessories", + "boxes": [ + 246, + 191, + 436, + 208 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 185, + 918, + 214 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 982, + 185, + 1202, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 185, + 1534, + 214 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 544, + 1058, + 568 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1349, + 541, + 1437, + 572 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 238, + 619, + 500, + 655 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 621, + 1510, + 657 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 651, + 352, + 681 + ] + }, + { + "text": "Explore all products in Kitcher", + "boxes": [ + 246, + 987, + 418, + 1005 + ] + }, + { + "text": "Shop the latest from Hom", + "boxes": [ + 614, + 987, + 766, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Shopnow", + "boxes": [ + 1354, + 987, + 1412, + 1007 + ] + } + ] + }, + { + "frame_id": "50_1_2", + "episode_id": 50, + "env_id": 1, + "step": 2, + "format_rewards": 0.0, + "rewards": 0.0, + "advantages": -4.660505771636963, + "generated_text": "{\n \"intent\": \"To locate and interact with apps and settings efficiently, I should start by navigating through the menu options.\",\n \"action\": \"Move(140, 40) Click(140, 40)\"\n}", + "action_string": "None", + "intent_string": null, + "before_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": " onen ile ", + "boxes": [ + 57, + 59, + 209, + 112 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "』 clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": " onen ile ", + "boxes": [ + 57, + 59, + 209, + 112 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "』 clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_2", + "episode_id": 50, + "env_id": 2, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.289260864257812, + "advantages": -0.08693013340234756, + "generated_text": "{\n \"intent\": \"Clicking the 'New Tab' button will allow me to open a new browser tab, which can be useful for exploring more websites or verifying bookmarked sites.\",\n \"action\": \"Click(85, 37)\"\n}", + "action_string": "Click(85, 37)", + "intent_string": "Clicking the 'New Tab' button will allow me to open a new browser tab, which can be useful for exploring more websites or verifying bookmarked sites.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 22, + 61, + 128, + 92 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 21, + 27, + 95, + 54 + ] + }, + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_3_2", + "episode_id": 50, + "env_id": 3, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.867382049560547, + "advantages": 0.19770832359790802, + "generated_text": "{\n \"intent\": \"To start exploring applications by clicking on one of the available application icons.\",\n \"action\": \"Click(35, 162)\"\n}", + "action_string": "Click(35, 162)", + "intent_string": "To start exploring applications by clicking on one of the available application icons.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 102, + 99 + ] + }, + { + "text": "1", + "boxes": [ + 34, + 119, + 82, + 165 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 264 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 3, + 162, + 117, + 205 + ] + }, + { + "text": "A", + "boxes": [ + 44, + 238, + 74, + 260 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "50_4_2", + "episode_id": 50, + "env_id": 4, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.503486633300781, + "advantages": 0.01854410581290722, + "generated_text": "{\n \"intent\": \"I am considering starting from the base of my exploration by interacting with a terminal since it usually allows access to system-level functionalities which could be useful for learning more about the environment.\",\n \"action\": \"DoubleClick(55, 293)\"\n}", + "action_string": "DoubleClick(55, 293)", + "intent_string": "I am considering starting from the base of my exploration by interacting with a terminal since it usually allows access to system-level functionalities which could be useful for learning more about the environment.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 38, + 129, + 78, + 155 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 112, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 9, + 271, + 111, + 316 + ] + }, + { + "text": "Terminal", + "boxes": [ + 64, + 310, + 144, + 336 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 16, + 498, + 96, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 32, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "1", + "boxes": [ + 34, + 119, + 82, + 165 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "Terginal", + "boxes": [ + 7, + 269, + 111, + 320 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "@02b525", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "50_5_2", + "episode_id": 50, + "env_id": 5, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.370196342468262, + "advantages": -0.04708148166537285, + "generated_text": "{\n \"intent\": \"To further explore the environment, I will look around the desktop by scanning areas where more applications or resources might be located.\",\n \"action\": \"Move(500, 500)\"\n}", + "action_string": "Move(500, 500)", + "intent_string": "To further explore the environment, I will look around the desktop by scanning areas where more applications or resources might be located.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C", + "boxes": [ + 54, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 40, + 66, + 128, + 89 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": " Search the Web", + "boxes": [ + 654, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 490, + 401, + 588, + 425 + ] + }, + { + "text": "*··", + "boxes": [ + 580, + 425, + 596, + 443 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_6_2", + "episode_id": 50, + "env_id": 6, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.088541030883789, + "advantages": -0.1857547163963318, + "generated_text": "{\n \"intent\": \"To continue exploring by interacting with an element on the screen.\",\n \"action\": \"DoubleClick(500, 200)\"\n}", + "action_string": "DoubleClick(500, 200)", + "intent_string": "To continue exploring by interacting with an element on the screen.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "File", + "boxes": [ + 8, + 26, + 36, + 46 + ] + }, + { + "text": "Edit View", + "boxes": [ + 42, + 26, + 118, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 524, + 30, + 550, + 44 + ] + }, + { + "text": "口-目·日@%自-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "上&", + "boxes": [ + 369, + 51, + 445, + 86 + ] + }, + { + "text": "勺·、", + "boxes": [ + 450, + 56, + 554, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "New (Ctrl+N)", + "boxes": [ + 12, + 89, + 102, + 113 + ] + }, + { + "text": "1aαaa-=要日日-%00+三至图-目", + "boxes": [ + 126, + 93, + 1208, + 123 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "K一→一", + "boxes": [ + 14, + 997, + 104, + 1014 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "··目·目@%自、 ", + "boxes": [ + 10, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "-、", + "boxes": [ + 444, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "F2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 256, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 197 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "→→+", + "boxes": [ + 14, + 995, + 132, + 1017 + ] + } + ] + }, + { + "frame_id": "50_7_2", + "episode_id": 50, + "env_id": 7, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.680074691772461, + "advantages": 0.1054873913526535, + "generated_text": "{\n \"intent\": \"To start exploring new content and websites by accessing a different webpage.\",\n \"action\": \"DoubleClick(798, 298)\"\n}", + "action_string": "DoubleClick(798, 298)", + "intent_string": "To start exploring new content and websites by accessing a different webpage.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "→C", + "boxes": [ + 42, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "6", + "boxes": [ + 1233, + 479, + 1271, + 506 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 166, + 123 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 642, + 286, + 772, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 524, + 488, + 562, + 528 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_0_3", + "episode_id": 50, + "env_id": 0, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.821083068847656, + "advantages": 0.17491298913955688, + "generated_text": "{\n \"intent\": \"If I want to explore more or perform any actions like searching or navigating further, interacting with the page by scrolling would be a reasonable choice.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "If I want to explore more or perform any actions like searching or navigating further, interacting with the page by scrolling would be a reasonable choice.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1606, + 69, + 1658, + 87 + ] + }, + { + "text": "Gamingaccessories", + "boxes": [ + 244, + 189, + 438, + 212 + ] + }, + { + "text": "Shopfor your home essentials", + "boxes": [ + 614, + 187, + 916, + 210 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 982, + 185, + 1202, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 185, + 1534, + 214 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 544, + 1058, + 568 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1349, + 541, + 1437, + 572 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 621, + 1510, + 657 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 653, + 354, + 683 + ] + }, + { + "text": "Explore all products in Kitcher", + "boxes": [ + 246, + 987, + 418, + 1005 + ] + }, + { + "text": "Shop the latest from Hom", + "boxes": [ + 614, + 987, + 766, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Shopnow", + "boxes": [ + 1354, + 987, + 1412, + 1007 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " under $50", + "boxes": [ + 1176, + 210, + 1240, + 230 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 262, + 300, + 280 + ] + }, + { + "text": "See all deals ", + "boxes": [ + 982, + 258, + 1062, + 284 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1351, + 258, + 1435, + 286 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 339, + 498, + 369 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 339, + 908, + 369 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 339, + 1220, + 369 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1350, + 339, + 1508, + 369 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 367, + 354, + 397 + ] + }, + { + "text": "See all deal", + "boxes": [ + 988, + 701, + 1052, + 719 + ] + }, + { + "text": "Continue shopping for", + "boxes": [ + 242, + 782, + 472, + 812 + ] + }, + { + "text": "See more", + "boxes": [ + 474, + 788, + 540, + 806 + ] + }, + { + "text": "", + "boxes": [ + 312, + 816, + 340, + 830 + ] + } + ] + }, + { + "frame_id": "50_1_3", + "episode_id": 50, + "env_id": 1, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.148218154907227, + "advantages": -0.15637263655662537, + "generated_text": "{\n \"intent\": \"To view the contents of the Home directory and explore files available there.\",\n \"action\": \"DoubleClick(367, 159)\"\n}", + "action_string": "DoubleClick(367, 159)", + "intent_string": "To view the contents of the Home directory and explore files available there.", + "before_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": " onen ile ", + "boxes": [ + 57, + 59, + 209, + 112 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "』 clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Desktop", + "boxes": [ + 399, + 172, + 493, + 209 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 104, + 183, + 228, + 212 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "』 clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "open", + "boxes": [ + 1442, + 949, + 1498, + 975 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_3", + "episode_id": 50, + "env_id": 2, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.315452575683594, + "advantages": -0.07403462380170822, + "generated_text": "{\n \"intent\": \"To move the cursor to an unvisited website icon in the 'Top Sites' section, which allows exploration of different websites.\",\n \"action\": \"Move(550, 485)\"\n}", + "action_string": "Move(550, 485)", + "intent_string": "To move the cursor to an unvisited website icon in the 'Top Sites' section, which allows exploration of different websites.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 21, + 27, + 95, + 54 + ] + }, + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "→C", + "boxes": [ + 44, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": " Search the Web", + "boxes": [ + 654, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 588, + 425 + ] + }, + { + "text": "*··", + "boxes": [ + 580, + 425, + 596, + 443 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 522, + 534, + 558, + 554 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_3_3", + "episode_id": 50, + "env_id": 3, + "step": 3, + "format_rewards": 1.0, + "rewards": 11.782666206359863, + "advantages": 1.140700101852417, + "generated_text": "{\n \"intent\": \"I will start by clicking on one of the applications on the desktop to investigate its functionality.\",\n \"action\": \"DoubleClick(50, 85)\"\n}", + "action_string": "DoubleClick(50, 85)", + "intent_string": "I will start by clicking on one of the applications on the desktop to investigate its functionality.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 3, + 162, + 117, + 205 + ] + }, + { + "text": "A", + "boxes": [ + 44, + 238, + 74, + 260 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 44, + 66, + 130, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 8, + 97, + 166, + 125 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_4_3", + "episode_id": 50, + "env_id": 4, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.065251350402832, + "advantages": -0.19722141325473785, + "generated_text": "{\n \"intent\": \"Clicking on the Firefox icon will allow me to browse the web, which will add variety to my exploration of the desktop environment by accessing external resources and websites.\",\n \"action\": \"Click(61, 30)\"\n}", + "action_string": "Click(61, 30)", + "intent_string": "Clicking on the Firefox icon will allow me to browse the web, which will add variety to my exploration of the desktop environment by accessing external resources and websites.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 6, + 169, + 114, + 201 + ] + }, + { + "text": "Terginal", + "boxes": [ + 10, + 274, + 110, + 318 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "Libre...", + "boxes": [ + 14, + 498, + 98, + 530 + ] + }, + { + "text": "@02b525", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "2", + "boxes": [ + 40, + 22, + 82, + 50 + ] + }, + { + "text": " Firefox", + "boxes": [ + 72, + 40, + 132, + 66 + ] + }, + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 102, + 99 + ] + }, + { + "text": "1", + "boxes": [ + 32, + 119, + 82, + 165 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 38, + 234, + 78, + 264 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "@02b525", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "50_5_3", + "episode_id": 50, + "env_id": 5, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.205345153808594, + "advantages": -0.12824611365795135, + "generated_text": "{\n \"intent\": \"To navigate outside the Firefox browser by moving the cursor away from the starting point could provide a broader overview of the desktop layout.\",\n \"action\": \"Move(900, 500)\"\n}", + "action_string": "Move(900, 500)", + "intent_string": "To navigate outside the Firefox browser by moving the cursor away from the starting point could provide a broader overview of the desktop layout.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "→C", + "boxes": [ + 44, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": " Search the Web", + "boxes": [ + 654, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 588, + 425 + ] + }, + { + "text": "*··", + "boxes": [ + 580, + 425, + 596, + 443 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "IQIY", + "boxes": [ + 860, + 457, + 934, + 506 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 996, + 534, + 1036, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_6_3", + "episode_id": 50, + "env_id": 6, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.382332801818848, + "advantages": -0.04110608622431755, + "generated_text": "{\n \"intent\": \"I will start by exploring the available functions and settings in the LibreOffice Calc application since I'm not sure how to begin.\",\n \"action\": \"DoubleClick(546, 76)\"\n}", + "action_string": "DoubleClick(546, 76)", + "intent_string": "I will start by exploring the available functions and settings in the LibreOffice Calc application since I'm not sure how to begin.", + "before_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "··目·目@%自、 ", + "boxes": [ + 10, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "-、", + "boxes": [ + 444, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "F2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 256, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 197 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K\\→→+", + "boxes": [ + 14, + 995, + 132, + 1017 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 524, + 30, + 552, + 44 + ] + }, + { + "text": "·-目·@%自- ", + "boxes": [ + 10, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "G--", + "boxes": [ + 448, + 56, + 542, + 81 + ] + }, + { + "text": "田-E", + "boxes": [ + 538, + 48, + 1264, + 87 + ] + }, + { + "text": "Liberation san10aαaa==", + "boxes": [ + 12, + 93, + 582, + 123 + ] + }, + { + "text": "E", + "boxes": [ + 584, + 97, + 616, + 117 + ] + }, + { + "text": "回日日-%00面t三至田-二-口-目", + "boxes": [ + 618, + 93, + 1208, + 123 + ] + }, + { + "text": "F2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 256, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "50_7_3", + "episode_id": 50, + "env_id": 7, + "step": 3, + "format_rewards": 1.0, + "rewards": 11.020000457763672, + "advantages": 0.7652009725570679, + "generated_text": "{\n \"intent\": \"To familiarize with different applications available on the browser by exploring bookmarks.\",\n \"action\": \"DoubleClick(650, 484)\"\n}", + "action_string": "DoubleClick(650, 484)", + "intent_string": "To familiarize with different applications available on the browser by exploring bookmarks.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 166, + 123 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web ", + "boxes": [ + 658, + 286, + 772, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 524, + 488, + 562, + 528 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1354, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 636, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 792, + 449, + 972, + 478 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 725, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1466, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1462, + 808, + 1502, + 838 + ] + }, + { + "text": "+2.0026 ", + "boxes": [ + 1636, + 802, + 1698, + 844 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": "261029 ", + "boxes": [ + 1632, + 846, + 1700, + 893 + ] + }, + { + "text": "13,67", + "boxes": [ + 1628, + 891, + 1698, + 941 + ] + }, + { + "text": "游戏天地·5天", + "boxes": [ + 830, + 921, + 942, + 945 + ] + }, + { + "text": "一点资讯·2周", + "boxes": [ + 1160, + 921, + 1254, + 945 + ] + }, + { + "text": "和平精英：3个提升听声辨位", + "boxes": [ + 824, + 941, + 1092, + 977 + ] + }, + { + "text": "霸王大陆属性趣谈：吕布两项", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+32829 ", + "boxes": [ + 1630, + 939, + 1700, + 989 + ] + }, + { + "text": "能力的方法，快试试！", + "boxes": [ + 824, + 967, + 1032, + 1003 + ] + }, + { + "text": "第一，但综合最强还是常山赵", + "boxes": [ + 1136, + 965, + 1408, + 1003 + ] + }, + { + "text": "适合近战法师的几把近战武器推荐", + "boxes": [ + 200, + 981, + 644, + 1022 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "子龙", + "boxes": [ + 1136, + 995, + 1192, + 1028 + ] + }, + { + "text": "3", + "boxes": [ + 1140, + 1028, + 1170, + 1042 + ] + } + ] + }, + { + "frame_id": "50_0_4", + "episode_id": 50, + "env_id": 0, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.322080612182617, + "advantages": 0.4215795695781708, + "generated_text": "{\n \"intent\": \"To explore more categories beyond the currently visible items by scrolling down.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore more categories beyond the currently visible items by scrolling down.", + "before_screen_ocr_results": [ + { + "text": "Shoes under $50", + "boxes": [ + 1150, + 208, + 1242, + 232 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 262, + 302, + 282 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 258, + 1062, + 284 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1352, + 260, + 1436, + 284 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 339, + 498, + 369 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 339, + 908, + 369 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 339, + 1222, + 369 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1350, + 339, + 1510, + 369 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 367, + 354, + 397 + ] + }, + { + "text": "Kettles", + "boxes": [ + 460, + 657, + 500, + 673 + ] + }, + { + "text": "Seealldeals", + "boxes": [ + 988, + 701, + 1054, + 719 + ] + }, + { + "text": "Continue shopping for", + "boxes": [ + 240, + 782, + 472, + 812 + ] + }, + { + "text": "See more", + "boxes": [ + 474, + 788, + 540, + 806 + ] + }, + { + "text": "", + "boxes": [ + 310, + 816, + 340, + 830 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "lucts in Kitcher", + "boxes": [ + 334, + 417, + 418, + 435 + ] + }, + { + "text": "hop the latest frc", + "boxes": [ + 620, + 417, + 720, + 435 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 415, + 1058, + 439 + ] + }, + { + "text": "Continue shopping for See more", + "boxes": [ + 238, + 494, + 544, + 526 + ] + }, + { + "text": "Pick up where you left off ", + "boxes": [ + 242, + 800, + 508, + 830 + ] + } + ] + }, + { + "frame_id": "50_1_4", + "episode_id": 50, + "env_id": 1, + "step": 4, + "format_rewards": 1.0, + "rewards": 8.986750602722168, + "advantages": -0.2358713299036026, + "generated_text": "{\n \"intent\": \"To explore different file locations and open files efficiently.\",\n \"action\": \"DoubleClick(876, 355)\"\n}", + "action_string": "DoubleClick(876, 355)", + "intent_string": "To explore different file locations and open files efficiently.", + "before_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Desktop", + "boxes": [ + 399, + 172, + 493, + 209 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 104, + 183, + 228, + 212 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "』 clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "open", + "boxes": [ + 1442, + 949, + 1498, + 975 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Libreoffic", + "boxes": [ + 918, + 10, + 996, + 28 + ] + }, + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 632, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "clipboard_app", + "boxes": [ + 556, + 220, + 668, + 242 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 242 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 220, + 981, + 294, + 1005 + ] + }, + { + "text": "ibreoffice", + "boxes": [ + 222, + 1050, + 288, + 1070 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_4", + "episode_id": 50, + "env_id": 2, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.634429931640625, + "advantages": 0.5753650069236755, + "generated_text": "{\n \"intent\": \"I need to explore different parts of the browser interface to understand its functionalities. Clicking on the 'Getting Started' link will help introduce me to Firefox's features.\",\n \"action\": \"DoubleClick(198, 102)\"\n}", + "action_string": "DoubleClick(198, 102)", + "intent_string": "I need to explore different parts of the browser interface to understand its functionalities. Clicking on the 'Getting Started' link will help introduce me to Firefox's features.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 40, + 66, + 128, + 89 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": " Search the Web", + "boxes": [ + 654, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 490, + 401, + 588, + 425 + ] + }, + { + "text": "*··", + "boxes": [ + 580, + 425, + 596, + 443 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 522, + 534, + 558, + 554 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 441, + 646, + 498 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 524, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "50_3_4", + "episode_id": 50, + "env_id": 3, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.965959548950195, + "advantages": 0.7385938763618469, + "generated_text": "{\n \"intent\": \"To start by interacting with the web browser to see if it opens any new websites or layouts.\",\n \"action\": \"DoubleClick(189, 110)\"\n}", + "action_string": "DoubleClick(189, 110)", + "intent_string": "To start by interacting with the web browser to see if it opens any new websites or layouts.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C", + "boxes": [ + 56, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 236, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 441, + 646, + 498 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 342, + 524, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "50_4_4", + "episode_id": 50, + "env_id": 4, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.073822021484375, + "advantages": -0.1930016279220581, + "generated_text": "{\n \"intent\": \"The terminal window appears active, allowing potential experimentation or exploration. Clicking on the title bar might reveal more actions related to managing terminals.\",\n \"action\": \"Click(783, 301)\"\n}", + "action_string": "Click(783, 301)", + "intent_string": "The terminal window appears active, allowing potential experimentation or exploration. Clicking on the title bar might reveal more actions related to managing terminals.", + "before_screen_ocr_results": [ + { + "text": "2", + "boxes": [ + 40, + 22, + 82, + 50 + ] + }, + { + "text": " Firefox", + "boxes": [ + 72, + 40, + 132, + 66 + ] + }, + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 102, + 99 + ] + }, + { + "text": "1", + "boxes": [ + 32, + 119, + 82, + 165 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 38, + 234, + 78, + 264 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "@02b525", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "1", + "boxes": [ + 34, + 119, + 82, + 165 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 38, + 234, + 78, + 264 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "Libre...", + "boxes": [ + 14, + 498, + 100, + 530 + ] + }, + { + "text": "root@02b525", + "boxes": [ + 216, + 1046, + 312, + 1072 + ] + }, + { + "text": "root@02b525..", + "boxes": [ + 336, + 1048, + 470, + 1072 + ] + } + ] + }, + { + "frame_id": "50_5_4", + "episode_id": 50, + "env_id": 5, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.203675270080566, + "advantages": -0.1290682703256607, + "generated_text": "{\n \"intent\": \"Since there's nothing else of interest visible in this Firefox browser tab, I will scroll down to see if more content related to browsing or web exploration is available.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "Since there's nothing else of interest visible in this Firefox browser tab, I will scroll down to see if more content related to browsing or web exploration is available.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "IQIY", + "boxes": [ + 860, + 457, + 934, + 506 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 996, + 534, + 1036, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "→C", + "boxes": [ + 42, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "6", + "boxes": [ + 1233, + 479, + 1271, + 506 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_6_4", + "episode_id": 50, + "env_id": 6, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.682913780212402, + "advantages": 0.10688521713018417, + "generated_text": "{\n \"intent\": \"To navigate through different tools or options available in LibreOffice Calc for further exploring the environment.\",\n \"action\": \"RightClick(500, 70)\"\n}", + "action_string": "RightClick(500, 70)", + "intent_string": "To navigate through different tools or options available in LibreOffice Calc for further exploring the environment.", + "before_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 524, + 30, + 552, + 44 + ] + }, + { + "text": "·-目·@%自- ", + "boxes": [ + 10, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "G--", + "boxes": [ + 448, + 56, + 542, + 81 + ] + }, + { + "text": "田-E", + "boxes": [ + 538, + 48, + 1264, + 87 + ] + }, + { + "text": "Liberation san10aαaa==", + "boxes": [ + 12, + 93, + 582, + 123 + ] + }, + { + "text": "E", + "boxes": [ + 584, + 97, + 616, + 117 + ] + }, + { + "text": "回日日-%00面t三至田-二-口-目", + "boxes": [ + 618, + 93, + 1208, + 123 + ] + }, + { + "text": "F2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 256, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOffice Calc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 8, + 26, + 36, + 46 + ] + }, + { + "text": "Styles", + "boxes": [ + 250, + 28, + 296, + 46 + ] + }, + { + "text": "Tools Windol", + "boxes": [ + 396, + 24, + 502, + 48 + ] + }, + { + "text": " Load URL", + "boxes": [ + 524, + 28, + 614, + 54 + ] + }, + { + "text": "--- 自-", + "boxes": [ + 8, + 48, + 386, + 87 + ] + }, + { + "text": "盒&", + "boxes": [ + 370, + 54, + 446, + 85 + ] + }, + { + "text": "", + "boxes": [ + 438, + 56, + 486, + 81 + ] + }, + { + "text": "↑E·B", + "boxes": [ + 774, + 46, + 1264, + 89 + ] + }, + { + "text": " Open Remote", + "boxes": [ + 522, + 81, + 644, + 111 + ] + }, + { + "text": "Liberation san10aαaa=", + "boxes": [ + 12, + 93, + 504, + 123 + ] + }, + { + "text": "%00正田二口目", + "boxes": [ + 774, + 93, + 1204, + 123 + ] + }, + { + "text": " save As..", + "boxes": [ + 522, + 105, + 610, + 137 + ] + }, + { + "text": "F2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "?", + "boxes": [ + 130, + 129, + 156, + 151 + ] + }, + { + "text": "=[", + "boxes": [ + 150, + 125, + 260, + 157 + ] + }, + { + "text": " Email", + "boxes": [ + 524, + 135, + 588, + 161 + ] + }, + { + "text": "4", + "boxes": [ + 524, + 191, + 548, + 214 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "A", + "boxes": [ + 506, + 220, + 548, + 240 + ] + }, + { + "text": " Navigator", + "boxes": [ + 520, + 244, + 618, + 270 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "", + "boxes": [ + 521, + 269, + 553, + 299 + ] + }, + { + "text": "Insert Rows Above", + "boxes": [ + 552, + 272, + 676, + 296 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 266, + 1914, + 312 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "日", + "boxes": [ + 534, + 359, + 546, + 371 + ] + }, + { + "text": "", + "boxes": [ + 528, + 463, + 554, + 482 + ] + }, + { + "text": "Advanced Filter", + "boxes": [ + 542, + 458, + 654, + 487 + ] + }, + { + "text": " Goal Seel", + "boxes": [ + 528, + 490, + 612, + 508 + ] + }, + { + "text": " Gallery ", + "boxes": [ + 526, + 514, + 602, + 540 + ] + }, + { + "text": " Formula", + "boxes": [ + 522, + 538, + 608, + 570 + ] + }, + { + "text": "Ω Symbol", + "boxes": [ + 524, + 596, + 602, + 619 + ] + }, + { + "text": "T Text Box", + "boxes": [ + 522, + 621, + 608, + 647 + ] + }, + { + "text": " Edit ", + "boxes": [ + 526, + 649, + 582, + 675 + ] + }, + { + "text": "备", + "boxes": [ + 530, + 679, + 546, + 695 + ] + }, + { + "text": "QZoom", + "boxes": [ + 528, + 786, + 586, + 804 + ] + }, + { + "text": " Libreoffice Help ", + "boxes": [ + 524, + 808, + 662, + 838 + ] + }, + { + "text": " What's This?", + "boxes": [ + 528, + 842, + 632, + 860 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 939 + ] + }, + { + "text": "Dock All Toolbars", + "boxes": [ + 528, + 957, + 640, + 981 + ] + }, + { + "text": "一→", + "boxes": [ + 20, + 997, + 92, + 1014 + ] + }, + { + "text": "Lock Toolbar Position", + "boxes": [ + 526, + 989, + 668, + 1012 + ] + } + ] + }, + { + "frame_id": "50_7_4", + "episode_id": 50, + "env_id": 7, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.508081436157227, + "advantages": 0.020806360989809036, + "generated_text": "{\n \"intent\": \"The current action will help the user explore different parts of the Bing homepage, which may include links and content on various interests.\",\n \"action\": \"Move(500, 314)\"\n}", + "action_string": "Move(500, 314)", + "intent_string": "The current action will help the user explore different parts of the Bing homepage, which may include links and content on various interests.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 636, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 792, + 449, + 972, + 478 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "194年6月6日", + "boxes": [ + 194, + 756, + 284, + 774 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1466, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1462, + 808, + 1502, + 838 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "和平精英", + "boxes": [ + 930, + 842, + 1004, + 866 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "游戏天地·5天", + "boxes": [ + 830, + 921, + 942, + 945 + ] + }, + { + "text": "1", + "boxes": [ + 1142, + 925, + 1162, + 941 + ] + }, + { + "text": "一点资讯·2周", + "boxes": [ + 1160, + 921, + 1256, + 945 + ] + }, + { + "text": "和平精英：3个提升听声辨位", + "boxes": [ + 824, + 941, + 1092, + 977 + ] + }, + { + "text": "霸王大陆属性趣谈：吕布两项", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.287", + "boxes": [ + 1626, + 939, + 1700, + 987 + ] + }, + { + "text": "游戏天", + "boxes": [ + 202, + 959, + 264, + 977 + ] + }, + { + "text": "能力的方法，快试试！", + "boxes": [ + 824, + 967, + 1032, + 1003 + ] + }, + { + "text": "第一，但综合最强还是常山赵", + "boxes": [ + 1136, + 967, + 1410, + 1003 + ] + }, + { + "text": "三国志10一特技详解", + "boxes": [ + 196, + 977, + 484, + 1024 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "子龙", + "boxes": [ + 1136, + 995, + 1192, + 1028 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "3", + "boxes": [ + 1140, + 1028, + 1170, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "", + "boxes": [ + 976, + 403, + 1000, + 425 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "取款身亡银行赔10万", + "boxes": [ + 802, + 453, + 968, + 476 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1466, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1462, + 808, + 1502, + 838 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "LDAT", + "boxes": [ + 598, + 836, + 810, + 925 + ] + }, + { + "text": "和平精英", + "boxes": [ + 928, + 842, + 1008, + 866 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": ",61023", + "boxes": [ + 1630, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "游戏天地·5天", + "boxes": [ + 828, + 919, + 944, + 949 + ] + }, + { + "text": "D", + "boxes": [ + 1140, + 923, + 1166, + 943 + ] + }, + { + "text": "一点资讯·2周", + "boxes": [ + 1160, + 921, + 1256, + 947 + ] + }, + { + "text": "和平精英：3个提升听声辨位", + "boxes": [ + 824, + 941, + 1092, + 977 + ] + }, + { + "text": "霸王大陆属性趣谈：吕布两项", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "OUNDING PARTI", + "boxes": [ + 590, + 947, + 804, + 989 + ] + }, + { + "text": "+3.2879", + "boxes": [ + 1626, + 939, + 1698, + 989 + ] + }, + { + "text": "潇湘晨报· 7 小时", + "boxes": [ + 198, + 953, + 330, + 983 + ] + }, + { + "text": "能力的方法，快试试！", + "boxes": [ + 824, + 967, + 1032, + 1003 + ] + }, + { + "text": "第一，但综合最强还是常山赵", + "boxes": [ + 1136, + 967, + 1410, + 1003 + ] + }, + { + "text": "吴艳妮打破这项29年以来纪录", + "boxes": [ + 200, + 979, + 600, + 1026 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "子龙", + "boxes": [ + 1136, + 995, + 1192, + 1028 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "3", + "boxes": [ + 1140, + 1028, + 1172, + 1042 + ] + } + ] + }, + { + "frame_id": "50_0_5", + "episode_id": 50, + "env_id": 0, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.494654655456543, + "advantages": 0.01419567409902811, + "generated_text": "{\n \"intent\": \"The current screen shows an Amazon homepage with various product categories visibly available. Since my goal is to explore as much of the desktop environment as possible, I will scroll down to view more content or options not currently visible.\",\n \"action\": \"ScrollDown(100, 50)\"\n}", + "action_string": "ScrollDown(100, 50)", + "intent_string": "The current screen shows an Amazon homepage with various product categories visibly available. Since my goal is to explore as much of the desktop environment as possible, I will scroll down to view more content or options not currently visible.", + "before_screen_ocr_results": [ + { + "text": "lucts in Kitcher", + "boxes": [ + 334, + 417, + 418, + 435 + ] + }, + { + "text": "hop the latest frc", + "boxes": [ + 620, + 417, + 720, + 435 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 415, + 1058, + 439 + ] + }, + { + "text": "Continue shopping for See more", + "boxes": [ + 238, + 494, + 544, + 526 + ] + }, + { + "text": "Pick up where you left off ", + "boxes": [ + 242, + 800, + 508, + 830 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": ". Spend less x", + "boxes": [ + 118, + 32, + 228, + 50 + ] + }, + { + "text": "lucts in Kitcher", + "boxes": [ + 334, + 417, + 418, + 435 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 415, + 1058, + 439 + ] + }, + { + "text": " Continue shopping for ", + "boxes": [ + 238, + 494, + 474, + 526 + ] + }, + { + "text": "See more", + "boxes": [ + 468, + 500, + 542, + 524 + ] + }, + { + "text": "Pick up where you left off", + "boxes": [ + 240, + 798, + 510, + 834 + ] + } + ] + }, + { + "frame_id": "50_1_5", + "episode_id": 50, + "env_id": 1, + "step": 5, + "format_rewards": 1.0, + "rewards": 8.84527587890625, + "advantages": -0.3055265247821808, + "generated_text": "{\n \"intent\": \"To start exploring additional file options by navigating the file selection window.\",\n \"action\": \"DoubleClick(870, 583)\"\n}", + "action_string": "DoubleClick(870, 583)", + "intent_string": "To start exploring additional file options by navigating the file selection window.", + "before_screen_ocr_results": [ + { + "text": "Libreoffic", + "boxes": [ + 918, + 10, + 996, + 28 + ] + }, + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 632, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "clipboard_app", + "boxes": [ + 556, + 220, + 668, + 242 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 220, + 981, + 294, + 1005 + ] + }, + { + "text": "ibreoffice", + "boxes": [ + 222, + 1050, + 288, + 1070 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "I clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates ", + "boxes": [ + 100, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 102, + 415, + 316, + 455 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 32, + 611, + 56, + 633 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "ibreoffice", + "boxes": [ + 222, + 1050, + 288, + 1070 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_5", + "episode_id": 50, + "env_id": 2, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.683058738708496, + "advantages": 0.1069565862417221, + "generated_text": "{\n \"intent\": \"To continue exploring by scrolling through the webpage to view more content and potentially discover additional resources or options not currently visible.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring by scrolling through the webpage to view more content and potentially discover additional resources or options not currently visible.", + "before_screen_ocr_results": [ + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 441, + 646, + 498 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 524, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "ns and other languages", + "boxes": [ + 440, + 367, + 576, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 492, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 800, + 917, + 1098, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ] + }, + { + "frame_id": "50_3_5", + "episode_id": 50, + "env_id": 3, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.837557792663574, + "advantages": 0.1830243319272995, + "generated_text": "{\n \"intent\": \"To continue exploring the interface further, I will scroll down slightly to see more content.\",\n \"action\": \"ScrollDown(741, 540)\"\n}", + "action_string": "ScrollDown(741, 540)", + "intent_string": "To continue exploring the interface further, I will scroll down slightly to see more content.", + "before_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 236, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 441, + 646, + 498 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 342, + 524, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "s and other languages", + "boxes": [ + 444, + 367, + 574, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 490, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 800, + 917, + 1098, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ] + }, + { + "frame_id": "50_4_5", + "episode_id": 50, + "env_id": 4, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.147838592529297, + "advantages": -0.1565595120191574, + "generated_text": "{\n \"intent\": \"I will start by clicking on one of the desktop icons to explore its functionality.\",\n \"action\": \"DoubleClick(54, 637)\"\n}", + "action_string": "DoubleClick(54, 637)", + "intent_string": "I will start by clicking on one of the desktop icons to explore its functionality.", + "before_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "1", + "boxes": [ + 34, + 119, + 82, + 165 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 38, + 234, + 78, + 264 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "Libre...", + "boxes": [ + 14, + 498, + 100, + 530 + ] + }, + { + "text": "root@02b525", + "boxes": [ + 216, + 1046, + 312, + 1072 + ] + }, + { + "text": "root@02b525..", + "boxes": [ + 336, + 1048, + 470, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "?", + "boxes": [ + 38, + 234, + 78, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "Tabs", + "boxes": [ + 84, + 608, + 124, + 623 + ] + }, + { + "text": "root@02b525", + "boxes": [ + 184, + 1048, + 312, + 1072 + ] + }, + { + "text": "root@02b525", + "boxes": [ + 368, + 1046, + 460, + 1072 + ] + } + ] + }, + { + "frame_id": "50_5_5", + "episode_id": 50, + "env_id": 5, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.255500793457031, + "advantages": -0.1035519391298294, + "generated_text": "{\n \"intent\": \"To open a new tab in the browser for further exploration.\",\n \"action\": \"DoubleClick(71, 44)\"\n}", + "action_string": "DoubleClick(71, 44)", + "intent_string": "To open a new tab in the browser for further exploration.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1270, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 74, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_6_5", + "episode_id": 50, + "env_id": 6, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.632614135742188, + "advantages": 0.0821201428771019, + "generated_text": "{\n \"intent\": \"To get an overview of available formats or templates as part of exploring LibreOffice Calc features.\",\n \"action\": \"DoubleClick(476, 77)\"\n}", + "action_string": "DoubleClick(476, 77)", + "intent_string": "To get an overview of available formats or templates as part of exploring LibreOffice Calc features.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOffice Calc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "File", + "boxes": [ + 8, + 26, + 36, + 46 + ] + }, + { + "text": "Styles", + "boxes": [ + 250, + 28, + 296, + 46 + ] + }, + { + "text": "Tools Windol", + "boxes": [ + 396, + 24, + 502, + 48 + ] + }, + { + "text": " Load URL", + "boxes": [ + 524, + 28, + 614, + 54 + ] + }, + { + "text": "--·自-", + "boxes": [ + 8, + 48, + 386, + 87 + ] + }, + { + "text": "盒&", + "boxes": [ + 370, + 54, + 446, + 85 + ] + }, + { + "text": "", + "boxes": [ + 438, + 56, + 486, + 81 + ] + }, + { + "text": "↑E·B", + "boxes": [ + 774, + 46, + 1264, + 89 + ] + }, + { + "text": " Open Remote", + "boxes": [ + 522, + 81, + 644, + 111 + ] + }, + { + "text": "Liberation san10aαaa=", + "boxes": [ + 12, + 93, + 504, + 123 + ] + }, + { + "text": "%00正至田二口目", + "boxes": [ + 774, + 93, + 1204, + 123 + ] + }, + { + "text": " save As..", + "boxes": [ + 522, + 105, + 610, + 137 + ] + }, + { + "text": "F2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "?", + "boxes": [ + 130, + 129, + 156, + 151 + ] + }, + { + "text": "=[", + "boxes": [ + 150, + 125, + 260, + 157 + ] + }, + { + "text": " Email", + "boxes": [ + 524, + 135, + 588, + 161 + ] + }, + { + "text": "4", + "boxes": [ + 524, + 191, + 548, + 214 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "A", + "boxes": [ + 506, + 220, + 548, + 240 + ] + }, + { + "text": " Navigator", + "boxes": [ + 520, + 244, + 618, + 270 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 270 + ] + }, + { + "text": "", + "boxes": [ + 521, + 269, + 553, + 299 + ] + }, + { + "text": "Insert Rows Above", + "boxes": [ + 552, + 272, + 676, + 296 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "日", + "boxes": [ + 534, + 359, + 546, + 371 + ] + }, + { + "text": "", + "boxes": [ + 528, + 463, + 554, + 482 + ] + }, + { + "text": "Advanced Filter", + "boxes": [ + 542, + 458, + 654, + 487 + ] + }, + { + "text": " Goal Seel", + "boxes": [ + 528, + 490, + 612, + 508 + ] + }, + { + "text": " Gallery ", + "boxes": [ + 526, + 514, + 602, + 540 + ] + }, + { + "text": " Formula", + "boxes": [ + 522, + 538, + 608, + 570 + ] + }, + { + "text": "Ω Symbol", + "boxes": [ + 524, + 596, + 602, + 619 + ] + }, + { + "text": "T Text Box", + "boxes": [ + 522, + 621, + 608, + 647 + ] + }, + { + "text": " Edit ", + "boxes": [ + 526, + 649, + 582, + 675 + ] + }, + { + "text": "备", + "boxes": [ + 530, + 679, + 546, + 695 + ] + }, + { + "text": "QZoom", + "boxes": [ + 528, + 786, + 586, + 804 + ] + }, + { + "text": " Libreoffice Help ", + "boxes": [ + 524, + 808, + 662, + 838 + ] + }, + { + "text": " What's This?", + "boxes": [ + 528, + 842, + 632, + 860 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 939 + ] + }, + { + "text": "Dock All Toolbars", + "boxes": [ + 528, + 957, + 640, + 981 + ] + }, + { + "text": "一→", + "boxes": [ + 20, + 997, + 92, + 1014 + ] + }, + { + "text": "Lock Toolbar Position", + "boxes": [ + 526, + 989, + 668, + 1012 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 522, + 26, + 556, + 46 + ] + }, + { + "text": "·-目·@自、 ", + "boxes": [ + 10, + 50, + 376, + 85 + ] + }, + { + "text": "盒&", + "boxes": [ + 374, + 54, + 444, + 85 + ] + }, + { + "text": "田、·", + "boxes": [ + 544, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberation san10aαaa=", + "boxes": [ + 12, + 93, + 554, + 123 + ] + }, + { + "text": "训", + "boxes": [ + 548, + 97, + 580, + 119 + ] + }, + { + "text": "E", + "boxes": [ + 584, + 97, + 616, + 117 + ] + }, + { + "text": "回日日-%00面t三至田-二-口-目", + "boxes": [ + 618, + 93, + 1208, + 123 + ] + }, + { + "text": "F2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 256, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "50_7_5", + "episode_id": 50, + "env_id": 7, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.99177360534668, + "advantages": 0.25895261764526367, + "generated_text": "{\n \"intent\": \"Clicking the search bar to start typing a query will allow me to explore more content or search specific information on Bing.\",\n \"action\": \"DoubleClick(518, 314)\"\n}", + "action_string": "DoubleClick(518, 314)", + "intent_string": "Clicking the search bar to start typing a query will allow me to explore more content or search specific information on Bing.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 62, + 132, + 93 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 492, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 562, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1342, + 302, + 1356, + 324 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "", + "boxes": [ + 1546, + 367, + 1562, + 381 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 486, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "", + "boxes": [ + 976, + 403, + 1000, + 425 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1108, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 796, + 453, + 968, + 476 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1520, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 727, + 270, + 752 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1466, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 840 + ] + }, + { + "text": "+2.09%", + "boxes": [ + 1640, + 808, + 1686, + 828 + ] + }, + { + "text": "和平精英", + "boxes": [ + 934, + 844, + 1004, + 866 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": "2,61029 ", + "boxes": [ + 1628, + 846, + 1700, + 893 + ] + }, + { + "text": "13,087%6", + "boxes": [ + 1626, + 891, + 1698, + 941 + ] + }, + { + "text": "游戏天地·5天", + "boxes": [ + 830, + 921, + 942, + 945 + ] + }, + { + "text": "1", + "boxes": [ + 1142, + 925, + 1162, + 941 + ] + }, + { + "text": "一点资讯·2周", + "boxes": [ + 1160, + 921, + 1254, + 945 + ] + }, + { + "text": "和平精英：3个提升听声辨位", + "boxes": [ + 824, + 941, + 1092, + 977 + ] + }, + { + "text": "霸王大陆属性趣谈：吕布两项", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "+3.2879", + "boxes": [ + 1626, + 939, + 1698, + 989 + ] + }, + { + "text": "游侠网·3周", + "boxes": [ + 202, + 955, + 306, + 979 + ] + }, + { + "text": "能力的方法，快试试！", + "boxes": [ + 822, + 965, + 1032, + 1005 + ] + }, + { + "text": "第一，但综合最强还是常山赵", + "boxes": [ + 1136, + 967, + 1410, + 1003 + ] + }, + { + "text": "我的世界大型地图种子，大到没边界", + "boxes": [ + 200, + 981, + 674, + 1022 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "-11420 ", + "boxes": [ + 1634, + 983, + 1700, + 1030 + ] + }, + { + "text": "子龙", + "boxes": [ + 1136, + 995, + 1192, + 1028 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "3", + "boxes": [ + 1140, + 1028, + 1172, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 40, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 28, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1607, + 65, + 1659, + 90 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Images", + "boxes": [ + 252, + 123, + 292, + 143 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 490, + 250, + 636, + 282 + ] + }, + { + "text": "雷军应SU7事故", + "boxes": [ + 478, + 300, + 614, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "电啤肉", + "boxes": [ + 1136, + 377, + 1176, + 414 + ] + }, + { + "text": "中享中", + "boxes": [ + 1320, + 377, + 1348, + 399 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 371, + 1422, + 405 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 486, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 918, + 441 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生蛆", + "boxes": [ + 486, + 451, + 636, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 790, + 449, + 968, + 478 + ] + }, + { + "text": "6“朱雀玄武救令“改名", + "boxes": [ + 1100, + 451, + 1270, + 474 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 729, + 268, + 746 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1458, + 766, + 1550, + 792 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 840 + ] + }, + { + "text": "+2.09 ", + "boxes": [ + 1626, + 798, + 1700, + 846 + ] + }, + { + "text": "Mwy", + "boxes": [ + 1591, + 803, + 1629, + 831 + ] + }, + { + "text": "和平精英", + "boxes": [ + 928, + 842, + 1008, + 866 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": ",114023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13.27", + "boxes": [ + 1628, + 891, + 1698, + 941 + ] + }, + { + "text": "游戏天地·5天", + "boxes": [ + 828, + 921, + 942, + 945 + ] + }, + { + "text": "1一点资讯·2周", + "boxes": [ + 1138, + 921, + 1256, + 945 + ] + }, + { + "text": "和平精英：3个提升听声辨位", + "boxes": [ + 824, + 941, + 1092, + 977 + ] + }, + { + "text": "霸王大陆属性趣谈：吕布两项", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "382", + "boxes": [ + 1620, + 935, + 1702, + 989 + ] + }, + { + "text": "NAS进阶秘籍：搭建私云，畅享数据自由与", + "boxes": [ + 204, + 947, + 768, + 983 + ] + }, + { + "text": "能力的方法，快试试！", + "boxes": [ + 826, + 969, + 1032, + 999 + ] + }, + { + "text": "第一，但综合最强还是常山赵", + "boxes": [ + 1136, + 969, + 1406, + 999 + ] + }, + { + "text": "安全！", + "boxes": [ + 202, + 983, + 298, + 1020 + ] + }, + { + "text": "中国安", + "boxes": [ + 1448, + 975, + 1524, + 1038 + ] + }, + { + "text": "-11620 ", + "boxes": [ + 1634, + 983, + 1702, + 1030 + ] + }, + { + "text": "子龙", + "boxes": [ + 1138, + 997, + 1188, + 1024 + ] + }, + { + "text": "WLWM", + "boxes": [ + 1596, + 1003, + 1628, + 1022 + ] + }, + { + "text": "22", + "boxes": [ + 830, + 1028, + 866, + 1042 + ] + }, + { + "text": "3", + "boxes": [ + 1134, + 1026, + 1200, + 1046 + ] + }, + { + "text": "奇丢白选股建", + "boxes": [ + 1630, + 1028, + 1714, + 1046 + ] + } + ] + }, + { + "frame_id": "50_0_6", + "episode_id": 50, + "env_id": 0, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.080906867980957, + "advantages": 0.30283746123313904, + "generated_text": "{\n \"intent\": \"Scrolling down the page to see more content related to the current focus area.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "Scrolling down the page to see more content related to the current focus area.", + "before_screen_ocr_results": [ + { + "text": ". Spend less x", + "boxes": [ + 118, + 32, + 228, + 50 + ] + }, + { + "text": "lucts in Kitcher", + "boxes": [ + 334, + 417, + 418, + 435 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 415, + 1058, + 439 + ] + }, + { + "text": " Continue shopping for ", + "boxes": [ + 238, + 494, + 474, + 526 + ] + }, + { + "text": "See more", + "boxes": [ + 468, + 500, + 542, + 524 + ] + }, + { + "text": "Pick up where you left off", + "boxes": [ + 240, + 798, + 510, + 834 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "See all deals", + "boxes": [ + 979, + 123, + 1059, + 153 + ] + }, + { + "text": "Continue shopping for", + "boxes": [ + 242, + 212, + 472, + 242 + ] + }, + { + "text": "See more", + "boxes": [ + 474, + 216, + 536, + 236 + ] + }, + { + "text": "Pick up where you left off", + "boxes": [ + 242, + 516, + 508, + 546 + ] + }, + { + "text": "Elevate Your Electronics", + "boxes": [ + 242, + 820, + 486, + 850 + ] + }, + { + "text": "Gear up to get fit ", + "boxes": [ + 610, + 816, + 790, + 852 + ] + }, + { + "text": " Most-loved travel essentials", + "boxes": [ + 978, + 814, + 1266, + 852 + ] + }, + { + "text": "Level up your PC here ", + "boxes": [ + 1348, + 816, + 1576, + 852 + ] + } + ] + }, + { + "frame_id": "50_1_6", + "episode_id": 50, + "env_id": 1, + "step": 6, + "format_rewards": 1.0, + "rewards": 8.760673522949219, + "advantages": -0.34718057513237, + "generated_text": "{\n \"intent\": \"Opening the Open File dialog to explore the contents of the Home directory.\",\n \"action\": \"DoubleClick(725, 136)\"\n}", + "action_string": "DoubleClick(725, 136)", + "intent_string": "Opening the Open File dialog to explore the contents of the Home directory.", + "before_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "I clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates ", + "boxes": [ + 100, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 102, + 415, + 316, + 455 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 32, + 611, + 56, + 633 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "ibreoffice", + "boxes": [ + 222, + 1050, + 288, + 1070 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Libreoffic", + "boxes": [ + 918, + 10, + 996, + 28 + ] + }, + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 632, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "1 clipboard_app", + "boxes": [ + 552, + 214, + 670, + 245 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "ibreoffice", + "boxes": [ + 222, + 1050, + 288, + 1070 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_6", + "episode_id": 50, + "env_id": 2, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.740386962890625, + "advantages": 0.6275330781936646, + "generated_text": "{\n \"intent\": \"The goal is to explore the website fully. Initializing a web browser session by launching Firefox is crucial. Clicking on the 'Download Firefox' button will initiate this process.\",\n \"action\": \"DoubleClick(438, 308)\"\n}", + "action_string": "DoubleClick(438, 308)", + "intent_string": "The goal is to explore the website fully. Initializing a web browser session by launching Firefox is crucial. Clicking on the 'Download Firefox' button will initiate this process.", + "before_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "ns and other languages", + "boxes": [ + 440, + 367, + 576, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 492, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 800, + 917, + 1098, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 415, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "50_3_6", + "episode_id": 50, + "env_id": 3, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.918216705322266, + "advantages": 0.22273682057857513, + "generated_text": "{\n \"intent\": \"To explore other parts of the webpage by scrolling down, to discover more content or options available.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore other parts of the webpage by scrolling down, to discover more content or options available.", + "before_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "s and other languages", + "boxes": [ + 444, + 367, + 574, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 490, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 800, + 917, + 1098, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 36, + 66, + 126, + 89 + ] + }, + { + "text": "①| http://www.mozilla.org/en-USs/firefox/new/", + "boxes": [ + 252, + 64, + 608, + 93 + ] + }, + { + "text": "目", + "boxes": [ + 1544, + 69, + 1562, + 85 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 119, + 490, + 137 + ] + }, + { + "text": "Dial down the", + "boxes": [ + 422, + 520, + 670, + 564 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 520, + 1028, + 564 + ] + }, + { + "text": " Keep tabs on all", + "boxes": [ + 1172, + 516, + 1462, + 570 + ] + }, + { + "text": "noise", + "boxes": [ + 422, + 568, + 528, + 606 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 568, + 994, + 604 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1180, + 568, + 1354, + 606 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 617, + 710, + 647 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 800, + 617, + 1096, + 647 + ] + }, + { + "text": "Close duplicates, search and pin open ", + "boxes": [ + 1176, + 611, + 1484, + 651 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 422, + 633, + 712, + 667 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 804, + 637, + 1096, + 667 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 657, + 650, + 681 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 651, + 928, + 687 + ] + }, + { + "text": "possibilities for the almost infinite", + "boxes": [ + 1182, + 653, + 1462, + 683 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1180, + 673, + 1424, + 703 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 878, + 725, + 1030, + 754 + ] + }, + { + "text": "Get the browser that helps you", + "boxes": [ + 610, + 991, + 1300, + 1042 + ] + } + ] + }, + { + "frame_id": "50_4_6", + "episode_id": 50, + "env_id": 4, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.801201820373535, + "advantages": 0.16512443125247955, + "generated_text": "{\n \"intent\": \"To start interacting with the desktop more deeply by exploring available applications.\",\n \"action\": \"DoubleClick(78, 283)\"\n}", + "action_string": "DoubleClick(78, 283)", + "intent_string": "To start interacting with the desktop more deeply by exploring available applications.", + "before_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "?", + "boxes": [ + 38, + 234, + 78, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "Tabs", + "boxes": [ + 84, + 608, + 124, + 623 + ] + }, + { + "text": "root@02b525", + "boxes": [ + 184, + 1048, + 312, + 1072 + ] + }, + { + "text": "root@02b525", + "boxes": [ + 368, + 1046, + 460, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 42, + 131, + 74, + 153 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 112, + 199 + ] + }, + { + "text": "Termirkal", + "boxes": [ + 9, + 271, + 109, + 316 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 608, + 34, + 621 + ] + }, + { + "text": "Edit Tabs ", + "boxes": [ + 42, + 606, + 128, + 623 + ] + }, + { + "text": "t@02b525", + "boxes": [ + 540, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "50_5_6", + "episode_id": 50, + "env_id": 5, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.306558609008789, + "advantages": -0.07841357588768005, + "generated_text": "{\n \"intent\": \"To start exploring other content by opening a new tab in the browser.\",\n \"action\": \"DoubleClick(270, 43)\"\n}", + "action_string": "DoubleClick(270, 43)", + "intent_string": "To start exploring other content by opening a new tab in the browser.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 74, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 54, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 658, + 290, + 766, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "50_6_6", + "episode_id": 50, + "env_id": 6, + "step": 6, + "format_rewards": 1.0, + "rewards": 8.872514724731445, + "advantages": -0.2921154499053955, + "generated_text": "{\n \"intent\": \"To ensure I am exploring different components of the LibreOffice Calc interface, I can start by clicking in an empty cell like F2.\",\n \"action\": \"Click(710, 214)\"\n}", + "action_string": "Click(710, 214)", + "intent_string": "To ensure I am exploring different components of the LibreOffice Calc interface, I can start by clicking in an empty cell like F2.", + "before_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 522, + 26, + 556, + 46 + ] + }, + { + "text": "·-目·@自、 ", + "boxes": [ + 10, + 50, + 376, + 85 + ] + }, + { + "text": "盒&", + "boxes": [ + 374, + 54, + 444, + 85 + ] + }, + { + "text": "田、·", + "boxes": [ + 544, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberation san10aαaa=", + "boxes": [ + 12, + 93, + 554, + 123 + ] + }, + { + "text": "训", + "boxes": [ + 548, + 97, + 580, + 119 + ] + }, + { + "text": "E", + "boxes": [ + 584, + 97, + 616, + 117 + ] + }, + { + "text": "回日日-%00面t三至田-二-口-目", + "boxes": [ + 618, + 93, + 1208, + 123 + ] + }, + { + "text": "F2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 256, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "9", + "boxes": [ + 20, + 824, + 32, + 836 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Styles", + "boxes": [ + 252, + 30, + 292, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 524, + 30, + 550, + 44 + ] + }, + { + "text": "··目·目@%自、 ", + "boxes": [ + 10, + 50, + 378, + 85 + ] + }, + { + "text": "上&", + "boxes": [ + 369, + 51, + 445, + 86 + ] + }, + { + "text": "G-.", + "boxes": [ + 450, + 56, + 550, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1886, + 302, + 1912, + 341 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "4", + "boxes": [ + 18, + 570, + 30, + 582 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "2", + "boxes": [ + 18, + 872, + 30, + 883 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "50_7_6", + "episode_id": 50, + "env_id": 7, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.873964309692383, + "advantages": 0.20094910264015198, + "generated_text": "{\n \"intent\": \"To navigate through the webpage and possibly open other sections or search options.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To navigate through the webpage and possibly open other sections or search options.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 40, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 28, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1607, + 65, + 1659, + 90 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Images", + "boxes": [ + 252, + 123, + 292, + 143 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 490, + 250, + 636, + 282 + ] + }, + { + "text": "雷军应SU7事故", + "boxes": [ + 478, + 300, + 614, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "啤", + "boxes": [ + 1136, + 376, + 1176, + 415 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 371, + 1422, + 405 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 918, + 441 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生蛆", + "boxes": [ + 486, + 451, + 636, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 790, + 449, + 968, + 478 + ] + }, + { + "text": "6“朱雀玄武救令“改名", + "boxes": [ + 1100, + 451, + 1270, + 474 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1522, + 677, + 1630, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 196, + 729, + 268, + 746 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1458, + 766, + 1550, + 792 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 840 + ] + }, + { + "text": "+2.09 ", + "boxes": [ + 1626, + 798, + 1700, + 846 + ] + }, + { + "text": "Whwr", + "boxes": [ + 1591, + 805, + 1629, + 831 + ] + }, + { + "text": "和平精英", + "boxes": [ + 928, + 842, + 1008, + 866 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1460, + 852, + 1512, + 887 + ] + }, + { + "text": ",114023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "139,87%", + "boxes": [ + 1626, + 891, + 1698, + 939 + ] + }, + { + "text": "游戏天地·5天", + "boxes": [ + 828, + 921, + 942, + 945 + ] + }, + { + "text": "1一点资讯·2周", + "boxes": [ + 1138, + 921, + 1256, + 945 + ] + }, + { + "text": "和平精英：3个提升听声辨位", + "boxes": [ + 826, + 941, + 1092, + 977 + ] + }, + { + "text": "霸王大陆属性趣谈：吕布两项", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "382", + "boxes": [ + 1620, + 935, + 1702, + 989 + ] + }, + { + "text": "能力的方法，快试试！", + "boxes": [ + 824, + 967, + 1034, + 1003 + ] + }, + { + "text": "第一，但综合最强还是常山赵", + "boxes": [ + 1136, + 969, + 1406, + 999 + ] + }, + { + "text": "适合近战法师的几把近战武器推荐", + "boxes": [ + 200, + 981, + 644, + 1022 + ] + }, + { + "text": "中国安", + "boxes": [ + 1448, + 975, + 1524, + 1038 + ] + }, + { + "text": "-11620 ", + "boxes": [ + 1634, + 983, + 1702, + 1030 + ] + }, + { + "text": "子龙", + "boxes": [ + 1138, + 997, + 1188, + 1024 + ] + }, + { + "text": "WLWM", + "boxes": [ + 1596, + 1003, + 1628, + 1022 + ] + }, + { + "text": "22", + "boxes": [ + 826, + 1026, + 868, + 1046 + ] + }, + { + "text": "3", + "boxes": [ + 1134, + 1026, + 1200, + 1046 + ] + }, + { + "text": "奇丢白选股建", + "boxes": [ + 1630, + 1028, + 1714, + 1046 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 62, + 446, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Images", + "boxes": [ + 252, + 123, + 292, + 143 + ] + }, + { + "text": "Translat", + "boxes": [ + 404, + 121, + 452, + 141 + ] + }, + { + "text": "国内版（国际版", + "boxes": [ + 488, + 189, + 636, + 218 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 236, + 614, + 266 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 185, + 434, + 275, + 465 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1460, + 482, + 1550, + 508 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 518, + 1504, + 556 + ] + }, + { + "text": "+2028", + "boxes": [ + 1630, + 514, + 1700, + 562 + ] + }, + { + "text": "Mwr", + "boxes": [ + 1590, + 519, + 1629, + 548 + ] + }, + { + "text": "贵州茶台", + "boxes": [ + 1458, + 562, + 1516, + 606 + ] + }, + { + "text": ".1023", + "boxes": [ + 1628, + 560, + 1700, + 609 + ] + }, + { + "text": "130870", + "boxes": [ + 1624, + 604, + 1704, + 657 + ] + }, + { + "text": "游戏天地·5天", + "boxes": [ + 828, + 635, + 944, + 659 + ] + }, + { + "text": "0", + "boxes": [ + 1138, + 637, + 1168, + 657 + ] + }, + { + "text": "一点资讯·2周", + "boxes": [ + 1160, + 635, + 1254, + 659 + ] + }, + { + "text": "李嫣出门聚会，遗传妈妈好身材手臂纤细白到", + "boxes": [ + 202, + 657, + 790, + 701 + ] + }, + { + "text": "和平精英：3个提升听声辨位", + "boxes": [ + 824, + 655, + 1092, + 691 + ] + }, + { + "text": "霸王大陆属性趣谈：吕布两项", + "boxes": [ + 1136, + 655, + 1408, + 691 + ] + }, + { + "text": "比", + "boxes": [ + 1456, + 655, + 1510, + 699 + ] + }, + { + "text": "+3.28%", + "boxes": [ + 1625, + 652, + 1697, + 692 + ] + }, + { + "text": "能力的方法，快试试！", + "boxes": [ + 826, + 683, + 1030, + 713 + ] + }, + { + "text": "第一，但综合最强还是常山赵", + "boxes": [ + 1136, + 685, + 1408, + 715 + ] + }, + { + "text": "发光", + "boxes": [ + 198, + 695, + 272, + 742 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1450, + 693, + 1522, + 748 + ] + }, + { + "text": "子龙", + "boxes": [ + 1138, + 711, + 1188, + 739 + ] + }, + { + "text": "WLMM", + "boxes": [ + 1596, + 717, + 1628, + 737 + ] + }, + { + "text": "22", + "boxes": [ + 824, + 741, + 896, + 766 + ] + }, + { + "text": "", + "boxes": [ + 1138, + 741, + 1202, + 766 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1620, + 744, + 1728, + 772 + ] + }, + { + "text": "车占花心", + "boxes": [ + 1201, + 791, + 1349, + 857 + ] + }, + { + "text": "粤象士好士象馬店", + "boxes": [ + 1218, + 788, + 1340, + 806 + ] + }, + { + "text": "长春市", + "boxes": [ + 1466, + 800, + 1524, + 820 + ] + }, + { + "text": "上观新闻", + "boxes": [ + 628, + 844, + 736, + 881 + ] + }, + { + "text": "刘殿中1赵国荣", + "boxes": [ + 1204, + 856, + 1348, + 885 + ] + }, + { + "text": "逸国酱园园", + "boxes": [ + 1204, + 879, + 1348, + 909 + ] + }, + { + "text": "历史名局", + "boxes": [ + 1204, + 897, + 1344, + 941 + ] + }, + { + "text": "康美物色勤世物美果", + "boxes": [ + 1200, + 933, + 1350, + 957 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 202, + 951, + 332, + 975 + ] + }, + { + "text": "解放日报·11小时", + "boxes": [ + 512, + 949, + 652, + 979 + ] + }, + { + "text": "ar玩一玩游戏网·2天", + "boxes": [ + 832, + 951, + 966, + 975 + ] + }, + { + "text": "一点视频/哈珀探索·3周", + "boxes": [ + 1136, + 961, + 1312, + 991 + ] + }, + { + "text": "多彩的天涯之国", + "boxes": [ + 202, + 975, + 352, + 1005 + ] + }, + { + "text": "每3000名新生儿中可能就有", + "boxes": [ + 512, + 971, + 782, + 1007 + ] + }, + { + "text": "流放之路2玻璃弹珠是什么玻", + "boxes": [ + 824, + 971, + 1098, + 1007 + ] + }, + { + "text": "刘殿中vs赵国荣弃子强攻组", + "boxes": [ + 1136, + 983, + 1408, + 1018 + ] + }, + { + "text": "一个“泡泡宝贝”，但疾病知晓", + "boxes": [ + 516, + 1001, + 790, + 1030 + ] + }, + { + "text": "璃弹珠用途与获取攻略", + "boxes": [ + 824, + 999, + 1036, + 1034 + ] + }, + { + "text": "合连招一套带走车占花心太经", + "boxes": [ + 1138, + 1012, + 1412, + 1042 + ] + }, + { + "text": "密很低", + "boxes": [ + 518, + 1026, + 580, + 1046 + ] + }, + { + "text": "arch - Micr", + "boxes": [ + 234, + 1050, + 308, + 1068 + ] + } + ] + }, + { + "frame_id": "50_0_7", + "episode_id": 50, + "env_id": 0, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.83631420135498, + "advantages": 0.1824120432138443, + "generated_text": "{\n \"intent\": \"To explore more of the Amazon web page by scrolling down to see additional content.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore more of the Amazon web page by scrolling down to see additional content.", + "before_screen_ocr_results": [ + { + "text": "See all deals", + "boxes": [ + 979, + 123, + 1059, + 153 + ] + }, + { + "text": "Continue shopping for", + "boxes": [ + 242, + 212, + 472, + 242 + ] + }, + { + "text": "See more", + "boxes": [ + 474, + 216, + 536, + 236 + ] + }, + { + "text": "Pick up where you left off", + "boxes": [ + 242, + 516, + 508, + 546 + ] + }, + { + "text": "Elevate Your Electronics", + "boxes": [ + 242, + 820, + 486, + 850 + ] + }, + { + "text": "Gear up to get fit ", + "boxes": [ + 610, + 816, + 790, + 852 + ] + }, + { + "text": " Most-loved travel essentials", + "boxes": [ + 978, + 814, + 1266, + 852 + ] + }, + { + "text": "Level up your PC here ", + "boxes": [ + 1348, + 816, + 1576, + 852 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Pick up where you left off", + "boxes": [ + 240, + 230, + 510, + 260 + ] + }, + { + "text": "Elevate Your Electronics", + "boxes": [ + 240, + 534, + 488, + 564 + ] + }, + { + "text": "Gear up to get fit", + "boxes": [ + 610, + 532, + 788, + 566 + ] + }, + { + "text": " Most-loved travel essentials", + "boxes": [ + 976, + 530, + 1264, + 564 + ] + }, + { + "text": "Level up your PC here", + "boxes": [ + 1350, + 534, + 1574, + 564 + ] + }, + { + "text": "Headphones", + "boxes": [ + 248, + 687, + 312, + 707 + ] + }, + { + "text": "Tablets", + "boxes": [ + 410, + 687, + 444, + 707 + ] + }, + { + "text": "Gift ideas inspired by your shopping history", + "boxes": [ + 240, + 973, + 682, + 1009 + ] + }, + { + "text": "YEOAY", + "boxes": [ + 264, + 1018, + 350, + 1044 + ] + } + ] + }, + { + "frame_id": "50_1_7", + "episode_id": 50, + "env_id": 1, + "step": 7, + "format_rewards": 1.0, + "rewards": 8.936880111694336, + "advantages": -0.2604250907897949, + "generated_text": "{\n \"intent\": \"To explore further by navigating through the 'Recent Files' section or creating a new document.\",\n \"action\": \"DoubleClick(164, 95)\"\n}", + "action_string": "DoubleClick(164, 95)", + "intent_string": "To explore further by navigating through the 'Recent Files' section or creating a new document.", + "before_screen_ocr_results": [ + { + "text": "Libreoffic", + "boxes": [ + 918, + 10, + 996, + 28 + ] + }, + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 632, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "1 clipboard_app", + "boxes": [ + 552, + 214, + 670, + 245 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "ibreoffice", + "boxes": [ + 222, + 1050, + 288, + 1070 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 58, + 89 + ] + }, + { + "text": " open File ", + "boxes": [ + 59, + 59, + 209, + 110 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "□□", + "boxes": [ + 1458, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 104, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 68, + 179, + 101, + 209 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "I clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": " Other Locatic", + "boxes": [ + 424, + 256, + 514, + 274 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_7", + "episode_id": 50, + "env_id": 2, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.252118110656738, + "advantages": 0.3871334493160248, + "generated_text": "{\n \"intent\": \"To navigate through different download options and potentially find the desired version of Firefox.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To navigate through different download options and potentially find the desired version of Firefox.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 415, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1874, + 89 + ] + }, + { + "text": "Mozilla /3", + "boxes": [ + 255, + 104, + 375, + 156 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1172, + 139 + ] + }, + { + "text": "Seiect d uuwiluau uptiui", + "boxes": [ + 674, + 147, + 1238, + 183 + ] + }, + { + "text": "below", + "boxes": [ + 873, + 178, + 1033, + 239 + ] + }, + { + "text": " Download for Linux 32-bit ", + "boxes": [ + 848, + 306, + 1060, + 330 + ] + }, + { + "text": " Download for Linux 64-bit ", + "boxes": [ + 846, + 355, + 1064, + 385 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 732, + 397, + 1176, + 427 + ] + }, + { + "text": "Youcansetupour", + "boxes": [ + 790, + 423, + 934, + 441 + ] + }, + { + "text": "r APT repositoryinstead.", + "boxes": [ + 926, + 421, + 1120, + 445 + ] + }, + { + "text": "Download in another Language or for another operating system", + "boxes": [ + 706, + 602, + 1200, + 631 + ] + }, + { + "text": "Mozilla", + "boxes": [ + 259, + 740, + 429, + 797 + ] + }, + { + "text": "Company", + "boxes": [ + 722, + 741, + 800, + 766 + ] + }, + { + "text": "support ", + "boxes": [ + 952, + 737, + 1028, + 768 + ] + }, + { + "text": " Developers", + "boxes": [ + 1181, + 732, + 1281, + 769 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 737, + 1646, + 768 + ] + }, + { + "text": " Leadership ", + "boxes": [ + 720, + 766, + 818, + 792 + ] + }, + { + "text": " Product Help", + "boxes": [ + 949, + 760, + 1065, + 797 + ] + }, + { + "text": " Developer Edition", + "boxes": [ + 1184, + 762, + 1332, + 796 + ] + }, + { + "text": "X@nd", + "boxes": [ + 1506, + 776, + 1644, + 806 + ] + }, + { + "text": "Advertising", + "boxes": [ + 257, + 792, + 529, + 854 + ] + }, + { + "text": " Press Center", + "boxes": [ + 718, + 792, + 830, + 822 + ] + }, + { + "text": "File a Bug ", + "boxes": [ + 954, + 794, + 1038, + 820 + ] + }, + { + "text": "Enterprise", + "boxes": [ + 1186, + 792, + 1278, + 824 + ] + }, + { + "text": "Careers", + "boxes": [ + 720, + 820, + 792, + 852 + ] + }, + { + "text": "Localize Mozilla", + "boxes": [ + 954, + 820, + 1086, + 850 + ] + }, + { + "text": "Tools", + "boxes": [ + 1186, + 818, + 1238, + 852 + ] + }, + { + "text": "Follow @Firefox", + "boxes": [ + 1514, + 826, + 1646, + 860 + ] + }, + { + "text": "Contact", + "boxes": [ + 724, + 852, + 788, + 872 + ] + }, + { + "text": "MDN ", + "boxes": [ + 1184, + 844, + 1234, + 878 + ] + }, + { + "text": " Privacy-first advertising solutions for", + "boxes": [ + 262, + 868, + 564, + 897 + ] + }, + { + "text": " Firefox Release Notes", + "boxes": [ + 1184, + 876, + 1362, + 905 + ] + }, + { + "text": "brands, publishers, and platforms.", + "boxes": [ + 264, + 889, + 532, + 913 + ] + }, + { + "text": "Learn more ", + "boxes": [ + 378, + 953, + 496, + 977 + ] + } + ] + }, + { + "frame_id": "50_3_7", + "episode_id": 50, + "env_id": 3, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.722614288330078, + "advantages": 0.12643179297447205, + "generated_text": "{\n \"intent\": \"To analyze more features or potentially scroll through the page content.\",\n \"action\": \"ScrollUp(960, 540)\"\n}", + "action_string": "ScrollUp(960, 540)", + "intent_string": "To analyze more features or potentially scroll through the page content.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 36, + 66, + 126, + 89 + ] + }, + { + "text": "①| http://www.mozilla.org/en-USs/firefox/new/", + "boxes": [ + 252, + 64, + 608, + 93 + ] + }, + { + "text": "目", + "boxes": [ + 1544, + 69, + 1562, + 85 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 119, + 490, + 137 + ] + }, + { + "text": "Dial down the", + "boxes": [ + 422, + 520, + 670, + 564 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 520, + 1028, + 564 + ] + }, + { + "text": " Keep tabs on all", + "boxes": [ + 1172, + 516, + 1462, + 570 + ] + }, + { + "text": "noise", + "boxes": [ + 422, + 568, + 528, + 606 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 568, + 994, + 604 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1180, + 568, + 1354, + 606 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 617, + 710, + 647 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 800, + 617, + 1096, + 647 + ] + }, + { + "text": "Close duplicates, search and pin open ", + "boxes": [ + 1176, + 611, + 1484, + 651 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 422, + 633, + 712, + 667 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 804, + 637, + 1096, + 667 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 657, + 650, + 681 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 651, + 928, + 687 + ] + }, + { + "text": "possibilities for the almost infinite", + "boxes": [ + 1182, + 653, + 1462, + 683 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1180, + 673, + 1424, + 703 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 878, + 725, + 1030, + 754 + ] + }, + { + "text": "Get the browser that helps you", + "boxes": [ + 610, + 991, + 1300, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "ns and other languages", + "boxes": [ + 440, + 367, + 576, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 492, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 800, + 917, + 1098, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ] + }, + { + "frame_id": "50_4_7", + "episode_id": 50, + "env_id": 4, + "step": 7, + "format_rewards": 0.0, + "rewards": 0.0, + "advantages": -4.660505771636963, + "generated_text": "{\n \"intent\": \"To start exploring further by accessing a terminal. This will allow interaction through shell commands.\",\n \"action\": \"CompoundAction(Terminal)\"\n}", + "action_string": "None", + "intent_string": null, + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 42, + 131, + 74, + 153 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 112, + 199 + ] + }, + { + "text": "Termirkal", + "boxes": [ + 9, + 271, + 109, + 316 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 608, + 34, + 621 + ] + }, + { + "text": "Edit Tabs ", + "boxes": [ + 42, + 606, + 128, + 623 + ] + }, + { + "text": "t@02b525", + "boxes": [ + 540, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 42, + 131, + 74, + 153 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 112, + 199 + ] + }, + { + "text": "Termirkal", + "boxes": [ + 9, + 271, + 109, + 316 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 608, + 34, + 621 + ] + }, + { + "text": "Edit Tabs ", + "boxes": [ + 42, + 606, + 128, + 623 + ] + }, + { + "text": "t@02b525", + "boxes": [ + 540, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "50_5_7", + "episode_id": 50, + "env_id": 5, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.338994979858398, + "advantages": 0.42990735173225403, + "generated_text": "{\n \"intent\": \"To explore various sites on the browser and gather more information about them.\",\n \"action\": \"DoubleClick(1072, 799)\"\n}", + "action_string": "DoubleClick(1072, 799)", + "intent_string": "To explore various sites on the browser and gather more information about them.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 230, + 30, + 316, + 62 + ] + }, + { + "text": "→e", + "boxes": [ + 46, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 658, + 290, + 766, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 24, + 32, + 84, + 50 + ] + }, + { + "text": "←→", + "boxes": [ + 4, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caivarta", + "boxes": [ + 134, + 109, + 218, + 149 + ] + }, + { + "text": " Search Amazon", + "boxes": [ + 300, + 119, + 410, + 137 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 121, + 1474, + 135 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 918, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "You are on amaz", + "boxes": [ + 510, + 461, + 612, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion ", + "boxes": [ + 980, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1220, + 999 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1352, + 969, + 1508, + 999 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + } + ] + }, + { + "frame_id": "50_6_7", + "episode_id": 50, + "env_id": 6, + "step": 7, + "format_rewards": 1.0, + "rewards": 8.683822631835938, + "advantages": -0.38501816987991333, + "generated_text": "{\n \"intent\": \"The goal is to explore the computer desktop environment effectively by accessing various applications and files. The current application appears to be an office software program. Scrolling may reveal additional content or make viewing elements within the application more convenient.\",\n \"action\": \"ScrollUp(960, 540)\"\n}", + "action_string": "ScrollUp(960, 540)", + "intent_string": "The goal is to explore the computer desktop environment effectively by accessing various applications and files. The current application appears to be an office software program. Scrolling may reveal additional content or make viewing elements within the application more convenient.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Styles", + "boxes": [ + 252, + 30, + 292, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 524, + 30, + 550, + 44 + ] + }, + { + "text": "--目· 自- ", + "boxes": [ + 10, + 50, + 378, + 85 + ] + }, + { + "text": "上&", + "boxes": [ + 369, + 51, + 445, + 86 + ] + }, + { + "text": "勺-、", + "boxes": [ + 450, + 56, + 550, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": " =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1886, + 302, + 1912, + 341 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": " Styles", + "boxes": [ + 246, + 28, + 292, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 524, + 30, + 552, + 44 + ] + }, + { + "text": "·目·@自、", + "boxes": [ + 10, + 50, + 388, + 85 + ] + }, + { + "text": "上&", + "boxes": [ + 369, + 51, + 445, + 86 + ] + }, + { + "text": "←-?、", + "boxes": [ + 444, + 56, + 552, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "2、 =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1886, + 302, + 1912, + 341 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": ".9", + "boxes": [ + 18, + 482, + 30, + 494 + ] + }, + { + "text": "21", + "boxes": [ + 16, + 512, + 30, + 528 + ] + }, + { + "text": "4", + "boxes": [ + 18, + 570, + 30, + 582 + ] + }, + { + "text": "25", + "boxes": [ + 16, + 584, + 30, + 602 + ] + }, + { + "text": "6", + "boxes": [ + 18, + 602, + 30, + 613 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 12, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "50_7_7", + "episode_id": 50, + "env_id": 7, + "step": 7, + "format_rewards": 1.0, + "rewards": 11.069819450378418, + "advantages": 0.7897294163703918, + "generated_text": "{\n \"intent\": \"The exploration goal involves browsing through the Bing website to gather more information. Clicking on a news article or a related element could provide additional context or details.\",\n \"action\": \"DoubleClick(1050, 667)\"\n}", + "action_string": "DoubleClick(1050, 667)", + "intent_string": "The exploration goal involves browsing through the Bing website to gather more information. Clicking on a news article or a related element could provide additional context or details.", + "before_screen_ocr_results": [ + { + "text": "Search - Microsoft Bing×十", + "boxes": [ + 36, + 32, + 244, + 50 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 62, + 446, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1600, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1872, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Images", + "boxes": [ + 252, + 123, + 292, + 143 + ] + }, + { + "text": "Translat", + "boxes": [ + 404, + 121, + 452, + 141 + ] + }, + { + "text": "国内版（国际版", + "boxes": [ + 488, + 189, + 636, + 218 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 236, + 614, + 266 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 439, + 272, + 465 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1460, + 482, + 1550, + 508 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 518, + 1504, + 558 + ] + }, + { + "text": "+2028", + "boxes": [ + 1630, + 514, + 1700, + 562 + ] + }, + { + "text": "Mwr", + "boxes": [ + 1590, + 519, + 1629, + 548 + ] + }, + { + "text": "LDA", + "boxes": [ + 590, + 554, + 808, + 637 + ] + }, + { + "text": "贵州茶台", + "boxes": [ + 1458, + 562, + 1516, + 606 + ] + }, + { + "text": ".1023", + "boxes": [ + 1628, + 560, + 1700, + 609 + ] + }, + { + "text": "游戏天地·5天", + "boxes": [ + 824, + 633, + 946, + 663 + ] + }, + { + "text": "0", + "boxes": [ + 1138, + 637, + 1168, + 657 + ] + }, + { + "text": "一点资讯·2周", + "boxes": [ + 1160, + 635, + 1254, + 659 + ] + }, + { + "text": "和平精英：3个提升听声辨位", + "boxes": [ + 824, + 655, + 1092, + 691 + ] + }, + { + "text": "霸王大陆属性趣谈：吕布两项", + "boxes": [ + 1136, + 655, + 1408, + 691 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 655, + 1510, + 701 + ] + }, + { + "text": "+3.28%", + "boxes": [ + 1625, + 652, + 1697, + 692 + ] + }, + { + "text": "潇湘晨报·7小时", + "boxes": [ + 198, + 667, + 328, + 697 + ] + }, + { + "text": "OUNDING PARTI", + "boxes": [ + 592, + 665, + 802, + 701 + ] + }, + { + "text": "能力的方法，快试试！", + "boxes": [ + 826, + 685, + 1030, + 715 + ] + }, + { + "text": "第一，但综合最强还是常山赵", + "boxes": [ + 1134, + 683, + 1406, + 713 + ] + }, + { + "text": "吴艳妮打破这项29年以来纪录", + "boxes": [ + 200, + 693, + 600, + 741 + ] + }, + { + "text": "子龙", + "boxes": [ + 1136, + 709, + 1190, + 742 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1450, + 693, + 1522, + 748 + ] + }, + { + "text": "WLMM", + "boxes": [ + 1596, + 717, + 1628, + 737 + ] + }, + { + "text": "10", + "boxes": [ + 208, + 748, + 244, + 762 + ] + }, + { + "text": "22", + "boxes": [ + 822, + 739, + 898, + 770 + ] + }, + { + "text": "", + "boxes": [ + 1138, + 741, + 1202, + 766 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1620, + 744, + 1728, + 772 + ] + }, + { + "text": "车占花心", + "boxes": [ + 1201, + 791, + 1349, + 857 + ] + }, + { + "text": "粤象士好士象馬店", + "boxes": [ + 1218, + 788, + 1340, + 806 + ] + }, + { + "text": "长春市", + "boxes": [ + 1466, + 800, + 1524, + 820 + ] + }, + { + "text": "上观新闻", + "boxes": [ + 628, + 844, + 736, + 881 + ] + }, + { + "text": "刘殿中1赵国荣", + "boxes": [ + 1204, + 856, + 1348, + 885 + ] + }, + { + "text": "逸国酱园园", + "boxes": [ + 1204, + 879, + 1348, + 909 + ] + }, + { + "text": "历史名局", + "boxes": [ + 1204, + 897, + 1344, + 941 + ] + }, + { + "text": "果国团团彩团出递型", + "boxes": [ + 1198, + 929, + 1352, + 959 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 202, + 951, + 332, + 975 + ] + }, + { + "text": "解放日报·11小时", + "boxes": [ + 512, + 949, + 652, + 979 + ] + }, + { + "text": "ar玩一玩游戏网·2天", + "boxes": [ + 832, + 951, + 966, + 975 + ] + }, + { + "text": "一点视频/哈珀探索·3周", + "boxes": [ + 1136, + 961, + 1312, + 991 + ] + }, + { + "text": "多彩的天涯之国", + "boxes": [ + 202, + 975, + 352, + 1005 + ] + }, + { + "text": "每3000名新生儿中可能就有", + "boxes": [ + 512, + 971, + 782, + 1007 + ] + }, + { + "text": "流放之路2玻璃弹珠是什么玻", + "boxes": [ + 824, + 971, + 1098, + 1007 + ] + }, + { + "text": "刘殿中vs赵国荣弃子强攻组", + "boxes": [ + 1136, + 983, + 1408, + 1018 + ] + }, + { + "text": "一个“泡泡宝贝”，但疾病知晓", + "boxes": [ + 514, + 1001, + 788, + 1030 + ] + }, + { + "text": "璃弹珠用途与获取攻略", + "boxes": [ + 824, + 999, + 1036, + 1034 + ] + }, + { + "text": "合连招一套带走车占花心太经", + "boxes": [ + 1136, + 1011, + 1414, + 1046 + ] + }, + { + "text": "密很低", + "boxes": [ + 518, + 1026, + 580, + 1046 + ] + }, + { + "text": "", + "boxes": [ + 100, + 1048, + 148, + 1066 + ] + }, + { + "text": "3", + "boxes": [ + 188, + 1052, + 204, + 1068 + ] + }, + { + "text": "arch - Micr.", + "boxes": [ + 232, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×", + "boxes": [ + 196, + 36, + 218, + 50 + ] + }, + { + "text": "和平精英：3个提升听", + "boxes": [ + 244, + 32, + 380, + 50 + ] + }, + { + "text": "Q白", + "boxes": [ + 64, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": "9", + "boxes": [ + 1854, + 71, + 1866, + 83 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 572, + 123, + 632, + 143 + ] + }, + { + "text": "中", + "boxes": [ + 1757, + 116, + 1793, + 150 + ] + }, + { + "text": "msn", + "boxes": [ + 60, + 129, + 102, + 149 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "关注", + "boxes": [ + 280, + 179, + 314, + 201 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1602, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "嘎嘎脆", + "boxes": [ + 1591, + 239, + 1689, + 287 + ] + }, + { + "text": "游戏天地十关注", + "boxes": [ + 386, + 264, + 576, + 306 + ] + }, + { + "text": "42.2K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "甜甜香", + "boxes": [ + 1618, + 274, + 1688, + 308 + ] + }, + { + "text": "想吃坚果蜜饯的", + "boxes": [ + 1424, + 292, + 1542, + 322 + ] + }, + { + "text": "点击领取", + "boxes": [ + 1454, + 336, + 1532, + 367 + ] + }, + { + "text": "和平精英：3个提升听声辨位能力的方法，快试试！", + "boxes": [ + 366, + 349, + 1104, + 391 + ] + }, + { + "text": "F", + "boxes": [ + 192, + 395, + 214, + 421 + ] + }, + { + "text": "闭眼冲", + "boxes": [ + 1618, + 391, + 1676, + 409 + ] + }, + { + "text": "游戏天地的报道·5天·①3分钟读取", + "boxes": [ + 364, + 413, + 666, + 449 + ] + }, + { + "text": "想吃坚果蜜饯的，来淘宝就GO", + "boxes": [ + 1422, + 411, + 1678, + 447 + ] + }, + { + "text": "了！", + "boxes": [ + 1428, + 443, + 1462, + 465 + ] + }, + { + "text": "广告淘宝网", + "boxes": [ + 1426, + 469, + 1508, + 492 + ] + }, + { + "text": "嘿，各位和平精英的玩家们！最近在游戏里是不是经常被敌人偷袭，却还搞不清楚人家从哪儿冒出来的？就像上次我", + "boxes": [ + 360, + 500, + 1240, + 538 + ] + }, + { + "text": "玩的时候，正美滋滋地搜着物资呢，突然就被人给阴了，连枪声方向都没搞明白，别提多郁闷了。其实啊，这听声辨", + "boxes": [ + 366, + 530, + 1236, + 560 + ] + }, + { + "text": "位的能力要是能提升上去，在游戏里那可就能抢占不少先机啦！今天就给大家分享3个提升听声辨位能力的方法，快", + "boxes": [ + 364, + 556, + 1230, + 586 + ] + }, + { + "text": "访问 游戏天地", + "boxes": [ + 1430, + 554, + 1552, + 584 + ] + }, + { + "text": "L", + "boxes": [ + 1680, + 558, + 1696, + 572 + ] + }, + { + "text": "试试哦！", + "boxes": [ + 366, + 586, + 432, + 611 + ] + }, + { + "text": "DNF手游确定无形自选和白色天三", + "boxes": [ + 1446, + 602, + 1664, + 631 + ] + }, + { + "text": "投票，别错过500万泰拉", + "boxes": [ + 1446, + 621, + 1604, + 651 + ] + }, + { + "text": "调大音量不一定就能听准位置", + "boxes": [ + 364, + 659, + 746, + 695 + ] + }, + { + "text": "梦幻西游夜叉改动！七八十个全是", + "boxes": [ + 1446, + 665, + 1664, + 695 + ] + }, + { + "text": "画魂，夜叉队准备转型", + "boxes": [ + 1446, + 685, + 1594, + 715 + ] + }, + { + "text": "梦幻西游：不战斗刷完慈心渡鬼，", + "boxes": [ + 1446, + 729, + 1654, + 758 + ] + }, + { + "text": "附任务领取坐标图", + "boxes": [ + 1446, + 748, + 1566, + 778 + ] + }, + { + "text": "淘", + "boxes": [ + 1422, + 836, + 1448, + 850 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 834, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 794, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置\"查看您的选项以及不", + "boxes": [ + 32, + 951, + 216, + 975 + ] + }, + { + "text": "在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的\"管理设置\"改变您的选择。欲了解", + "boxes": [ + 206, + 951, + 794, + 975 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "平精英：3个", + "boxes": [ + 224, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "50_0_8", + "episode_id": 50, + "env_id": 0, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.859996795654297, + "advantages": 0.1940721869468689, + "generated_text": "{\n \"intent\": \"To scroll down the webpage to see more products or information.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To scroll down the webpage to see more products or information.", + "before_screen_ocr_results": [ + { + "text": "Pick up where you left off", + "boxes": [ + 240, + 230, + 510, + 260 + ] + }, + { + "text": "Elevate Your Electronics", + "boxes": [ + 240, + 534, + 488, + 564 + ] + }, + { + "text": "Gear up to get fit", + "boxes": [ + 610, + 532, + 788, + 566 + ] + }, + { + "text": " Most-loved travel essentials", + "boxes": [ + 976, + 530, + 1264, + 564 + ] + }, + { + "text": "Level up your PC here", + "boxes": [ + 1350, + 534, + 1574, + 564 + ] + }, + { + "text": "Headphones", + "boxes": [ + 248, + 687, + 312, + 707 + ] + }, + { + "text": "Tablets", + "boxes": [ + 410, + 687, + 444, + 707 + ] + }, + { + "text": "Gift ideas inspired by your shopping history", + "boxes": [ + 240, + 973, + 682, + 1009 + ] + }, + { + "text": "YEOAY", + "boxes": [ + 264, + 1018, + 350, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Elevate Your Electronics", + "boxes": [ + 242, + 248, + 486, + 278 + ] + }, + { + "text": "Gear up to get fit", + "boxes": [ + 608, + 246, + 788, + 280 + ] + }, + { + "text": "Most-loved travel essentials", + "boxes": [ + 982, + 248, + 1264, + 278 + ] + }, + { + "text": "Level up your PC here", + "boxes": [ + 1350, + 250, + 1574, + 280 + ] + }, + { + "text": "Speakers", + "boxes": [ + 406, + 558, + 456, + 584 + ] + }, + { + "text": "Hard Drives", + "boxes": [ + 1356, + 562, + 1414, + 580 + ] + }, + { + "text": "Discover more", + "boxes": [ + 614, + 611, + 694, + 629 + ] + }, + { + "text": "Discover more", + "boxes": [ + 1354, + 611, + 1436, + 629 + ] + }, + { + "text": "Gift ideas inspired by your shopping history", + "boxes": [ + 240, + 689, + 682, + 725 + ] + }, + { + "text": "", + "boxes": [ + 878, + 758, + 910, + 792 + ] + }, + { + "text": "Additional items to explore See more", + "boxes": [ + 242, + 991, + 588, + 1020 + ] + }, + { + "text": "A", + "boxes": [ + 254, + 1026, + 336, + 1044 + ] + } + ] + }, + { + "frame_id": "50_1_8", + "episode_id": 50, + "env_id": 1, + "step": 8, + "format_rewards": 0.0, + "rewards": 0.0, + "advantages": -4.660505771636963, + "generated_text": "{\n \"intent\": \"To proceed with exploring the file system and potentially open files from the recent list.\",\n \"action\": \"DoubleClick(327, 98) Click(327, 98)\"\n}", + "action_string": "None", + "intent_string": null, + "before_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 58, + 89 + ] + }, + { + "text": " open File ", + "boxes": [ + 59, + 59, + 209, + 110 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "□□", + "boxes": [ + 1458, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 104, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 68, + 179, + 101, + 209 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "I clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": " Other Locatic", + "boxes": [ + 424, + 256, + 514, + 274 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 58, + 89 + ] + }, + { + "text": " open File ", + "boxes": [ + 59, + 59, + 209, + 110 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "□□", + "boxes": [ + 1458, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 104, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 68, + 179, + 101, + 209 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "I clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": " Other Locatic", + "boxes": [ + 424, + 256, + 514, + 274 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_8", + "episode_id": 50, + "env_id": 2, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.383322715759277, + "advantages": 0.45173215866088867, + "generated_text": "{\n \"intent\": \"To continue exploring further by navigating the web page, clicking the 'Download for Linux 32-bit' button will accomplish this goal.\",\n \"action\": \"DoubleClick(956, 315)\"\n}", + "action_string": "DoubleClick(956, 315)", + "intent_string": "To continue exploring further by navigating the web page, clicking the 'Download for Linux 32-bit' button will accomplish this goal.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1874, + 89 + ] + }, + { + "text": "Mozilla /3", + "boxes": [ + 255, + 104, + 375, + 156 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1172, + 139 + ] + }, + { + "text": "Seiect d uuwiluau uptiui", + "boxes": [ + 674, + 147, + 1238, + 183 + ] + }, + { + "text": "below", + "boxes": [ + 873, + 178, + 1033, + 239 + ] + }, + { + "text": " Download for Linux 32-bit ", + "boxes": [ + 848, + 306, + 1060, + 330 + ] + }, + { + "text": " Download for Linux 64-bit ", + "boxes": [ + 846, + 355, + 1064, + 385 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 732, + 397, + 1176, + 427 + ] + }, + { + "text": "Youcansetupour", + "boxes": [ + 790, + 423, + 934, + 441 + ] + }, + { + "text": "r APT repositoryinstead.", + "boxes": [ + 926, + 421, + 1120, + 445 + ] + }, + { + "text": "Download in another Language or for another operating system", + "boxes": [ + 706, + 602, + 1200, + 631 + ] + }, + { + "text": "Mozilla", + "boxes": [ + 259, + 740, + 429, + 797 + ] + }, + { + "text": "Company", + "boxes": [ + 722, + 741, + 800, + 766 + ] + }, + { + "text": "support ", + "boxes": [ + 952, + 737, + 1028, + 768 + ] + }, + { + "text": " Developers", + "boxes": [ + 1181, + 732, + 1281, + 769 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 737, + 1646, + 768 + ] + }, + { + "text": " Leadership ", + "boxes": [ + 720, + 766, + 818, + 792 + ] + }, + { + "text": " Product Help", + "boxes": [ + 949, + 760, + 1065, + 797 + ] + }, + { + "text": " Developer Edition", + "boxes": [ + 1184, + 762, + 1332, + 796 + ] + }, + { + "text": "X@nd", + "boxes": [ + 1506, + 776, + 1644, + 806 + ] + }, + { + "text": "Advertising", + "boxes": [ + 257, + 792, + 529, + 854 + ] + }, + { + "text": " Press Center", + "boxes": [ + 718, + 792, + 830, + 822 + ] + }, + { + "text": "File a Bug ", + "boxes": [ + 954, + 794, + 1038, + 820 + ] + }, + { + "text": "Enterprise", + "boxes": [ + 1186, + 792, + 1278, + 824 + ] + }, + { + "text": "Careers", + "boxes": [ + 720, + 820, + 792, + 852 + ] + }, + { + "text": "Localize Mozilla", + "boxes": [ + 954, + 820, + 1086, + 850 + ] + }, + { + "text": "Tools", + "boxes": [ + 1186, + 818, + 1238, + 852 + ] + }, + { + "text": "Follow @Firefox", + "boxes": [ + 1514, + 826, + 1646, + 860 + ] + }, + { + "text": "Contact", + "boxes": [ + 724, + 852, + 788, + 872 + ] + }, + { + "text": "MDN ", + "boxes": [ + 1184, + 844, + 1234, + 878 + ] + }, + { + "text": " Privacy-first advertising solutions for", + "boxes": [ + 262, + 868, + 564, + 897 + ] + }, + { + "text": " Firefox Release Notes", + "boxes": [ + 1184, + 876, + 1362, + 905 + ] + }, + { + "text": "brands, publishers, and platforms.", + "boxes": [ + 264, + 889, + 532, + 913 + ] + }, + { + "text": "Learn more ", + "boxes": [ + 378, + 953, + 496, + 977 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口回", + "boxes": [ + 1853, + 0, + 1919, + 31 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozilα 3", + "boxes": [ + 253, + 102, + 377, + 156 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1172, + 139 + ] + }, + { + "text": "seiect d uuwiliuau upliul", + "boxes": [ + 676, + 147, + 1238, + 183 + ] + }, + { + "text": "below", + "boxes": [ + 873, + 178, + 1033, + 239 + ] + }, + { + "text": " Download for inux 32-bit", + "boxes": [ + 846, + 304, + 1062, + 334 + ] + }, + { + "text": "Opening firefox-138.0.3.tar.xz", + "boxes": [ + 828, + 357, + 1094, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1122, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open:", + "boxes": [ + 742, + 383, + 922, + 411 + ] + }, + { + "text": " firefox-138.0.3.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Which is: TAR Archive (72.6 MB)", + "boxes": [ + 772, + 433, + 998, + 469 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": "Dov", + "boxes": [ + 701, + 599, + 735, + 632 + ] + }, + { + "text": " Do this automatically for fles like this fron", + "boxes": [ + 764, + 594, + 1062, + 617 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Mozilla", + "boxes": [ + 260, + 741, + 432, + 796 + ] + }, + { + "text": " Company", + "boxes": [ + 717, + 734, + 803, + 771 + ] + }, + { + "text": "support ", + "boxes": [ + 954, + 737, + 1028, + 768 + ] + }, + { + "text": " Developers", + "boxes": [ + 1181, + 732, + 1281, + 769 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 737, + 1646, + 768 + ] + }, + { + "text": " Leadership ", + "boxes": [ + 720, + 768, + 818, + 792 + ] + }, + { + "text": " Product Help", + "boxes": [ + 949, + 760, + 1065, + 797 + ] + }, + { + "text": "Developer Edition", + "boxes": [ + 1186, + 766, + 1332, + 796 + ] + }, + { + "text": "X@Ed", + "boxes": [ + 1504, + 776, + 1642, + 806 + ] + }, + { + "text": "Advertising", + "boxes": [ + 257, + 792, + 527, + 854 + ] + }, + { + "text": " Press Center", + "boxes": [ + 718, + 792, + 830, + 822 + ] + }, + { + "text": "File a Bug ", + "boxes": [ + 954, + 794, + 1038, + 820 + ] + }, + { + "text": "Enterprise", + "boxes": [ + 1186, + 792, + 1278, + 824 + ] + }, + { + "text": "Careers", + "boxes": [ + 720, + 820, + 792, + 852 + ] + }, + { + "text": "Localize Mozilla", + "boxes": [ + 954, + 820, + 1086, + 850 + ] + }, + { + "text": "Tools", + "boxes": [ + 1186, + 818, + 1238, + 852 + ] + }, + { + "text": "Follow @Firefox", + "boxes": [ + 1514, + 826, + 1646, + 860 + ] + }, + { + "text": "Contact", + "boxes": [ + 724, + 852, + 788, + 870 + ] + }, + { + "text": "MDN ", + "boxes": [ + 1184, + 844, + 1234, + 878 + ] + }, + { + "text": "Privacy-first advertising solutions for", + "boxes": [ + 262, + 868, + 564, + 897 + ] + }, + { + "text": "回小", + "boxes": [ + 1548, + 868, + 1638, + 893 + ] + }, + { + "text": " Firefox Release Notes", + "boxes": [ + 1186, + 878, + 1360, + 901 + ] + }, + { + "text": "brands, publishers, and platforms.", + "boxes": [ + 264, + 889, + 534, + 915 + ] + }, + { + "text": "Learn more -", + "boxes": [ + 376, + 949, + 496, + 977 + ] + }, + { + "text": "ng firef", + "boxes": [ + 410, + 1050, + 458, + 1070 + ] + } + ] + }, + { + "frame_id": "50_3_8", + "episode_id": 50, + "env_id": 3, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.84316635131836, + "advantages": 0.185785710811615, + "generated_text": "{\n \"intent\": \"To explore different browser options or settings by scrolling down the page to access more information.\",\n \"action\": \"ScrollDown(990, 540)\"\n}", + "action_string": "ScrollDown(990, 540)", + "intent_string": "To explore different browser options or settings by scrolling down the page to access more information.", + "before_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "ns and other languages", + "boxes": [ + 440, + 367, + 576, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 492, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 800, + 917, + 1098, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 36, + 66, + 126, + 89 + ] + }, + { + "text": "①| http://www.mozilla.org/en-USs/firefox/new/", + "boxes": [ + 252, + 64, + 608, + 93 + ] + }, + { + "text": "目", + "boxes": [ + 1544, + 69, + 1562, + 85 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 119, + 490, + 137 + ] + }, + { + "text": "Dial down the", + "boxes": [ + 422, + 520, + 670, + 564 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 520, + 1028, + 564 + ] + }, + { + "text": " Keep tabs on all", + "boxes": [ + 1172, + 516, + 1462, + 570 + ] + }, + { + "text": "noise", + "boxes": [ + 422, + 568, + 528, + 606 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 568, + 994, + 604 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1180, + 568, + 1354, + 606 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 617, + 710, + 647 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 800, + 617, + 1096, + 647 + ] + }, + { + "text": "Close duplicates, search and pin open ", + "boxes": [ + 1176, + 611, + 1484, + 651 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 422, + 633, + 712, + 667 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 804, + 637, + 1096, + 667 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 657, + 650, + 681 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 651, + 928, + 687 + ] + }, + { + "text": "possibilities for the almost infinite", + "boxes": [ + 1182, + 653, + 1462, + 683 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1180, + 673, + 1424, + 703 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 878, + 725, + 1030, + 754 + ] + }, + { + "text": "Get the browser that helps you", + "boxes": [ + 610, + 991, + 1300, + 1042 + ] + } + ] + }, + { + "frame_id": "50_4_8", + "episode_id": 50, + "env_id": 4, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.363313674926758, + "advantages": 0.4418807029724121, + "generated_text": "{\n \"intent\": \"To continue exploring by interacting with applications available on the desktop. Clicking on an application icon will open it and allow further interactions such as navigating menus or entering search terms.\",\n \"action\": \"DoubleClick(55, 492)\"\n}", + "action_string": "DoubleClick(55, 492)", + "intent_string": "To continue exploring by interacting with applications available on the desktop. Clicking on an application icon will open it and allow further interactions such as navigating menus or entering search terms.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "", + "boxes": [ + 42, + 131, + 74, + 153 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 112, + 199 + ] + }, + { + "text": "Termirkal", + "boxes": [ + 9, + 271, + 109, + 316 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 116, + 421 + ] + }, + { + "text": "LibreO.,", + "boxes": [ + 14, + 498, + 94, + 530 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 608, + 34, + 621 + ] + }, + { + "text": "Edit Tabs ", + "boxes": [ + 42, + 606, + 128, + 623 + ] + }, + { + "text": "t@02b525", + "boxes": [ + 540, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "FTre", + "boxes": [ + 19, + 56, + 64, + 100 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 98, + 115, + 236, + 149 + ] + }, + { + "text": "Mous", + "boxes": [ + 10, + 171, + 56, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 72, + 183, + 98, + 204 + ] + }, + { + "text": "Recent Files", + "boxes": [ + 104, + 181, + 226, + 212 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Teri", + "boxes": [ + 22, + 284, + 52, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "雪", + "boxes": [ + 70, + 326, + 98, + 353 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 106, + 325, + 274, + 354 + ] + }, + { + "text": "國", + "boxes": [ + 70, + 373, + 98, + 401 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 106, + 369, + 280, + 405 + ] + }, + { + "text": "Calo", + "boxes": [ + 16, + 393, + 50, + 413 + ] + }, + { + "text": "", + "boxes": [ + 68, + 413, + 98, + 449 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 108, + 423, + 310, + 447 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 54, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 98, + 514, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 104, + 558, + 258, + 592 + ] + }, + { + "text": "福", + "boxes": [ + 5, + 578, + 25, + 598 + ] + }, + { + "text": "0", + "boxes": [ + 1862, + 574, + 1920, + 606 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1372, + 699 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "0ot@02b525", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + }, + { + "text": "reoffice", + "boxes": [ + 690, + 1050, + 744, + 1070 + ] + } + ] + }, + { + "frame_id": "50_5_8", + "episode_id": 50, + "env_id": 5, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.310099601745605, + "advantages": 0.4156807065010071, + "generated_text": "{\n \"intent\": \"To explore other parts of the website, which might include various sections and products.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore other parts of the website, which might include various sections and products.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 296, + 115, + 412, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1452, + 115, + 1476, + 137 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "Account&Lists", + "boxes": [ + 1598, + 127, + 1702, + 145 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 414, + 187 + ] + }, + { + "text": "Beauty products", + "boxes": [ + 612, + 280, + 876, + 328 + ] + }, + { + "text": "Explore top sellers", + "boxes": [ + 644, + 322, + 846, + 359 + ] + }, + { + "text": "8", + "boxes": [ + 1107, + 404, + 1227, + 451 + ] + }, + { + "text": "R", + "boxes": [ + 1330, + 425, + 1378, + 451 + ] + }, + { + "text": "ns of products withfast local delivery. Click here to go to ar", + "boxes": [ + 972, + 453, + 1344, + 484 + ] + }, + { + "text": "oom", + "boxes": [ + 1016, + 486, + 1312, + 510 + ] + }, + { + "text": "", + "boxes": [ + 1346, + 486, + 1466, + 510 + ] + }, + { + "text": "", + "boxes": [ + 1456, + 486, + 1534, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1534, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1352, + 969, + 1510, + 999 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "UTF8&k", + "boxes": [ + 266, + 1022, + 332, + 1040 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1606, + 69, + 1658, + 87 + ] + }, + { + "text": "Gamingaccessories", + "boxes": [ + 244, + 189, + 438, + 212 + ] + }, + { + "text": "Shopfor your home essentials", + "boxes": [ + 614, + 187, + 916, + 210 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 982, + 185, + 1202, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 185, + 1534, + 214 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 544, + 1058, + 568 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1349, + 541, + 1437, + 572 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 621, + 1510, + 657 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 653, + 354, + 683 + ] + }, + { + "text": "Explore all products in Kitcher", + "boxes": [ + 246, + 987, + 418, + 1005 + ] + }, + { + "text": "Shop the latest from Hom", + "boxes": [ + 614, + 987, + 766, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Shopnow", + "boxes": [ + 1354, + 987, + 1412, + 1007 + ] + } + ] + }, + { + "frame_id": "50_6_8", + "episode_id": 50, + "env_id": 6, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.177717208862305, + "advantages": -0.14184874296188354, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment, I will start by viewing a list of files by clicking on the folder icon located at the top left corner.\",\n \"action\": \"Click(38, 70)\"\n}", + "action_string": "Click(38, 70)", + "intent_string": "To continue exploring the desktop environment, I will start by viewing a list of files by clicking on the folder icon located at the top left corner.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Styles", + "boxes": [ + 246, + 28, + 292, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 524, + 30, + 552, + 44 + ] + }, + { + "text": "--目· 自、", + "boxes": [ + 10, + 50, + 388, + 85 + ] + }, + { + "text": "上&", + "boxes": [ + 369, + 51, + 445, + 86 + ] + }, + { + "text": "勺-、", + "boxes": [ + 444, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": " =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1886, + 302, + 1912, + 341 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": ".9", + "boxes": [ + 18, + 482, + 30, + 494 + ] + }, + { + "text": "21", + "boxes": [ + 16, + 512, + 30, + 528 + ] + }, + { + "text": "4", + "boxes": [ + 18, + 570, + 30, + 582 + ] + }, + { + "text": "25", + "boxes": [ + 16, + 584, + 30, + 602 + ] + }, + { + "text": "6", + "boxes": [ + 18, + 602, + 30, + 613 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "42", + "boxes": [ + 14, + 870, + 30, + 883 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 32, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 12, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 10, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled2-LibreOfficeCalc", + "boxes": [ + 846, + 4, + 1074, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "--目-目@%电自-", + "boxes": [ + 14, + 54, + 374, + 83 + ] + }, + { + "text": "上&", + "boxes": [ + 373, + 53, + 443, + 84 + ] + }, + { + "text": "-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberation San", + "boxes": [ + 14, + 95, + 150, + 119 + ] + }, + { + "text": "10→", + "boxes": [ + 140, + 95, + 230, + 119 + ] + }, + { + "text": "Jaaa.", + "boxes": [ + 218, + 93, + 352, + 123 + ] + }, + { + "text": "", + "boxes": [ + 336, + 93, + 590, + 123 + ] + }, + { + "text": "国", + "boxes": [ + 580, + 99, + 616, + 113 + ] + }, + { + "text": "□日日-%00+三至田二-口-目", + "boxes": [ + 606, + 93, + 1204, + 123 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "▼", + "boxes": [ + 132, + 133, + 154, + 147 + ] + }, + { + "text": "、=", + "boxes": [ + 152, + 129, + 256, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "+5", + "boxes": [ + 16, + 927, + 28, + 939 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 464, + 1072 + ] + } + ] + }, + { + "frame_id": "50_7_8", + "episode_id": 50, + "env_id": 7, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.30919075012207, + "advantages": -0.07711763679981232, + "generated_text": "{\n \"intent\": \"Scrolling down could provide more content from the webpage which might include deeper insights or additional links related to the topic.\",\n \"action\": \"ScrollUp(960, 540)\"\n}", + "action_string": "ScrollUp(960, 540)", + "intent_string": "Scrolling down could provide more content from the webpage which might include deeper insights or additional links related to the topic.", + "before_screen_ocr_results": [ + { + "text": "×", + "boxes": [ + 196, + 36, + 218, + 50 + ] + }, + { + "text": "·和平精英：3个提升", + "boxes": [ + 244, + 32, + 374, + 50 + ] + }, + { + "text": "Q白", + "boxes": [ + 64, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": "9", + "boxes": [ + 1854, + 71, + 1866, + 83 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 572, + 123, + 632, + 143 + ] + }, + { + "text": "中", + "boxes": [ + 1757, + 116, + 1793, + 150 + ] + }, + { + "text": "msn", + "boxes": [ + 60, + 129, + 102, + 149 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "关注", + "boxes": [ + 280, + 179, + 314, + 201 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1602, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "嘎嘎脆", + "boxes": [ + 1591, + 239, + 1689, + 287 + ] + }, + { + "text": "游戏天地十关注", + "boxes": [ + 386, + 264, + 576, + 306 + ] + }, + { + "text": "42.2K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "甜甜香", + "boxes": [ + 1618, + 274, + 1688, + 308 + ] + }, + { + "text": "想吃坚果蜜饯的", + "boxes": [ + 1424, + 292, + 1542, + 322 + ] + }, + { + "text": "点击领取", + "boxes": [ + 1454, + 336, + 1532, + 367 + ] + }, + { + "text": "和平精英：3个提升听声辨位能力的方法，快试试！", + "boxes": [ + 366, + 349, + 1104, + 391 + ] + }, + { + "text": "F", + "boxes": [ + 192, + 395, + 214, + 421 + ] + }, + { + "text": "闭眼冲", + "boxes": [ + 1618, + 391, + 1676, + 409 + ] + }, + { + "text": "游戏天地的报道·5天·①3分钟读取", + "boxes": [ + 364, + 413, + 666, + 449 + ] + }, + { + "text": "想吃坚果蜜饯的，来淘宝就GO", + "boxes": [ + 1422, + 411, + 1678, + 447 + ] + }, + { + "text": "了！", + "boxes": [ + 1428, + 443, + 1462, + 465 + ] + }, + { + "text": "广告淘宝网", + "boxes": [ + 1426, + 469, + 1508, + 492 + ] + }, + { + "text": "嘿，各位和平精英的玩家们！最近在游戏里是不是经常被敌人偷袭，却还搞不清楚人家从哪儿冒出来的？就像上次我", + "boxes": [ + 360, + 500, + 1240, + 538 + ] + }, + { + "text": "玩的时候，正美滋滋地搜着物资呢，突然就被人给阴了，连枪声方向都没搞明白，别提多郁闷了。其实啊，这听声辨", + "boxes": [ + 366, + 530, + 1236, + 560 + ] + }, + { + "text": "位的能力要是能提升上去，在游戏里那可就能抢占不少先机啦！今天就给大家分享3个提升听声辨位能力的方法，快", + "boxes": [ + 364, + 556, + 1230, + 586 + ] + }, + { + "text": "访问 游戏天地", + "boxes": [ + 1430, + 554, + 1552, + 584 + ] + }, + { + "text": "L", + "boxes": [ + 1680, + 558, + 1696, + 572 + ] + }, + { + "text": "试试哦！", + "boxes": [ + 366, + 586, + 432, + 611 + ] + }, + { + "text": "DNF手游确定无形自选和白色天三", + "boxes": [ + 1446, + 602, + 1664, + 631 + ] + }, + { + "text": "投票，别错过500万泰拉", + "boxes": [ + 1446, + 621, + 1604, + 651 + ] + }, + { + "text": "调大音量不一定就能听准位置", + "boxes": [ + 364, + 659, + 746, + 695 + ] + }, + { + "text": "梦幻西游夜叉改动！七八十个全是", + "boxes": [ + 1446, + 665, + 1664, + 695 + ] + }, + { + "text": "画魂，夜叉队准备转型", + "boxes": [ + 1446, + 685, + 1594, + 715 + ] + }, + { + "text": "梦幻西游：不战斗刷完慈心渡鬼，", + "boxes": [ + 1446, + 729, + 1654, + 758 + ] + }, + { + "text": "附任务领取坐标图", + "boxes": [ + 1446, + 748, + 1566, + 778 + ] + }, + { + "text": "淘", + "boxes": [ + 1422, + 836, + 1448, + 850 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 834, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 794, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置\"查看您的选项以及不", + "boxes": [ + 32, + 951, + 216, + 975 + ] + }, + { + "text": "在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的\"管理设置\"改变您的选择。欲了解", + "boxes": [ + 206, + 951, + 794, + 975 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "平精英：3个", + "boxes": [ + 224, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×", + "boxes": [ + 196, + 36, + 218, + 50 + ] + }, + { + "text": "·和平精英：3个提升", + "boxes": [ + 244, + 32, + 374, + 50 + ] + }, + { + "text": "Q白", + "boxes": [ + 64, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": "?", + "boxes": [ + 1850, + 69, + 1868, + 83 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 572, + 123, + 632, + 143 + ] + }, + { + "text": "中", + "boxes": [ + 1757, + 116, + 1793, + 150 + ] + }, + { + "text": "msn", + "boxes": [ + 60, + 129, + 102, + 149 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "关注", + "boxes": [ + 280, + 179, + 314, + 201 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1602, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "嘎嘎脆", + "boxes": [ + 1591, + 239, + 1689, + 287 + ] + }, + { + "text": "游戏天地十关注", + "boxes": [ + 386, + 264, + 576, + 306 + ] + }, + { + "text": "42.2K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "甜甜香", + "boxes": [ + 1618, + 274, + 1688, + 308 + ] + }, + { + "text": "想吃坚果蜜饯的", + "boxes": [ + 1424, + 292, + 1542, + 322 + ] + }, + { + "text": "点击领取", + "boxes": [ + 1454, + 336, + 1532, + 367 + ] + }, + { + "text": "和平精英：3个提升听声辨位能力的方法，快试试！", + "boxes": [ + 366, + 349, + 1104, + 391 + ] + }, + { + "text": "F", + "boxes": [ + 192, + 395, + 214, + 421 + ] + }, + { + "text": "闭眼冲", + "boxes": [ + 1618, + 391, + 1676, + 409 + ] + }, + { + "text": "游戏天地的报道·5天·①3分钟读取", + "boxes": [ + 364, + 413, + 666, + 449 + ] + }, + { + "text": "想吃坚果蜜饯的，来淘宝就GO", + "boxes": [ + 1422, + 411, + 1678, + 447 + ] + }, + { + "text": "了！", + "boxes": [ + 1428, + 443, + 1462, + 465 + ] + }, + { + "text": "广告淘宝网", + "boxes": [ + 1426, + 469, + 1508, + 492 + ] + }, + { + "text": "嘿，各位和平精英的玩家们！最近在游戏里是不是经常被敌人偷袭，却还搞不清楚人家从哪儿冒出来的？就像上次我", + "boxes": [ + 364, + 506, + 1238, + 536 + ] + }, + { + "text": "玩的时候，正美滋滋地搜着物资呢，突然就被人给阴了，连枪声方向都没搞明白，别提多郁闷了。其实啊，这听声辨", + "boxes": [ + 366, + 530, + 1236, + 560 + ] + }, + { + "text": "位的能力要是能提升上去，在游戏里那可就能抢占不少先机啦！今天就给大家分享3个提升听声辨位能力的方法，快", + "boxes": [ + 364, + 556, + 1230, + 586 + ] + }, + { + "text": "访问 游戏天地", + "boxes": [ + 1430, + 554, + 1552, + 584 + ] + }, + { + "text": "L", + "boxes": [ + 1680, + 558, + 1696, + 572 + ] + }, + { + "text": "试试哦！", + "boxes": [ + 366, + 586, + 432, + 611 + ] + }, + { + "text": "DNF手游确定无形自选和白色天三", + "boxes": [ + 1446, + 602, + 1664, + 631 + ] + }, + { + "text": "投票，别错过500万泰拉", + "boxes": [ + 1446, + 621, + 1604, + 651 + ] + }, + { + "text": "调大音量不一定就能听准位置", + "boxes": [ + 364, + 659, + 746, + 695 + ] + }, + { + "text": "梦幻西游夜叉改动！七八十个全是", + "boxes": [ + 1446, + 665, + 1664, + 695 + ] + }, + { + "text": "画魂，夜叉队准备转型", + "boxes": [ + 1446, + 685, + 1594, + 715 + ] + }, + { + "text": "梦幻西游：不战斗刷完慈心渡鬼，", + "boxes": [ + 1446, + 729, + 1656, + 758 + ] + }, + { + "text": "附任务领取坐标图", + "boxes": [ + 1446, + 748, + 1566, + 778 + ] + }, + { + "text": "淘", + "boxes": [ + 1422, + 836, + 1448, + 850 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 834, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 794, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置\"查看您的选项以及不", + "boxes": [ + 32, + 951, + 216, + 975 + ] + }, + { + "text": "在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的\"管理设置\"改变您的选择。欲了解", + "boxes": [ + 206, + 951, + 794, + 975 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "平精英：3个", + "boxes": [ + 224, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "50_0_9", + "episode_id": 50, + "env_id": 0, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.570523262023926, + "advantages": 0.05154964700341225, + "generated_text": "{\n \"intent\": \"To continue interacting with the environment by exploring more products or sections%\",\n \"action\": \"DoubleClick(946, 76)\"\n}", + "action_string": "DoubleClick(946, 76)", + "intent_string": "To continue interacting with the environment by exploring more products or sections%", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Elevate Your Electronics", + "boxes": [ + 242, + 248, + 486, + 278 + ] + }, + { + "text": "Gear up to get fit", + "boxes": [ + 608, + 246, + 788, + 280 + ] + }, + { + "text": "Most-loved travel essentials", + "boxes": [ + 982, + 248, + 1264, + 278 + ] + }, + { + "text": "Level up your PC here", + "boxes": [ + 1350, + 250, + 1574, + 280 + ] + }, + { + "text": "Speakers", + "boxes": [ + 406, + 558, + 456, + 584 + ] + }, + { + "text": "Hard Drives", + "boxes": [ + 1356, + 562, + 1414, + 580 + ] + }, + { + "text": "Discover more", + "boxes": [ + 614, + 611, + 694, + 629 + ] + }, + { + "text": "Discover more", + "boxes": [ + 1354, + 611, + 1436, + 629 + ] + }, + { + "text": "Gift ideas inspired by your shopping history", + "boxes": [ + 240, + 689, + 682, + 725 + ] + }, + { + "text": "", + "boxes": [ + 878, + 758, + 910, + 792 + ] + }, + { + "text": "Additional items to explore See more", + "boxes": [ + 242, + 991, + 588, + 1020 + ] + }, + { + "text": "A", + "boxes": [ + 254, + 1026, + 336, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "http://www.apple.c", + "boxes": [ + 340, + 113, + 460, + 131 + ] + }, + { + "text": "bi", + "boxes": [ + 260, + 149, + 294, + 163 + ] + }, + { + "text": "奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 302, + 216, + 586, + 234 + ] + }, + { + "text": "Speakers", + "boxes": [ + 408, + 562, + 452, + 580 + ] + }, + { + "text": "Hard Drives", + "boxes": [ + 1356, + 562, + 1414, + 580 + ] + }, + { + "text": "Discover more", + "boxes": [ + 614, + 611, + 694, + 629 + ] + }, + { + "text": "Discover more", + "boxes": [ + 1356, + 611, + 1436, + 629 + ] + }, + { + "text": " Gift ideas inspired by your shopping history", + "boxes": [ + 238, + 687, + 682, + 725 + ] + }, + { + "text": "C", + "boxes": [ + 880, + 758, + 908, + 792 + ] + }, + { + "text": "Additional items to explore See more", + "boxes": [ + 242, + 991, + 584, + 1020 + ] + }, + { + "text": "A", + "boxes": [ + 254, + 1026, + 334, + 1044 + ] + } + ] + }, + { + "frame_id": "50_1_9", + "episode_id": 50, + "env_id": 1, + "step": 9, + "format_rewards": 1.0, + "rewards": 8.656644821166992, + "advantages": -0.3983992040157318, + "generated_text": "{\n \"intent\": \"To locate and open files from the home directory, I will begin by clicking on the 'Home' option in the directory list.\",\n \"action\": \"Click(536, 97)\"\n}", + "action_string": "Click(536, 97)", + "intent_string": "To locate and open files from the home directory, I will begin by clicking on the 'Home' option in the directory list.", + "before_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 58, + 89 + ] + }, + { + "text": " open File ", + "boxes": [ + 59, + 59, + 209, + 110 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "□□", + "boxes": [ + 1458, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 104, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 68, + 179, + 101, + 209 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "I clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 357 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "LibreOffice", + "boxes": [ + 218, + 1048, + 292, + 1072 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 28, + 68, + 56, + 89 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "口", + "boxes": [ + 1460, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": " Recent Files ", + "boxes": [ + 102, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 552, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": " clipboard_app", + "boxes": [ + 556, + 218, + 670, + 242 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 244 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 326, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 106, + 326, + 278, + 355 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 391, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": "Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 512, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "1open", + "boxes": [ + 1439, + 946, + 1499, + 976 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 294, + 1005 + ] + }, + { + "text": "ibreoffice", + "boxes": [ + 222, + 1050, + 288, + 1070 + ] + }, + { + "text": "Open", + "boxes": [ + 328, + 1048, + 388, + 1074 + ] + } + ] + }, + { + "frame_id": "50_2_9", + "episode_id": 50, + "env_id": 2, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.857449531555176, + "advantages": 0.1928180456161499, + "generated_text": "{\n \"intent\": \"To explore further options by clicking the download button.\",\n \"action\": \"DoubleClick(957, 330)\"\n}", + "action_string": "DoubleClick(957, 330)", + "intent_string": "To explore further options by clicking the download button.", + "before_screen_ocr_results": [ + { + "text": "口回", + "boxes": [ + 1853, + 0, + 1919, + 31 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozilα 3", + "boxes": [ + 253, + 102, + 377, + 156 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1172, + 139 + ] + }, + { + "text": "seiect d uuwiliuau upliul", + "boxes": [ + 676, + 147, + 1238, + 183 + ] + }, + { + "text": "below", + "boxes": [ + 873, + 178, + 1033, + 239 + ] + }, + { + "text": " Download for inux 32-bit", + "boxes": [ + 846, + 304, + 1062, + 334 + ] + }, + { + "text": "Opening firefox-138.0.3.tar.xz", + "boxes": [ + 828, + 357, + 1094, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1122, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open:", + "boxes": [ + 742, + 383, + 922, + 411 + ] + }, + { + "text": " firefox-138.0.3.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Which is: TAR Archive (72.6 MB)", + "boxes": [ + 772, + 433, + 998, + 469 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": "Dov", + "boxes": [ + 701, + 599, + 735, + 632 + ] + }, + { + "text": " Do this automatically for fles like this fron", + "boxes": [ + 764, + 594, + 1062, + 617 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Mozilla", + "boxes": [ + 260, + 741, + 432, + 796 + ] + }, + { + "text": " Company", + "boxes": [ + 717, + 734, + 803, + 771 + ] + }, + { + "text": "support ", + "boxes": [ + 954, + 737, + 1028, + 768 + ] + }, + { + "text": " Developers", + "boxes": [ + 1181, + 732, + 1281, + 769 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 737, + 1646, + 768 + ] + }, + { + "text": " Leadership ", + "boxes": [ + 720, + 768, + 818, + 792 + ] + }, + { + "text": " Product Help", + "boxes": [ + 949, + 760, + 1065, + 797 + ] + }, + { + "text": "Developer Edition", + "boxes": [ + 1186, + 766, + 1332, + 796 + ] + }, + { + "text": "X@Ed", + "boxes": [ + 1504, + 776, + 1642, + 806 + ] + }, + { + "text": "Advertising", + "boxes": [ + 257, + 792, + 527, + 854 + ] + }, + { + "text": " Press Center", + "boxes": [ + 718, + 792, + 830, + 822 + ] + }, + { + "text": "File a Bug ", + "boxes": [ + 954, + 794, + 1038, + 820 + ] + }, + { + "text": "Enterprise", + "boxes": [ + 1186, + 792, + 1278, + 824 + ] + }, + { + "text": "Careers", + "boxes": [ + 720, + 820, + 792, + 852 + ] + }, + { + "text": "Localize Mozilla", + "boxes": [ + 954, + 820, + 1086, + 850 + ] + }, + { + "text": "Tools", + "boxes": [ + 1186, + 818, + 1238, + 852 + ] + }, + { + "text": "Follow @Firefox", + "boxes": [ + 1514, + 826, + 1646, + 860 + ] + }, + { + "text": "Contact", + "boxes": [ + 724, + 852, + 788, + 870 + ] + }, + { + "text": "MDN ", + "boxes": [ + 1184, + 844, + 1234, + 878 + ] + }, + { + "text": "Privacy-first advertising solutions for", + "boxes": [ + 262, + 868, + 564, + 897 + ] + }, + { + "text": "回小", + "boxes": [ + 1548, + 868, + 1638, + 893 + ] + }, + { + "text": " Firefox Release Notes", + "boxes": [ + 1186, + 878, + 1360, + 901 + ] + }, + { + "text": "brands, publishers, and platforms.", + "boxes": [ + 264, + 889, + 534, + 915 + ] + }, + { + "text": "Learn more -", + "boxes": [ + 376, + 949, + 496, + 977 + ] + }, + { + "text": "ng firef", + "boxes": [ + 408, + 1050, + 458, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口回", + "boxes": [ + 1853, + 0, + 1919, + 31 + ] + }, + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozilα 3", + "boxes": [ + 253, + 102, + 377, + 156 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1172, + 139 + ] + }, + { + "text": "seiect d uuwiliuau upliul", + "boxes": [ + 676, + 147, + 1238, + 183 + ] + }, + { + "text": "below", + "boxes": [ + 873, + 178, + 1033, + 239 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 842, + 300, + 1064, + 336 + ] + }, + { + "text": "Opening firefox-138.0.3.tar.xz", + "boxes": [ + 828, + 357, + 1094, + 381 + ] + }, + { + "text": "回区", + "boxes": [ + 1122, + 359, + 1186, + 379 + ] + }, + { + "text": " You have chosen to open:", + "boxes": [ + 742, + 383, + 922, + 411 + ] + }, + { + "text": " firefox-138.0.3.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Which is: TAR Archive (72.6 MB)", + "boxes": [ + 772, + 433, + 998, + 469 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1036, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": "DOy", + "boxes": [ + 701, + 602, + 733, + 629 + ] + }, + { + "text": " Do this automatically for fles like this fron", + "boxes": [ + 764, + 594, + 1062, + 617 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Mozilla", + "boxes": [ + 258, + 741, + 430, + 796 + ] + }, + { + "text": " Company", + "boxes": [ + 717, + 734, + 803, + 771 + ] + }, + { + "text": "support ", + "boxes": [ + 954, + 737, + 1028, + 768 + ] + }, + { + "text": " Developers", + "boxes": [ + 1181, + 732, + 1281, + 769 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 737, + 1646, + 768 + ] + }, + { + "text": " Leadership ", + "boxes": [ + 720, + 768, + 818, + 792 + ] + }, + { + "text": " Product Help", + "boxes": [ + 949, + 760, + 1065, + 797 + ] + }, + { + "text": "Developer Edition", + "boxes": [ + 1186, + 766, + 1332, + 796 + ] + }, + { + "text": "X@Ed", + "boxes": [ + 1504, + 776, + 1642, + 806 + ] + }, + { + "text": "Advertising", + "boxes": [ + 257, + 792, + 527, + 854 + ] + }, + { + "text": " Press Center", + "boxes": [ + 718, + 792, + 830, + 822 + ] + }, + { + "text": "File a Bug ", + "boxes": [ + 954, + 794, + 1038, + 820 + ] + }, + { + "text": "Enterprise", + "boxes": [ + 1186, + 792, + 1278, + 824 + ] + }, + { + "text": "Careers", + "boxes": [ + 720, + 820, + 792, + 852 + ] + }, + { + "text": "Localize Mozilla", + "boxes": [ + 954, + 820, + 1086, + 850 + ] + }, + { + "text": "Tools", + "boxes": [ + 1186, + 818, + 1238, + 852 + ] + }, + { + "text": "Follow @Firefox", + "boxes": [ + 1514, + 826, + 1646, + 860 + ] + }, + { + "text": "Contact", + "boxes": [ + 724, + 852, + 788, + 872 + ] + }, + { + "text": "MDN ", + "boxes": [ + 1184, + 844, + 1234, + 878 + ] + }, + { + "text": " Privacy-first advertising solutions for", + "boxes": [ + 262, + 868, + 564, + 897 + ] + }, + { + "text": "回小", + "boxes": [ + 1548, + 868, + 1638, + 893 + ] + }, + { + "text": " Firefox Release Notes", + "boxes": [ + 1186, + 878, + 1360, + 901 + ] + }, + { + "text": "brands, publishers, and platforms.", + "boxes": [ + 266, + 891, + 534, + 915 + ] + }, + { + "text": "Learn more -", + "boxes": [ + 376, + 949, + 496, + 977 + ] + } + ] + }, + { + "frame_id": "50_3_9", + "episode_id": 50, + "env_id": 3, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.664595603942871, + "advantages": 0.0978662446141243, + "generated_text": "{\n \"intent\": \"To explore further details about Firefox based on the current page.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore further details about Firefox based on the current page.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 36, + 66, + 126, + 89 + ] + }, + { + "text": "①| http://www.mozilla.org/en-USs/firefox/new/", + "boxes": [ + 252, + 64, + 608, + 93 + ] + }, + { + "text": "目", + "boxes": [ + 1544, + 69, + 1562, + 85 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 119, + 490, + 137 + ] + }, + { + "text": "Dial down the", + "boxes": [ + 422, + 520, + 670, + 564 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 520, + 1028, + 564 + ] + }, + { + "text": " Keep tabs on all", + "boxes": [ + 1172, + 516, + 1462, + 570 + ] + }, + { + "text": "noise", + "boxes": [ + 422, + 568, + 528, + 606 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 568, + 994, + 604 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1180, + 568, + 1354, + 606 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 617, + 710, + 647 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 800, + 617, + 1096, + 647 + ] + }, + { + "text": "Close duplicates, search and pin open ", + "boxes": [ + 1176, + 611, + 1484, + 651 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 422, + 633, + 712, + 667 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 804, + 637, + 1096, + 667 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 657, + 650, + 681 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 651, + 928, + 687 + ] + }, + { + "text": "possibilities for the almost infinite", + "boxes": [ + 1182, + 653, + 1462, + 683 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1180, + 673, + 1424, + 703 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 878, + 725, + 1030, + 754 + ] + }, + { + "text": "Get the browser that helps you", + "boxes": [ + 610, + 991, + 1300, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "e", + "boxes": [ + 64, + 69, + 124, + 87 + ] + }, + { + "text": "目", + "boxes": [ + 1544, + 69, + 1562, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1566, + 64, + 1660, + 89 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 230, + 672, + 284 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 798, + 232, + 1030, + 280 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1176, + 234, + 1462, + 282 + ] + }, + { + "text": "noise", + "boxes": [ + 422, + 282, + 528, + 320 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 282, + 996, + 320 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1180, + 282, + 1354, + 320 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 330, + 712, + 365 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 332, + 1096, + 361 + ] + }, + { + "text": "Close dulicatarchand inn", + "boxes": [ + 1176, + 327, + 1482, + 365 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 351, + 712, + 381 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 804, + 351, + 1096, + 381 + ] + }, + { + "text": "tabs-thereareamostinfinite", + "boxes": [ + 1180, + 347, + 1434, + 383 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 422, + 367, + 650, + 399 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 802, + 367, + 924, + 403 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1182, + 369, + 1460, + 399 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 391, + 1426, + 421 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 443, + 1030, + 472 + ] + }, + { + "text": "Get the browser that helps you", + "boxes": [ + 610, + 705, + 1300, + 756 + ] + }, + { + "text": "get sh*t done", + "boxes": [ + 800, + 756, + 1108, + 814 + ] + }, + { + "text": "Block ad trackers", + "boxes": [ + 994, + 1014, + 1304, + 1050 + ] + } + ] + }, + { + "frame_id": "50_4_9", + "episode_id": 50, + "env_id": 4, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.62407112121582, + "advantages": 0.5702648162841797, + "generated_text": "{\n \"intent\": \"To explore the functionality available under the 'File' menu by interacting with the file management options.\",\n \"action\": \"DoubleClick(171, 80)\"\n}", + "action_string": "DoubleClick(171, 80)", + "intent_string": "To explore the functionality available under the 'File' menu by interacting with the file management options.", + "before_screen_ocr_results": [ + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "FTre", + "boxes": [ + 19, + 56, + 64, + 100 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 98, + 115, + 236, + 149 + ] + }, + { + "text": "Mous", + "boxes": [ + 10, + 171, + 56, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 72, + 183, + 98, + 204 + ] + }, + { + "text": "Recent Files", + "boxes": [ + 104, + 181, + 226, + 212 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Teri", + "boxes": [ + 22, + 284, + 52, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "雪", + "boxes": [ + 70, + 326, + 98, + 353 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 106, + 325, + 274, + 354 + ] + }, + { + "text": "國", + "boxes": [ + 70, + 373, + 98, + 401 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 106, + 369, + 280, + 405 + ] + }, + { + "text": "Calo", + "boxes": [ + 16, + 393, + 50, + 413 + ] + }, + { + "text": "", + "boxes": [ + 68, + 413, + 98, + 449 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 108, + 423, + 310, + 447 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 54, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 98, + 514, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 104, + 558, + 258, + 592 + ] + }, + { + "text": "福", + "boxes": [ + 5, + 578, + 25, + 598 + ] + }, + { + "text": "0", + "boxes": [ + 1862, + 574, + 1920, + 606 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1372, + 699 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "0ot@02b525", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + }, + { + "text": "reoffice", + "boxes": [ + 690, + 1050, + 744, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "口口", + "boxes": [ + 1798, + 6, + 1864, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 20, + 60, + 62, + 95 + ] + }, + { + "text": " Open Nle ", + "boxes": [ + 62, + 62, + 208, + 105 + ] + }, + { + "text": "Open", + "boxes": [ + 940, + 91, + 980, + 107 + ] + }, + { + "text": "□□", + "boxes": [ + 1458, + 85, + 1522, + 111 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 115, + 236, + 151 + ] + }, + { + "text": " Recent", + "boxes": [ + 400, + 115, + 482, + 141 + ] + }, + { + "text": "root", + "boxes": [ + 576, + 115, + 638, + 141 + ] + }, + { + "text": "Name", + "boxes": [ + 554, + 153, + 596, + 173 + ] + }, + { + "text": "| Size | Type Modifed", + "boxes": [ + 1334, + 149, + 1512, + 179 + ] + }, + { + "text": "Mous", + "boxes": [ + 8, + 171, + 56, + 199 + ] + }, + { + "text": "3", + "boxes": [ + 72, + 183, + 96, + 204 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 104, + 183, + 228, + 212 + ] + }, + { + "text": " Desktop", + "boxes": [ + 404, + 181, + 490, + 206 + ] + }, + { + "text": "Desktop", + "boxes": [ + 550, + 175, + 636, + 199 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 177, + 1492, + 197 + ] + }, + { + "text": "Downloads", + "boxes": [ + 552, + 197, + 650, + 220 + ] + }, + { + "text": "7 May", + "boxes": [ + 1442, + 197, + 1486, + 222 + ] + }, + { + "text": " Trash", + "boxes": [ + 406, + 214, + 466, + 234 + ] + }, + { + "text": "I clipboard_app", + "boxes": [ + 552, + 216, + 672, + 244 + ] + }, + { + "text": "7 Feb 2024", + "boxes": [ + 1442, + 218, + 1518, + 242 + ] + }, + { + "text": " Templates", + "boxes": [ + 102, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 106, + 326, + 278, + 355 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": " Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "回", + "boxes": [ + 66, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 103, + 462, + 251, + 501 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 64, + 508, + 104, + 548 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 98, + 514, + 242, + 544 + ] + }, + { + "text": " Base Database", + "boxes": [ + 102, + 558, + 258, + 594 + ] + }, + { + "text": "3", + "boxes": [ + 2, + 576, + 28, + 600 + ] + }, + { + "text": "J00", + "boxes": [ + 1862, + 572, + 1920, + 606 + ] + }, + { + "text": "1open", + "boxes": [ + 1444, + 951, + 1494, + 971 + ] + }, + { + "text": "Help ", + "boxes": [ + 108, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 222, + 985, + 292, + 1003 + ] + }, + { + "text": "oot@02b525...", + "boxes": [ + 374, + 1050, + 476, + 1068 + ] + }, + { + "text": "root@02b525.", + "boxes": [ + 482, + 1048, + 620, + 1072 + ] + }, + { + "text": "reoffice", + "boxes": [ + 688, + 1050, + 740, + 1070 + ] + } + ] + }, + { + "frame_id": "50_5_9", + "episode_id": 50, + "env_id": 5, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.565585136413574, + "advantages": 0.5414692163467407, + "generated_text": "{\n \"intent\": \"Double-click one of the categories or deals to explore more products.\",\n \"action\": \"DoubleClick(1478, 276)\"\n}", + "action_string": "DoubleClick(1478, 276)", + "intent_string": "Double-click one of the categories or deals to explore more products.", + "before_screen_ocr_results": [ + { + "text": "Gamingaccessories", + "boxes": [ + 246, + 191, + 434, + 208 + ] + }, + { + "text": "Shopfor your home essentials", + "boxes": [ + 614, + 187, + 916, + 210 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 982, + 185, + 1202, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 185, + 1534, + 214 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 544, + 1058, + 568 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1349, + 541, + 1437, + 572 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 621, + 1510, + 657 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 653, + 354, + 683 + ] + }, + { + "text": "Explore all products in Kitchel", + "boxes": [ + 246, + 987, + 418, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Shopnow", + "boxes": [ + 1354, + 987, + 1412, + 1007 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "<", + "boxes": [ + 6, + 60, + 41, + 94 + ] + }, + { + "text": "→×", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "", + "boxes": [ + 251, + 59, + 311, + 96 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "gaming", + "boxes": [ + 294, + 113, + 364, + 147 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Explore Related Products", + "boxes": [ + 10, + 256, + 184, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "gaming headset ", + "boxes": [ + 10, + 280, + 122, + 310 + ] + }, + { + "text": "gaming keyboard", + "boxes": [ + 12, + 304, + 126, + 328 + ] + }, + { + "text": "gaming pc", + "boxes": [ + 12, + 324, + 86, + 349 + ] + }, + { + "text": "gaming mouse", + "boxes": [ + 14, + 343, + 110, + 367 + ] + }, + { + "text": "See", + "boxes": [ + 20, + 367, + 50, + 381 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 409, + 50, + 435 + ] + }, + { + "text": "$0 - $8,900+", + "boxes": [ + 10, + 435, + 100, + 459 + ] + }, + { + "text": "120", + "boxes": [ + 480, + 478, + 532, + 498 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 10, + 514, + 136, + 544 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 540, + 98, + 564 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 560, + 100, + 586 + ] + }, + { + "text": "red 6", + "boxes": [ + 352, + 596, + 384, + 609 + ] + }, + { + "text": "ed6", + "boxes": [ + 944, + 596, + 970, + 609 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 617, + 144, + 647 + ] + }, + { + "text": "16\" 120Hz Portable Gaming", + "boxes": [ + 314, + 613, + 526, + 641 + ] + }, + { + "text": "Amazon Basics IPS Monitor 75 Hz ", + "boxes": [ + 610, + 617, + 858, + 641 + ] + }, + { + "text": "Magnetic Dart Board - 12pcs ", + "boxes": [ + 904, + 617, + 1118, + 641 + ] + }, + { + "text": "ASUS ROG Strix G16 Gaming ", + "boxes": [ + 1488, + 613, + 1708, + 643 + ] + }, + { + "text": "Monitor, 100% sRGB 2.5K QHD", + "boxes": [ + 312, + 637, + 546, + 667 + ] + }, + { + "text": "Powered with AOC Technology FHD", + "boxes": [ + 606, + 639, + 872, + 669 + ] + }, + { + "text": "Magnetic Darts - Excellent Indoor ", + "boxes": [ + 900, + 639, + 1152, + 669 + ] + }, + { + "text": "Nex Playground - The Active Play", + "boxes": [ + 1192, + 639, + 1442, + 669 + ] + }, + { + "text": "Laptop, 165Hz Display, NVIDIA@", + "boxes": [ + 1486, + 637, + 1726, + 667 + ] + }, + { + "text": "Department", + "boxes": [ + 12, + 655, + 98, + 681 + ] + }, + { + "text": "400nits Portable Monitor with HDR,", + "boxes": [ + 314, + 663, + 578, + 687 + ] + }, + { + "text": "1080P HDMI, Display Port and VGA", + "boxes": [ + 608, + 663, + 872, + 693 + ] + }, + { + "text": "Game and Party Games - Gifts for 5", + "boxes": [ + 904, + 665, + 1166, + 689 + ] + }, + { + "text": " System for Kids & Families Where", + "boxes": [ + 1192, + 663, + 1446, + 693 + ] + }, + { + "text": "GeForce RTXTM 4060,Intel Core", + "boxes": [ + 1488, + 661, + 1724, + 691 + ] + }, + { + "text": "Computers & Accssories", + "boxes": [ + 10, + 677, + 176, + 707 + ] + }, + { + "text": "Ultra Slim, Eye Care, External..", + "boxes": [ + 314, + 689, + 540, + 713 + ] + }, + { + "text": "Input VESA Compatible Built-in...", + "boxes": [ + 608, + 689, + 854, + 713 + ] + }, + { + "text": "6 7 8 9 10 11 12 Year Old Boy Kids", + "boxes": [ + 902, + 689, + 1160, + 713 + ] + }, + { + "text": "Indoor Physical Activity Meets", + "boxes": [ + 1194, + 687, + 1422, + 717 + ] + }, + { + "text": "i7-13650HX, 16GB DDR5, 1TB PCI..", + "boxes": [ + 1486, + 685, + 1756, + 715 + ] + }, + { + "text": " Computers & Tablets", + "boxes": [ + 18, + 699, + 164, + 723 + ] + }, + { + "text": "PC Games & Accessories", + "boxes": [ + 12, + 719, + 168, + 742 + ] + }, + { + "text": "★★★★☆", + "boxes": [ + 318, + 715, + 416, + 739 + ] + }, + { + "text": "Options: 2 sizes ", + "boxes": [ + 610, + 717, + 700, + 741 + ] + }, + { + "text": "★★★★5,961", + "boxes": [ + 902, + 713, + 1044, + 742 + ] + }, + { + "text": "Interactive Family Fun and is Great.", + "boxes": [ + 1194, + 711, + 1456, + 741 + ] + }, + { + "text": "★★★★☆1,895", + "boxes": [ + 1490, + 715, + 1630, + 739 + ] + }, + { + "text": "★★★★★1.028", + "boxes": [ + 608, + 739, + 750, + 768 + ] + }, + { + "text": " 2K+ bought in past month", + "boxes": [ + 900, + 732, + 1074, + 760 + ] + }, + { + "text": "$14999", + "boxes": [ + 309, + 742, + 395, + 783 + ] + }, + { + "text": "Apr 23, 2024", + "boxes": [ + 1196, + 735, + 1286, + 760 + ] + }, + { + "text": "3K+ bought in past month", + "boxes": [ + 1490, + 735, + 1658, + 758 + ] + }, + { + "text": "Brands", + "boxes": [ + 8, + 748, + 68, + 780 + ] + }, + { + "text": "ESRB Rating: Everyone", + "boxes": [ + 1196, + 752, + 1342, + 780 + ] + }, + { + "text": "ASUS", + "boxes": [ + 5, + 766, + 79, + 803 + ] + }, + { + "text": "$2279 List: 29.99", + "boxes": [ + 897, + 762, + 1043, + 807 + ] + }, + { + "text": "$1,24478", + "boxes": [ + 1484, + 762, + 1594, + 800 + ] + }, + { + "text": "Save $20.00 with coupon", + "boxes": [ + 316, + 776, + 482, + 806 + ] + }, + { + "text": "★★★★☆1.459", + "boxes": [ + 1194, + 776, + 1338, + 806 + ] + }, + { + "text": " VISOONE", + "boxes": [ + 10, + 794, + 100, + 818 + ] + }, + { + "text": "$9999", + "boxes": [ + 606, + 794, + 672, + 826 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 898, + 798, + 1044, + 834 + ] + }, + { + "text": " 8K+ bought in past month", + "boxes": [ + 1192, + 792, + 1370, + 826 + ] + }, + { + "text": " Delivery Tue, May 20 ", + "boxes": [ + 312, + 802, + 456, + 832 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": " Logitech G", + "boxes": [ + 10, + 812, + 108, + 844 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 314, + 826, + 406, + 850 + ] + }, + { + "text": "Amazor", + "boxes": [ + 34, + 836, + 84, + 854 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 604, + 828, + 752, + 858 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 906, + 828, + 992, + 846 + ] + }, + { + "text": "Androic", + "boxes": [ + 1200, + 830, + 1248, + 848 + ] + }, + { + "text": "Ships to Cana", + "boxes": [ + 1492, + 828, + 1566, + 846 + ] + }, + { + "text": "=", + "boxes": [ + 14, + 838, + 38, + 852 + ] + }, + { + "text": "$199% Lst: +249.00", + "boxes": [ + 1189, + 845, + 1361, + 890 + ] + }, + { + "text": "F", + "boxes": [ + 16, + 858, + 38, + 872 + ] + }, + { + "text": "Add to car", + "boxes": [ + 328, + 860, + 388, + 878 + ] + }, + { + "text": "Ships to Cana", + "boxes": [ + 612, + 856, + 682, + 874 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": " FIFINE", + "boxes": [ + 9, + 871, + 75, + 896 + ] + }, + { + "text": "Ad to cart", + "boxes": [ + 612, + 881, + 698, + 913 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 1196, + 889, + 1334, + 913 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1486, + 883, + 1630, + 913 + ] + }, + { + "text": "Redrago", + "boxes": [ + 9, + 893, + 85, + 920 + ] + }, + { + "text": "$1,219.99 (6 used & new offers)", + "boxes": [ + 1488, + 907, + 1698, + 931 + ] + }, + { + "text": "Condition", + "boxes": [ + 14, + 951, + 78, + 969 + ] + }, + { + "text": "New", + "boxes": [ + 11, + 968, + 67, + 996 + ] + }, + { + "text": "$10", + "boxes": [ + 1370, + 979, + 1412, + 1007 + ] + }, + { + "text": "Used", + "boxes": [ + 16, + 993, + 68, + 1012 + ] + }, + { + "text": "RDBLOX", + "boxes": [ + 1260, + 1014, + 1400, + 1046 + ] + } + ] + }, + { + "frame_id": "50_6_9", + "episode_id": 50, + "env_id": 6, + "step": 9, + "format_rewards": 1.0, + "rewards": 8.744401931762695, + "advantages": -0.3551918864250183, + "generated_text": "{\n \"intent\": \"The goal here is to explore the environment by interacting with elements visible on the desktop. Since there are no desktop icons immediately visible in the current application window, scrolling or clicking might help reveal more options or icons further down the desktop. Scrolling up can show more of the hidden desktop area.\",\n \"action\": \"ScrollUp(960, 540)\"\n}", + "action_string": "ScrollUp(960, 540)", + "intent_string": "The goal here is to explore the environment by interacting with elements visible on the desktop. Since there are no desktop icons immediately visible in the current application window, scrolling or clicking might help reveal more options or icons further down the desktop. Scrolling up can show more of the hidden desktop area.", + "before_screen_ocr_results": [ + { + "text": "Untitled2-LibreOfficeCalc", + "boxes": [ + 846, + 4, + 1074, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "--目-目@%电自-", + "boxes": [ + 14, + 54, + 374, + 83 + ] + }, + { + "text": "上&", + "boxes": [ + 373, + 53, + 443, + 84 + ] + }, + { + "text": "-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberation San", + "boxes": [ + 14, + 95, + 150, + 119 + ] + }, + { + "text": "10→", + "boxes": [ + 140, + 95, + 230, + 119 + ] + }, + { + "text": "Jaaa.", + "boxes": [ + 218, + 93, + 352, + 123 + ] + }, + { + "text": "", + "boxes": [ + 336, + 93, + 590, + 123 + ] + }, + { + "text": "国", + "boxes": [ + 580, + 99, + 616, + 113 + ] + }, + { + "text": "□日日-%00+三至田二-口-目", + "boxes": [ + 606, + 93, + 1204, + 123 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "▼", + "boxes": [ + 132, + 133, + 154, + 147 + ] + }, + { + "text": "、=", + "boxes": [ + 152, + 129, + 256, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "+5", + "boxes": [ + 16, + 927, + 28, + 939 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 464, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "--目-X电自- ", + "boxes": [ + 8, + 50, + 376, + 83 + ] + }, + { + "text": "上&", + "boxes": [ + 375, + 53, + 443, + 84 + ] + }, + { + "text": "勺·、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberation San", + "boxes": [ + 14, + 95, + 150, + 119 + ] + }, + { + "text": "10→", + "boxes": [ + 142, + 95, + 230, + 119 + ] + }, + { + "text": "aaa.", + "boxes": [ + 218, + 93, + 350, + 123 + ] + }, + { + "text": "a·旦·", + "boxes": [ + 340, + 93, + 558, + 123 + ] + }, + { + "text": "", + "boxes": [ + 550, + 101, + 576, + 115 + ] + }, + { + "text": "回日日-%00+正至田二-口-目", + "boxes": [ + 614, + 93, + 1202, + 123 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "▼", + "boxes": [ + 132, + 133, + 154, + 147 + ] + }, + { + "text": "=", + "boxes": [ + 150, + 129, + 258, + 157 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "+5", + "boxes": [ + 16, + 927, + 28, + 939 + ] + } + ] + }, + { + "frame_id": "50_7_9", + "episode_id": 50, + "env_id": 7, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.490511894226074, + "advantages": 0.012155982665717602, + "generated_text": "{\n \"intent\": \"To explore more content or open potential links by interacting with the page elements.\",\n \"action\": \"ScrollDown(1000, 540)\"\n}", + "action_string": "ScrollDown(1000, 540)", + "intent_string": "To explore more content or open potential links by interacting with the page elements.", + "before_screen_ocr_results": [ + { + "text": "×", + "boxes": [ + 196, + 36, + 218, + 50 + ] + }, + { + "text": "·和平精英：3个提升", + "boxes": [ + 244, + 32, + 374, + 50 + ] + }, + { + "text": "Q白", + "boxes": [ + 64, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": "?", + "boxes": [ + 1850, + 69, + 1868, + 83 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 572, + 123, + 632, + 143 + ] + }, + { + "text": "中", + "boxes": [ + 1757, + 116, + 1793, + 150 + ] + }, + { + "text": "msn", + "boxes": [ + 60, + 129, + 102, + 149 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "关注", + "boxes": [ + 280, + 179, + 314, + 201 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1602, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "嘎嘎脆", + "boxes": [ + 1591, + 239, + 1689, + 287 + ] + }, + { + "text": "游戏天地十关注", + "boxes": [ + 386, + 264, + 576, + 306 + ] + }, + { + "text": "42.2K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "甜甜香", + "boxes": [ + 1618, + 274, + 1688, + 308 + ] + }, + { + "text": "想吃坚果蜜饯的", + "boxes": [ + 1424, + 292, + 1542, + 322 + ] + }, + { + "text": "点击领取", + "boxes": [ + 1454, + 336, + 1532, + 367 + ] + }, + { + "text": "和平精英：3个提升听声辨位能力的方法，快试试！", + "boxes": [ + 366, + 349, + 1104, + 391 + ] + }, + { + "text": "F", + "boxes": [ + 192, + 395, + 214, + 421 + ] + }, + { + "text": "闭眼冲", + "boxes": [ + 1618, + 391, + 1676, + 409 + ] + }, + { + "text": "游戏天地的报道·5天·①3分钟读取", + "boxes": [ + 364, + 413, + 666, + 449 + ] + }, + { + "text": "想吃坚果蜜饯的，来淘宝就GO", + "boxes": [ + 1422, + 411, + 1678, + 447 + ] + }, + { + "text": "了！", + "boxes": [ + 1428, + 443, + 1462, + 465 + ] + }, + { + "text": "广告淘宝网", + "boxes": [ + 1426, + 469, + 1508, + 492 + ] + }, + { + "text": "嘿，各位和平精英的玩家们！最近在游戏里是不是经常被敌人偷袭，却还搞不清楚人家从哪儿冒出来的？就像上次我", + "boxes": [ + 364, + 506, + 1238, + 536 + ] + }, + { + "text": "玩的时候，正美滋滋地搜着物资呢，突然就被人给阴了，连枪声方向都没搞明白，别提多郁闷了。其实啊，这听声辨", + "boxes": [ + 366, + 530, + 1236, + 560 + ] + }, + { + "text": "位的能力要是能提升上去，在游戏里那可就能抢占不少先机啦！今天就给大家分享3个提升听声辨位能力的方法，快", + "boxes": [ + 364, + 556, + 1230, + 586 + ] + }, + { + "text": "访问 游戏天地", + "boxes": [ + 1430, + 554, + 1552, + 584 + ] + }, + { + "text": "L", + "boxes": [ + 1680, + 558, + 1696, + 572 + ] + }, + { + "text": "试试哦！", + "boxes": [ + 366, + 586, + 432, + 611 + ] + }, + { + "text": "DNF手游确定无形自选和白色天三", + "boxes": [ + 1446, + 602, + 1664, + 631 + ] + }, + { + "text": "投票，别错过500万泰拉", + "boxes": [ + 1446, + 621, + 1604, + 651 + ] + }, + { + "text": "调大音量不一定就能听准位置", + "boxes": [ + 364, + 659, + 746, + 695 + ] + }, + { + "text": "梦幻西游夜叉改动！七八十个全是", + "boxes": [ + 1446, + 665, + 1664, + 695 + ] + }, + { + "text": "画魂，夜叉队准备转型", + "boxes": [ + 1446, + 685, + 1594, + 715 + ] + }, + { + "text": "梦幻西游：不战斗刷完慈心渡鬼，", + "boxes": [ + 1446, + 729, + 1656, + 758 + ] + }, + { + "text": "附任务领取坐标图", + "boxes": [ + 1446, + 748, + 1566, + 778 + ] + }, + { + "text": "淘", + "boxes": [ + 1422, + 836, + 1448, + 850 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 834, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 794, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置\"查看您的选项以及不", + "boxes": [ + 32, + 951, + 216, + 975 + ] + }, + { + "text": "在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的\"管理设置\"改变您的选择。欲了解", + "boxes": [ + 206, + 951, + 794, + 975 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "平精英：3个", + "boxes": [ + 224, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 74, + 68, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四@", + "boxes": [ + 1784, + 62, + 1880, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 566, + 121, + 636, + 147 + ] + }, + { + "text": "长春市6°c", + "boxes": [ + 1630, + 119, + 1744, + 149 + ] + }, + { + "text": "中", + "boxes": [ + 1762, + 121, + 1788, + 145 + ] + }, + { + "text": "登录", + "boxes": [ + 1830, + 123, + 1858, + 145 + ] + }, + { + "text": "嘿，各位和平精英的玩家们！最近在游戏里是不是经常被敌人偷袭，却还搞不清楚人家从哪儿冒出来的？就像上次我", + "boxes": [ + 362, + 171, + 1240, + 206 + ] + }, + { + "text": "嘎嘎脆", + "boxes": [ + 1589, + 179, + 1693, + 232 + ] + }, + { + "text": "玩的时候，正美滋滋地搜着物资呢，突然就被人给阴了，连枪声方向都没搞明白，别提多郁闷了。其实啊，这听声辨", + "boxes": [ + 366, + 199, + 1236, + 228 + ] + }, + { + "text": "甜甜香", + "boxes": [ + 1610, + 210, + 1692, + 256 + ] + }, + { + "text": "位的能力要是能提升上去，在游戏里那可就能抢占不少先机啦！今天就给大家分享3个提升听声辨位能力的方法，快", + "boxes": [ + 366, + 228, + 1228, + 252 + ] + }, + { + "text": "想吃坚果蜜饯的", + "boxes": [ + 1424, + 236, + 1544, + 266 + ] + }, + { + "text": "试试哦！", + "boxes": [ + 366, + 254, + 432, + 280 + ] + }, + { + "text": "点击领取", + "boxes": [ + 1456, + 280, + 1534, + 306 + ] + }, + { + "text": "Ke", + "boxes": [ + 1496, + 310, + 1540, + 330 + ] + }, + { + "text": "调大音量不一定就能听准位置", + "boxes": [ + 364, + 328, + 746, + 363 + ] + }, + { + "text": "闭眼冲", + "boxes": [ + 1610, + 336, + 1676, + 353 + ] + }, + { + "text": "想吃坚果蜜饯的，来淘宝就GO", + "boxes": [ + 1424, + 355, + 1676, + 385 + ] + }, + { + "text": "P", + "boxes": [ + 192, + 373, + 214, + 397 + ] + }, + { + "text": "了！", + "boxes": [ + 1426, + 381, + 1466, + 409 + ] + }, + { + "text": "广告淘宝网", + "boxes": [ + 1428, + 411, + 1506, + 437 + ] + }, + { + "text": "?", + "boxes": [ + 194, + 433, + 210, + 447 + ] + }, + { + "text": "访问游戏天地", + "boxes": [ + 1428, + 496, + 1552, + 526 + ] + }, + { + "text": "DNF手游确定无形自选和白色天三", + "boxes": [ + 1444, + 544, + 1664, + 574 + ] + }, + { + "text": "投票，别错过500万泰拉", + "boxes": [ + 1446, + 564, + 1606, + 594 + ] + }, + { + "text": "梦幻西游夜叉改动！七八十个全是", + "boxes": [ + 1446, + 608, + 1664, + 637 + ] + }, + { + "text": "画魂，夜叉队准备转型", + "boxes": [ + 1446, + 627, + 1594, + 657 + ] + }, + { + "text": "梦幻西游：不战斗刷完慈心渡鬼，", + "boxes": [ + 1446, + 671, + 1654, + 701 + ] + }, + { + "text": "附任务领取坐标图", + "boxes": [ + 1446, + 691, + 1566, + 721 + ] + }, + { + "text": "正在检查更新", + "boxes": [ + 526, + 715, + 608, + 741 + ] + }, + { + "text": "#", + "boxes": [ + 432, + 762, + 468, + 790 + ] + }, + { + "text": "机械五金无门槛", + "boxes": [ + 1462, + 790, + 1664, + 828 + ] + }, + { + "text": "和平精英：3个提升听声辨位能力的方法，快试试！", + "boxes": [ + 382, + 816, + 660, + 846 + ] + }, + { + "text": "单件直享批发价", + "boxes": [ + 1480, + 836, + 1648, + 860 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 28, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 834, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1670, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 794, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击\"管理设置\"查看您的选项以及不", + "boxes": [ + 30, + 951, + 216, + 975 + ] + }, + { + "text": "在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的\"管理设置\"改变您的选择。欲了解", + "boxes": [ + 206, + 951, + 794, + 975 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "平精英：3个", + "boxes": [ + 224, + 1048, + 310, + 1072 + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_50/scores_v3.json b/ScreenExplorer-7B-E1/episode_50/scores_v3.json new file mode 100644 index 0000000000000000000000000000000000000000..10d2ce31366c31bb16dcde3b8e049e31c08fd4f6 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_50/scores_v3.json @@ -0,0 +1,19 @@ +{ + "directory_path": "logs/ScreenExplorer_2025-05-17_22-11-27ab2072/episode_50", + "single_trajectory_image_diversity": 0.3832573880208656, + "single_trajectory_text_diversity": 0.25408529583364725, + "single_trajectory_image_diversity_max": 0.81103515625, + "single_trajectory_image_diversity_min": 0.00341796875, + "single_trajectory_image_diversity_std": 0.2801513671875, + "single_trajectory_text_diversity_max": 0.45086669921875, + "single_trajectory_text_diversity_min": 0.02130126953125, + "single_trajectory_text_diversity_std": 0.13287353515625, + "group_trajectories_image_diversity": 0.6682555675506592, + "group_trajectories_text_diversity": 0.4170882999897003, + "group_trajectories_image_diversity_max": 1.2890625, + "group_trajectories_image_diversity_min": -0.0078125, + "group_trajectories_image_diversity_std": 0.3671875, + "group_trajectories_text_diversity_max": 0.6328125, + "group_trajectories_text_diversity_min": -0.0009765625, + "group_trajectories_text_diversity_std": 0.1280517578125 +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_60/game_record_20250518_045841.json b/ScreenExplorer-7B-E1/episode_60/game_record_20250518_045841.json new file mode 100644 index 0000000000000000000000000000000000000000..9d5d4fb867dd8c32e579096bf0c65ada5cd4da89 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_60/game_record_20250518_045841.json @@ -0,0 +1,47061 @@ +{ + "start_time": "2025-05-18T04:58:41.955561", + "end_time": "2025-05-18T05:05:55.782083", + "buffer_size": 80, + "all_records": [ + { + "frame_id": "60_0_0", + "episode_id": 60, + "env_id": 0, + "step": 0, + "format_rewards": 1.0, + "rewards": 10.278768539428711, + "advantages": 0.4448094367980957, + "generated_text": "{\n \"intent\": \"To begin exploring the applications available on the desktop by opening one of the icons.\",\n \"action\": \"DoubleClick(73, 484)\"\n}", + "action_string": "DoubleClick(73, 484)", + "intent_string": "To begin exploring the applications available on the desktop by opening one of the icons.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 26, + 64, + 60, + 93 + ] + }, + { + "text": " open File", + "boxes": [ + 60, + 62, + 208, + 105 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 92, + 115, + 236, + 149 + ] + }, + { + "text": "Mous", + "boxes": [ + 10, + 171, + 56, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 66, + 173, + 104, + 210 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 102, + 181, + 226, + 212 + ] + }, + { + "text": " Templates ", + "boxes": [ + 92, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 63, + 279, + 141, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 102, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 403 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 389, + 54, + 417 + ] + }, + { + "text": "", + "boxes": [ + 64, + 411, + 102, + 451 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 419, + 314, + 449 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 102, + 462, + 252, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "圆", + "boxes": [ + 64, + 508, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 96, + 508, + 244, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 32, + 609, + 56, + 631 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1374, + 699 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "Extensions", + "boxes": [ + 220, + 981, + 294, + 1005 + ] + }, + { + "text": "breoffice", + "boxes": [ + 226, + 1050, + 288, + 1068 + ] + } + ] + }, + { + "frame_id": "60_1_0", + "episode_id": 60, + "env_id": 1, + "step": 0, + "format_rewards": 1.0, + "rewards": 9.753464698791504, + "advantages": 0.05807042494416237, + "generated_text": "{\n \"intent\": \"To start interacting with one of the available applications on the desktop, which could provide insights or lead to further exploration.\",\n \"action\": \"DoubleClick(63, 371)\"\n}", + "action_string": "DoubleClick(63, 371)", + "intent_string": "To start interacting with one of the available applications on the desktop, which could provide insights or lead to further exploration.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "", + "boxes": [ + 42, + 131, + 72, + 153 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "酒", + "boxes": [ + 24, + 334, + 90, + 385 + ] + }, + { + "text": "galculator", + "boxes": [ + 906, + 355, + 1032, + 379 + ] + }, + { + "text": "□□", + "boxes": [ + 1028, + 353, + 1108, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 4, + 385, + 118, + 423 + ] + }, + { + "text": "File", + "boxes": [ + 820, + 381, + 854, + 401 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 848, + 379, + 1064, + 403 + ] + }, + { + "text": "O", + "boxes": [ + 1060, + 416, + 1095, + 451 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 18, + 502, + 96, + 528 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "國", + "boxes": [ + 34, + 675, + 82, + 721 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 86, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "galculator", + "boxes": [ + 216, + 1050, + 286, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_0", + "episode_id": 60, + "env_id": 2, + "step": 0, + "format_rewards": 1.0, + "rewards": 9.512029647827148, + "advantages": -0.1196788027882576, + "generated_text": "{\n \"intent\": \"The action `DoubleClick(123, 456)` will allow us to open an application, which aligns with the goal of exploring various programs on the desktop.\",\n \"action\": \"DoubleClick(123, 456)\"\n}", + "action_string": "DoubleClick(123, 456)", + "intent_string": "The action `DoubleClick(123, 456)` will allow us to open an application, which aligns with the goal of exploring various programs on the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 78, + 264 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 114, + 421 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 16, + 498, + 96, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "60_3_0", + "episode_id": 60, + "env_id": 3, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.670243263244629, + "advantages": 1.469240427017212, + "generated_text": "{\n \"intent\": \"To start the exploration by interacting with one of the applications available on the desktop.\",\n \"action\": \"DoubleClick(50, 77)\"\n}", + "action_string": "DoubleClick(50, 77)", + "intent_string": "To start the exploration by interacting with one of the applications available on the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "√C", + "boxes": [ + 42, + 66, + 130, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 524, + 488, + 562, + 528 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 988, + 671, + 1172, + 719 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_4_0", + "episode_id": 60, + "env_id": 4, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.300804138183594, + "advantages": 1.1972521543502808, + "generated_text": "{\n \"intent\": \"To explore the desktop further by opening an application. Clicking on Firefox will allow me to browse the internet.\",\n \"action\": \"DoubleClick(51, 47)\"\n}", + "action_string": "DoubleClick(51, 47)", + "intent_string": "To explore the desktop further by opening an application. Clicking on Firefox will allow me to browse the internet.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_5_0", + "episode_id": 60, + "env_id": 5, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.241949081420898, + "advantages": 1.1539218425750732, + "generated_text": "{\n \"intent\": \"To start exploring further by selecting an application, I choose to open the Firefox browser from the icons on the desktop.\",\n \"action\": \"DoubleClick(56, 49)\"\n}", + "action_string": "DoubleClick(56, 49)", + "intent_string": "To start exploring further by selecting an application, I choose to open the Firefox browser from the icons on the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_6_0", + "episode_id": 60, + "env_id": 6, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.391969680786133, + "advantages": 1.2643699645996094, + "generated_text": "{\n \"intent\": \"Opening Firefox browser will allow me to search for more information or perform various tasks online.\",\n \"action\": \"DoubleClick(56, 47)\"\n}", + "action_string": "DoubleClick(56, 47)", + "intent_string": "Opening Firefox browser will allow me to search for more information or perform various tasks online.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_7_0", + "episode_id": 60, + "env_id": 7, + "step": 0, + "format_rewards": 1.0, + "rewards": 10.418322563171387, + "advantages": 0.5475518107414246, + "generated_text": "{\n \"intent\": \"To start exploring by focusing on one of the applications available on the desktop.\",\n \"action\": \"DoubleClick(73, 474)\"\n}", + "action_string": "DoubleClick(73, 474)", + "intent_string": "To start exploring by focusing on one of the applications available on the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 26, + 66, + 60, + 93 + ] + }, + { + "text": " open File", + "boxes": [ + 60, + 62, + 208, + 105 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 92, + 115, + 236, + 149 + ] + }, + { + "text": "Mous", + "boxes": [ + 10, + 171, + 56, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 66, + 173, + 104, + 210 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 102, + 181, + 226, + 212 + ] + }, + { + "text": " Templates ", + "boxes": [ + 92, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 63, + 279, + 141, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 102, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 403 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 389, + 54, + 415 + ] + }, + { + "text": "宣", + "boxes": [ + 68, + 413, + 100, + 449 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 104, + 415, + 314, + 453 + ] + }, + { + "text": "D", + "boxes": [ + 64, + 459, + 102, + 500 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 102, + 462, + 252, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": "Math Formula", + "boxes": [ + 96, + 510, + 242, + 546 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 32, + 609, + 56, + 631 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1374, + 699 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "Extensions", + "boxes": [ + 220, + 981, + 294, + 1005 + ] + }, + { + "text": "breoffice", + "boxes": [ + 226, + 1050, + 286, + 1068 + ] + } + ] + }, + { + "frame_id": "60_0_1", + "episode_id": 60, + "env_id": 0, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.667791366577148, + "advantages": -0.005003964528441429, + "generated_text": "{\n \"intent\": \"To interact with the interface more directly and possibly find hidden files or settings, I will right-click near the top left corner of the application window.\",\n \"action\": \"RightClick(10, 10)\"\n}", + "action_string": "RightClick(10, 10)", + "intent_string": "To interact with the interface more directly and possibly find hidden files or settings, I will right-click near the top left corner of the application window.", + "before_screen_ocr_results": [ + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 26, + 64, + 60, + 93 + ] + }, + { + "text": " open File", + "boxes": [ + 60, + 62, + 208, + 105 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 92, + 115, + 236, + 149 + ] + }, + { + "text": "Mous", + "boxes": [ + 10, + 171, + 56, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 66, + 173, + 104, + 210 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 102, + 181, + 226, + 212 + ] + }, + { + "text": " Templates ", + "boxes": [ + 92, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 63, + 279, + 141, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 102, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 403 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 389, + 54, + 417 + ] + }, + { + "text": "", + "boxes": [ + 64, + 411, + 102, + 451 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 419, + 314, + 449 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 102, + 462, + 252, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "圆", + "boxes": [ + 64, + 508, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 96, + 508, + 244, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 32, + 609, + 56, + 631 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1374, + 699 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "Extensions", + "boxes": [ + 220, + 981, + 294, + 1005 + ] + }, + { + "text": "breoffice", + "boxes": [ + 226, + 1050, + 288, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Folder", + "boxes": [ + 217, + 7, + 295, + 38 + ] + }, + { + "text": "□0", + "boxes": [ + 1798, + 6, + 1862, + 38 + ] + }, + { + "text": "ate Nel", + "boxes": [ + 58, + 16, + 110, + 34 + ] + }, + { + "text": " Paste", + "boxes": [ + 12, + 56, + 74, + 81 + ] + }, + { + "text": " Empty File ", + "boxes": [ + 242, + 56, + 324, + 81 + ] + }, + { + "text": " Select All", + "boxes": [ + 14, + 99, + 106, + 125 + ] + }, + { + "text": " Invert Selection", + "boxes": [ + 32, + 127, + 150, + 151 + ] + }, + { + "text": "Sort Files", + "boxes": [ + 32, + 169, + 104, + 195 + ] + }, + { + "text": " Desktop Preferences", + "boxes": [ + 12, + 208, + 192, + 238 + ] + }, + { + "text": "", + "boxes": [ + 56, + 236, + 110, + 256 + ] + }, + { + "text": "Templates", + "boxes": [ + 100, + 234, + 210, + 258 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 56, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 66, + 284, + 140, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 276, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 98, + 403 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 106, + 369, + 280, + 405 + ] + }, + { + "text": "回", + "boxes": [ + 64, + 409, + 104, + 455 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 108, + 423, + 310, + 447 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 496 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 514, + 240, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 558, + 254, + 590 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1372, + 699 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "Extensions", + "boxes": [ + 222, + 985, + 290, + 1003 + ] + } + ] + }, + { + "frame_id": "60_1_1", + "episode_id": 60, + "env_id": 1, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.768086433410645, + "advantages": 0.06883523613214493, + "generated_text": "{\n \"intent\": \"To continue exploring by accessing and interacting with applications present on the desktop.\",\n \"action\": \"DoubleClick(57, 368)\"\n}", + "action_string": "DoubleClick(57, 368)", + "intent_string": "To continue exploring by accessing and interacting with applications present on the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "", + "boxes": [ + 42, + 131, + 72, + 153 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 8, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "酒", + "boxes": [ + 24, + 334, + 90, + 385 + ] + }, + { + "text": "galculator", + "boxes": [ + 906, + 355, + 1032, + 379 + ] + }, + { + "text": "□□", + "boxes": [ + 1028, + 353, + 1108, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 4, + 385, + 118, + 423 + ] + }, + { + "text": "File", + "boxes": [ + 820, + 381, + 854, + 401 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 848, + 379, + 1064, + 403 + ] + }, + { + "text": "O", + "boxes": [ + 1060, + 416, + 1095, + 451 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 18, + 502, + 96, + 528 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "國", + "boxes": [ + 34, + 675, + 82, + 721 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 86, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "galculator", + "boxes": [ + 216, + 1050, + 286, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "jalculatol", + "boxes": [ + 370, + 12, + 444, + 30 + ] + }, + { + "text": "日回×", + "boxes": [ + 490, + 12, + 548, + 30 + ] + }, + { + "text": "File Edit View", + "boxes": [ + 266, + 38, + 380, + 56 + ] + }, + { + "text": "Calculator Help", + "boxes": [ + 392, + 38, + 506, + 56 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "?", + "boxes": [ + 514, + 81, + 530, + 97 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 102, + 308 + ] + }, + { + "text": "", + "boxes": [ + 20, + 334, + 89, + 395 + ] + }, + { + "text": "国", + "boxes": [ + 818, + 359, + 834, + 375 + ] + }, + { + "text": "galculator", + "boxes": [ + 910, + 351, + 1014, + 379 + ] + }, + { + "text": "口", + "boxes": [ + 1036, + 353, + 1106, + 379 + ] + }, + { + "text": "File", + "boxes": [ + 822, + 381, + 852, + 401 + ] + }, + { + "text": "Edit View", + "boxes": [ + 852, + 379, + 946, + 403 + ] + }, + { + "text": "Calculator Help", + "boxes": [ + 934, + 379, + 1062, + 403 + ] + }, + { + "text": "Calculator", + "boxes": [ + 6, + 389, + 114, + 421 + ] + }, + { + "text": "0", + "boxes": [ + 1068, + 421, + 1090, + 447 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 18, + 502, + 96, + 528 + ] + }, + { + "text": "", + "boxes": [ + 25, + 547, + 88, + 616 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "围", + "boxes": [ + 36, + 679, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 32, + 772, + 84, + 830 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "galculator", + "boxes": [ + 364, + 1048, + 438, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_1", + "episode_id": 60, + "env_id": 2, + "step": 1, + "format_rewards": 1.0, + "rewards": 11.443639755249023, + "advantages": 1.3024104833602905, + "generated_text": "{\n \"intent\": \"Starting by opening Firefox. This will allow me to explore various web resources, which could provide more context about the computer system.\",\n \"action\": \"DoubleClick(55, 55)\"\n}", + "action_string": "DoubleClick(55, 55)", + "intent_string": "Starting by opening Firefox. This will allow me to explore various web resources, which could provide more context about the computer system.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 78, + 264 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 8, + 389, + 114, + 421 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 16, + 498, + 96, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_3_1", + "episode_id": 60, + "env_id": 3, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.109641075134277, + "advantages": -0.41592517495155334, + "generated_text": "{\n \"intent\": \"By clicking on 'New Tab' at the top left corner, I aim to explore different websites and gather more information about the desktop and browser functionalities.\",\n \"action\": \"DoubleClick(378, 103)\"\n}", + "action_string": "DoubleClick(378, 103)", + "intent_string": "By clicking on 'New Tab' at the top left corner, I aim to explore different websites and gather more information about the desktop and browser functionalities.", + "before_screen_ocr_results": [ + { + "text": "√C合", + "boxes": [ + 44, + 68, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 42, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_4_1", + "episode_id": 60, + "env_id": 4, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.170984268188477, + "advantages": -0.3707630932331085, + "generated_text": "{\n \"intent\": \"To maximize interaction with different web pages by starting with the 'New Tab' feature, which allows exploration of new sites.\",\n \"action\": \"DoubleClick(34, 43)\"\n}", + "action_string": "DoubleClick(34, 43)", + "intent_string": "To maximize interaction with different web pages by starting with the 'New Tab' feature, which allows exploration of new sites.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 74, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_5_1", + "episode_id": 60, + "env_id": 5, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.543681144714355, + "advantages": -0.09637634456157684, + "generated_text": "{\n \"intent\": \"Moving the cursor over different pages allows me to explore various resources available on the internet by accessing different websites suggested here.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "Moving the cursor over different pages allows me to explore various resources available on the internet by accessing different websites suggested here.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1270, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_6_1", + "episode_id": 60, + "env_id": 6, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.534183502197266, + "advantages": -0.10336869955062866, + "generated_text": "{\n \"intent\": \"To start exploring by opening a new tab where I can test more functionalities or search for interesting resources.\",\n \"action\": \"DoubleClick(570, 74)\"\n}", + "action_string": "DoubleClick(570, 74)", + "intent_string": "To start exploring by opening a new tab where I can test more functionalities or search for interesting resources.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 36, + 68, + 128, + 91 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing 一 bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "iqiyi.com", + "boxes": [ + 602, + 220, + 662, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 516 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1223, + 668, + 1419, + 737 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 748, + 772, + 792, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ] + }, + { + "frame_id": "60_7_1", + "episode_id": 60, + "env_id": 7, + "step": 1, + "format_rewards": 1.0, + "rewards": 11.897305488586426, + "advantages": 1.6364080905914307, + "generated_text": "{\n \"intent\": \"To explore more features of the LibreOffice environment by creating a new document. This step will help us understand how different types of documents can be created.\",\n \"action\": \"DoubleClick(95, 540)\"\n}", + "action_string": "DoubleClick(95, 540)", + "intent_string": "To explore more features of the LibreOffice environment by creating a new document. This step will help us understand how different types of documents can be created.", + "before_screen_ocr_results": [ + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": "Fire", + "boxes": [ + 26, + 66, + 60, + 93 + ] + }, + { + "text": " open File", + "boxes": [ + 60, + 62, + 208, + 105 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 92, + 115, + 236, + 149 + ] + }, + { + "text": "Mous", + "boxes": [ + 10, + 171, + 56, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 66, + 173, + 104, + 210 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 102, + 181, + 226, + 212 + ] + }, + { + "text": " Templates ", + "boxes": [ + 92, + 228, + 212, + 260 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 63, + 279, + 141, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 102, + 321, + 278, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 403 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 104, + 369, + 280, + 403 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 389, + 54, + 415 + ] + }, + { + "text": "宣", + "boxes": [ + 68, + 413, + 100, + 449 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 104, + 415, + 314, + 453 + ] + }, + { + "text": "D", + "boxes": [ + 64, + 459, + 102, + 500 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 102, + 462, + 252, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": "Math Formula", + "boxes": [ + 96, + 510, + 242, + 546 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 258, + 590 + ] + }, + { + "text": "Wr", + "boxes": [ + 32, + 609, + 56, + 631 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1374, + 699 + ] + }, + { + "text": "D", + "boxes": [ + 38, + 830, + 52, + 852 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "Extensions", + "boxes": [ + 220, + 981, + 294, + 1005 + ] + }, + { + "text": "breoffice", + "boxes": [ + 226, + 1050, + 286, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 8, + 28, + 34, + 42 + ] + }, + { + "text": " Edit View", + "boxes": [ + 32, + 26, + 122, + 44 + ] + }, + { + "text": "Tools Window Help", + "boxes": [ + 190, + 24, + 348, + 48 + ] + }, + { + "text": "- . ", + "boxes": [ + 2, + 46, + 474, + 89 + ] + }, + { + "text": "SI@0DB", + "boxes": [ + 454, + 50, + 648, + 87 + ] + }, + { + "text": "T", + "boxes": [ + 686, + 60, + 714, + 73 + ] + }, + { + "text": "Ω", + "boxes": [ + 714, + 58, + 742, + 79 + ] + }, + { + "text": "Unary/Binary Operators", + "boxes": [ + 12, + 97, + 178, + 131 + ] + }, + { + "text": "+口", + "boxes": [ + 30, + 153, + 72, + 179 + ] + }, + { + "text": "冏-口", + "boxes": [ + 72, + 149, + 144, + 181 + ] + }, + { + "text": "+口", + "boxes": [ + 164, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 151, + 282, + 179 + ] + }, + { + "text": "□+-□·×", + "boxes": [ + 16, + 204, + 294, + 240 + ] + }, + { + "text": "+口口", + "boxes": [ + 128, + 250, + 290, + 286 + ] + }, + { + "text": "", + "boxes": [ + 14, + 296, + 292, + 336 + ] + }, + { + "text": "0 0%", + "boxes": [ + 8, + 332, + 294, + 393 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 407, + 78, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "Untitled 2 - Li.", + "boxes": [ + 372, + 1048, + 466, + 1072 + ] + } + ] + }, + { + "frame_id": "60_0_2", + "episode_id": 60, + "env_id": 0, + "step": 2, + "format_rewards": 1.0, + "rewards": 11.248809814453125, + "advantages": 1.1589728593826294, + "generated_text": "{\n \"intent\": \"To begin exploring the available applications by selecting an option from the Menu bar, specifically targeting file or application creation.\",\n \"action\": \"DoubleClick(107, 225)\"\n}", + "action_string": "DoubleClick(107, 225)", + "intent_string": "To begin exploring the available applications by selecting an option from the Menu bar, specifically targeting file or application creation.", + "before_screen_ocr_results": [ + { + "text": " Folder", + "boxes": [ + 217, + 7, + 295, + 38 + ] + }, + { + "text": "□0", + "boxes": [ + 1798, + 6, + 1862, + 38 + ] + }, + { + "text": "ate Nel", + "boxes": [ + 58, + 16, + 110, + 34 + ] + }, + { + "text": " Paste", + "boxes": [ + 12, + 56, + 74, + 81 + ] + }, + { + "text": " Empty File ", + "boxes": [ + 242, + 56, + 324, + 81 + ] + }, + { + "text": " Select All", + "boxes": [ + 14, + 99, + 106, + 125 + ] + }, + { + "text": " Invert Selection", + "boxes": [ + 32, + 127, + 150, + 151 + ] + }, + { + "text": "Sort Files", + "boxes": [ + 32, + 169, + 104, + 195 + ] + }, + { + "text": " Desktop Preferences", + "boxes": [ + 12, + 208, + 192, + 238 + ] + }, + { + "text": "", + "boxes": [ + 56, + 236, + 110, + 256 + ] + }, + { + "text": "Templates", + "boxes": [ + 100, + 234, + 210, + 258 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 56, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 66, + 284, + 140, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 68, + 324, + 100, + 355 + ] + }, + { + "text": "Writer Document", + "boxes": [ + 104, + 321, + 276, + 360 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 98, + 403 + ] + }, + { + "text": "Calc Spreadsheet", + "boxes": [ + 106, + 369, + 280, + 405 + ] + }, + { + "text": "回", + "boxes": [ + 64, + 409, + 102, + 453 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 108, + 423, + 310, + 447 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 496 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 104, + 464, + 250, + 499 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": "", + "boxes": [ + 68, + 510, + 104, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 100, + 514, + 240, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 558, + 254, + 590 + ] + }, + { + "text": "Welcome to LibreOffice.", + "boxes": [ + 994, + 637, + 1192, + 671 + ] + }, + { + "text": "Drop a document here or pick an app from the left side to create one.", + "boxes": [ + 812, + 669, + 1372, + 699 + ] + }, + { + "text": "Help", + "boxes": [ + 110, + 981, + 144, + 1003 + ] + }, + { + "text": "Extensions", + "boxes": [ + 222, + 985, + 290, + 1003 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Libreoffic", + "boxes": [ + 916, + 8, + 994, + 32 + ] + }, + { + "text": "□0", + "boxes": [ + 1798, + 6, + 1862, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "", + "boxes": [ + 1644, + 103, + 1688, + 141 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 117, + 236, + 149 + ] + }, + { + "text": "", + "boxes": [ + 1646, + 141, + 1728, + 173 + ] + }, + { + "text": "Mh", + "boxes": [ + 1602, + 157, + 1642, + 238 + ] + }, + { + "text": "Mou", + "boxes": [ + 8, + 171, + 54, + 199 + ] + }, + { + "text": "", + "boxes": [ + 1650, + 171, + 1690, + 197 + ] + }, + { + "text": "?", + "boxes": [ + 72, + 185, + 100, + 208 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 98, + 183, + 228, + 214 + ] + }, + { + "text": "Templates", + "boxes": [ + 65, + 220, + 215, + 265 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 326, + 100, + 357 + ] + }, + { + "text": "Writer Document ", + "boxes": [ + 104, + 319, + 280, + 360 + ] + }, + { + "text": "Businesscard with logo", + "boxes": [ + 1308, + 331, + 1462, + 363 + ] + }, + { + "text": "口", + "boxes": [ + 1190, + 353, + 1248, + 373 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": "Calc Spreadsheet ", + "boxes": [ + 106, + 369, + 284, + 405 + ] + }, + { + "text": "Appearance Desktop lcons Advanced ", + "boxes": [ + 696, + 385, + 992, + 409 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "Background", + "boxes": [ + 704, + 419, + 806, + 445 + ] + }, + { + "text": "", + "boxes": [ + 412, + 431, + 436, + 453 + ] + }, + { + "text": "Wallpaper mode:", + "boxes": [ + 714, + 447, + 846, + 475 + ] + }, + { + "text": "Stretch to fill the entire monitor area", + "boxes": [ + 844, + 449, + 1118, + 472 + ] + }, + { + "text": " Draw Drawing ", + "boxes": [ + 101, + 460, + 255, + 503 + ] + }, + { + "text": "·Door si met, cnsetetradipisin e.", + "boxes": [ + 1272, + 461, + 1476, + 484 + ] + }, + { + "text": "Lorem Ipsum", + "boxes": [ + 1626, + 459, + 1742, + 488 + ] + }, + { + "text": "2", + "boxes": [ + 72, + 471, + 98, + 494 + ] + }, + { + "text": "Wallpaper:", + "boxes": [ + 718, + 486, + 798, + 510 + ] + }, + { + "text": "bg1.jpg", + "boxes": [ + 810, + 486, + 892, + 512 + ] + }, + { + "text": "Vestibulu", + "boxes": [ + 1280, + 478, + 1324, + 496 + ] + }, + { + "text": "m consequat mi quis pretium semper", + "boxes": [ + 1326, + 478, + 1490, + 496 + ] + }, + { + "text": "Libr", + "boxes": [ + 18, + 500, + 54, + 528 + ] + }, + { + "text": " Proin luctus orci ac neque venenatis", + "boxes": [ + 1276, + 494, + 1446, + 512 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 98, + 514, + 242, + 544 + ] + }, + { + "text": "Use the same", + "boxes": [ + 746, + 522, + 862, + 540 + ] + }, + { + "text": " on all desktops", + "boxes": [ + 936, + 520, + 1048, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 104, + 556, + 258, + 592 + ] + }, + { + "text": "Background color:", + "boxes": [ + 720, + 556, + 846, + 574 + ] + }, + { + "text": "Text", + "boxes": [ + 708, + 598, + 740, + 619 + ] + }, + { + "text": "Font of label text:", + "boxes": [ + 716, + 629, + 842, + 647 + ] + }, + { + "text": "Sans", + "boxes": [ + 1006, + 629, + 1044, + 649 + ] + }, + { + "text": "[14 ", + "boxes": [ + 1188, + 623, + 1224, + 651 + ] + }, + { + "text": "Blue Curve", + "boxes": [ + 1350, + 653, + 1420, + 671 + ] + }, + { + "text": "Color of label text:", + "boxes": [ + 716, + 665, + 852, + 689 + ] + }, + { + "text": "Color of shadow:", + "boxes": [ + 982, + 665, + 1102, + 689 + ] + }, + { + "text": "Close", + "boxes": [ + 1162, + 713, + 1232, + 744 + ] + }, + { + "text": "Lorem ipsum", + "boxes": [ + 1342, + 748, + 1430, + 774 + ] + }, + { + "text": "Lorem Ipsum", + "boxes": [ + 1640, + 746, + 1728, + 770 + ] + }, + { + "text": "dolor sit amet", + "boxes": [ + 1286, + 774, + 1362, + 796 + ] + }, + { + "text": "Vestibulu", + "boxes": [ + 1286, + 810, + 1334, + 838 + ] + }, + { + "text": "Dr", + "boxes": [ + 36, + 828, + 56, + 854 + ] + }, + { + "text": "n Dolor staemnci", + "boxes": [ + 765, + 839, + 855, + 870 + ] + }, + { + "text": "Bright Blue", + "boxes": [ + 454, + 959, + 532, + 991 + ] + }, + { + "text": " Classy Red", + "boxes": [ + 748, + 959, + 832, + 991 + ] + }, + { + "text": "DNA", + "boxes": [ + 1068, + 961, + 1104, + 989 + ] + }, + { + "text": "Focus", + "boxes": [ + 1366, + 963, + 1406, + 989 + ] + }, + { + "text": "Forestbird", + "boxes": [ + 1650, + 961, + 1718, + 987 + ] + }, + { + "text": "Help ", + "boxes": [ + 106, + 979, + 148, + 1007 + ] + }, + { + "text": " Extensions", + "boxes": [ + 218, + 981, + 296, + 1005 + ] + }, + { + "text": "ibreOffice", + "boxes": [ + 222, + 1050, + 286, + 1070 + ] + }, + { + "text": "esktop Pref.", + "boxes": [ + 376, + 1048, + 462, + 1072 + ] + } + ] + }, + { + "frame_id": "60_1_2", + "episode_id": 60, + "env_id": 1, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.810291290283203, + "advantages": 0.8361269235610962, + "generated_text": "{\n \"intent\": \"To start exploring different applications and their functionalities, I will open Firefox by clicking on its icon.\",\n \"action\": \"DoubleClick(72, 48)\"\n}", + "action_string": "DoubleClick(72, 48)", + "intent_string": "To start exploring different applications and their functionalities, I will open Firefox by clicking on its icon.", + "before_screen_ocr_results": [ + { + "text": "jalculatol", + "boxes": [ + 370, + 12, + 444, + 30 + ] + }, + { + "text": "日回×", + "boxes": [ + 490, + 12, + 548, + 30 + ] + }, + { + "text": "File Edit View", + "boxes": [ + 266, + 38, + 380, + 56 + ] + }, + { + "text": "Calculator Help", + "boxes": [ + 392, + 38, + 506, + 56 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "?", + "boxes": [ + 514, + 81, + 530, + 97 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 102, + 308 + ] + }, + { + "text": "", + "boxes": [ + 20, + 334, + 89, + 395 + ] + }, + { + "text": "国", + "boxes": [ + 818, + 359, + 834, + 375 + ] + }, + { + "text": "galculator", + "boxes": [ + 910, + 351, + 1014, + 379 + ] + }, + { + "text": "口", + "boxes": [ + 1036, + 353, + 1106, + 379 + ] + }, + { + "text": "File", + "boxes": [ + 822, + 381, + 852, + 401 + ] + }, + { + "text": "Edit View", + "boxes": [ + 852, + 379, + 946, + 403 + ] + }, + { + "text": "Calculator Help", + "boxes": [ + 934, + 379, + 1062, + 403 + ] + }, + { + "text": "Calculator", + "boxes": [ + 6, + 389, + 114, + 421 + ] + }, + { + "text": "0", + "boxes": [ + 1068, + 421, + 1090, + 447 + ] + }, + { + "text": "LibreO..", + "boxes": [ + 18, + 502, + 96, + 528 + ] + }, + { + "text": "", + "boxes": [ + 26, + 547, + 88, + 616 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "围", + "boxes": [ + 36, + 679, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 32, + 772, + 84, + 830 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "galculator", + "boxes": [ + 364, + 1048, + 438, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 538, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 816 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 522, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_2", + "episode_id": 60, + "env_id": 2, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.539591789245605, + "advantages": -0.09938701242208481, + "generated_text": "{\n \"intent\": \"The action `DoubleClick(740, 220)` is chosen to explore navigation by opening a new tab with Widows Search (usually part of Windows Start menu). This allows us to access different features or information sources.\",\n \"action\": \"DoubleClick(740, 220)\"\n}", + "action_string": "DoubleClick(740, 220)", + "intent_string": "The action `DoubleClick(740, 220)` is chosen to explore navigation by opening a new tab with Widows Search (usually part of Windows Start menu). This allows us to access different features or information sources.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFiref", + "boxes": [ + 904, + 4, + 1004, + 22 + ] + }, + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 660, + 288, + 764, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_3_2", + "episode_id": 60, + "env_id": 3, + "step": 2, + "format_rewards": 1.0, + "rewards": 8.99020004272461, + "advantages": -0.5038599967956543, + "generated_text": "{\n \"intent\": \"To open a new tab by clicking the 'New Tab' button, which will give more options for exploring web pages.\",\n \"action\": \"DoubleClick(111, 37)\"\n}", + "action_string": "DoubleClick(111, 37)", + "intent_string": "To open a new tab by clicking the 'New Tab' button, which will give more options for exploring web pages.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 42, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_4_2", + "episode_id": 60, + "env_id": 4, + "step": 2, + "format_rewards": 1.0, + "rewards": 11.11727237701416, + "advantages": 1.0621323585510254, + "generated_text": "{\n \"intent\": \"Opening a new tab will allow me to explore more search results or bookmarksшла изучить больше результатов поиска или закладок.\",\n \"action\": \"DoubleClick(211, 38)\"\n}", + "action_string": "DoubleClick(211, 38)", + "intent_string": "Opening a new tab will allow me to explore more search results or bookmarksшла изучить больше результатов поиска или закладок.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 22, + 332, + 86, + 381 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 86, + 969 + ] + } + ] + }, + { + "frame_id": "60_5_2", + "episode_id": 60, + "env_id": 5, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.673822402954102, + "advantages": 0.7356558442115784, + "generated_text": "{\n \"intent\": \"To explore further, double-clicking an icon will help navigate to different websites or applications.\",\n \"action\": \"DoubleClick(667, 497)\"\n}", + "action_string": "DoubleClick(667, 497)", + "intent_string": "To explore further, double-clicking an icon will help navigate to different websites or applications.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "6", + "boxes": [ + 1233, + 479, + 1271, + 506 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "ransla", + "boxes": [ + 410, + 125, + 444, + 139 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 558, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1342, + 302, + 1356, + 324 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1354, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "", + "boxes": [ + 1542, + 361, + 1566, + 385 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 634, + 474 + ] + }, + { + "text": "取款身亡银行赔10万", + "boxes": [ + 802, + 453, + 968, + 476 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "一点视频/九尾说游戏", + "boxes": [ + 851, + 754, + 980, + 791 + ] + }, + { + "text": "金融界汽车·9小时", + "boxes": [ + 1142, + 760, + 1280, + 792 + ] + }, + { + "text": "挑战烽火荣都绿门或白门里的", + "boxes": [ + 824, + 780, + 1096, + 818 + ] + }, + { + "text": "赛力斯申请高速过弯转向不足", + "boxes": [ + 1136, + 780, + 1406, + 820 + ] + }, + { + "text": "枪，千万别是M24和98K！", + "boxes": [ + 826, + 814, + 1082, + 844 + ] + }, + { + "text": "控制专利，提高稳定性和安全", + "boxes": [ + 1134, + 808, + 1408, + 846 + ] + }, + { + "text": "性", + "boxes": [ + 1140, + 842, + 1166, + 866 + ] + }, + { + "text": "河融媒科技密探", + "boxes": [ + 290, + 925, + 374, + 943 + ] + }, + { + "text": "国外牛人发明不会翻的船，就算船头朝下也能", + "boxes": [ + 202, + 945, + 794, + 987 + ] + }, + { + "text": "自己翻回来，长见识了", + "boxes": [ + 198, + 981, + 502, + 1022 + ] + }, + { + "text": "21", + "boxes": [ + 210, + 1026, + 264, + 1044 + ] + } + ] + }, + { + "frame_id": "60_6_2", + "episode_id": 60, + "env_id": 6, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.173648834228516, + "advantages": -0.36880138516426086, + "generated_text": "{\n \"intent\": \"To explore the width of the webpage and gather more information about the website's content.\",\n \"action\": \"ScrollUp(960, 540)\"\n}", + "action_string": "ScrollUp(960, 540)", + "intent_string": "To explore the width of the webpage and gather more information about the website's content.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 40, + 69, + 126, + 87 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "iqiyi.com", + "boxes": [ + 602, + 220, + 662, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 516 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1223, + 668, + 1419, + 737 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 748, + 772, + 792, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 36, + 68, + 128, + 91 + ] + }, + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 62, + 522, + 91 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "IQIYI", + "boxes": [ + 860, + 459, + 936, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1062, + 516 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 884, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1225, + 673, + 1417, + 735 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ] + }, + { + "frame_id": "60_7_2", + "episode_id": 60, + "env_id": 7, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.911489486694336, + "advantages": 0.1744113713502884, + "generated_text": "{\n \"intent\": \"Clicking on an operator from the Unary/Binary Operators palette will demonstrate interaction with the application interface and help assess its responsiveness.\",\n \"action\": \"DoubleClick(100, 150)\"\n}", + "action_string": "DoubleClick(100, 150)", + "intent_string": "Clicking on an operator from the Unary/Binary Operators palette will demonstrate interaction with the application interface and help assess its responsiveness.", + "before_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 8, + 28, + 34, + 42 + ] + }, + { + "text": " Edit View", + "boxes": [ + 32, + 26, + 118, + 44 + ] + }, + { + "text": "Tools Window Help", + "boxes": [ + 190, + 24, + 348, + 48 + ] + }, + { + "text": "- . ", + "boxes": [ + 2, + 46, + 474, + 89 + ] + }, + { + "text": "SI@0DB", + "boxes": [ + 454, + 50, + 648, + 87 + ] + }, + { + "text": "T", + "boxes": [ + 686, + 60, + 714, + 73 + ] + }, + { + "text": "Ω", + "boxes": [ + 714, + 58, + 742, + 79 + ] + }, + { + "text": "Unary/Binary Operators", + "boxes": [ + 12, + 97, + 178, + 131 + ] + }, + { + "text": "+口", + "boxes": [ + 30, + 153, + 72, + 179 + ] + }, + { + "text": "冏-口", + "boxes": [ + 72, + 149, + 144, + 181 + ] + }, + { + "text": "+口", + "boxes": [ + 164, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 151, + 282, + 179 + ] + }, + { + "text": "□+-□·×", + "boxes": [ + 16, + 204, + 294, + 240 + ] + }, + { + "text": "+口口", + "boxes": [ + 128, + 250, + 290, + 286 + ] + }, + { + "text": "", + "boxes": [ + 14, + 296, + 292, + 336 + ] + }, + { + "text": "0 0%", + "boxes": [ + 8, + 332, + 294, + 393 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 407, + 78, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "Untitled 2 - Li.", + "boxes": [ + 372, + 1048, + 466, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": "---- I", + "boxes": [ + 6, + 50, + 646, + 85 + ] + }, + { + "text": "S", + "boxes": [ + 714, + 54, + 746, + 85 + ] + }, + { + "text": "Unary/Binary Operators", + "boxes": [ + 18, + 105, + 172, + 123 + ] + }, + { + "text": "r□", + "boxes": [ + 88, + 143, + 144, + 178 + ] + }, + { + "text": "+口", + "boxes": [ + 36, + 153, + 64, + 175 + ] + }, + { + "text": "+口", + "boxes": [ + 174, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 153, + 282, + 179 + ] + }, + { + "text": "□+□冏□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 86, + 282 + ] + }, + { + "text": "号", + "boxes": [ + 104, + 252, + 134, + 286 + ] + }, + { + "text": "□÷□口", + "boxes": [ + 156, + 254, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": " %", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 407, + 78, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ] + }, + { + "frame_id": "60_0_3", + "episode_id": 60, + "env_id": 0, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.605916023254395, + "advantages": 0.6856618523597717, + "generated_text": "{\n \"intent\": \"To explore available templates or features within LibreOffice by interacting with displayed options\",\n \"action\": \"DoubleClick(200, 250)\"\n}", + "action_string": "DoubleClick(200, 250)", + "intent_string": "To explore available templates or features within LibreOffice by interacting with displayed options", + "before_screen_ocr_results": [ + { + "text": "Libreoffic", + "boxes": [ + 916, + 8, + 994, + 32 + ] + }, + { + "text": "□0", + "boxes": [ + 1798, + 6, + 1862, + 38 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 102, + 32, + 190, + 58 + ] + }, + { + "text": "", + "boxes": [ + 1644, + 103, + 1688, + 141 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 117, + 236, + 149 + ] + }, + { + "text": "", + "boxes": [ + 1646, + 141, + 1728, + 173 + ] + }, + { + "text": "Mh", + "boxes": [ + 1602, + 157, + 1642, + 238 + ] + }, + { + "text": "Mou", + "boxes": [ + 8, + 171, + 54, + 199 + ] + }, + { + "text": "", + "boxes": [ + 1650, + 171, + 1690, + 197 + ] + }, + { + "text": "?", + "boxes": [ + 72, + 185, + 100, + 208 + ] + }, + { + "text": " Recent Files", + "boxes": [ + 98, + 183, + 228, + 214 + ] + }, + { + "text": "Templates", + "boxes": [ + 65, + 220, + 215, + 265 + ] + }, + { + "text": "Terr", + "boxes": [ + 22, + 284, + 54, + 306 + ] + }, + { + "text": "Create:", + "boxes": [ + 68, + 286, + 138, + 312 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 326, + 100, + 357 + ] + }, + { + "text": "Writer Document ", + "boxes": [ + 104, + 319, + 280, + 360 + ] + }, + { + "text": "Businesscard with logo", + "boxes": [ + 1308, + 331, + 1462, + 363 + ] + }, + { + "text": "口", + "boxes": [ + 1190, + 353, + 1248, + 373 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": "Calc Spreadsheet ", + "boxes": [ + 106, + 369, + 284, + 405 + ] + }, + { + "text": "Appearance Desktop lcons Advanced ", + "boxes": [ + 696, + 385, + 992, + 409 + ] + }, + { + "text": "Impress Presentation", + "boxes": [ + 106, + 421, + 314, + 451 + ] + }, + { + "text": "Background", + "boxes": [ + 704, + 419, + 806, + 445 + ] + }, + { + "text": "", + "boxes": [ + 412, + 431, + 436, + 453 + ] + }, + { + "text": "Wallpaper mode:", + "boxes": [ + 714, + 447, + 846, + 475 + ] + }, + { + "text": "Stretch to fill the entire monitor area", + "boxes": [ + 844, + 449, + 1118, + 472 + ] + }, + { + "text": " Draw Drawing ", + "boxes": [ + 101, + 460, + 255, + 503 + ] + }, + { + "text": "·Door si met, cnsetetradipisin e.", + "boxes": [ + 1272, + 461, + 1476, + 484 + ] + }, + { + "text": "Lorem Ipsum", + "boxes": [ + 1626, + 459, + 1742, + 488 + ] + }, + { + "text": "2", + "boxes": [ + 72, + 471, + 98, + 494 + ] + }, + { + "text": "Wallpaper:", + "boxes": [ + 718, + 486, + 798, + 510 + ] + }, + { + "text": "bg1.jpg", + "boxes": [ + 810, + 486, + 892, + 512 + ] + }, + { + "text": "Vestibulu", + "boxes": [ + 1280, + 478, + 1324, + 496 + ] + }, + { + "text": "m consequat mi quis pretium semper", + "boxes": [ + 1326, + 478, + 1490, + 496 + ] + }, + { + "text": "Libr", + "boxes": [ + 18, + 500, + 54, + 528 + ] + }, + { + "text": " Proin luctus orci ac neque venenatis", + "boxes": [ + 1276, + 494, + 1446, + 512 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 98, + 514, + 242, + 544 + ] + }, + { + "text": " Use the same", + "boxes": [ + 746, + 522, + 864, + 540 + ] + }, + { + "text": " on all desktops", + "boxes": [ + 936, + 520, + 1048, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 104, + 556, + 258, + 592 + ] + }, + { + "text": "Background color:", + "boxes": [ + 720, + 556, + 846, + 574 + ] + }, + { + "text": "Text", + "boxes": [ + 708, + 598, + 740, + 619 + ] + }, + { + "text": "Font of label text:", + "boxes": [ + 716, + 629, + 842, + 647 + ] + }, + { + "text": "Sans", + "boxes": [ + 1006, + 629, + 1044, + 649 + ] + }, + { + "text": "[14 ", + "boxes": [ + 1188, + 623, + 1224, + 651 + ] + }, + { + "text": "Blue Curve", + "boxes": [ + 1350, + 653, + 1420, + 671 + ] + }, + { + "text": "Color of label text:", + "boxes": [ + 716, + 665, + 852, + 689 + ] + }, + { + "text": "Color of shadow:", + "boxes": [ + 982, + 665, + 1102, + 689 + ] + }, + { + "text": "Close", + "boxes": [ + 1162, + 713, + 1232, + 744 + ] + }, + { + "text": "Lorem ipsum", + "boxes": [ + 1342, + 748, + 1430, + 774 + ] + }, + { + "text": "Lorem Ipsum", + "boxes": [ + 1640, + 746, + 1728, + 770 + ] + }, + { + "text": "dolor sit amet", + "boxes": [ + 1286, + 774, + 1362, + 796 + ] + }, + { + "text": "Vestibulu", + "boxes": [ + 1286, + 810, + 1334, + 838 + ] + }, + { + "text": "Dr", + "boxes": [ + 36, + 828, + 56, + 854 + ] + }, + { + "text": "n Dolor staemnci", + "boxes": [ + 765, + 839, + 855, + 870 + ] + }, + { + "text": "Bright Blue", + "boxes": [ + 454, + 959, + 532, + 991 + ] + }, + { + "text": " Classy Red", + "boxes": [ + 748, + 959, + 832, + 991 + ] + }, + { + "text": "DNA", + "boxes": [ + 1068, + 961, + 1104, + 989 + ] + }, + { + "text": "Focus", + "boxes": [ + 1366, + 963, + 1406, + 989 + ] + }, + { + "text": "Forestbird", + "boxes": [ + 1650, + 961, + 1718, + 987 + ] + }, + { + "text": "Help ", + "boxes": [ + 106, + 979, + 148, + 1007 + ] + }, + { + "text": " Extensions", + "boxes": [ + 218, + 981, + 296, + 1005 + ] + }, + { + "text": "ibreOffice", + "boxes": [ + 222, + 1050, + 286, + 1070 + ] + }, + { + "text": "esktop Pref.", + "boxes": [ + 376, + 1048, + 462, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Libreoffic", + "boxes": [ + 914, + 12, + 992, + 30 + ] + }, + { + "text": "口", + "boxes": [ + 1800, + 8, + 1860, + 34 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": " Open File ", + "boxes": [ + 60, + 62, + 208, + 105 + ] + }, + { + "text": "", + "boxes": [ + 1606, + 111, + 1636, + 147 + ] + }, + { + "text": "", + "boxes": [ + 1644, + 105, + 1688, + 141 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 117, + 236, + 149 + ] + }, + { + "text": "", + "boxes": [ + 1600, + 127, + 1648, + 170 + ] + }, + { + "text": "", + "boxes": [ + 1646, + 141, + 1728, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1602, + 159, + 1642, + 218 + ] + }, + { + "text": "Mou", + "boxes": [ + 8, + 171, + 54, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 68, + 183, + 102, + 210 + ] + }, + { + "text": "Recent Files ", + "boxes": [ + 94, + 183, + 228, + 214 + ] + }, + { + "text": "三", + "boxes": [ + 1608, + 206, + 1634, + 230 + ] + }, + { + "text": " Templates", + "boxes": [ + 85, + 219, + 217, + 273 + ] + }, + { + "text": "口", + "boxes": [ + 74, + 230, + 102, + 258 + ] + }, + { + "text": "Terr", + "boxes": [ + 20, + 280, + 54, + 308 + ] + }, + { + "text": "Create:", + "boxes": [ + 66, + 284, + 140, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 324, + 102, + 359 + ] + }, + { + "text": "Writer Document ", + "boxes": [ + 102, + 323, + 280, + 358 + ] + }, + { + "text": "scard with log", + "boxes": [ + 1360, + 336, + 1454, + 359 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": "Calc Spreadsheet ", + "boxes": [ + 104, + 369, + 282, + 405 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 389, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 64, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation ", + "boxes": [ + 102, + 415, + 316, + 453 + ] + }, + { + "text": "", + "boxes": [ + 414, + 433, + 432, + 447 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 102, + 464, + 250, + 501 + ] + }, + { + "text": "·Door si met, cnsetetradipisi e", + "boxes": [ + 1272, + 461, + 1474, + 484 + ] + }, + { + "text": "Lorem Ipsum", + "boxes": [ + 1626, + 459, + 1742, + 488 + ] + }, + { + "text": "Vestil", + "boxes": [ + 1284, + 480, + 1306, + 494 + ] + }, + { + "text": "m consequat mi quis pretiun", + "boxes": [ + 1324, + 478, + 1452, + 496 + ] + }, + { + "text": "sempe", + "boxes": [ + 1454, + 480, + 1486, + 494 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": " orci ac neque venenalis", + "boxes": [ + 1332, + 494, + 1446, + 512 + ] + }, + { + "text": "", + "boxes": [ + 66, + 508, + 106, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 98, + 514, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 256, + 592 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "Lorem ipsum", + "boxes": [ + 1342, + 748, + 1430, + 774 + ] + }, + { + "text": "Lorem Ipsum", + "boxes": [ + 1640, + 746, + 1726, + 772 + ] + }, + { + "text": "dolor sit amet", + "boxes": [ + 1288, + 778, + 1362, + 796 + ] + }, + { + "text": "consectetur adipiscing elit ", + "boxes": [ + 1286, + 794, + 1428, + 812 + ] + }, + { + "text": "Vestiblu", + "boxes": [ + 1286, + 812, + 1334, + 838 + ] + }, + { + "text": " consequat mi quis pretiun", + "boxes": [ + 1344, + 812, + 1482, + 830 + ] + }, + { + "text": "Bright Blue", + "boxes": [ + 452, + 959, + 532, + 991 + ] + }, + { + "text": " Classy Red", + "boxes": [ + 748, + 959, + 832, + 991 + ] + }, + { + "text": "DNA", + "boxes": [ + 1068, + 961, + 1104, + 989 + ] + }, + { + "text": "Focus", + "boxes": [ + 1366, + 963, + 1406, + 989 + ] + }, + { + "text": "Forestbird", + "boxes": [ + 1650, + 963, + 1718, + 989 + ] + }, + { + "text": "Help ", + "boxes": [ + 106, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 296, + 1007 + ] + }, + { + "text": "Libreoffice", + "boxes": [ + 218, + 1050, + 288, + 1068 + ] + } + ] + }, + { + "frame_id": "60_1_3", + "episode_id": 60, + "env_id": 1, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.22494888305664, + "advantages": -0.33103328943252563, + "generated_text": "{\n \"intent\": \"To open a new tab which will provide an opportunity to further interact with different web pages or applications.\",\n \"action\": \"DoubleClick(259, 44)\"\n}", + "action_string": "DoubleClick(259, 44)", + "intent_string": "To open a new tab which will provide an opportunity to further interact with different web pages or applications.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Qbearch with Bing orenteraddress", + "boxes": [ + 244, + 57, + 522, + 100 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 538, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 816 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 522, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 227, + 25, + 317, + 62 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 202, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 556, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 538, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 816 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 522, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_3", + "episode_id": 60, + "env_id": 2, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.58216667175293, + "advantages": -0.06804254651069641, + "generated_text": "{\n \"intent\": \"To explore more web content or bookmarks, I'll try scrolling through available options.\",\n \"action\": \"ScrollDown(1066, 572)\"\n}", + "action_string": "ScrollDown(1066, 572)", + "intent_string": "To explore more web content or bookmarks, I'll try scrolling through available options.", + "before_screen_ocr_results": [ + { + "text": "MozillaFiref", + "boxes": [ + 904, + 4, + 1004, + 22 + ] + }, + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 660, + 288, + 764, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1000, + 536, + 1034, + 550 + ] + }, + { + "text": "ifeng", + "boxes": [ + 1118, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_3_3", + "episode_id": 60, + "env_id": 3, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.053621292114258, + "advantages": -0.4571680426597595, + "generated_text": "{\n \"intent\": \"To start exploring web content by opening a new tab.\",\n \"action\": \"DoubleClick(87, 35)\"\n}", + "action_string": "DoubleClick(87, 35)", + "intent_string": "To start exploring web content by opening a new tab.", + "before_screen_ocr_results": [ + { + "text": "Q仓", + "boxes": [ + 68, + 68, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 19, + 23, + 99, + 56 + ] + }, + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 14, + 103, + 158, + 121 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_4_3", + "episode_id": 60, + "env_id": 4, + "step": 3, + "format_rewards": 1.0, + "rewards": 11.418378829956055, + "advantages": 1.2838128805160522, + "generated_text": "{\n \"intent\": \"To explore further into one of the applications available on the desktop, starting with selecting the Firefox browser.\",\n \"action\": \"DoubleClick(53, 74)\"\n}", + "action_string": "DoubleClick(53, 74)", + "intent_string": "To explore further into one of the applications available on the desktop, starting with selecting the Firefox browser.", + "before_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 22, + 332, + 86, + 381 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 86, + 969 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "√C合", + "boxes": [ + 44, + 68, + 128, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_5_3", + "episode_id": 60, + "env_id": 5, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.055065155029297, + "advantages": 0.2801145911216736, + "generated_text": "{\n \"intent\": \"To scroll down the webpage to explore more content\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To scroll down the webpage to explore more content", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 558, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "攻", + "boxes": [ + 560, + 834, + 586, + 848 + ] + }, + { + "text": "击资质", + "boxes": [ + 575, + 821, + 681, + 858 + ] + }, + { + "text": "1320", + "boxes": [ + 690, + 828, + 732, + 848 + ] + }, + { + "text": "挑战烽火绿巾白内里的枪", + "boxes": [ + 805, + 813, + 1121, + 879 + ] + }, + { + "text": "御资质", + "boxes": [ + 584, + 856, + 678, + 881 + ] + }, + { + "text": "1140", + "boxes": [ + 690, + 858, + 730, + 878 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "力资质", + "boxes": [ + 582, + 881, + 678, + 907 + ] + }, + { + "text": "4200", + "boxes": [ + 690, + 883, + 732, + 903 + ] + }, + { + "text": "130287%", + "boxes": [ + 1623, + 888, + 1701, + 942 + ] + }, + { + "text": "力资质", + "boxes": [ + 578, + 909, + 678, + 933 + ] + }, + { + "text": "2400", + "boxes": [ + 692, + 915, + 728, + 929 + ] + }, + { + "text": "金融界汽车·9小时", + "boxes": [ + 1138, + 917, + 1280, + 947 + ] + }, + { + "text": "梦幻西游：梦幻不该修改的地方", + "boxes": [ + 198, + 943, + 618, + 987 + ] + }, + { + "text": "赛力斯申请高速过弯转向不足", + "boxes": [ + 1138, + 941, + 1408, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "把玩家的游", + "boxes": [ + 614, + 945, + 792, + 987 + ] + }, + { + "text": "+3.287", + "boxes": [ + 1626, + 939, + 1700, + 987 + ] + }, + { + "text": "日", + "boxes": [ + 868, + 959, + 902, + 973 + ] + }, + { + "text": "控制专利，提高稳定性和安全", + "boxes": [ + 1136, + 967, + 1410, + 1003 + ] + }, + { + "text": "戏情怀都抹掉了", + "boxes": [ + 200, + 985, + 410, + 1020 + ] + }, + { + "text": "一点视频/九尾说游戏", + "boxes": [ + 844, + 982, + 972, + 1019 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 993, + 850, + 1009 + ] + }, + { + "text": "性", + "boxes": [ + 1136, + 999, + 1172, + 1026 + ] + }, + { + "text": "挑战烽火荣都绿门或白门里的", + "boxes": [ + 822, + 1009, + 1098, + 1044 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "1.142", + "boxes": [ + 700, + 1028, + 740, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Search - Microsoft Bing×十", + "boxes": [ + 36, + 32, + 244, + 50 + ] + }, + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1608, + 68, + 1658, + 87 + ] + }, + { + "text": "三目9", + "boxes": [ + 1786, + 66, + 1866, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "mages", + "boxes": [ + 254, + 123, + 292, + 143 + ] + }, + { + "text": "Video", + "boxes": [ + 334, + 125, + 362, + 139 + ] + }, + { + "text": "Translate", + "boxes": [ + 402, + 119, + 454, + 145 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 490, + 189, + 638, + 220 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 236, + 614, + 266 + ] + }, + { + "text": "中忠空", + "boxes": [ + 1538, + 357, + 1574, + 387 + ] + }, + { + "text": "中", + "boxes": [ + 1882, + 377, + 1894, + 389 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 389, + 1634, + 421 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 185, + 436, + 273, + 467 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 480, + 1552, + 512 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1456, + 516, + 1506, + 560 + ] + }, + { + "text": "Mw", + "boxes": [ + 1588, + 519, + 1632, + 552 + ] + }, + { + "text": "+ 4998", + "boxes": [ + 1634, + 518, + 1698, + 560 + ] + }, + { + "text": "挑战烽火绿巾白内里的枪", + "boxes": [ + 807, + 529, + 1119, + 595 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1458, + 564, + 1516, + 606 + ] + }, + { + "text": "#6123 ", + "boxes": [ + 1624, + 558, + 1704, + 611 + ] + }, + { + "text": "", + "boxes": [ + 1592, + 572, + 1638, + 600 + ] + }, + { + "text": "时中国A50", + "boxes": [ + 1449, + 601, + 1551, + 658 + ] + }, + { + "text": "L", + "boxes": [ + 1594, + 623, + 1610, + 641 + ] + }, + { + "text": "金融界汽车·9小时", + "boxes": [ + 1138, + 633, + 1280, + 663 + ] + }, + { + "text": "赛力斯申请高速过弯转向不足", + "boxes": [ + 1136, + 655, + 1408, + 691 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 655, + 1510, + 701 + ] + }, + { + "text": "r.", + "boxes": [ + 1585, + 653, + 1634, + 697 + ] + }, + { + "text": "+3.28%", + "boxes": [ + 1626, + 653, + 1700, + 701 + ] + }, + { + "text": "生活研究所", + "boxes": [ + 230, + 673, + 292, + 691 + ] + }, + { + "text": "控制专利，提高稳定性和安全", + "boxes": [ + 1134, + 683, + 1404, + 713 + ] + }, + { + "text": "北欧史诗级的浪漫岛屿", + "boxes": [ + 200, + 697, + 504, + 739 + ] + }, + { + "text": "", + "boxes": [ + 824, + 703, + 853, + 730 + ] + }, + { + "text": "一点视频/九尾说游戏", + "boxes": [ + 846, + 701, + 972, + 731 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 699, + 1522, + 748 + ] + }, + { + "text": "1140 ", + "boxes": [ + 1630, + 699, + 1700, + 746 + ] + }, + { + "text": "性", + "boxes": [ + 1136, + 713, + 1170, + 741 + ] + }, + { + "text": "挑战烽火荣都绿门或白门里的", + "boxes": [ + 824, + 723, + 1096, + 756 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 717, + 1628, + 739 + ] + }, + { + "text": "+", + "boxes": [ + 1688, + 717, + 1712, + 731 + ] + }, + { + "text": "", + "boxes": [ + 204, + 746, + 264, + 764 + ] + }, + { + "text": "枪，千万别是M24和98K！", + "boxes": [ + 818, + 748, + 1082, + 786 + ] + }, + { + "text": "", + "boxes": [ + 1134, + 739, + 1192, + 770 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1622, + 744, + 1726, + 770 + ] + }, + { + "text": "长春市", + "boxes": [ + 1462, + 800, + 1524, + 820 + ] + }, + { + "text": "小白", + "boxes": [ + 502, + 856, + 604, + 887 + ] + }, + { + "text": "更预示着全球汽车", + "boxes": [ + 865, + 923, + 951, + 951 + ] + }, + { + "text": "米拍摄影社区·15 小时", + "boxes": [ + 198, + 949, + 362, + 979 + ] + }, + { + "text": "一点资讯·3周", + "boxes": [ + 512, + 949, + 634, + 979 + ] + }, + { + "text": "生活研究所", + "boxes": [ + 1138, + 951, + 1228, + 977 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 967, + 850, + 983 + ] + }, + { + "text": "一点视频/防务急先锋", + "boxes": [ + 844, + 961, + 972, + 991 + ] + }, + { + "text": "国境之南-南极村", + "boxes": [ + 202, + 975, + 358, + 1005 + ] + }, + { + "text": "穿越火线老玩家记忆里的那些", + "boxes": [ + 510, + 971, + 784, + 1007 + ] + }, + { + "text": "地球上的天赐谷地", + "boxes": [ + 1134, + 966, + 1308, + 1009 + ] + }, + { + "text": "爆破步枪现在怀旧服也多少", + "boxes": [ + 508, + 993, + 788, + 1030 + ] + }, + { + "text": "中国汽车技术崛起", + "boxes": [ + 824, + 987, + 994, + 1016 + ] + } + ] + }, + { + "frame_id": "60_6_3", + "episode_id": 60, + "env_id": 6, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.126218795776367, + "advantages": -0.40372031927108765, + "generated_text": "{\n \"intent\": \"To open one of the frequently accessed websites by right-clicking on the tab at the bottom of the browser window.\",\n \"action\": \"RightClick(900, 395)\"\n}", + "action_string": "RightClick(900, 395)", + "intent_string": "To open one of the frequently accessed websites by right-clicking on the tab at the bottom of the browser window.", + "before_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 36, + 68, + 128, + 91 + ] + }, + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 62, + 522, + 91 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "IQIYI", + "boxes": [ + 860, + 459, + 936, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1062, + 516 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 884, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1225, + 673, + 1417, + 735 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 36, + 68, + 128, + 91 + ] + }, + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 62, + 522, + 91 + ] + }, + { + "text": " Apple --htt://ww.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 252, + 210, + 592, + 242 + ] + }, + { + "text": "iqiyi.com", + "boxes": [ + 602, + 220, + 662, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1223, + 668, + 1419, + 737 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 748, + 772, + 792, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ] + }, + { + "frame_id": "60_7_3", + "episode_id": 60, + "env_id": 7, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.097358703613281, + "advantages": -0.4249677062034607, + "generated_text": "{\n \"intent\": \"To start interacting with the contents of the LibreOffice Math application by exploring the symbols on the left sidebar.\",\n \"action\": \"DoubleClick(100, 135)\"\n}", + "action_string": "DoubleClick(100, 135)", + "intent_string": "To start interacting with the contents of the LibreOffice Math application by exploring the symbols on the left sidebar.", + "before_screen_ocr_results": [ + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": "---- I", + "boxes": [ + 6, + 50, + 646, + 85 + ] + }, + { + "text": "S", + "boxes": [ + 714, + 54, + 746, + 85 + ] + }, + { + "text": "Unary/Binary Operators", + "boxes": [ + 18, + 105, + 172, + 123 + ] + }, + { + "text": "r□", + "boxes": [ + 88, + 143, + 144, + 178 + ] + }, + { + "text": "+口", + "boxes": [ + 36, + 153, + 64, + 175 + ] + }, + { + "text": "+口", + "boxes": [ + 174, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 153, + 282, + 179 + ] + }, + { + "text": "□+□冏□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 86, + 282 + ] + }, + { + "text": "号", + "boxes": [ + 104, + 252, + 134, + 286 + ] + }, + { + "text": "□÷□口", + "boxes": [ + 156, + 254, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": " %", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 407, + 78, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": "----- I", + "boxes": [ + 6, + 50, + 644, + 85 + ] + }, + { + "text": "S", + "boxes": [ + 715, + 55, + 742, + 83 + ] + }, + { + "text": " Unary/Binary Operators", + "boxes": [ + 16, + 103, + 174, + 127 + ] + }, + { + "text": "+口", + "boxes": [ + 34, + 155, + 66, + 175 + ] + }, + { + "text": "-□", + "boxes": [ + 98, + 151, + 140, + 177 + ] + }, + { + "text": "+口", + "boxes": [ + 170, + 151, + 212, + 177 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 151, + 282, + 179 + ] + }, + { + "text": "□+□□-□□·口□×□", + "boxes": [ + 16, + 204, + 294, + 240 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 96, + 282 + ] + }, + { + "text": "廿", + "boxes": [ + 102, + 252, + 138, + 288 + ] + }, + { + "text": "□÷口口", + "boxes": [ + 156, + 252, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": " %", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 407, + 78, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ] + }, + { + "frame_id": "60_0_4", + "episode_id": 60, + "env_id": 0, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.839771270751953, + "advantages": 0.8578306436538696, + "generated_text": "{\n \"intent\": \"The user wants to explore one of the templates available in LibreOffice.\",\n \"action\": \"DoubleClick(570, 250)\"\n}", + "action_string": "DoubleClick(570, 250)", + "intent_string": "The user wants to explore one of the templates available in LibreOffice.", + "before_screen_ocr_results": [ + { + "text": "Libreoffic", + "boxes": [ + 914, + 12, + 992, + 30 + ] + }, + { + "text": "口", + "boxes": [ + 1800, + 8, + 1860, + 34 + ] + }, + { + "text": "Tools Help ", + "boxes": [ + 100, + 32, + 190, + 58 + ] + }, + { + "text": " Open File ", + "boxes": [ + 60, + 62, + 208, + 105 + ] + }, + { + "text": "", + "boxes": [ + 1606, + 111, + 1636, + 147 + ] + }, + { + "text": "", + "boxes": [ + 1644, + 105, + 1688, + 141 + ] + }, + { + "text": " Remote Files", + "boxes": [ + 100, + 117, + 236, + 149 + ] + }, + { + "text": "", + "boxes": [ + 1608, + 135, + 1644, + 163 + ] + }, + { + "text": "", + "boxes": [ + 1646, + 141, + 1728, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1602, + 159, + 1642, + 218 + ] + }, + { + "text": "Mou", + "boxes": [ + 8, + 171, + 54, + 199 + ] + }, + { + "text": "?", + "boxes": [ + 68, + 183, + 102, + 210 + ] + }, + { + "text": "Recent Files ", + "boxes": [ + 94, + 183, + 228, + 214 + ] + }, + { + "text": "三", + "boxes": [ + 1608, + 206, + 1634, + 230 + ] + }, + { + "text": " Templates", + "boxes": [ + 85, + 219, + 217, + 273 + ] + }, + { + "text": "口", + "boxes": [ + 74, + 230, + 102, + 258 + ] + }, + { + "text": "Terr", + "boxes": [ + 20, + 280, + 54, + 308 + ] + }, + { + "text": "Create:", + "boxes": [ + 66, + 284, + 140, + 316 + ] + }, + { + "text": "国", + "boxes": [ + 66, + 324, + 102, + 359 + ] + }, + { + "text": "Writer Document ", + "boxes": [ + 102, + 323, + 280, + 358 + ] + }, + { + "text": "scard with log", + "boxes": [ + 1360, + 336, + 1454, + 359 + ] + }, + { + "text": "國", + "boxes": [ + 68, + 369, + 100, + 405 + ] + }, + { + "text": "Calc Spreadsheet ", + "boxes": [ + 104, + 369, + 282, + 405 + ] + }, + { + "text": "Calc", + "boxes": [ + 12, + 389, + 54, + 417 + ] + }, + { + "text": "回", + "boxes": [ + 64, + 411, + 102, + 453 + ] + }, + { + "text": "Impress Presentation ", + "boxes": [ + 102, + 415, + 316, + 453 + ] + }, + { + "text": "", + "boxes": [ + 414, + 433, + 432, + 447 + ] + }, + { + "text": "D", + "boxes": [ + 70, + 461, + 100, + 498 + ] + }, + { + "text": " Draw Drawing", + "boxes": [ + 102, + 464, + 250, + 501 + ] + }, + { + "text": "·Door si met, cnsetetradipisi e", + "boxes": [ + 1272, + 461, + 1474, + 484 + ] + }, + { + "text": "Lorem Ipsum", + "boxes": [ + 1626, + 459, + 1742, + 488 + ] + }, + { + "text": "Vestil", + "boxes": [ + 1284, + 480, + 1306, + 494 + ] + }, + { + "text": "m consequat mi quis pretiun", + "boxes": [ + 1324, + 478, + 1452, + 496 + ] + }, + { + "text": "sempe", + "boxes": [ + 1454, + 480, + 1486, + 494 + ] + }, + { + "text": "Libr", + "boxes": [ + 16, + 500, + 56, + 528 + ] + }, + { + "text": " orci ac neque venenalis", + "boxes": [ + 1332, + 494, + 1446, + 512 + ] + }, + { + "text": "", + "boxes": [ + 66, + 508, + 106, + 546 + ] + }, + { + "text": " Math Formula", + "boxes": [ + 98, + 514, + 242, + 544 + ] + }, + { + "text": "Base Database", + "boxes": [ + 106, + 560, + 256, + 592 + ] + }, + { + "text": "Wr", + "boxes": [ + 34, + 613, + 54, + 629 + ] + }, + { + "text": "Lorem ipsum", + "boxes": [ + 1342, + 748, + 1428, + 774 + ] + }, + { + "text": "Lorem Ipsum", + "boxes": [ + 1640, + 746, + 1726, + 772 + ] + }, + { + "text": "dolor sit amet", + "boxes": [ + 1288, + 778, + 1362, + 796 + ] + }, + { + "text": "consectetur adipiscing elit ", + "boxes": [ + 1286, + 794, + 1428, + 812 + ] + }, + { + "text": "Vestiblu", + "boxes": [ + 1286, + 812, + 1334, + 838 + ] + }, + { + "text": " consequat mi quis pretiun", + "boxes": [ + 1344, + 812, + 1482, + 830 + ] + }, + { + "text": "Bright Blue", + "boxes": [ + 452, + 959, + 532, + 991 + ] + }, + { + "text": " Classy Red", + "boxes": [ + 748, + 959, + 832, + 991 + ] + }, + { + "text": "DNA", + "boxes": [ + 1068, + 961, + 1104, + 989 + ] + }, + { + "text": "Focus", + "boxes": [ + 1366, + 963, + 1406, + 989 + ] + }, + { + "text": "Forestbird", + "boxes": [ + 1650, + 963, + 1718, + 989 + ] + }, + { + "text": "Help ", + "boxes": [ + 106, + 979, + 148, + 1007 + ] + }, + { + "text": "Extensions", + "boxes": [ + 218, + 981, + 296, + 1007 + ] + }, + { + "text": "Libreoffice", + "boxes": [ + 218, + 1050, + 288, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "·目X自", + "boxes": [ + 12, + 52, + 362, + 87 + ] + }, + { + "text": "2", + "boxes": [ + 842, + 60, + 876, + 79 + ] + }, + { + "text": "Noto san", + "boxes": [ + 142, + 93, + 290, + 123 + ] + }, + { + "text": "a", + "boxes": [ + 484, + 101, + 512, + 115 + ] + }, + { + "text": "D", + "boxes": [ + 1890, + 208, + 1912, + 234 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": " Put", + "boxes": [ + 1176, + 252, + 1222, + 280 + ] + }, + { + "text": "your", + "boxes": [ + 1172, + 280, + 1230, + 314 + ] + }, + { + "text": "LOGO", + "boxes": [ + 1164, + 306, + 1240, + 343 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 310, + 1914, + 343 + ] + }, + { + "text": "here", + "boxes": [ + 1174, + 339, + 1228, + 369 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 484, + 698, + 510 + ] + }, + { + "text": "Dear <RECIPIENT's NAME>", + "boxes": [ + 608, + 526, + 802, + 550 + ] + }, + { + "text": "Loremipsum dolorsit amet. consectetur adiniscng eit sed do giusmodteupr nididut ut labore et", + "boxes": [ + 606, + 566, + 1256, + 596 + ] + }, + { + "text": "dolore magna aliqua Ut enim ad minim yeniam, guis nostrud exercitation ylamc laboris nisi yt", + "boxes": [ + 606, + 586, + 1256, + 615 + ] + }, + { + "text": "lujexttdte", + "boxes": [ + 604, + 606, + 1258, + 641 + ] + }, + { + "text": "dfed", + "boxes": [ + 604, + 627, + 1258, + 663 + ] + }, + { + "text": "deserunt molit anim id est Jaborum.", + "boxes": [ + 606, + 653, + 842, + 683 + ] + }, + { + "text": "Loremipsum dolorsit amet. consectetur adiiscina elt sed do eiusmod temprncididut ut labore et", + "boxes": [ + 606, + 693, + 1256, + 723 + ] + }, + { + "text": "solore magna slaua. Ut eni ad minim yeniam, guis nstrud exeritation wanc aboris ni t", + "boxes": [ + 602, + 711, + 1258, + 748 + ] + }, + { + "text": "aliquip exea commdo sonsequat.", + "boxes": [ + 604, + 734, + 832, + 768 + ] + }, + { + "text": "We are looking forward to hearing from you soon.", + "boxes": [ + 606, + 774, + 926, + 806 + ] + } + ] + }, + { + "frame_id": "60_1_4", + "episode_id": 60, + "env_id": 1, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.969725608825684, + "advantages": 0.21728594601154327, + "generated_text": "{\n \"intent\": \"To maximize screen utilization and explore various web content, clicking on the 'New Tab' may present new browser pages to browse.\",\n \"action\": \"DoubleClick(370, 19)\"\n}", + "action_string": "DoubleClick(370, 19)", + "intent_string": "To maximize screen utilization and explore various web content, clicking on the 'New Tab' may present new browser pages to browse.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 227, + 25, + 317, + 62 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 202, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 556, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 538, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 816 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 522, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 358, + 6, + 452, + 36 + ] + }, + { + "text": "口", + "boxes": [ + 482, + 10, + 554, + 36 + ] + }, + { + "text": "Flle", + "boxes": [ + 268, + 40, + 294, + 54 + ] + }, + { + "text": "Edit View", + "boxes": [ + 298, + 36, + 380, + 58 + ] + }, + { + "text": "Calculator Help ", + "boxes": [ + 390, + 36, + 510, + 60 + ] + }, + { + "text": "New Tab", + "boxes": [ + 572, + 34, + 642, + 52 + ] + }, + { + "text": "3 New Tab", + "boxes": [ + 784, + 32, + 870, + 56 + ] + }, + { + "text": "Firefox", + "boxes": [ + 19, + 59, + 97, + 98 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 568, + 103, + 714, + 121 + ] + }, + { + "text": "etting Started", + "boxes": [ + 768, + 101, + 868, + 125 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 898, + 250, + 1002, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 806, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "N", + "boxes": [ + 1094, + 415, + 1174, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 16, + 500, + 92, + 532 + ] + }, + { + "text": "Apple", + "boxes": [ + 752, + 492, + 796, + 518 + ] + }, + { + "text": "bing", + "boxes": [ + 882, + 498, + 908, + 512 + ] + }, + { + "text": "Amazon", + "boxes": [ + 984, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": "", + "boxes": [ + 26, + 546, + 88, + 616 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 822, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1464, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search - Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1622, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_4", + "episode_id": 60, + "env_id": 2, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.3976411819458, + "advantages": -0.20389384031295776, + "generated_text": "{\n \"intent\": \"By clicking the search bar, I aim to explore search functionality and understand the browser's capabilities.\",\n \"action\": \"DoubleClick(795, 302)\"\n}", + "action_string": "DoubleClick(795, 302)", + "intent_string": "By clicking the search bar, I aim to explore search functionality and understand the browser's capabilities.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1000, + 536, + 1034, + 550 + ] + }, + { + "text": "ifeng", + "boxes": [ + 1118, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFiref", + "boxes": [ + 900, + 2, + 1006, + 26 + ] + }, + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks...", + "boxes": [ + 12, + 99, + 166, + 123 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web ", + "boxes": [ + 658, + 286, + 774, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 524, + 488, + 562, + 528 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_3_4", + "episode_id": 60, + "env_id": 3, + "step": 4, + "format_rewards": 1.0, + "rewards": 8.915294647216797, + "advantages": -0.5590068101882935, + "generated_text": "{\n \"intent\": \"To begin interacting with the browser more extensively, I will open a new tab by clicking on the '+' button to gain access to different web pages and functionalities.\",\n \"action\": \"DoubleClick(268, 40)\"\n}", + "action_string": "DoubleClick(268, 40)", + "intent_string": "To begin interacting with the browser more extensively, I will open a new tab by clicking on the '+' button to gain access to different web pages and functionalities.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 19, + 23, + 99, + 56 + ] + }, + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 14, + 103, + 158, + 121 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Ngw Tab", + "boxes": [ + 227, + 25, + 319, + 62 + ] + }, + { + "text": "→e", + "boxes": [ + 46, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_4_4", + "episode_id": 60, + "env_id": 4, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.06021785736084, + "advantages": 0.2839081287384033, + "generated_text": "{\n \"intent\": \"I will start by clicking on one of the top sites or highlights to see a different webpage, expanding my exploration of the current application.\",\n \"action\": \"DoubleClick(773, 482)\"\n}", + "action_string": "DoubleClick(773, 482)", + "intent_string": "I will start by clicking on one of the top sites or highlights to see a different webpage, expanding my exploration of the current application.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1028 + ] + } + ] + }, + { + "frame_id": "60_5_4", + "episode_id": 60, + "env_id": 5, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.94420051574707, + "advantages": 0.9347135424613953, + "generated_text": "{\n \"intent\": \"The current screenshot shows a web browser window open to Microsoft Bing. Besides exploring different parts of the Bing website, we could also considering clicking on other elements like search results or relevant articles, which might offer further exploration opportunities.\",\n \"action\": \"DoubleClick(576, 205)\"\n}", + "action_string": "DoubleClick(576, 205)", + "intent_string": "The current screenshot shows a web browser window open to Microsoft Bing. Besides exploring different parts of the Bing website, we could also considering clicking on other elements like search results or relevant articles, which might offer further exploration opportunities.", + "before_screen_ocr_results": [ + { + "text": "Search-Microsoft Bing×+", + "boxes": [ + 26, + 32, + 244, + 50 + ] + }, + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1872, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "mages", + "boxes": [ + 254, + 123, + 292, + 143 + ] + }, + { + "text": "Translate", + "boxes": [ + 402, + 119, + 454, + 145 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 490, + 189, + 638, + 220 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 236, + 614, + 266 + ] + }, + { + "text": "中忠空", + "boxes": [ + 1538, + 357, + 1574, + 387 + ] + }, + { + "text": "福", + "boxes": [ + 1875, + 371, + 1897, + 393 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 389, + 1634, + 421 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 439, + 274, + 465 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 480, + 1552, + 512 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1456, + 516, + 1506, + 560 + ] + }, + { + "text": "K", + "boxes": [ + 1592, + 522, + 1630, + 550 + ] + }, + { + "text": "+2.0936 ", + "boxes": [ + 1632, + 514, + 1700, + 562 + ] + }, + { + "text": "挑战烽火绿巾白内里的枪", + "boxes": [ + 805, + 525, + 1121, + 597 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1456, + 564, + 1516, + 608 + ] + }, + { + "text": " 6123 ", + "boxes": [ + 1624, + 558, + 1706, + 611 + ] + }, + { + "text": "", + "boxes": [ + 1592, + 572, + 1638, + 600 + ] + }, + { + "text": "?", + "boxes": [ + 1686, + 576, + 1712, + 596 + ] + }, + { + "text": "时中国A50", + "boxes": [ + 1449, + 601, + 1551, + 658 + ] + }, + { + "text": "1387% ", + "boxes": [ + 1624, + 604, + 1722, + 657 + ] + }, + { + "text": "W", + "boxes": [ + 1589, + 618, + 1615, + 647 + ] + }, + { + "text": "E", + "boxes": [ + 204, + 639, + 226, + 653 + ] + }, + { + "text": "科普先锋·17小时", + "boxes": [ + 224, + 635, + 334, + 659 + ] + }, + { + "text": "金融界汽车·9小时", + "boxes": [ + 1136, + 633, + 1280, + 663 + ] + }, + { + "text": "太阳系大秘密！恒星擦肩乱轨道，第九行星真", + "boxes": [ + 202, + 661, + 794, + 703 + ] + }, + { + "text": "赛力斯申请高速过弯转向不足", + "boxes": [ + 1136, + 655, + 1408, + 691 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 655, + 1510, + 701 + ] + }, + { + "text": "[", + "boxes": [ + 1590, + 657, + 1630, + 695 + ] + }, + { + "text": "+3.28%", + "boxes": [ + 1626, + 653, + 1700, + 701 + ] + }, + { + "text": "控制专利，提高稳定性和安全", + "boxes": [ + 1134, + 683, + 1404, + 713 + ] + }, + { + "text": "不存在？", + "boxes": [ + 198, + 697, + 326, + 741 + ] + }, + { + "text": "D", + "boxes": [ + 828, + 705, + 854, + 727 + ] + }, + { + "text": "一点视频/九尾说游戏", + "boxes": [ + 842, + 701, + 972, + 731 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 699, + 1522, + 748 + ] + }, + { + "text": "性", + "boxes": [ + 1136, + 711, + 1168, + 741 + ] + }, + { + "text": "挑战烽火荣都绿门或白门里的", + "boxes": [ + 822, + 721, + 1098, + 758 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 717, + 1628, + 739 + ] + }, + { + "text": "1", + "boxes": [ + 200, + 742, + 276, + 768 + ] + }, + { + "text": "", + "boxes": [ + 1134, + 739, + 1192, + 770 + ] + }, + { + "text": "枪，千万别是M24和98K ！", + "boxes": [ + 818, + 750, + 1082, + 786 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1620, + 741, + 1728, + 772 + ] + }, + { + "text": "长春市", + "boxes": [ + 1462, + 800, + 1526, + 820 + ] + }, + { + "text": "江", + "boxes": [ + 502, + 824, + 562, + 864 + ] + }, + { + "text": "心", + "boxes": [ + 502, + 856, + 610, + 887 + ] + }, + { + "text": "更预示着全球汽车", + "boxes": [ + 868, + 929, + 954, + 947 + ] + }, + { + "text": "米拍摄影社区·15 小时", + "boxes": [ + 198, + 949, + 362, + 979 + ] + }, + { + "text": "一点资讯·3周", + "boxes": [ + 512, + 949, + 634, + 979 + ] + }, + { + "text": "生活研究所", + "boxes": [ + 1138, + 951, + 1228, + 977 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 967, + 850, + 983 + ] + }, + { + "text": "一点视频/防务急先锋", + "boxes": [ + 844, + 961, + 972, + 991 + ] + }, + { + "text": "国境之南-南极村", + "boxes": [ + 202, + 975, + 358, + 1005 + ] + }, + { + "text": "穿越火线老玩家记忆里的那些", + "boxes": [ + 510, + 969, + 786, + 1009 + ] + }, + { + "text": "地球上的天赐谷地", + "boxes": [ + 1134, + 966, + 1308, + 1009 + ] + }, + { + "text": "爆破步枪现在怀旧服也多少", + "boxes": [ + 508, + 993, + 788, + 1032 + ] + }, + { + "text": "中国汽车技术崛起", + "boxes": [ + 824, + 987, + 994, + 1016 + ] + }, + { + "text": "t/vi-GWaTLWQYysjtTw?vid=vi-GWaTLWQYysJtTW&pr", + "boxes": [ + 440, + 1018, + 782, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Search - Microsoft Bing", + "boxes": [ + 26, + 32, + 188, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 10, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 18, + 109, + 184, + 151 + ] + }, + { + "text": " The Venetia dolce vit?", + "boxes": [ + 1454, + 671, + 1634, + 707 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 186, + 723, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1454, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1456, + 802, + 1506, + 846 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 891 + ] + }, + { + "text": "6102 ", + "boxes": [ + 1622, + 844, + 1702, + 897 + ] + }, + { + "text": "+", + "boxes": [ + 1692, + 862, + 1710, + 878 + ] + }, + { + "text": "13.87% ", + "boxes": [ + 1626, + 893, + 1700, + 943 + ] + }, + { + "text": "智作说车·23小时", + "boxes": [ + 826, + 917, + 964, + 947 + ] + }, + { + "text": "生活研究所", + "boxes": [ + 1131, + 913, + 1231, + 950 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 824, + 941, + 1076, + 977 + ] + }, + { + "text": "地球上的天赐谷地", + "boxes": [ + 1134, + 941, + 1308, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1510, + 991 + ] + }, + { + "text": "我国最窑的领土宽度仅有8米米距离日本海", + "boxes": [ + 196, + 945, + 782, + 987 + ] + }, + { + "text": "[", + "boxes": [ + 1588, + 939, + 1638, + 983 + ] + }, + { + "text": "( +32629", + "boxes": [ + 1619, + 932, + 1703, + 992 + ] + }, + { + "text": "只有10公里！", + "boxes": [ + 192, + 983, + 404, + 1018 + ] + }, + { + "text": "中国安", + "boxes": [ + 1452, + 981, + 1522, + 1040 + ] + }, + { + "text": "L", + "boxes": [ + 1585, + 990, + 1635, + 1034 + ] + }, + { + "text": "14%0 ", + "boxes": [ + 1632, + 983, + 1722, + 1032 + ] + }, + { + "text": "本海只有10公里！", + "boxes": [ + 186, + 1004, + 470, + 1049 + ] + }, + { + "text": "李丢白选股建议", + "boxes": [ + 1628, + 1028, + 1716, + 1046 + ] + }, + { + "text": "earch - Micr.", + "boxes": [ + 224, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "60_6_4", + "episode_id": 60, + "env_id": 6, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.847640037536621, + "advantages": 0.12740415334701538, + "generated_text": "{\n \"intent\": \"Exploring web content by visiting popular websites.\",\n \"action\": \"DoubleClick(531, 117)\"\n}", + "action_string": "DoubleClick(531, 117)", + "intent_string": "Exploring web content by visiting popular websites.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 60, + 524, + 95 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 14, + 103, + 156, + 121 + ] + }, + { + "text": " Apple - htt:/w.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "a Amazon — http://www.amazon.com", + "boxes": [ + 256, + 179, + 510, + 202 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 252, + 210, + 592, + 242 + ] + }, + { + "text": "iqiyi.com", + "boxes": [ + 602, + 220, + 662, + 238 + ] + }, + { + "text": "知zhihu -", + "boxes": [ + 254, + 250, + 328, + 274 + ] + }, + { + "text": "G ifeng - ifeng.com", + "boxes": [ + 254, + 284, + 402, + 314 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1270, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1223, + 668, + 1419, + 737 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 748, + 772, + 792, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 822, + 318, + 932, + 361 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "60_7_4", + "episode_id": 60, + "env_id": 7, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.006380081176758, + "advantages": -0.4919479489326477, + "generated_text": "{\n \"intent\": \"To start interacting with the content by selecting an element from the toolbar or creating something in the document, moving the mouse cursor to a common location would allow access to functionality.\",\n \"action\": \"Move(100, 50)\"\n}", + "action_string": "Move(100, 50)", + "intent_string": "To start interacting with the content by selecting an element from the toolbar or creating something in the document, moving the mouse cursor to a common location would allow access to functionality.", + "before_screen_ocr_results": [ + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": "----- I", + "boxes": [ + 6, + 50, + 644, + 85 + ] + }, + { + "text": "S", + "boxes": [ + 715, + 55, + 742, + 83 + ] + }, + { + "text": " Unary/Binary Operators", + "boxes": [ + 16, + 103, + 174, + 127 + ] + }, + { + "text": "+口", + "boxes": [ + 34, + 155, + 66, + 175 + ] + }, + { + "text": "-□", + "boxes": [ + 98, + 151, + 140, + 177 + ] + }, + { + "text": "+口", + "boxes": [ + 170, + 151, + 212, + 177 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 151, + 282, + 179 + ] + }, + { + "text": "□+□□-□□·口□×□", + "boxes": [ + 16, + 204, + 294, + 240 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 94, + 282 + ] + }, + { + "text": "廿", + "boxes": [ + 102, + 252, + 138, + 288 + ] + }, + { + "text": "□÷口口", + "boxes": [ + 156, + 252, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": " %", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 407, + 78, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": "-m e XEnGd sI @@", + "boxes": [ + 4, + 48, + 648, + 89 + ] + }, + { + "text": " Unary/Binary Operators", + "boxes": [ + 16, + 103, + 174, + 127 + ] + }, + { + "text": "+口", + "boxes": [ + 30, + 153, + 70, + 179 + ] + }, + { + "text": "-□", + "boxes": [ + 96, + 153, + 142, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 168, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 151, + 282, + 179 + ] + }, + { + "text": "□+□冏□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*口冏", + "boxes": [ + 22, + 256, + 106, + 282 + ] + }, + { + "text": "品", + "boxes": [ + 100, + 250, + 140, + 288 + ] + }, + { + "text": "÷□□口", + "boxes": [ + 154, + 250, + 286, + 282 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": "00 0。%", + "boxes": [ + 8, + 331, + 292, + 393 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 407, + 78, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ] + }, + { + "frame_id": "60_0_5", + "episode_id": 60, + "env_id": 0, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.800586700439453, + "advantages": 0.09276256710290909, + "generated_text": "{\n \"intent\": \"To continue exploring the LibreOffice Writer interface and possibly locate more features or options.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring the LibreOffice Writer interface and possibly locate more features or options.", + "before_screen_ocr_results": [ + { + "text": "·目X自", + "boxes": [ + 12, + 52, + 362, + 87 + ] + }, + { + "text": "2", + "boxes": [ + 842, + 60, + 876, + 79 + ] + }, + { + "text": "Noto san", + "boxes": [ + 142, + 93, + 290, + 123 + ] + }, + { + "text": "a", + "boxes": [ + 484, + 101, + 512, + 115 + ] + }, + { + "text": "D", + "boxes": [ + 1890, + 208, + 1912, + 234 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": " Put", + "boxes": [ + 1176, + 252, + 1222, + 280 + ] + }, + { + "text": "your", + "boxes": [ + 1172, + 280, + 1230, + 314 + ] + }, + { + "text": "LOGO", + "boxes": [ + 1164, + 306, + 1240, + 343 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 310, + 1914, + 343 + ] + }, + { + "text": "here", + "boxes": [ + 1174, + 339, + 1228, + 369 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 484, + 698, + 510 + ] + }, + { + "text": "Dear <TITLE><RECIPIENT's NAME>", + "boxes": [ + 608, + 526, + 802, + 550 + ] + }, + { + "text": "Loremipsum dolorsit amet. consectetur adiniscng eit sed do giusmodteupr nididut ut labore et", + "boxes": [ + 606, + 566, + 1256, + 596 + ] + }, + { + "text": "dolore magna aliqua Ut enim ad minim yeniam, guis nostrud exercitation ylamc laboris nisi yt", + "boxes": [ + 606, + 586, + 1256, + 615 + ] + }, + { + "text": "lujexttdte", + "boxes": [ + 604, + 606, + 1258, + 641 + ] + }, + { + "text": "dfed", + "boxes": [ + 604, + 627, + 1258, + 663 + ] + }, + { + "text": "deserunt molit anim id est Jaborum.", + "boxes": [ + 606, + 653, + 842, + 683 + ] + }, + { + "text": "Loremipsum dolorsit amet. consectetur adiiscina elt sed do eiusmod temprncididut ut labore et", + "boxes": [ + 606, + 693, + 1256, + 723 + ] + }, + { + "text": "solore magna slaua. Ut eni ad minim yeniam, guis nstrud exeritation wanc aboris ni t", + "boxes": [ + 602, + 711, + 1258, + 748 + ] + }, + { + "text": "aliquip exea commdo sonsequat.", + "boxes": [ + 604, + 734, + 832, + 768 + ] + }, + { + "text": "We are looking forward to hearing from you soon.", + "boxes": [ + 606, + 774, + 926, + 806 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit I", + "boxes": [ + 8, + 26, + 92, + 44 + ] + }, + { + "text": "Styles", + "boxes": [ + 252, + 30, + 292, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 508, + 26, + 550, + 46 + ] + }, + { + "text": "--目·@x自 ", + "boxes": [ + 10, + 52, + 378, + 87 + ] + }, + { + "text": "二", + "boxes": [ + 380, + 60, + 408, + 79 + ] + }, + { + "text": "-、", + "boxes": [ + 410, + 56, + 520, + 79 + ] + }, + { + "text": " Abc", + "boxes": [ + 514, + 56, + 590, + 81 + ] + }, + { + "text": "m·", + "boxes": [ + 608, + 54, + 728, + 83 + ] + }, + { + "text": "·国→田", + "boxes": [ + 716, + 52, + 1256, + 87 + ] + }, + { + "text": "Noto sans", + "boxes": [ + 138, + 93, + 296, + 123 + ] + }, + { + "text": "aaa a ab a,", + "boxes": [ + 438, + 93, + 654, + 123 + ] + }, + { + "text": "一", + "boxes": [ + 804, + 101, + 820, + 115 + ] + }, + { + "text": "小山三··mm三·州", + "boxes": [ + 830, + 93, + 1212, + 123 + ] + }, + { + "text": "D", + "boxes": [ + 1885, + 205, + 1913, + 235 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 236, + 698, + 262 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": "Dea", + "boxes": [ + 612, + 282, + 634, + 296 + ] + }, + { + "text": "Loremipm dolrst me sonectetracpisinlt seddo gusmod teurnciddt tlaeet", + "boxes": [ + 604, + 314, + 1258, + 349 + ] + }, + { + "text": "O", + "boxes": [ + 1878, + 306, + 1918, + 346 + ] + }, + { + "text": "sloremagn3lcuaUt erimadminimyenimsuisnstrudxeritinencisniit", + "boxes": [ + 602, + 333, + 1256, + 371 + ] + }, + { + "text": "xtt", + "boxes": [ + 604, + 357, + 1258, + 393 + ] + }, + { + "text": "dlrefgitaraturxceter sintaeat cuidatat non pridet st inuauiffc", + "boxes": [ + 606, + 383, + 1256, + 413 + ] + }, + { + "text": "deserwnt molitam id est lbowm.", + "boxes": [ + 602, + 401, + 844, + 439 + ] + }, + { + "text": "Loremipsm dolorsi me. consectetr acinsin lt sedo gusmod teuprnciddut utlareet", + "boxes": [ + 604, + 441, + 1258, + 476 + ] + }, + { + "text": "solore magna sliaua. ut eni ad minim yeniam, guis nostrud exeritation wlanc laboris nisi yt", + "boxes": [ + 602, + 463, + 1260, + 500 + ] + }, + { + "text": "aliquip ex ea sommodo. consequat.", + "boxes": [ + 606, + 490, + 832, + 520 + ] + }, + { + "text": "We are looking forward to hearing from you soon.", + "boxes": [ + 606, + 528, + 934, + 556 + ] + }, + { + "text": "Yours sincerely.", + "boxes": [ + 608, + 572, + 710, + 598 + ] + }, + { + "text": "<SENDER'S NAME>", + "boxes": [ + 608, + 611, + 710, + 635 + ] + } + ] + }, + { + "frame_id": "60_1_5", + "episode_id": 60, + "env_id": 1, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.200587272644043, + "advantages": -0.3489688038825989, + "generated_text": "{\n \"intent\": \"To explore various applications on the desktop by interacting with their icons, starting with 'Firefox'.\",\n \"action\": \"DoubleClick(358, 456)\"\n}", + "action_string": "DoubleClick(358, 456)", + "intent_string": "To explore various applications on the desktop by interacting with their icons, starting with 'Firefox'.", + "before_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 358, + 6, + 452, + 36 + ] + }, + { + "text": "口", + "boxes": [ + 482, + 10, + 554, + 36 + ] + }, + { + "text": "Flle", + "boxes": [ + 268, + 40, + 294, + 54 + ] + }, + { + "text": "Edit View", + "boxes": [ + 298, + 36, + 380, + 58 + ] + }, + { + "text": "Calculator Help ", + "boxes": [ + 390, + 36, + 510, + 60 + ] + }, + { + "text": "New Tab", + "boxes": [ + 572, + 34, + 642, + 52 + ] + }, + { + "text": "3 New Tab", + "boxes": [ + 784, + 32, + 870, + 56 + ] + }, + { + "text": "Firefox", + "boxes": [ + 19, + 59, + 97, + 98 + ] + }, + { + "text": "Q pearch with Bing or er", + "boxes": [ + 784, + 68, + 968, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 568, + 103, + 714, + 121 + ] + }, + { + "text": "etting Started", + "boxes": [ + 768, + 101, + 868, + 125 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 898, + 250, + 1002, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 808, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "N", + "boxes": [ + 1094, + 415, + 1174, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "“", + "boxes": [ + 789, + 457, + 818, + 488 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 16, + 500, + 92, + 532 + ] + }, + { + "text": "Apple", + "boxes": [ + 752, + 492, + 796, + 518 + ] + }, + { + "text": "bing", + "boxes": [ + 882, + 498, + 908, + 512 + ] + }, + { + "text": "Amazon", + "boxes": [ + 984, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": "", + "boxes": [ + 26, + 546, + 88, + 616 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 822, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1464, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search -Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 284, + 34 + ] + }, + { + "text": "Flle", + "boxes": [ + 268, + 40, + 292, + 54 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 298, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→", + "boxes": [ + 584, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 904, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 808, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "Apple", + "boxes": [ + 752, + 492, + 796, + 518 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 90, + 528 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 822, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "?", + "boxes": [ + 1621, + 703, + 1650, + 732 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search -Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 29, + 825, + 89, + 867 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "alculato", + "boxes": [ + 380, + 1052, + 428, + 1070 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_5", + "episode_id": 60, + "env_id": 2, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.49228286743164, + "advantages": 0.6020028591156006, + "generated_text": "{\n \"intent\": \"To start investigating one of the popular websites listed for Top Sites, which can provide valuable content such as news, entertainment, or tools.\",\n \"action\": \"DoubleClick(782, 499)\"\n}", + "action_string": "DoubleClick(782, 499)", + "intent_string": "To start investigating one of the popular websites listed for Top Sites, which can provide valuable content such as news, entertainment, or tools.", + "before_screen_ocr_results": [ + { + "text": "MozillaFiref", + "boxes": [ + 900, + 2, + 1006, + 26 + ] + }, + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks...", + "boxes": [ + 12, + 99, + 166, + 123 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web ", + "boxes": [ + 658, + 286, + 774, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 524, + 488, + 562, + 528 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "You are on an", + "boxes": [ + 510, + 461, + 596, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1336, + 486, + 1418, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 612, + 530, + 792, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 530, + 1288, + 560 + ] + }, + { + "text": " Shop deals in Fashion ", + "boxes": [ + 1348, + 526, + 1574, + 562 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 610, + 889, + 696, + 915 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 1350, + 889, + 1430, + 913 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1352, + 969, + 1508, + 999 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1028 + ] + } + ] + }, + { + "frame_id": "60_3_5", + "episode_id": 60, + "env_id": 3, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.066974639892578, + "advantages": -0.4473370313644409, + "generated_text": "{\n \"intent\": \"To start exploring different websites by accessing one of the top or highlights via a double-click.\",\n \"action\": \"DoubleClick(710, 773)\"\n}", + "action_string": "DoubleClick(710, 773)", + "intent_string": "To start exploring different websites by accessing one of the top or highlights via a double-click.", + "before_screen_ocr_results": [ + { + "text": " Ngw Tab", + "boxes": [ + 227, + 25, + 317, + 62 + ] + }, + { + "text": "→C", + "boxes": [ + 42, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFiref", + "boxes": [ + 904, + 4, + 1004, + 22 + ] + }, + { + "text": "Q Search with Bing or ente", + "boxes": [ + 254, + 66, + 452, + 89 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 14, + 103, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 996, + 534, + 1036, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1226, + 673, + 1418, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 674, + 750, + 688, + 766 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 746, + 770, + 792, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_4_5", + "episode_id": 60, + "env_id": 4, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.153491973876953, + "advantages": 0.35257837176322937, + "generated_text": "{\n \"intent\": \"Clicking on the 'Hello, sign in' button will allow me to explore account-related features or additional pages available only after logging in.\",\n \"action\": \"Click(1648, 107)\"\n}", + "action_string": "Click(1648, 107)", + "intent_string": "Clicking on the 'Hello, sign in' button will allow me to explore account-related features or additional pages available only after logging in.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 249, + 63, + 313, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop for your home essentials", + "boxes": [ + 608, + 528, + 916, + 558 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1599, + 65, + 1659, + 90 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "60_5_5", + "episode_id": 60, + "env_id": 5, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.041971206665039, + "advantages": 0.2704745829105377, + "generated_text": "{\n \"intent\": \"The goal is to explore more of the desktop environment. Since I am already on a webpage, checking the navigation options or possibly interacting further within the search bar would be valuable, like scrolling or searching.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "The goal is to explore more of the desktop environment. Since I am already on a webpage, checking the navigation options or possibly interacting further within the search bar would be valuable, like scrolling or searching.", + "before_screen_ocr_results": [ + { + "text": " Search - Microsoft Bing", + "boxes": [ + 24, + 32, + 188, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 10, + 66, + 36, + 89 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 252, + 64, + 280, + 92 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 18, + 109, + 186, + 151 + ] + }, + { + "text": " The Venetian'dolce vita?", + "boxes": [ + 1454, + 671, + 1632, + 709 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 186, + 725, + 272, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1452, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1456, + 802, + 1506, + 846 + ] + }, + { + "text": "2093", + "boxes": [ + 1624, + 798, + 1700, + 846 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 891 + ] + }, + { + "text": "6102 ", + "boxes": [ + 1622, + 844, + 1702, + 897 + ] + }, + { + "text": "13, 87%", + "boxes": [ + 1626, + 891, + 1700, + 941 + ] + }, + { + "text": "智作说车·23小时", + "boxes": [ + 826, + 917, + 964, + 947 + ] + }, + { + "text": "生活研究所", + "boxes": [ + 1136, + 919, + 1228, + 945 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 824, + 941, + 1078, + 977 + ] + }, + { + "text": "地球上的天赐谷地", + "boxes": [ + 1134, + 941, + 1310, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 991 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "+329 ", + "boxes": [ + 1622, + 935, + 1702, + 991 + ] + }, + { + "text": "B", + "boxes": [ + 198, + 957, + 228, + 979 + ] + }, + { + "text": "人民资讯17小", + "boxes": [ + 228, + 955, + 324, + 977 + ] + }, + { + "text": "生态水库景如画", + "boxes": [ + 198, + 979, + 412, + 1022 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1452, + 981, + 1524, + 1038 + ] + }, + { + "text": "1420 ", + "boxes": [ + 1632, + 983, + 1700, + 1032 + ] + }, + { + "text": "Lr", + "boxes": [ + 1590, + 997, + 1630, + 1026 + ] + }, + { + "text": "查丢白选股建试", + "boxes": [ + 1626, + 1026, + 1718, + 1050 + ] + }, + { + "text": "?", + "boxes": [ + 178, + 1048, + 206, + 1070 + ] + }, + { + "text": "earch - Micr.", + "boxes": [ + 224, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Search - Microsoft Bing", + "boxes": [ + 26, + 32, + 188, + 50 + ] + }, + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 494, + 191, + 558, + 216 + ] + }, + { + "text": "国际版", + "boxes": [ + 576, + 191, + 634, + 216 + ] + }, + { + "text": " DISCOVER ", + "boxes": [ + 181, + 432, + 277, + 469 + ] + }, + { + "text": "×", + "boxes": [ + 1044, + 488, + 1066, + 502 + ] + }, + { + "text": "M自选股建议", + "boxes": [ + 1454, + 480, + 1554, + 512 + ] + }, + { + "text": "CF老玩家记忆里", + "boxes": [ + 312, + 516, + 660, + 578 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1456, + 516, + 1506, + 560 + ] + }, + { + "text": "K", + "boxes": [ + 1592, + 522, + 1630, + 550 + ] + }, + { + "text": "+ 0028 ", + "boxes": [ + 1630, + 514, + 1700, + 562 + ] + }, + { + "text": "+", + "boxes": [ + 1692, + 530, + 1710, + 546 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1456, + 562, + 1516, + 606 + ] + }, + { + "text": ",1024 ", + "boxes": [ + 1628, + 560, + 1702, + 609 + ] + }, + { + "text": "的那些爆破步枪", + "boxes": [ + 374, + 571, + 664, + 632 + ] + }, + { + "text": "", + "boxes": [ + 1592, + 572, + 1638, + 600 + ] + }, + { + "text": "?", + "boxes": [ + 364, + 584, + 398, + 611 + ] + }, + { + "text": "+", + "boxes": [ + 1690, + 578, + 1710, + 592 + ] + }, + { + "text": "时中国A50", + "boxes": [ + 1452, + 604, + 1550, + 657 + ] + }, + { + "text": "13 87%", + "boxes": [ + 1624, + 604, + 1704, + 657 + ] + }, + { + "text": "L", + "boxes": [ + 1594, + 623, + 1610, + 641 + ] + }, + { + "text": "智作说车+关注·23小时", + "boxes": [ + 828, + 635, + 1006, + 659 + ] + }, + { + "text": "生活研究所", + "boxes": [ + 1136, + 635, + 1228, + 661 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 822, + 655, + 1078, + 691 + ] + }, + { + "text": "地球上的天赐谷地", + "boxes": [ + 1134, + 655, + 1310, + 693 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 655, + 1510, + 701 + ] + }, + { + "text": "", + "boxes": [ + 1592, + 661, + 1628, + 691 + ] + }, + { + "text": "+3.28", + "boxes": [ + 1628, + 653, + 1700, + 701 + ] + }, + { + "text": "穿越火线老玩家记忆里的那些爆破步枪，现在", + "boxes": [ + 202, + 663, + 794, + 699 + ] + }, + { + "text": "怀旧服也多少玩家用了", + "boxes": [ + 202, + 697, + 496, + 739 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 699, + 1522, + 748 + ] + }, + { + "text": "-1.14%6 ±", + "boxes": [ + 1636, + 701, + 1722, + 746 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 717, + 1628, + 739 + ] + }, + { + "text": "21", + "boxes": [ + 1138, + 741, + 1182, + 766 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1622, + 744, + 1726, + 770 + ] + }, + { + "text": "长春市", + "boxes": [ + 1464, + 798, + 1526, + 824 + ] + }, + { + "text": "ACACMPSIT", + "boxes": [ + 1149, + 871, + 1237, + 906 + ] + }, + { + "text": "", + "boxes": [ + 1154, + 919, + 1306, + 949 + ] + }, + { + "text": "生活·家·20小时", + "boxes": [ + 202, + 949, + 328, + 977 + ] + }, + { + "text": "娱乐扒客·16小时", + "boxes": [ + 512, + 949, + 650, + 979 + ] + }, + { + "text": "盖世汽车·1周", + "boxes": [ + 1138, + 949, + 1256, + 979 + ] + }, + { + "text": "D", + "boxes": [ + 826, + 965, + 852, + 985 + ] + }, + { + "text": "一点视频/南哥说异·2天", + "boxes": [ + 846, + 961, + 998, + 991 + ] + }, + { + "text": "7万余亩土地喝上“营养水”", + "boxes": [ + 204, + 975, + 448, + 1005 + ] + }, + { + "text": "47岁温峥嵘高开叉亮片裙闪耀", + "boxes": [ + 512, + 971, + 792, + 1007 + ] + }, + { + "text": "汽车轻量化角逐，艾可迅碳纤", + "boxes": [ + 1134, + 969, + 1408, + 1009 + ] + }, + { + "text": "红军缴获鬼子10吨武器，无人", + "boxes": [ + 822, + 983, + 1102, + 1018 + ] + }, + { + "text": "戛纳！", + "boxes": [ + 514, + 999, + 578, + 1032 + ] + }, + { + "text": "维“亮剑”", + "boxes": [ + 1138, + 1001, + 1230, + 1032 + ] + }, + { + "text": "ews/other/理性看待风阻系数测试争", + "boxes": [ + 192, + 1018, + 400, + 1042 + ] + }, + { + "text": "会用，谁知一个新兵全会操作", + "boxes": [ + 828, + 1012, + 1096, + 1042 + ] + } + ] + }, + { + "frame_id": "60_6_5", + "episode_id": 60, + "env_id": 6, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.705842018127441, + "advantages": 0.02300967276096344, + "generated_text": "{\n \"intent\": \"To explore more products or sections of the Apple website by scrolling down.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore more products or sections of the Apple website by scrolling down.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 822, + 318, + 932, + 361 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Appl", + "boxes": [ + 28, + 34, + 62, + 48 + ] + }, + { + "text": "→", + "boxes": [ + 32, + 66, + 126, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Visiont for AirPodse Intv & Homece.", + "boxes": [ + 836, + 105, + 1080, + 129 + ] + }, + { + "text": "Entertainmer", + "boxes": [ + 1070, + 111, + 1150, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1370, + 113, + 1392, + 127 + ] + }, + { + "text": "3", + "boxes": [ + 1419, + 108, + 1442, + 131 + ] + }, + { + "text": "WATCH", + "boxes": [ + 872, + 621, + 1072, + 677 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 904, + 677, + 1030, + 709 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 846, + 710, + 1062, + 753 + ] + }, + { + "text": " Learn more", + "boxes": [ + 858, + 774, + 958, + 800 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 774, + 1048, + 800 + ] + } + ] + }, + { + "frame_id": "60_7_5", + "episode_id": 60, + "env_id": 7, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.441400527954102, + "advantages": -0.1716773509979248, + "generated_text": "{\n \"intent\": \"To explore different functionalities available in this environment, starting by interacting with the while lists and help menus.\",\n \"action\": \"DoubleClick(50, 177)\"\n}", + "action_string": "DoubleClick(50, 177)", + "intent_string": "To explore different functionalities available in this environment, starting by interacting with the while lists and help menus.", + "before_screen_ocr_results": [ + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": "-m e XEnGd sI @@", + "boxes": [ + 4, + 48, + 648, + 89 + ] + }, + { + "text": " Unary/Binary Operators", + "boxes": [ + 16, + 103, + 174, + 127 + ] + }, + { + "text": "+口", + "boxes": [ + 30, + 153, + 70, + 179 + ] + }, + { + "text": "-□", + "boxes": [ + 96, + 153, + 142, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 168, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 151, + 282, + 179 + ] + }, + { + "text": "□+□冏□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*口冏", + "boxes": [ + 22, + 256, + 106, + 282 + ] + }, + { + "text": "品", + "boxes": [ + 100, + 250, + 140, + 288 + ] + }, + { + "text": "÷□□口", + "boxes": [ + 154, + 250, + 286, + 282 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": "00 0。%", + "boxes": [ + 8, + 331, + 292, + 393 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 407, + 78, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Help", + "boxes": [ + 300, + 26, + 344, + 46 + ] + }, + { + "text": "-----", + "boxes": [ + 6, + 52, + 470, + 87 + ] + }, + { + "text": "SID@D", + "boxes": [ + 454, + 50, + 644, + 85 + ] + }, + { + "text": "A", + "boxes": [ + 724, + 62, + 740, + 77 + ] + }, + { + "text": "±口", + "boxes": [ + 174, + 155, + 208, + 175 + ] + }, + { + "text": "口", + "boxes": [ + 242, + 153, + 282, + 179 + ] + }, + { + "text": "□+□□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 290, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 94, + 282 + ] + }, + { + "text": "号", + "boxes": [ + 106, + 254, + 134, + 282 + ] + }, + { + "text": "□÷□□口", + "boxes": [ + 158, + 254, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 290, + 332 + ] + }, + { + "text": "00%品", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 409, + 72, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 70, + 405, + 218, + 437 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ] + }, + { + "frame_id": "60_0_6", + "episode_id": 60, + "env_id": 0, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.56149673461914, + "advantages": -0.08326015621423721, + "generated_text": "{\n \"intent\": \"To start exploring the LibreOffice Writer application by interacting with different elements, such as headers, tools, or sections of the document.\",\n \"action\": \"DoubleClick(300, 120)\"\n}", + "action_string": "DoubleClick(300, 120)", + "intent_string": "To start exploring the LibreOffice Writer application by interacting with different elements, such as headers, tools, or sections of the document.", + "before_screen_ocr_results": [ + { + "text": "File Edit I", + "boxes": [ + 8, + 26, + 92, + 44 + ] + }, + { + "text": "Styles", + "boxes": [ + 252, + 30, + 292, + 46 + ] + }, + { + "text": "Help", + "boxes": [ + 508, + 26, + 550, + 46 + ] + }, + { + "text": "--目·@x自 ", + "boxes": [ + 10, + 52, + 378, + 87 + ] + }, + { + "text": "二", + "boxes": [ + 380, + 60, + 408, + 79 + ] + }, + { + "text": "-、", + "boxes": [ + 410, + 56, + 520, + 79 + ] + }, + { + "text": " Abc", + "boxes": [ + 514, + 56, + 590, + 81 + ] + }, + { + "text": "m·", + "boxes": [ + 608, + 54, + 728, + 83 + ] + }, + { + "text": "·国→田", + "boxes": [ + 716, + 52, + 1256, + 87 + ] + }, + { + "text": "Noto sans", + "boxes": [ + 138, + 93, + 296, + 123 + ] + }, + { + "text": "aaa a ab a,", + "boxes": [ + 438, + 93, + 654, + 123 + ] + }, + { + "text": "一", + "boxes": [ + 804, + 101, + 820, + 115 + ] + }, + { + "text": "小山三··mm三·州", + "boxes": [ + 830, + 93, + 1212, + 123 + ] + }, + { + "text": "D", + "boxes": [ + 1885, + 205, + 1913, + 235 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 236, + 698, + 262 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": "Dea", + "boxes": [ + 612, + 282, + 634, + 296 + ] + }, + { + "text": "Loremipm dolrst me sonectetracpisinlt seddo gusmod teurnciddt tlaeet", + "boxes": [ + 604, + 314, + 1258, + 349 + ] + }, + { + "text": "O", + "boxes": [ + 1878, + 306, + 1918, + 346 + ] + }, + { + "text": "sloremagn3lcuaUt erimadminimyenimsuisnstrudxeritinencisniit", + "boxes": [ + 602, + 333, + 1256, + 371 + ] + }, + { + "text": "xtt", + "boxes": [ + 604, + 357, + 1258, + 393 + ] + }, + { + "text": "dlrefgitaraturxceter sintaeat cuidatat non pridet st inuauiffc", + "boxes": [ + 606, + 383, + 1256, + 413 + ] + }, + { + "text": "deserwnt molitam id est lbowm.", + "boxes": [ + 602, + 401, + 844, + 439 + ] + }, + { + "text": "Loremipsm dolorsi me. consectetr acinsin lt sedo gusmod teuprnciddut utlareet", + "boxes": [ + 604, + 441, + 1258, + 476 + ] + }, + { + "text": "solore magna sliaua. ut eni ad minim yeniam, guis nostrud exeritation wlanc laboris nisi yt", + "boxes": [ + 602, + 463, + 1260, + 500 + ] + }, + { + "text": "aliquip ex ea sommodo. consequat.", + "boxes": [ + 606, + 490, + 832, + 520 + ] + }, + { + "text": "We are looking forward to hearing from you soon.", + "boxes": [ + 606, + 528, + 934, + 556 + ] + }, + { + "text": "Yours sincerely.", + "boxes": [ + 608, + 572, + 710, + 598 + ] + }, + { + "text": "<SENDER'S NAME>", + "boxes": [ + 608, + 611, + 710, + 635 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit I", + "boxes": [ + 8, + 26, + 92, + 44 + ] + }, + { + "text": "Styles", + "boxes": [ + 252, + 30, + 292, + 46 + ] + }, + { + "text": "--目·@x自、 ", + "boxes": [ + 10, + 52, + 376, + 87 + ] + }, + { + "text": "G-.", + "boxes": [ + 412, + 56, + 514, + 79 + ] + }, + { + "text": " Abc", + "boxes": [ + 514, + 56, + 590, + 81 + ] + }, + { + "text": "m·", + "boxes": [ + 614, + 56, + 710, + 81 + ] + }, + { + "text": "·国→田", + "boxes": [ + 716, + 52, + 1256, + 87 + ] + }, + { + "text": " Noto sa", + "boxes": [ + 144, + 93, + 282, + 123 + ] + }, + { + "text": "aaaaab ab", + "boxes": [ + 438, + 93, + 654, + 123 + ] + }, + { + "text": "αa-.", + "boxes": [ + 644, + 93, + 786, + 123 + ] + }, + { + "text": "一", + "boxes": [ + 804, + 101, + 820, + 115 + ] + }, + { + "text": "小山三··m三三·州", + "boxes": [ + 830, + 93, + 1212, + 123 + ] + }, + { + "text": "D", + "boxes": [ + 1888, + 208, + 1912, + 234 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 236, + 698, + 262 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": "Dea", + "boxes": [ + 612, + 282, + 634, + 296 + ] + }, + { + "text": "Loridlsteesedddtl", + "boxes": [ + 604, + 314, + 1258, + 349 + ] + }, + { + "text": "O", + "boxes": [ + 1878, + 306, + 1918, + 346 + ] + }, + { + "text": "dolore magna aliqua Ut enim ad minim yeniam, guis nostrud exercitation ylamc laboris nisi yt", + "boxes": [ + 606, + 338, + 1256, + 367 + ] + }, + { + "text": "ljexeterit", + "boxes": [ + 604, + 357, + 1258, + 393 + ] + }, + { + "text": "dlreefugitne Rariatur Exceter sint aeat cuidatat non proident st inua qui ffca", + "boxes": [ + 606, + 383, + 1256, + 413 + ] + }, + { + "text": "deserwnt molit aim id est labowm.", + "boxes": [ + 602, + 401, + 844, + 439 + ] + }, + { + "text": "Lorem jsum dolrsit amet. consectetur adiiscina elt sed do eiumed teukr jncididut ut laboeet", + "boxes": [ + 606, + 445, + 1256, + 474 + ] + }, + { + "text": "solore magna sliaua. Ut enim ad minim yeniam, guis nostrud exeritation wlanc laboris nisi yt", + "boxes": [ + 602, + 463, + 1260, + 500 + ] + }, + { + "text": "alquip ex ea commod conseguat.", + "boxes": [ + 606, + 490, + 832, + 520 + ] + }, + { + "text": "We are looking forward to hearing from you soon.", + "boxes": [ + 606, + 528, + 926, + 556 + ] + }, + { + "text": "Yours sincerely.", + "boxes": [ + 608, + 572, + 710, + 598 + ] + }, + { + "text": "<SENDER's NAME>", + "boxes": [ + 608, + 611, + 710, + 635 + ] + } + ] + }, + { + "frame_id": "60_1_6", + "episode_id": 60, + "env_id": 1, + "step": 6, + "format_rewards": 1.0, + "rewards": 8.803727149963379, + "advantages": -0.6411449909210205, + "generated_text": "{\n \"intent\": \"To further explore the desktop by interacting with an application icon.\",\n \"action\": \"DoubleClick(123, 456)\"\n}", + "action_string": "DoubleClick(123, 456)", + "intent_string": "To further explore the desktop by interacting with an application icon.", + "before_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 284, + 34 + ] + }, + { + "text": "Flle", + "boxes": [ + 268, + 40, + 292, + 54 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 298, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→", + "boxes": [ + 584, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 900, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 808, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "Apple", + "boxes": [ + 752, + 492, + 796, + 518 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 90, + 528 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 822, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search -Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 29, + 825, + 89, + 867 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "alculato", + "boxes": [ + 380, + 1052, + 428, + 1070 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 284, + 34 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 298, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→", + "boxes": [ + 584, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 904, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 806, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 112, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "“", + "boxes": [ + 792, + 463, + 816, + 486 + ] + }, + { + "text": "Apple", + "boxes": [ + 754, + 492, + 796, + 518 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 88, + 528 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 820, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1220, + 627, + 1410, + 683 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search -Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 29, + 825, + 89, + 867 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_6", + "episode_id": 60, + "env_id": 2, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.283141136169434, + "advantages": 0.4480286240577698, + "generated_text": "{\n \"intent\": \"To continue exploring the environment by accessing different sections or features of the Amazon website.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring the environment by accessing different sections or features of the Amazon website.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 249, + 63, + 313, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "is of pre", + "boxes": [ + 974, + 461, + 1020, + 478 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go toamazon.ca", + "boxes": [ + 1010, + 455, + 1398, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1226, + 488, + 1276, + 508 + ] + }, + { + "text": "1", + "boxes": [ + 1434, + 488, + 1468, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 612, + 530, + 792, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 980, + 530, + 1288, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 1346, + 524, + 1572, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 610, + 889, + 696, + 915 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 982, + 891, + 1116, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 1350, + 889, + 1430, + 913 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 910, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1352, + 969, + 1508, + 999 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Gaming accessories", + "boxes": [ + 240, + 183, + 440, + 215 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 612, + 185, + 794, + 214 + ] + }, + { + "text": "Shopfor your home", + "boxes": [ + 984, + 187, + 1184, + 210 + ] + }, + { + "text": "ssentials", + "boxes": [ + 1196, + 189, + 1282, + 206 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 1350, + 181, + 1576, + 216 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 610, + 544, + 694, + 568 + ] + }, + { + "text": "See all deals", + "boxes": [ + 1354, + 546, + 1428, + 564 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 621, + 1510, + 657 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 653, + 354, + 683 + ] + }, + { + "text": "Explore all products in Kitchel", + "boxes": [ + 246, + 987, + 418, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Shopnow", + "boxes": [ + 1354, + 987, + 1412, + 1007 + ] + } + ] + }, + { + "frame_id": "60_3_6", + "episode_id": 60, + "env_id": 3, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.010627746582031, + "advantages": -0.48882073163986206, + "generated_text": "{\n \"intent\": \"Exploring more options by scrolling down the current page.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "Exploring more options by scrolling down the current page.", + "before_screen_ocr_results": [ + { + "text": "MozillaFiref", + "boxes": [ + 904, + 4, + 1004, + 22 + ] + }, + { + "text": "Q Search with Bing or ente", + "boxes": [ + 254, + 66, + 452, + 89 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 14, + 103, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 996, + 534, + 1036, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1226, + 673, + 1418, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 674, + 750, + 688, + 766 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 746, + 770, + 792, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or ente", + "boxes": [ + 254, + 66, + 452, + 89 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 14, + 103, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 658, + 290, + 766, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "6", + "boxes": [ + 1233, + 479, + 1271, + 506 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 884, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1284, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_4_6", + "episode_id": 60, + "env_id": 4, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.080744743347168, + "advantages": -0.437199205160141, + "generated_text": "{\n \"intent\": \"Expanding the current view may reveal more content or additional options underneath.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "Expanding the current view may reveal more content or additional options underneath.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1599, + 65, + 1659, + 90 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1384, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 836, + 582, + 936, + 606 + ] + }, + { + "text": "Privacy Notice", + "boxes": [ + 944, + 582, + 1046, + 606 + ] + } + ] + }, + { + "frame_id": "60_5_6", + "episode_id": 60, + "env_id": 5, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.423383712768555, + "advantages": 0.5512779355049133, + "generated_text": "{\n \"intent\": \"To explore the Bing search engine more thoroughly by navigating through the discover section to gain insights or read trending articles.\",\n \"action\": \"DoubleClick(972, 587)\"\n}", + "action_string": "DoubleClick(972, 587)", + "intent_string": "To explore the Bing search engine more thoroughly by navigating through the discover section to gain insights or read trending articles.", + "before_screen_ocr_results": [ + { + "text": " Search - Microsoft Bing", + "boxes": [ + 26, + 32, + 188, + 50 + ] + }, + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1872, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 494, + 191, + 558, + 216 + ] + }, + { + "text": "国际版", + "boxes": [ + 568, + 191, + 636, + 216 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 186, + 439, + 274, + 465 + ] + }, + { + "text": "×", + "boxes": [ + 1044, + 488, + 1068, + 502 + ] + }, + { + "text": "M自选股建议", + "boxes": [ + 1454, + 480, + 1554, + 512 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1456, + 516, + 1506, + 560 + ] + }, + { + "text": "K", + "boxes": [ + 1592, + 522, + 1630, + 550 + ] + }, + { + "text": "+2.0", + "boxes": [ + 1628, + 514, + 1700, + 562 + ] + }, + { + "text": "+", + "boxes": [ + 1692, + 530, + 1710, + 546 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1450, + 556, + 1520, + 613 + ] + }, + { + "text": "161029 ", + "boxes": [ + 1626, + 558, + 1706, + 611 + ] + }, + { + "text": "", + "boxes": [ + 1592, + 572, + 1638, + 600 + ] + }, + { + "text": "+", + "boxes": [ + 1690, + 578, + 1710, + 592 + ] + }, + { + "text": "时中国A50", + "boxes": [ + 1449, + 601, + 1551, + 658 + ] + }, + { + "text": "13 87% ", + "boxes": [ + 1624, + 604, + 1716, + 657 + ] + }, + { + "text": "智作说车+关注·23小时", + "boxes": [ + 826, + 633, + 1008, + 663 + ] + }, + { + "text": "生活研究所", + "boxes": [ + 1136, + 635, + 1228, + 661 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 822, + 655, + 1078, + 691 + ] + }, + { + "text": "地球上的天赐谷地", + "boxes": [ + 1134, + 655, + 1310, + 693 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 655, + 1510, + 701 + ] + }, + { + "text": "", + "boxes": [ + 1592, + 661, + 1628, + 691 + ] + }, + { + "text": "+3.28%", + "boxes": [ + 1626, + 653, + 1700, + 701 + ] + }, + { + "text": "点视频/防务急先锋", + "boxes": [ + 232, + 667, + 348, + 697 + ] + }, + { + "text": "中国汽车技术崛起", + "boxes": [ + 196, + 693, + 444, + 741 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 699, + 1522, + 748 + ] + }, + { + "text": "-1.14% ±", + "boxes": [ + 1634, + 701, + 1722, + 746 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 717, + 1628, + 739 + ] + }, + { + "text": "7", + "boxes": [ + 204, + 746, + 266, + 766 + ] + }, + { + "text": "更预示着全球汽", + "boxes": [ + 294, + 742, + 462, + 772 + ] + }, + { + "text": "21", + "boxes": [ + 1138, + 741, + 1182, + 766 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1622, + 744, + 1726, + 770 + ] + }, + { + "text": "长春市", + "boxes": [ + 1464, + 798, + 1526, + 824 + ] + }, + { + "text": "下周五气温骤降", + "boxes": [ + 1616, + 852, + 1714, + 876 + ] + }, + { + "text": "ACACMPSIT", + "boxes": [ + 1149, + 871, + 1237, + 906 + ] + }, + { + "text": "", + "boxes": [ + 1156, + 919, + 1308, + 949 + ] + }, + { + "text": "生活家·20小时", + "boxes": [ + 202, + 945, + 328, + 979 + ] + }, + { + "text": "娱乐扒客·16小时", + "boxes": [ + 512, + 949, + 650, + 979 + ] + }, + { + "text": "盖世汽车·1周", + "boxes": [ + 1138, + 949, + 1256, + 979 + ] + }, + { + "text": "D", + "boxes": [ + 826, + 965, + 852, + 985 + ] + }, + { + "text": "一点视频/南哥说异·2天", + "boxes": [ + 846, + 961, + 998, + 991 + ] + }, + { + "text": "7万余亩土地喝上“营养水\"", + "boxes": [ + 202, + 973, + 450, + 1009 + ] + }, + { + "text": "47岁温峥嵘高开叉亮片裙闪耀", + "boxes": [ + 512, + 971, + 792, + 1007 + ] + }, + { + "text": "汽车轻量化角逐，艾可迅碳纤", + "boxes": [ + 1134, + 969, + 1408, + 1009 + ] + }, + { + "text": "红军缴获鬼子10吨武器，无人", + "boxes": [ + 824, + 983, + 1102, + 1018 + ] + }, + { + "text": "戛纳！", + "boxes": [ + 514, + 999, + 578, + 1032 + ] + }, + { + "text": "维“亮剑”", + "boxes": [ + 1138, + 1001, + 1230, + 1032 + ] + }, + { + "text": "ews/other/理性看待风阻系数测试争", + "boxes": [ + 192, + 1018, + 406, + 1042 + ] + }, + { + "text": "会用，谁知一个新兵全会操作", + "boxes": [ + 828, + 1012, + 1096, + 1042 + ] + }, + { + "text": "earch - Micr", + "boxes": [ + 224, + 1048, + 308, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 240, + 26, + 416, + 52 + ] + }, + { + "text": "C", + "boxes": [ + 74, + 68, + 124, + 87 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 62, + 1664, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 566, + 117, + 638, + 149 + ] + }, + { + "text": "长春市（", + "boxes": [ + 1632, + 117, + 1696, + 149 + ] + }, + { + "text": "msn", + "boxes": [ + 62, + 129, + 102, + 149 + ] + }, + { + "text": "资讯娱乐时尚教育科技 萌宠汽车", + "boxes": [ + 346, + 175, + 690, + 204 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1606, + 170, + 1724, + 209 + ] + }, + { + "text": "智作说车+关注", + "boxes": [ + 410, + 264, + 574, + 302 + ] + }, + { + "text": "豆包AI", + "boxes": [ + 1568, + 258, + 1708, + 316 + ] + }, + { + "text": "瓷车", + "boxes": [ + 392, + 274, + 420, + 296 + ] + }, + { + "text": "4.3K关注者", + "boxes": [ + 1106, + 276, + 1172, + 296 + ] + }, + { + "text": "", + "boxes": [ + 194, + 288, + 210, + 306 + ] + }, + { + "text": "全能高效", + "boxes": [ + 1565, + 305, + 1707, + 354 + ] + }, + { + "text": "", + "boxes": [ + 188, + 336, + 218, + 371 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 364, + 349, + 764, + 391 + ] + }, + { + "text": "写作助手", + "boxes": [ + 1565, + 339, + 1707, + 388 + ] + }, + { + "text": "#", + "boxes": [ + 182, + 387, + 222, + 427 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1604, + 383, + 1670, + 409 + ] + }, + { + "text": "经济日报的报道·23小时·① 2 分钟读取", + "boxes": [ + 360, + 411, + 690, + 451 + ] + }, + { + "text": "豆包PC版_操作便捷_免费下载安", + "boxes": [ + 1420, + 411, + 1690, + 447 + ] + }, + { + "text": "希", + "boxes": [ + 1422, + 436, + 1455, + 469 + ] + }, + { + "text": "?", + "boxes": [ + 194, + 457, + 210, + 471 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 469, + 1528, + 492 + ] + }, + { + "text": "本文转自：经济日报", + "boxes": [ + 362, + 500, + 524, + 536 + ] + }, + { + "text": "访问智作说车", + "boxes": [ + 1456, + 556, + 1548, + 582 + ] + }, + { + "text": "L", + "boxes": [ + 1680, + 558, + 1696, + 572 + ] + }, + { + "text": "长安汽车首个海外整车工厂在泰国", + "boxes": [ + 1450, + 606, + 1662, + 629 + ] + }, + { + "text": "$", + "boxes": [ + 1445, + 619, + 1477, + 651 + ] + }, + { + "text": "中国东部小县城撬动全球轮胎市场", + "boxes": [ + 1444, + 677, + 1662, + 709 + ] + }, + { + "text": "长安汽车泰国工厂峻工投产", + "boxes": [ + 1450, + 739, + 1614, + 762 + ] + }, + { + "text": "®", + "boxes": [ + 1414, + 830, + 1444, + 852 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 832, + 873, + 1092, + 907 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 30, + 911, + 794, + 935 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1526, + 935 + ] + }, + { + "text": "ID）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意”从而同意上述使用场景，或者点", + "boxes": [ + 28, + 927, + 798, + 957 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1670, + 937, + 1726, + 963 + ] + }, + { + "text": "击\"管理设置\"查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的\"管理设置\"改变您的选择。欲了解", + "boxes": [ + 32, + 951, + 794, + 975 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 130, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 118, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "理性看待风阻··", + "boxes": [ + 216, + 1048, + 306, + 1072 + ] + } + ] + }, + { + "frame_id": "60_6_6", + "episode_id": 60, + "env_id": 6, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.575727462768555, + "advantages": -0.0727832168340683, + "generated_text": "{\n \"intent\": \"To explore more parts of the website by scrolling down.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore more parts of the website by scrolling down.", + "before_screen_ocr_results": [ + { + "text": "Appl", + "boxes": [ + 28, + 34, + 62, + 48 + ] + }, + { + "text": "→", + "boxes": [ + 32, + 66, + 126, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Visiont for AirPodse Intv & Homece.", + "boxes": [ + 836, + 105, + 1080, + 129 + ] + }, + { + "text": "Entertainmer", + "boxes": [ + 1070, + 111, + 1150, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1370, + 113, + 1392, + 127 + ] + }, + { + "text": "3", + "boxes": [ + 1419, + 108, + 1442, + 131 + ] + }, + { + "text": "WATCH", + "boxes": [ + 872, + 621, + 1072, + 677 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 904, + 677, + 1030, + 709 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 846, + 710, + 1062, + 753 + ] + }, + { + "text": " Learn more", + "boxes": [ + 858, + 774, + 958, + 800 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 774, + 1048, + 800 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Appi", + "boxes": [ + 866, + 8, + 898, + 22 + ] + }, + { + "text": "Appl", + "boxes": [ + 28, + 34, + 62, + 48 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 944, + 127 + ] + }, + { + "text": "WATCH", + "boxes": [ + 886, + 336, + 1072, + 391 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 389, + 1032, + 427 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 846, + 425, + 1060, + 467 + ] + }, + { + "text": " Learn more ", + "boxes": [ + 856, + 486, + 960, + 518 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 490, + 1048, + 516 + ] + }, + { + "text": "vatch-series-10/", + "boxes": [ + 208, + 1018, + 316, + 1042 + ] + } + ] + }, + { + "frame_id": "60_7_6", + "episode_id": 60, + "env_id": 7, + "step": 6, + "format_rewards": 1.0, + "rewards": 8.74730110168457, + "advantages": -0.682686984539032, + "generated_text": "{\n \"intent\": \"To explore the LibreOffice Math environment by interacting with the left pane where various mathematical symbols are available.\",\n \"action\": \"DoubleClick(100, 709)\"\n}", + "action_string": "DoubleClick(100, 709)", + "intent_string": "To explore the LibreOffice Math environment by interacting with the left pane where various mathematical symbols are available.", + "before_screen_ocr_results": [ + { + "text": "Help", + "boxes": [ + 300, + 26, + 344, + 46 + ] + }, + { + "text": "-----", + "boxes": [ + 6, + 52, + 470, + 87 + ] + }, + { + "text": "SID@D", + "boxes": [ + 454, + 50, + 644, + 85 + ] + }, + { + "text": "A", + "boxes": [ + 724, + 62, + 740, + 77 + ] + }, + { + "text": "±口", + "boxes": [ + 174, + 155, + 208, + 175 + ] + }, + { + "text": "口", + "boxes": [ + 242, + 153, + 282, + 179 + ] + }, + { + "text": "□+□□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 290, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 94, + 282 + ] + }, + { + "text": "号", + "boxes": [ + 106, + 254, + 134, + 282 + ] + }, + { + "text": "□÷□□口", + "boxes": [ + 158, + 254, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 290, + 332 + ] + }, + { + "text": "00%品", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 409, + 72, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 70, + 405, + 218, + 437 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": " -. n-- I@", + "boxes": [ + 4, + 48, + 644, + 87 + ] + }, + { + "text": "A", + "boxes": [ + 724, + 62, + 740, + 77 + ] + }, + { + "text": "Unary/Binary Operators", + "boxes": [ + 20, + 105, + 172, + 123 + ] + }, + { + "text": "一", + "boxes": [ + 46, + 159, + 58, + 171 + ] + }, + { + "text": "口", + "boxes": [ + 114, + 157, + 136, + 171 + ] + }, + { + "text": "±口", + "boxes": [ + 174, + 155, + 208, + 175 + ] + }, + { + "text": "口", + "boxes": [ + 242, + 153, + 282, + 179 + ] + }, + { + "text": "□+□冏□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 86, + 282 + ] + }, + { + "text": "号", + "boxes": [ + 104, + 252, + 134, + 286 + ] + }, + { + "text": "□÷□□口", + "boxes": [ + 154, + 252, + 286, + 282 + ] + }, + { + "text": "□", + "boxes": [ + 16, + 296, + 292, + 334 + ] + }, + { + "text": " %", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 409, + 72, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ] + }, + { + "frame_id": "60_0_7", + "episode_id": 60, + "env_id": 0, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.526978492736816, + "advantages": -0.10867317020893097, + "generated_text": "{\n \"intent\": \"To start exploring or interacting with elements off-screen, I will scroll down slightly to see more content.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To start exploring or interacting with elements off-screen, I will scroll down slightly to see more content.", + "before_screen_ocr_results": [ + { + "text": "File Edit I", + "boxes": [ + 8, + 26, + 92, + 44 + ] + }, + { + "text": "Styles", + "boxes": [ + 252, + 30, + 292, + 46 + ] + }, + { + "text": "--目·@x自、 ", + "boxes": [ + 10, + 52, + 376, + 87 + ] + }, + { + "text": "G-.", + "boxes": [ + 412, + 56, + 514, + 79 + ] + }, + { + "text": " Abc", + "boxes": [ + 514, + 56, + 590, + 81 + ] + }, + { + "text": "m·", + "boxes": [ + 614, + 56, + 710, + 81 + ] + }, + { + "text": "·国→田", + "boxes": [ + 716, + 52, + 1256, + 87 + ] + }, + { + "text": " Noto sa", + "boxes": [ + 144, + 93, + 282, + 123 + ] + }, + { + "text": "aaaaab ab", + "boxes": [ + 438, + 93, + 654, + 123 + ] + }, + { + "text": "αa-.", + "boxes": [ + 644, + 93, + 786, + 123 + ] + }, + { + "text": "一", + "boxes": [ + 804, + 101, + 820, + 115 + ] + }, + { + "text": "小山三··m三三·州", + "boxes": [ + 830, + 93, + 1212, + 123 + ] + }, + { + "text": "D", + "boxes": [ + 1888, + 208, + 1912, + 234 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 236, + 698, + 262 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": "Dea", + "boxes": [ + 612, + 282, + 634, + 296 + ] + }, + { + "text": "Loridlsteesedddtl", + "boxes": [ + 604, + 314, + 1258, + 349 + ] + }, + { + "text": "O", + "boxes": [ + 1878, + 306, + 1918, + 346 + ] + }, + { + "text": "dolore magna aliqua Ut enim ad minim yeniam, guis nostrud exercitation ylamc laboris nisi yt", + "boxes": [ + 606, + 338, + 1256, + 367 + ] + }, + { + "text": "ljexeterit", + "boxes": [ + 604, + 357, + 1258, + 393 + ] + }, + { + "text": "dlreefugitne Rariatur Exceter sint aeat cuidatat non proident st inua qui ffca", + "boxes": [ + 606, + 383, + 1256, + 413 + ] + }, + { + "text": "deserwnt molit aim id est labowm.", + "boxes": [ + 602, + 401, + 844, + 439 + ] + }, + { + "text": "Lorem jsum dolrsit amet. consectetur adiiscina elt sed do eiumed teukr jncididut ut laboeet", + "boxes": [ + 606, + 445, + 1256, + 474 + ] + }, + { + "text": "solore magna sliaua. Ut enim ad minim yeniam, guis nostrud exeritation wlanc laboris nisi yt", + "boxes": [ + 602, + 463, + 1260, + 500 + ] + }, + { + "text": "alquip ex ea commod conseguat.", + "boxes": [ + 606, + 490, + 832, + 520 + ] + }, + { + "text": "We are looking forward to hearing from you soon.", + "boxes": [ + 606, + 528, + 926, + 556 + ] + }, + { + "text": "Yours sincerely.", + "boxes": [ + 608, + 572, + 710, + 598 + ] + }, + { + "text": "<SENDER's NAME>", + "boxes": [ + 608, + 611, + 710, + 635 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit View", + "boxes": [ + 8, + 26, + 116, + 44 + ] + }, + { + "text": "Styles", + "boxes": [ + 252, + 30, + 292, + 46 + ] + }, + { + "text": "--目- @x自- ", + "boxes": [ + 10, + 52, + 378, + 87 + ] + }, + { + "text": "G--", + "boxes": [ + 402, + 54, + 524, + 83 + ] + }, + { + "text": "AbcT", + "boxes": [ + 514, + 56, + 598, + 81 + ] + }, + { + "text": "m·", + "boxes": [ + 608, + 54, + 728, + 83 + ] + }, + { + "text": "·国一→田", + "boxes": [ + 716, + 52, + 1254, + 87 + ] + }, + { + "text": "Noto sans", + "boxes": [ + 136, + 93, + 298, + 123 + ] + }, + { + "text": "J aaa aab ab", + "boxes": [ + 426, + 93, + 654, + 123 + ] + }, + { + "text": "D", + "boxes": [ + 744, + 101, + 770, + 115 + ] + }, + { + "text": "川", + "boxes": [ + 834, + 99, + 874, + 119 + ] + }, + { + "text": "三", + "boxes": [ + 850, + 95, + 930, + 121 + ] + }, + { + "text": "··m国三·", + "boxes": [ + 928, + 93, + 1212, + 123 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 173, + 1912, + 199 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 606, + 193, + 700, + 224 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 204, + 1914, + 238 + ] + }, + { + "text": "Dear <TITLE><RECIPIENT's NAME>", + "boxes": [ + 606, + 238, + 802, + 262 + ] + }, + { + "text": "E", + "boxes": [ + 1884, + 238, + 1910, + 278 + ] + }, + { + "text": "Loreujudorsit met sonectetrcinscitseddgdteurnidntet", + "boxes": [ + 604, + 274, + 1258, + 310 + ] + }, + { + "text": "dlore magna aliaua Ut enim ad minim yenim. guis nostud eeritatin yamco laboris nisi yt", + "boxes": [ + 606, + 298, + 1256, + 328 + ] + }, + { + "text": "slguip exeaodsoseuatusteiedlorinerhendrtinyolupteylteseslum", + "boxes": [ + 602, + 314, + 1260, + 355 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 310, + 1916, + 343 + ] + }, + { + "text": "dlrefugitaatxceteurintaeatidanroett inifcia", + "boxes": [ + 602, + 337, + 1258, + 375 + ] + }, + { + "text": "deserut molt aidestlam.", + "boxes": [ + 604, + 359, + 844, + 397 + ] + }, + { + "text": "Loreujumdorsit met sonectetrcinscietseddgiodteurnidnttet", + "boxes": [ + 604, + 401, + 1258, + 437 + ] + }, + { + "text": "dlore mana aliaua Ut enim ad minim yenim. auis nostrud exercitatin wamco labois nisi t", + "boxes": [ + 606, + 427, + 1258, + 457 + ] + }, + { + "text": "aliquip ex ea commodo consequat.", + "boxes": [ + 606, + 451, + 832, + 480 + ] + }, + { + "text": "We are looking forward to hearing from you soon.", + "boxes": [ + 608, + 490, + 922, + 514 + ] + }, + { + "text": "Yours sincerely.", + "boxes": [ + 608, + 532, + 710, + 558 + ] + }, + { + "text": "<SENDER's NAME>", + "boxes": [ + 608, + 572, + 710, + 596 + ] + }, + { + "text": "First Page", + "boxes": [ + 668, + 1018, + 732, + 1044 + ] + } + ] + }, + { + "frame_id": "60_1_7", + "episode_id": 60, + "env_id": 1, + "step": 7, + "format_rewards": 1.0, + "rewards": 8.818347930908203, + "advantages": -0.6303809285163879, + "generated_text": "{\n \"intent\": \"To start exploring various applications by interacting with different shortcuts.\",\n \"action\": \"DoubleClick(250, 70)\"\n}", + "action_string": "DoubleClick(250, 70)", + "intent_string": "To start exploring various applications by interacting with different shortcuts.", + "before_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 284, + 34 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 298, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→", + "boxes": [ + 584, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 904, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 806, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 112, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "“", + "boxes": [ + 792, + 463, + 816, + 486 + ] + }, + { + "text": "Apple", + "boxes": [ + 754, + 492, + 796, + 518 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 88, + 528 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 820, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1220, + 627, + 1410, + 683 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search -Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 29, + 825, + 89, + 867 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 286, + 34 + ] + }, + { + "text": "Edit View", + "boxes": [ + 298, + 35, + 388, + 58 + ] + }, + { + "text": " Calculator Help", + "boxes": [ + 382, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→", + "boxes": [ + 584, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 904, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 806, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "Apple", + "boxes": [ + 752, + 492, + 796, + 518 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 90, + 528 + ] + }, + { + "text": "bing", + "boxes": [ + 879, + 493, + 909, + 514 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 820, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search -Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 29, + 825, + 89, + 867 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_7", + "episode_id": 60, + "env_id": 2, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.301553726196289, + "advantages": 0.46158432960510254, + "generated_text": "{\n \"intent\": \"The user seems to be exploring the Amazon website, likely to find specific products or browse different sections. To achieve this, clicking on one of the featured sections would provide more detailed information about available items in that category.\",\n \"action\": \"DoubleClick(776, 357)\"\n}", + "action_string": "DoubleClick(776, 357)", + "intent_string": "The user seems to be exploring the Amazon website, likely to find specific products or browse different sections. To achieve this, clicking on one of the featured sections would provide more detailed information about available items in that category.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1598, + 68, + 1660, + 87 + ] + }, + { + "text": "S", + "boxes": [ + 1346, + 141, + 1534, + 165 + ] + }, + { + "text": "Gamingaccessories", + "boxes": [ + 248, + 191, + 434, + 208 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 612, + 183, + 790, + 215 + ] + }, + { + "text": "Shopforyourhome", + "boxes": [ + 984, + 187, + 1188, + 210 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 1350, + 185, + 1574, + 214 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 610, + 544, + 694, + 568 + ] + }, + { + "text": "See all deals", + "boxes": [ + 1354, + 546, + 1428, + 564 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 621, + 1508, + 655 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 651, + 352, + 681 + ] + }, + { + "text": "Explore all products in Kitcher", + "boxes": [ + 246, + 987, + 418, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Shopnow", + "boxes": [ + 1354, + 987, + 1412, + 1007 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→×", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 251, + 61, + 311, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "gaming", + "boxes": [ + 291, + 108, + 365, + 148 + ] + }, + { + "text": "", + "boxes": [ + 1690, + 113, + 1732, + 139 + ] + }, + { + "text": "EN", + "boxes": [ + 1408, + 127, + 1432, + 141 + ] + }, + { + "text": "三 AIl", + "boxes": [ + 22, + 167, + 68, + 195 + ] + }, + { + "text": "Explore Related Products", + "boxes": [ + 10, + 256, + 184, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 378, + 290 + ] + }, + { + "text": " gaming headset", + "boxes": [ + 10, + 280, + 120, + 308 + ] + }, + { + "text": "gaming keyboard", + "boxes": [ + 12, + 304, + 126, + 328 + ] + }, + { + "text": "gaming pc", + "boxes": [ + 12, + 324, + 86, + 347 + ] + }, + { + "text": "gaming mouse", + "boxes": [ + 14, + 343, + 108, + 367 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 409, + 50, + 435 + ] + }, + { + "text": "$0 - $8,900+", + "boxes": [ + 10, + 435, + 100, + 459 + ] + }, + { + "text": "Deals & Discou", + "boxes": [ + 12, + 516, + 110, + 542 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 540, + 98, + 566 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 560, + 100, + 586 + ] + }, + { + "text": "Sponsored ", + "boxes": [ + 1195, + 583, + 1271, + 616 + ] + }, + { + "text": "por", + "boxes": [ + 322, + 596, + 338, + 609 + ] + }, + { + "text": "red 6", + "boxes": [ + 352, + 596, + 382, + 609 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 617, + 144, + 647 + ] + }, + { + "text": "Magnetic Dart Board - 12pcs", + "boxes": [ + 314, + 615, + 528, + 639 + ] + }, + { + "text": "Amazon Basics IPS Monitor 75 Hz ", + "boxes": [ + 610, + 617, + 858, + 641 + ] + }, + { + "text": "Mini projector portable projector", + "boxes": [ + 1194, + 615, + 1440, + 645 + ] + }, + { + "text": "ASUS ROG Strix G16 Gaming ", + "boxes": [ + 1488, + 613, + 1708, + 643 + ] + }, + { + "text": "Magnetic Darts - Excellent Indoor ", + "boxes": [ + 312, + 639, + 568, + 669 + ] + }, + { + "text": " Powered with AOC Technology FHD", + "boxes": [ + 606, + 639, + 872, + 669 + ] + }, + { + "text": "Wireless Bluetooth Desktop", + "boxes": [ + 900, + 635, + 1112, + 665 + ] + }, + { + "text": "movie home cinema bedroom", + "boxes": [ + 1196, + 641, + 1418, + 665 + ] + }, + { + "text": "Laptop, 165Hz Display, NVIDIA@", + "boxes": [ + 1486, + 637, + 1726, + 667 + ] + }, + { + "text": "Department", + "boxes": [ + 12, + 655, + 98, + 681 + ] + }, + { + "text": "Game and Party Games - Gifts for 5", + "boxes": [ + 314, + 663, + 578, + 687 + ] + }, + { + "text": "1080P HDMI, Display Port and VGA", + "boxes": [ + 608, + 663, + 872, + 693 + ] + }, + { + "text": "Speakers - 60w Computer Speak", + "boxes": [ + 902, + 665, + 1142, + 689 + ] + }, + { + "text": "student dorm projector watch soccer", + "boxes": [ + 1194, + 665, + 1464, + 689 + ] + }, + { + "text": "GeForce RTXTM 4060, Intel Core", + "boxes": [ + 1486, + 661, + 1724, + 691 + ] + }, + { + "text": "Computers & Accssories", + "boxes": [ + 10, + 677, + 176, + 707 + ] + }, + { + "text": "6 78 9 10 11 12 Year Old Boy Kids", + "boxes": [ + 312, + 687, + 570, + 711 + ] + }, + { + "text": "Input VESA Compatible Built-in...", + "boxes": [ + 608, + 689, + 854, + 713 + ] + }, + { + "text": "for Music and Gaming ", + "boxes": [ + 900, + 685, + 1068, + 715 + ] + }, + { + "text": "competition basketball match for..", + "boxes": [ + 1194, + 687, + 1464, + 717 + ] + }, + { + "text": " i7-13650HX, 16GB DDR5, 1TB PCI..", + "boxes": [ + 1486, + 685, + 1756, + 715 + ] + }, + { + "text": " Computers & Tablets", + "boxes": [ + 20, + 703, + 162, + 721 + ] + }, + { + "text": "PC Games & Accessories", + "boxes": [ + 12, + 719, + 168, + 742 + ] + }, + { + "text": "★★★★5,961", + "boxes": [ + 318, + 715, + 454, + 739 + ] + }, + { + "text": "Options: 2 sizes ", + "boxes": [ + 610, + 717, + 702, + 741 + ] + }, + { + "text": "★★★★☆3,323", + "boxes": [ + 904, + 715, + 1044, + 739 + ] + }, + { + "text": "★★★★☆127", + "boxes": [ + 1196, + 713, + 1324, + 742 + ] + }, + { + "text": "★★★★☆1,895", + "boxes": [ + 1490, + 715, + 1630, + 739 + ] + }, + { + "text": "★★★★★1.028", + "boxes": [ + 608, + 739, + 752, + 768 + ] + }, + { + "text": "3K+ bought in past month", + "boxes": [ + 1490, + 735, + 1660, + 758 + ] + }, + { + "text": "Brands", + "boxes": [ + 8, + 748, + 68, + 780 + ] + }, + { + "text": "$8999", + "boxes": [ + 1194, + 746, + 1262, + 778 + ] + }, + { + "text": "ASUS", + "boxes": [ + 5, + 765, + 79, + 805 + ] + }, + { + "text": "$2279 List: $29.9", + "boxes": [ + 313, + 762, + 455, + 805 + ] + }, + { + "text": "3K+ bought in past month", + "boxes": [ + 610, + 764, + 778, + 788 + ] + }, + { + "text": "$27900", + "boxes": [ + 900, + 764, + 984, + 802 + ] + }, + { + "text": "$1,24478", + "boxes": [ + 1484, + 762, + 1594, + 800 + ] + }, + { + "text": "Save 10% with coupon", + "boxes": [ + 1194, + 776, + 1346, + 806 + ] + }, + { + "text": " vISOONE ", + "boxes": [ + 10, + 794, + 102, + 818 + ] + }, + { + "text": "$9999", + "boxes": [ + 606, + 794, + 672, + 826 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 316, + 804, + 454, + 828 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 900, + 800, + 1048, + 830 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 1196, + 804, + 1338, + 828 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": "Logitech G", + "boxes": [ + 12, + 814, + 106, + 840 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 606, + 828, + 752, + 858 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 903, + 822, + 991, + 850 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 826, + 1286, + 850 + ] + }, + { + "text": "Ships to Cana", + "boxes": [ + 1492, + 828, + 1564, + 846 + ] + }, + { + "text": "=", + "boxes": [ + 14, + 838, + 38, + 852 + ] + }, + { + "text": "F", + "boxes": [ + 16, + 858, + 38, + 872 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 612, + 856, + 688, + 874 + ] + }, + { + "text": "Add to ca", + "boxes": [ + 914, + 860, + 970, + 878 + ] + }, + { + "text": "Add to car", + "boxes": [ + 1206, + 858, + 1272, + 878 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1494, + 856, + 1578, + 881 + ] + }, + { + "text": " FIFINE", + "boxes": [ + 11, + 871, + 75, + 896 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 614, + 881, + 698, + 913 + ] + }, + { + "text": " More Buying Choices", + "boxes": [ + 1486, + 881, + 1628, + 916 + ] + }, + { + "text": "$1,219.99 (7 used & new offers)", + "boxes": [ + 1488, + 907, + 1698, + 931 + ] + }, + { + "text": "Condition", + "boxes": [ + 14, + 951, + 78, + 969 + ] + }, + { + "text": "Used", + "boxes": [ + 16, + 993, + 68, + 1012 + ] + }, + { + "text": "RO", + "boxes": [ + 970, + 997, + 1018, + 1016 + ] + }, + { + "text": "BLOX", + "boxes": [ + 1016, + 995, + 1110, + 1018 + ] + } + ] + }, + { + "frame_id": "60_3_7", + "episode_id": 60, + "env_id": 3, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.15705394744873, + "advantages": -0.38101887702941895, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment, I will begin by interacting with an icon to access more information or a web page.\",\n \"action\": \"DoubleClick(7, 19)\"\n}", + "action_string": "DoubleClick(7, 19)", + "intent_string": "To continue exploring the desktop environment, I will begin by interacting with an icon to access more information or a web page.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or ente", + "boxes": [ + 254, + 66, + 452, + 89 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 14, + 103, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 658, + 290, + 766, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "6", + "boxes": [ + 1233, + 479, + 1271, + 506 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1284, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 42, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or ente", + "boxes": [ + 254, + 66, + 452, + 89 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 190, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 658, + 290, + 766, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_4_7", + "episode_id": 60, + "env_id": 4, + "step": 7, + "format_rewards": 1.0, + "rewards": 8.966629028320312, + "advantages": -0.5212134718894958, + "generated_text": "{\n \"intent\": \"To explore the web page, such as looking for more elements or useful information.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore the web page, such as looking for more elements or useful information.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1384, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 836, + 582, + 936, + 606 + ] + }, + { + "text": "Privacy Notice", + "boxes": [ + 944, + 582, + 1046, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1384, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 836, + 582, + 936, + 606 + ] + }, + { + "text": "Privacy Notice", + "boxes": [ + 944, + 582, + 1046, + 606 + ] + } + ] + }, + { + "frame_id": "60_5_7", + "episode_id": 60, + "env_id": 5, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.481966018676758, + "advantages": -0.14181223511695862, + "generated_text": "{\n \"intent\": \"To read more about the topic being displayed on the news article.\",\n \"action\": \"ScrollDown(480, 540)\"\n}", + "action_string": "ScrollDown(480, 540)", + "intent_string": "To read more about the topic being displayed on the news article.", + "before_screen_ocr_results": [ + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 240, + 26, + 416, + 52 + ] + }, + { + "text": "C", + "boxes": [ + 74, + 68, + 124, + 87 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 62, + 1664, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 566, + 117, + 638, + 149 + ] + }, + { + "text": "长春市（", + "boxes": [ + 1632, + 117, + 1696, + 149 + ] + }, + { + "text": "msn", + "boxes": [ + 62, + 129, + 102, + 149 + ] + }, + { + "text": "资讯娱乐时尚教育科技 萌宠汽车", + "boxes": [ + 346, + 175, + 690, + 204 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1606, + 170, + 1724, + 209 + ] + }, + { + "text": "智作说车+关注", + "boxes": [ + 410, + 264, + 574, + 302 + ] + }, + { + "text": "豆包AI", + "boxes": [ + 1568, + 258, + 1708, + 316 + ] + }, + { + "text": "瓷车", + "boxes": [ + 392, + 274, + 420, + 296 + ] + }, + { + "text": "4.3K关注者", + "boxes": [ + 1106, + 276, + 1172, + 296 + ] + }, + { + "text": "", + "boxes": [ + 194, + 288, + 210, + 306 + ] + }, + { + "text": "全能高效", + "boxes": [ + 1565, + 305, + 1707, + 354 + ] + }, + { + "text": "", + "boxes": [ + 188, + 336, + 218, + 371 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 364, + 349, + 764, + 391 + ] + }, + { + "text": "写作助手", + "boxes": [ + 1565, + 339, + 1707, + 388 + ] + }, + { + "text": "#", + "boxes": [ + 182, + 387, + 222, + 427 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1604, + 383, + 1670, + 409 + ] + }, + { + "text": "经济日报的报道·23小时·① 2 分钟读取", + "boxes": [ + 360, + 411, + 690, + 451 + ] + }, + { + "text": "豆包PC版_操作便捷_免费下载安", + "boxes": [ + 1420, + 411, + 1690, + 447 + ] + }, + { + "text": "希", + "boxes": [ + 1422, + 436, + 1455, + 469 + ] + }, + { + "text": "?", + "boxes": [ + 194, + 457, + 210, + 471 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 469, + 1528, + 492 + ] + }, + { + "text": "本文转自：经济日报", + "boxes": [ + 362, + 500, + 524, + 536 + ] + }, + { + "text": "访问智作说车", + "boxes": [ + 1456, + 556, + 1548, + 582 + ] + }, + { + "text": "L", + "boxes": [ + 1680, + 558, + 1696, + 572 + ] + }, + { + "text": "长安汽车首个海外整车工厂在泰国", + "boxes": [ + 1450, + 606, + 1662, + 629 + ] + }, + { + "text": "$", + "boxes": [ + 1445, + 619, + 1477, + 651 + ] + }, + { + "text": "中国东部小县城撬动全球轮胎市场", + "boxes": [ + 1444, + 677, + 1662, + 709 + ] + }, + { + "text": "长安汽车泰国工厂峻工投产", + "boxes": [ + 1450, + 739, + 1614, + 762 + ] + }, + { + "text": "®", + "boxes": [ + 1414, + 830, + 1444, + 852 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 832, + 873, + 1092, + 907 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 30, + 911, + 794, + 935 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1526, + 935 + ] + }, + { + "text": "ID）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意”从而同意上述使用场景，或者点", + "boxes": [ + 28, + 927, + 798, + 957 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1670, + 937, + 1726, + 963 + ] + }, + { + "text": "击\"管理设置\"查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的\"管理设置\"改变您的选择。欲了解", + "boxes": [ + 32, + 951, + 794, + 975 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 130, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 118, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "理性看待风阻··", + "boxes": [ + 216, + 1048, + 306, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 252, + 32, + 412, + 50 + ] + }, + { + "text": "C", + "boxes": [ + 74, + 68, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1872, + 89 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 568, + 121, + 636, + 147 + ] + }, + { + "text": "长春市", + "boxes": [ + 1636, + 125, + 1684, + 143 + ] + }, + { + "text": "本文转自：经济日报", + "boxes": [ + 364, + 173, + 524, + 202 + ] + }, + { + "text": "豆包AI", + "boxes": [ + 1568, + 199, + 1708, + 256 + ] + }, + { + "text": "全能高效", + "boxes": [ + 1568, + 246, + 1706, + 296 + ] + }, + { + "text": "小", + "boxes": [ + 196, + 264, + 210, + 280 + ] + }, + { + "text": "写作助手", + "boxes": [ + 1564, + 282, + 1706, + 332 + ] + }, + { + "text": "", + "boxes": [ + 192, + 318, + 214, + 341 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1608, + 326, + 1668, + 351 + ] + }, + { + "text": "豆包PC版_操作便捷_免费下载安", + "boxes": [ + 1422, + 353, + 1688, + 389 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 369, + 218, + 401 + ] + }, + { + "text": "装", + "boxes": [ + 1428, + 383, + 1452, + 405 + ] + }, + { + "text": "?", + "boxes": [ + 854, + 413, + 902, + 441 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1430, + 411, + 1528, + 435 + ] + }, + { + "text": "?", + "boxes": [ + 194, + 433, + 210, + 451 + ] + }, + { + "text": "方问智作说车", + "boxes": [ + 1464, + 498, + 1548, + 522 + ] + }, + { + "text": "长安汽车首个海外整车工厂在泰国", + "boxes": [ + 1444, + 542, + 1662, + 572 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 382, + 558, + 536, + 586 + ] + }, + { + "text": "投产", + "boxes": [ + 1450, + 568, + 1476, + 588 + ] + }, + { + "text": "最近，一个冷门技术参数上了热搜。起因是有汽车博主自费测试阿维塔12风阻系数，得出的数据与企业公布的数据差", + "boxes": [ + 360, + 590, + 1238, + 625 + ] + }, + { + "text": "距不小；而企业在相关公证机构见证下，通过中国汽车工程研究院在风洞实验室全程直播测试，得出的数据并不存在", + "boxes": [ + 364, + 621, + 1238, + 651 + ] + }, + { + "text": "中国东部小县城撬动全球轮胎市场", + "boxes": [ + 1448, + 623, + 1662, + 647 + ] + }, + { + "text": "虚标，从而引发舆论关注。", + "boxes": [ + 366, + 651, + 560, + 675 + ] + }, + { + "text": "汽车风阻系数是衡量车辆行驶时空气阻力大小的关键参数，直接影响燃油经济性、续航能力、高速稳定性和噪声水", + "boxes": [ + 364, + 689, + 1222, + 719 + ] + }, + { + "text": "长安汽车泰国工厂峻工投产", + "boxes": [ + 1450, + 683, + 1610, + 701 + ] + }, + { + "text": "平。数值越低，说明空气阻力越小。新能源汽车的加速普及和补能焦虑，使得低风阻成为新能源车企竞争中少有的\"有", + "boxes": [ + 364, + 717, + 1244, + 746 + ] + }, + { + "text": "感数据”：不仅能影响消费者认知，还能提升续航里程。", + "boxes": [ + 364, + 742, + 774, + 772 + ] + }, + { + "text": "电", + "boxes": [ + 1422, + 774, + 1442, + 794 + ] + }, + { + "text": "要了解风阻系数在新能源赛道是如何\"吹\"起来的，首先还得回溯到2012年，Model S横空出世，特斯拉便将风阻系数", + "boxes": [ + 362, + 784, + 1226, + 814 + ] + }, + { + "text": "打工人", + "boxes": [ + 1426, + 800, + 1502, + 832 + ] + }, + { + "text": "与科技含量进行了大量捆绑营销，不仅重塑了行业以及消费者对\"低风阻\"的认知，也强化了特斯拉富有科技感的品牌", + "boxes": [ + 364, + 810, + 1232, + 840 + ] + }, + { + "text": "高效工作的", + "boxes": [ + 1430, + 828, + 1568, + 860 + ] + }, + { + "text": "形象。此后，诸多新能源车企开始争相效仿，风阻系数纪录也不断被刷新。", + "boxes": [ + 362, + 838, + 918, + 862 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 28, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 836, + 879, + 1090, + 903 + ] + }, + { + "text": "我接受", + "boxes": [ + 1670, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 798, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1526, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置“改变您的选择。欲了解", + "boxes": [ + 28, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "理性看待风阻··", + "boxes": [ + 216, + 1048, + 306, + 1072 + ] + } + ] + }, + { + "frame_id": "60_6_7", + "episode_id": 60, + "env_id": 6, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.801910400390625, + "advantages": 0.09373710304498672, + "generated_text": "{\n \"intent\": \"To scroll down the page to view more content and get a broader understanding of the website's features and products.\",\n \"action\": \"ScrollDown(100, 100)\"\n}", + "action_string": "ScrollDown(100, 100)", + "intent_string": "To scroll down the page to view more content and get a broader understanding of the website's features and products.", + "before_screen_ocr_results": [ + { + "text": "Appi", + "boxes": [ + 866, + 8, + 898, + 22 + ] + }, + { + "text": "Appl", + "boxes": [ + 28, + 34, + 62, + 48 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 944, + 127 + ] + }, + { + "text": "WATCH", + "boxes": [ + 886, + 336, + 1072, + 391 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 389, + 1032, + 427 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 846, + 425, + 1060, + 467 + ] + }, + { + "text": " Learn more ", + "boxes": [ + 856, + 486, + 960, + 518 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 490, + 1048, + 516 + ] + }, + { + "text": "vatch-series-10/", + "boxes": [ + 208, + 1018, + 316, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 262, + 73, + 274, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Visior", + "boxes": [ + 838, + 113, + 868, + 127 + ] + }, + { + "text": "AirPods I E TV&Home", + "boxes": [ + 904, + 109, + 1046, + 133 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1396, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1438, + 129 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 141, + 1060, + 177 + ] + }, + { + "text": "Learn more", + "boxes": [ + 864, + 204, + 954, + 230 + ] + }, + { + "text": "Buy", + "boxes": [ + 1010, + 206, + 1044, + 228 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 796, + 752, + 1116, + 808 + ] + }, + { + "text": " Sky blue color.", + "boxes": [ + 860, + 810, + 1048, + 858 + ] + }, + { + "text": "Sky high performance with M4.", + "boxes": [ + 764, + 846, + 1144, + 887 + ] + }, + { + "text": "Learn more ", + "boxes": [ + 852, + 907, + 960, + 939 + ] + }, + { + "text": "Buy", + "boxes": [ + 1006, + 909, + 1048, + 935 + ] + } + ] + }, + { + "frame_id": "60_7_7", + "episode_id": 60, + "env_id": 7, + "step": 7, + "format_rewards": 1.0, + "rewards": 8.765989303588867, + "advantages": -0.668928325176239, + "generated_text": "{\n \"intent\": \"To start exploring the features available in LibreOffice Math by clicking an icon or option on the toolbar.\",\n \"action\": \"DoubleClick(500, 110)\"\n}", + "action_string": "DoubleClick(500, 110)", + "intent_string": "To start exploring the features available in LibreOffice Math by clicking an icon or option on the toolbar.", + "before_screen_ocr_results": [ + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": " -. n-- I@", + "boxes": [ + 4, + 48, + 644, + 87 + ] + }, + { + "text": "A", + "boxes": [ + 724, + 62, + 740, + 77 + ] + }, + { + "text": "Unary/Binary Operators", + "boxes": [ + 20, + 105, + 172, + 123 + ] + }, + { + "text": "一", + "boxes": [ + 46, + 159, + 58, + 171 + ] + }, + { + "text": "口", + "boxes": [ + 114, + 157, + 136, + 171 + ] + }, + { + "text": "±口", + "boxes": [ + 174, + 155, + 208, + 175 + ] + }, + { + "text": "口", + "boxes": [ + 242, + 153, + 282, + 179 + ] + }, + { + "text": "□+□冏□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 86, + 282 + ] + }, + { + "text": "号", + "boxes": [ + 104, + 252, + 134, + 286 + ] + }, + { + "text": "□÷□□口", + "boxes": [ + 154, + 252, + 286, + 282 + ] + }, + { + "text": "□", + "boxes": [ + 16, + 296, + 292, + 334 + ] + }, + { + "text": " %", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 409, + 72, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 10, + 28, + 34, + 42 + ] + }, + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": "----.", + "boxes": [ + 6, + 52, + 458, + 87 + ] + }, + { + "text": "SI④口回", + "boxes": [ + 456, + 49, + 642, + 84 + ] + }, + { + "text": " Unary/Binary Operators", + "boxes": [ + 16, + 103, + 174, + 127 + ] + }, + { + "text": "+口", + "boxes": [ + 30, + 153, + 70, + 179 + ] + }, + { + "text": "-□", + "boxes": [ + 96, + 153, + 142, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 168, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 153, + 282, + 179 + ] + }, + { + "text": "□+□□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 96, + 282 + ] + }, + { + "text": "廿", + "boxes": [ + 102, + 252, + 138, + 288 + ] + }, + { + "text": "□÷口口", + "boxes": [ + 156, + 252, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": "0 %品", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 409, + 72, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ] + }, + { + "frame_id": "60_0_8", + "episode_id": 60, + "env_id": 0, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.480428695678711, + "advantages": -0.14294403791427612, + "generated_text": "{\n \"intent\": \"To scroll upwards to see more of the document content and potentially find elements or further interactions.\",\n \"action\": \"ScrollUp(100, 500)\"\n}", + "action_string": "ScrollUp(100, 500)", + "intent_string": "To scroll upwards to see more of the document content and potentially find elements or further interactions.", + "before_screen_ocr_results": [ + { + "text": "File Edit View", + "boxes": [ + 8, + 26, + 116, + 44 + ] + }, + { + "text": "Styles", + "boxes": [ + 252, + 30, + 292, + 46 + ] + }, + { + "text": "--- @x自- ", + "boxes": [ + 10, + 52, + 378, + 87 + ] + }, + { + "text": "G--", + "boxes": [ + 402, + 54, + 524, + 83 + ] + }, + { + "text": "AbcT", + "boxes": [ + 514, + 56, + 598, + 81 + ] + }, + { + "text": "m·", + "boxes": [ + 608, + 54, + 728, + 83 + ] + }, + { + "text": "·国→田", + "boxes": [ + 716, + 52, + 1254, + 87 + ] + }, + { + "text": "Noto sans", + "boxes": [ + 136, + 93, + 298, + 123 + ] + }, + { + "text": "J aaa aab ab", + "boxes": [ + 426, + 93, + 654, + 123 + ] + }, + { + "text": "、", + "boxes": [ + 724, + 101, + 750, + 115 + ] + }, + { + "text": "D", + "boxes": [ + 744, + 101, + 770, + 115 + ] + }, + { + "text": "川", + "boxes": [ + 834, + 99, + 874, + 119 + ] + }, + { + "text": "三", + "boxes": [ + 850, + 95, + 930, + 121 + ] + }, + { + "text": "·m国三·", + "boxes": [ + 928, + 93, + 1212, + 123 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 173, + 1912, + 199 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 606, + 193, + 700, + 224 + ] + }, + { + "text": "D", + "boxes": [ + 1886, + 204, + 1914, + 238 + ] + }, + { + "text": "Dear <TITLE><RECIPIENT's NAME>", + "boxes": [ + 606, + 238, + 802, + 262 + ] + }, + { + "text": "E", + "boxes": [ + 1884, + 238, + 1910, + 278 + ] + }, + { + "text": "Lejsteestseddgdtrtt", + "boxes": [ + 604, + 274, + 1258, + 310 + ] + }, + { + "text": "dlore magna aliaua Ut enim ad minim yenim. guis nostud eeritatin yamco laboris nisi yt", + "boxes": [ + 606, + 298, + 1256, + 328 + ] + }, + { + "text": "lipexetteedderinlteltsl", + "boxes": [ + 602, + 314, + 1260, + 355 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 310, + 1916, + 343 + ] + }, + { + "text": "fd", + "boxes": [ + 602, + 337, + 1258, + 375 + ] + }, + { + "text": "deserunt molt aiun id est laherwm.", + "boxes": [ + 604, + 359, + 844, + 397 + ] + }, + { + "text": "Lejstoeecstseddgdtertt", + "boxes": [ + 604, + 401, + 1258, + 437 + ] + }, + { + "text": "dlore mana aliaua Ut enim ad minim yenim. auis nostrud exercitatin wamco labois nisi t", + "boxes": [ + 606, + 427, + 1258, + 457 + ] + }, + { + "text": "aliquip ex ea commodo consequat.", + "boxes": [ + 606, + 451, + 832, + 480 + ] + }, + { + "text": "We are looking forward to hearing from you soon.", + "boxes": [ + 608, + 490, + 922, + 514 + ] + }, + { + "text": "Yours sincerely.", + "boxes": [ + 608, + 532, + 710, + 558 + ] + }, + { + "text": "<SENDER's NAME>", + "boxes": [ + 608, + 572, + 710, + 596 + ] + }, + { + "text": "First Page", + "boxes": [ + 668, + 1018, + 732, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "·目X自", + "boxes": [ + 12, + 52, + 364, + 87 + ] + }, + { + "text": " Abc", + "boxes": [ + 514, + 56, + 582, + 81 + ] + }, + { + "text": "Noto sans", + "boxes": [ + 144, + 93, + 296, + 123 + ] + }, + { + "text": "aaaaabab", + "boxes": [ + 448, + 93, + 650, + 123 + ] + }, + { + "text": "10", + "boxes": [ + 374, + 103, + 386, + 115 + ] + }, + { + "text": "X", + "boxes": [ + 666, + 99, + 692, + 119 + ] + }, + { + "text": "=", + "boxes": [ + 870, + 101, + 890, + 115 + ] + }, + { + "text": "正正E·手", + "boxes": [ + 1034, + 95, + 1194, + 119 + ] + }, + { + "text": "E", + "boxes": [ + 1186, + 101, + 1204, + 115 + ] + }, + { + "text": "SENDER'S NAME", + "boxes": [ + 1036, + 201, + 1128, + 218 + ] + }, + { + "text": " Put ", + "boxes": [ + 1174, + 202, + 1224, + 236 + ] + }, + { + "text": "D", + "boxes": [ + 1890, + 208, + 1912, + 234 + ] + }, + { + "text": " your", + "boxes": [ + 1170, + 232, + 1230, + 266 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": "LOGO", + "boxes": [ + 1164, + 256, + 1242, + 294 + ] + }, + { + "text": "here", + "boxes": [ + 1167, + 287, + 1231, + 325 + ] + }, + { + "text": "V", + "boxes": [ + 1890, + 282, + 1906, + 300 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 310, + 1914, + 343 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 437, + 698, + 463 + ] + }, + { + "text": "Dear <TITLE><RECIPIENT's NAME>", + "boxes": [ + 608, + 478, + 802, + 502 + ] + }, + { + "text": "Leteesedddrt", + "boxes": [ + 604, + 514, + 1258, + 550 + ] + }, + { + "text": "dlore magna aliaua Ut enim ad minim yenim. guis nostrud eeritatin yamco laboris nisi yt", + "boxes": [ + 606, + 538, + 1256, + 568 + ] + }, + { + "text": "sluip exeasomudnseuatuisuteiuredolrineprehendertinyoluteteeit esseilm", + "boxes": [ + 602, + 556, + 1256, + 592 + ] + }, + { + "text": "dQlre ufugiat nu ariatu ceter sintcaecat supidatat non proident, sunt in sua guigffcia", + "boxes": [ + 606, + 582, + 1256, + 611 + ] + }, + { + "text": "deserunt molit anim id est laborum.", + "boxes": [ + 608, + 607, + 838, + 631 + ] + }, + { + "text": "Loreujumdorsit met sonectetr aciniscieltseddogiumodteurniduntytloeet", + "boxes": [ + 604, + 641, + 1258, + 677 + ] + }, + { + "text": "dlore mana aliaua. Ut enim ad minim yenim. auis nostrud exercitation wamco Jaboris nisi ut", + "boxes": [ + 606, + 667, + 1256, + 697 + ] + }, + { + "text": " aliquip ex ea sommodo sonsequat.", + "boxes": [ + 604, + 687, + 832, + 721 + ] + }, + { + "text": "urs sincerely,", + "boxes": [ + 626, + 772, + 708, + 796 + ] + } + ] + }, + { + "frame_id": "60_1_8", + "episode_id": 60, + "env_id": 1, + "step": 8, + "format_rewards": 1.0, + "rewards": 8.776427268981934, + "advantages": -0.661243736743927, + "generated_text": "{\n \"intent\": \"Double-clicking on an icon will help me open an application and explore its functionality, such as calculator in this case.\",\n \"action\": \"DoubleClick(210, 257)\"\n}", + "action_string": "DoubleClick(210, 257)", + "intent_string": "Double-clicking on an icon will help me open an application and explore its functionality, such as calculator in this case.", + "before_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 286, + 34 + ] + }, + { + "text": "Edit View", + "boxes": [ + 298, + 35, + 388, + 58 + ] + }, + { + "text": " Calculator Help", + "boxes": [ + 382, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→", + "boxes": [ + 584, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 904, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 806, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "Apple", + "boxes": [ + 752, + 492, + 796, + 518 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 90, + 528 + ] + }, + { + "text": "bing", + "boxes": [ + 879, + 493, + 909, + 514 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 820, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search -Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 284, + 34 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 298, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→C", + "boxes": [ + 582, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 904, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 806, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "Apple", + "boxes": [ + 754, + 492, + 796, + 518 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 90, + 528 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 820, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search - Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_8", + "episode_id": 60, + "env_id": 2, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.181102752685547, + "advantages": 0.372905969619751, + "generated_text": "{\n \"intent\": \"To scroll down the webpage to view more gaming products.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To scroll down the webpage to view more gaming products.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "曰", + "boxes": [ + 254, + 66, + 310, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "gaming", + "boxes": [ + 294, + 113, + 364, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1804, + 127, + 1884, + 151 + ] + }, + { + "text": "三 AIl", + "boxes": [ + 22, + 167, + 68, + 195 + ] + }, + { + "text": "Explore Related Products", + "boxes": [ + 10, + 256, + 184, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "gaming headset ", + "boxes": [ + 10, + 280, + 122, + 310 + ] + }, + { + "text": "gaming keyboard", + "boxes": [ + 12, + 304, + 126, + 328 + ] + }, + { + "text": "gaming pc", + "boxes": [ + 12, + 324, + 86, + 347 + ] + }, + { + "text": "gaming mouse", + "boxes": [ + 14, + 343, + 108, + 367 + ] + }, + { + "text": "See", + "boxes": [ + 20, + 367, + 50, + 381 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 409, + 50, + 435 + ] + }, + { + "text": "$0 - $8,900+", + "boxes": [ + 10, + 435, + 100, + 459 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1808, + 502, + 1866, + 528 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 10, + 514, + 136, + 544 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 540, + 98, + 566 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1794, + 550, + 1860, + 570 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 560, + 100, + 586 + ] + }, + { + "text": "rReviews", + "boxes": [ + 70, + 596, + 130, + 613 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 617, + 144, + 647 + ] + }, + { + "text": "ASUS ROG Strix G16 Gaming ", + "boxes": [ + 1488, + 613, + 1708, + 643 + ] + }, + { + "text": "Magnetic Dart Board - 12pcs", + "boxes": [ + 310, + 635, + 532, + 669 + ] + }, + { + "text": "Amazon Basics IPS Monitor 75 Hz ", + "boxes": [ + 608, + 637, + 860, + 667 + ] + }, + { + "text": "Audioengine A2+ Desktop 24 Bit", + "boxes": [ + 900, + 635, + 1146, + 671 + ] + }, + { + "text": "Mini projector portable projector", + "boxes": [ + 1194, + 639, + 1440, + 669 + ] + }, + { + "text": "Laptop, 165Hz Display, NVIDIA@", + "boxes": [ + 1486, + 637, + 1726, + 667 + ] + }, + { + "text": "Department", + "boxes": [ + 10, + 653, + 100, + 685 + ] + }, + { + "text": "Magnetic Darts - Excellent Indoor", + "boxes": [ + 316, + 665, + 562, + 689 + ] + }, + { + "text": "Powered with AOC Technology FHD", + "boxes": [ + 608, + 663, + 872, + 693 + ] + }, + { + "text": "Wireless Bluetooth Desktop", + "boxes": [ + 900, + 659, + 1112, + 693 + ] + }, + { + "text": "movie home cinema bedroom", + "boxes": [ + 1194, + 663, + 1418, + 691 + ] + }, + { + "text": "GeForce RTXTM 4060,Intel Core", + "boxes": [ + 1488, + 661, + 1724, + 691 + ] + }, + { + "text": "Computers & Accssories", + "boxes": [ + 10, + 677, + 176, + 707 + ] + }, + { + "text": "Game and Party Games - Gifts for 5", + "boxes": [ + 314, + 687, + 580, + 717 + ] + }, + { + "text": "1080P HDMI, Display Port and VGA", + "boxes": [ + 608, + 687, + 872, + 717 + ] + }, + { + "text": "Speakers - 60W Computer Speakers ", + "boxes": [ + 900, + 687, + 1168, + 717 + ] + }, + { + "text": "student dorm projector watch soccer", + "boxes": [ + 1192, + 687, + 1468, + 717 + ] + }, + { + "text": "i7-13650HX, 16GB DDR5, 1TB PCI...", + "boxes": [ + 1482, + 683, + 1754, + 713 + ] + }, + { + "text": " Computers & Tablets", + "boxes": [ + 20, + 703, + 162, + 721 + ] + }, + { + "text": "678 9 1011 12 Year Old Boy Kids", + "boxes": [ + 312, + 709, + 574, + 741 + ] + }, + { + "text": "PC Games & Accessories", + "boxes": [ + 12, + 719, + 168, + 742 + ] + }, + { + "text": "Input VESA Compatible Built-i...", + "boxes": [ + 608, + 711, + 854, + 741 + ] + }, + { + "text": "for Music and Gaming ", + "boxes": [ + 902, + 711, + 1068, + 741 + ] + }, + { + "text": " competition basketball match for..", + "boxes": [ + 1192, + 709, + 1452, + 741 + ] + }, + { + "text": "★★★★☆1,895", + "boxes": [ + 1490, + 715, + 1630, + 739 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 721, + 1860, + 739 + ] + }, + { + "text": "★★★★☆5,961", + "boxes": [ + 318, + 739, + 454, + 762 + ] + }, + { + "text": "ons: 2 sizes", + "boxes": [ + 634, + 742, + 696, + 760 + ] + }, + { + "text": "★★★★☆3,323", + "boxes": [ + 904, + 739, + 1042, + 762 + ] + }, + { + "text": "★★★★☆127", + "boxes": [ + 1196, + 737, + 1326, + 766 + ] + }, + { + "text": "3K+ bought in past month", + "boxes": [ + 1490, + 735, + 1660, + 758 + ] + }, + { + "text": "Brands", + "boxes": [ + 8, + 748, + 68, + 780 + ] + }, + { + "text": "ASUS", + "boxes": [ + 5, + 766, + 79, + 803 + ] + }, + { + "text": "ught in past month", + "boxes": [ + 362, + 762, + 484, + 780 + ] + }, + { + "text": "★★★★★1.028", + "boxes": [ + 606, + 760, + 750, + 794 + ] + }, + { + "text": "300+", + "boxes": [ + 906, + 760, + 936, + 780 + ] + }, + { + "text": "$8999", + "boxes": [ + 1194, + 770, + 1262, + 802 + ] + }, + { + "text": "$1,24478", + "boxes": [ + 1484, + 762, + 1594, + 800 + ] + }, + { + "text": "立", + "boxes": [ + 1792, + 768, + 1820, + 788 + ] + }, + { + "text": "1", + "boxes": [ + 1818, + 768, + 1868, + 786 + ] + }, + { + "text": " VISOONE", + "boxes": [ + 10, + 794, + 100, + 818 + ] + }, + { + "text": "$2279 List: 529.99", + "boxes": [ + 309, + 785, + 455, + 833 + ] + }, + { + "text": "ught in past month", + "boxes": [ + 658, + 790, + 776, + 808 + ] + }, + { + "text": "$279", + "boxes": [ + 900, + 788, + 982, + 826 + ] + }, + { + "text": "Save 10% with coupon", + "boxes": [ + 1196, + 800, + 1344, + 830 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": " Logitech G", + "boxes": [ + 10, + 812, + 108, + 844 + ] + }, + { + "text": "$9999", + "boxes": [ + 606, + 816, + 672, + 850 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 316, + 826, + 454, + 856 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 902, + 822, + 1044, + 856 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 1194, + 824, + 1338, + 854 + ] + }, + { + "text": "Ships to Cana", + "boxes": [ + 1492, + 828, + 1564, + 846 + ] + }, + { + "text": "=", + "boxes": [ + 14, + 838, + 38, + 852 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 606, + 852, + 750, + 881 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 902, + 850, + 992, + 874 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 848, + 1288, + 874 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": " FIFINE", + "boxes": [ + 9, + 871, + 75, + 896 + ] + }, + { + "text": "Ships to Cana", + "boxes": [ + 610, + 878, + 684, + 901 + ] + }, + { + "text": "Add to cat", + "boxes": [ + 901, + 873, + 993, + 910 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1198, + 876, + 1286, + 907 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1486, + 883, + 1632, + 913 + ] + }, + { + "text": "Redrago", + "boxes": [ + 11, + 893, + 85, + 920 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 616, + 907, + 696, + 933 + ] + }, + { + "text": "$1,219.99 (7 used & new offers)", + "boxes": [ + 1490, + 907, + 1698, + 931 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1814, + 927, + 1858, + 945 + ] + }, + { + "text": "Condition", + "boxes": [ + 14, + 951, + 78, + 969 + ] + }, + { + "text": "Best Seller", + "boxes": [ + 612, + 963, + 670, + 983 + ] + }, + { + "text": "1", + "boxes": [ + 1820, + 977, + 1852, + 991 + ] + }, + { + "text": "Used", + "boxes": [ + 16, + 993, + 68, + 1012 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "（", + "boxes": [ + 16, + 68, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 30, + 61, + 126, + 92 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1878, + 91 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 197, + 94, + 220 + ] + }, + { + "text": "Today's Deal", + "boxes": [ + 18, + 220, + 94, + 238 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 272, + 144, + 302 + ] + }, + { + "text": "Sponsored ", + "boxes": [ + 318, + 272, + 388, + 294 + ] + }, + { + "text": "Sponsored", + "boxes": [ + 906, + 272, + 972, + 292 + ] + }, + { + "text": "Sponsored ", + "boxes": [ + 1196, + 270, + 1270, + 294 + ] + }, + { + "text": "ASUS ROG Strix G16 Gaming ", + "boxes": [ + 1488, + 268, + 1706, + 298 + ] + }, + { + "text": "Magnetic Dart Board - 12pcs", + "boxes": [ + 310, + 290, + 532, + 324 + ] + }, + { + "text": "Amazon Basics IPS Monitor 75 Hz ", + "boxes": [ + 608, + 292, + 860, + 322 + ] + }, + { + "text": "Audioengine A2+ Desktop 24 Bit", + "boxes": [ + 898, + 288, + 1144, + 326 + ] + }, + { + "text": "Mini projector portable projector", + "boxes": [ + 1190, + 290, + 1442, + 326 + ] + }, + { + "text": "Laptop, 165Hz Display, NVIDIA", + "boxes": [ + 1486, + 288, + 1722, + 324 + ] + }, + { + "text": "Department", + "boxes": [ + 12, + 310, + 98, + 336 + ] + }, + { + "text": "Magnetic Darts - Excellent Indoor", + "boxes": [ + 314, + 318, + 564, + 347 + ] + }, + { + "text": "Powered with AOC Technology FHD", + "boxes": [ + 608, + 318, + 872, + 347 + ] + }, + { + "text": "Wireless Bluetooth Desktop", + "boxes": [ + 900, + 316, + 1112, + 350 + ] + }, + { + "text": " movie home cinema bedroom", + "boxes": [ + 1192, + 315, + 1420, + 350 + ] + }, + { + "text": "GeForce RTXTM 4060, Intel Core", + "boxes": [ + 1488, + 316, + 1720, + 345 + ] + }, + { + "text": "Computers&Accessories", + "boxes": [ + 18, + 338, + 172, + 355 + ] + }, + { + "text": "Game and Party Games - Gifts for 5", + "boxes": [ + 312, + 339, + 578, + 369 + ] + }, + { + "text": "1080P HDMI, Display Port and VGA", + "boxes": [ + 610, + 341, + 870, + 371 + ] + }, + { + "text": "Speakers - 60w Comput", + "boxes": [ + 902, + 341, + 1076, + 371 + ] + }, + { + "text": "student dorm projector watch soccer", + "boxes": [ + 1194, + 341, + 1464, + 371 + ] + }, + { + "text": "i7-13650HX, 16GB DDR5, 1TB PCI..", + "boxes": [ + 1486, + 339, + 1754, + 369 + ] + }, + { + "text": " Computers & Tablets", + "boxes": [ + 14, + 351, + 162, + 381 + ] + }, + { + "text": "立", + "boxes": [ + 1794, + 347, + 1820, + 369 + ] + }, + { + "text": "678 9 10 11 12 Year Old Boy Kids", + "boxes": [ + 314, + 363, + 570, + 395 + ] + }, + { + "text": "Input VESA Compatible Built-i...", + "boxes": [ + 608, + 365, + 852, + 395 + ] + }, + { + "text": "for Music and Gaming", + "boxes": [ + 902, + 365, + 1066, + 395 + ] + }, + { + "text": "competition basketball match for...", + "boxes": [ + 1194, + 365, + 1456, + 395 + ] + }, + { + "text": "★★★★1.895", + "boxes": [ + 1488, + 367, + 1632, + 397 + ] + }, + { + "text": "★★★★☆5,961", + "boxes": [ + 316, + 395, + 452, + 419 + ] + }, + { + "text": "Options: 2 sizes", + "boxes": [ + 612, + 399, + 696, + 417 + ] + }, + { + "text": "★★★★☆3.323", + "boxes": [ + 904, + 391, + 1044, + 421 + ] + }, + { + "text": "★★★★☆127", + "boxes": [ + 1194, + 393, + 1324, + 423 + ] + }, + { + "text": "in past month", + "boxes": [ + 1566, + 393, + 1656, + 411 + ] + }, + { + "text": "Brands", + "boxes": [ + 10, + 405, + 62, + 431 + ] + }, + { + "text": "ASUS", + "boxes": [ + 7, + 424, + 77, + 457 + ] + }, + { + "text": "★★★★★1.028", + "boxes": [ + 608, + 419, + 750, + 449 + ] + }, + { + "text": "$8999", + "boxes": [ + 1191, + 422, + 1261, + 459 + ] + }, + { + "text": "$1,24478", + "boxes": [ + 1486, + 419, + 1592, + 457 + ] + }, + { + "text": "VISOONE", + "boxes": [ + 5, + 442, + 99, + 479 + ] + }, + { + "text": "$2279 List: $29.99", + "boxes": [ + 311, + 442, + 453, + 483 + ] + }, + { + "text": "3K+ bought in past month", + "boxes": [ + 606, + 439, + 776, + 469 + ] + }, + { + "text": "$27900", + "boxes": [ + 897, + 440, + 983, + 483 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 455, + 1628, + 484 + ] + }, + { + "text": "$9999", + "boxes": [ + 606, + 472, + 672, + 506 + ] + }, + { + "text": "口A", + "boxes": [ + 14, + 488, + 40, + 510 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 316, + 484, + 452, + 508 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 900, + 480, + 1042, + 510 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 1194, + 480, + 1336, + 510 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1492, + 480, + 1578, + 504 + ] + }, + { + "text": "Amazon Basic", + "boxes": [ + 34, + 494, + 124, + 512 + ] + }, + { + "text": "口R", + "boxes": [ + 12, + 510, + 42, + 530 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 606, + 506, + 750, + 536 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 504, + 990, + 530 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 504, + 1282, + 530 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 506, + 1580, + 538 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1808, + 502, + 1864, + 528 + ] + }, + { + "text": "Razer", + "boxes": [ + 34, + 512, + 68, + 532 + ] + }, + { + "text": "FIFINE", + "boxes": [ + 7, + 525, + 79, + 559 + ] + }, + { + "text": "Ssee optlons", + "boxes": [ + 315, + 531, + 403, + 568 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 608, + 532, + 700, + 556 + ] + }, + { + "text": "Add to t", + "boxes": [ + 903, + 529, + 993, + 566 + ] + }, + { + "text": "Ad t cat", + "boxes": [ + 1198, + 530, + 1288, + 568 + ] + }, + { + "text": " Redragon", + "boxes": [ + 5, + 541, + 97, + 581 + ] + }, + { + "text": "More Buying Choices", + "boxes": [ + 1486, + 538, + 1624, + 568 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 607, + 557, + 703, + 592 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1792, + 548, + 1862, + 574 + ] + }, + { + "text": "$1,219.99 (7 used & ne", + "boxes": [ + 1488, + 562, + 1638, + 586 + ] + }, + { + "text": "V See more", + "boxes": [ + 14, + 572, + 90, + 596 + ] + }, + { + "text": "Condition", + "boxes": [ + 12, + 606, + 82, + 627 + ] + }, + { + "text": "New", + "boxes": [ + 10, + 623, + 68, + 649 + ] + }, + { + "text": "used ", + "boxes": [ + 8, + 641, + 72, + 673 + ] + }, + { + "text": "$10", + "boxes": [ + 1080, + 641, + 1118, + 671 + ] + }, + { + "text": "Seller", + "boxes": [ + 10, + 675, + 54, + 703 + ] + }, + { + "text": "ROBLOX", + "boxes": [ + 964, + 677, + 1114, + 715 + ] + }, + { + "text": " The Wasteland G", + "boxes": [ + 8, + 717, + 144, + 746 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 719, + 1860, + 739 + ] + }, + { + "text": "5070", + "boxes": [ + 692, + 758, + 726, + 772 + ] + }, + { + "text": "New Releases", + "boxes": [ + 12, + 774, + 100, + 798 + ] + }, + { + "text": "口", + "boxes": [ + 1796, + 770, + 1810, + 786 + ] + }, + { + "text": "Last 30 days ", + "boxes": [ + 7, + 794, + 101, + 830 + ] + }, + { + "text": "Last 90 days ", + "boxes": [ + 12, + 818, + 98, + 844 + ] + }, + { + "text": "", + "boxes": [ + 986, + 816, + 1034, + 836 + ] + }, + { + "text": "Coming Soon", + "boxes": [ + 12, + 840, + 102, + 864 + ] + }, + { + "text": "1,000 Robux", + "boxes": [ + 974, + 844, + 1096, + 874 + ] + }, + { + "text": "Video Game DRM", + "boxes": [ + 8, + 868, + 132, + 897 + ] + }, + { + "text": "800+200Bonus Robux", + "boxes": [ + 970, + 883, + 1100, + 901 + ] + }, + { + "text": " DRM-Free", + "boxes": [ + 10, + 893, + 102, + 919 + ] + }, + { + "text": "□orgin", + "boxes": [ + 10, + 911, + 80, + 943 + ] + }, + { + "text": "●●+2", + "boxes": [ + 426, + 917, + 494, + 935 + ] + }, + { + "text": "ASUS TUF Gaming GeForce RTX TM", + "boxes": [ + 606, + 911, + 860, + 941 + ] + }, + { + "text": "Roblox Digital Gift Card - 1,000", + "boxes": [ + 900, + 911, + 1136, + 941 + ] + }, + { + "text": "SAMSUNG 990 PRO SSD 4TB PCle", + "boxes": [ + 1488, + 913, + 1738, + 937 + ] + }, + { + "text": " steam", + "boxes": [ + 8, + 927, + 78, + 961 + ] + }, + { + "text": "5070 12GB GDDR7 OC Edition", + "boxes": [ + 606, + 931, + 840, + 965 + ] + }, + { + "text": "Robux [Includes Exclusive Virtual", + "boxes": [ + 900, + 933, + 1144, + 965 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1814, + 925, + 1860, + 945 + ] + }, + { + "text": "BENGOO G9000 Stereo Gaming ", + "boxes": [ + 312, + 941, + 556, + 975 + ] + }, + { + "text": "Razer BlackShark V2 X Gaming ", + "boxes": [ + 1192, + 941, + 1428, + 977 + ] + }, + { + "text": "4.0 M.2 2280 Internal Solid State", + "boxes": [ + 1490, + 937, + 1736, + 961 + ] + }, + { + "text": "uplay", + "boxes": [ + 7, + 950, + 75, + 981 + ] + }, + { + "text": "Gaming Graphics Card (PCle@ 5.0,", + "boxes": [ + 608, + 959, + 856, + 989 + ] + }, + { + "text": "Item] [Digital Code] ", + "boxes": [ + 902, + 959, + 1052, + 989 + ] + }, + { + "text": "Hard Drive, Seq. Read Speeds Upto", + "boxes": [ + 1486, + 959, + 1752, + 989 + ] + }, + { + "text": "Headset for PS4 PC Xbox One PS5 ", + "boxes": [ + 316, + 971, + 570, + 995 + ] + }, + { + "text": "Headset: 7.1 Surround Sound -", + "boxes": [ + 1194, + 969, + 1424, + 999 + ] + }, + { + "text": "廿1+", + "boxes": [ + 1790, + 971, + 1886, + 997 + ] + }, + { + "text": "HDMI/DP 2.1, 3.125-slot, Military..", + "boxes": [ + 608, + 983, + 874, + 1012 + ] + }, + { + "text": "May 15, 2019 ", + "boxes": [ + 904, + 985, + 996, + 1009 + ] + }, + { + "text": "7,450 MB/s for High End Computi.", + "boxes": [ + 1488, + 983, + 1750, + 1012 + ] + }, + { + "text": "Controlle, Noise Cancelling Over Ear", + "boxes": [ + 314, + 993, + 588, + 1022 + ] + }, + { + "text": "50mm Drivers - Memory Foam", + "boxes": [ + 1194, + 993, + 1426, + 1022 + ] + }, + { + "text": "s with Mic, LED Light, 7...", + "boxes": [ + 404, + 1012, + 584, + 1047 + ] + }, + { + "text": "★★★★★683", + "boxes": [ + 606, + 1006, + 740, + 1040 + ] + }, + { + "text": "★★★★☆171,494", + "boxes": [ + 902, + 1007, + 1062, + 1036 + ] + }, + { + "text": "Cushion - For PC, PS4, PS5, Switch", + "boxes": [ + 1194, + 1016, + 1446, + 1040 + ] + }, + { + "text": "★★★★★12,593", + "boxes": [ + 1486, + 1007, + 1640, + 1040 + ] + } + ] + }, + { + "frame_id": "60_3_8", + "episode_id": 60, + "env_id": 3, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.646425247192383, + "advantages": 0.7154855132102966, + "generated_text": "{\n \"intent\": \"Clicking on one of the website shortcuts will provide navigation possibilities and allow further exploration of various resources.\",\n \"action\": \"DoubleClick(779, 735)\"\n}", + "action_string": "DoubleClick(779, 735)", + "intent_string": "Clicking on one of the website shortcuts will provide navigation possibilities and allow further exploration of various resources.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 42, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or ente", + "boxes": [ + 254, + 66, + 452, + 89 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 190, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 658, + 290, + 766, + 314 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 10, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 440, + 89 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1358, + 373, + 1420, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 484, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13, 287", + "boxes": [ + 1626, + 893, + 1700, + 941 + ] + }, + { + "text": "点资讯·1天", + "boxes": [ + 240, + 921, + 316, + 945 + ] + }, + { + "text": "04128", + "boxes": [ + 456, + 927, + 550, + 951 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 925, + 850, + 941 + ] + }, + { + "text": "一点资讯·1天", + "boxes": [ + 850, + 921, + 942, + 945 + ] + }, + { + "text": "金融界汽车·9小时", + "boxes": [ + 1138, + 921, + 1278, + 945 + ] + }, + { + "text": "梦幻西游：万元赐福，数亿成", + "boxes": [ + 824, + 941, + 1098, + 977 + ] + }, + { + "text": "赛力斯申请高速过弯转向不足", + "boxes": [ + 1136, + 941, + 1408, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "梦幻西游：梦幻也进入数据膨胀时代了，物理", + "boxes": [ + 202, + 945, + 794, + 987 + ] + }, + { + "text": "r'.", + "boxes": [ + 1584, + 937, + 1636, + 983 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "本改书，号主被称为梦幻玩家", + "boxes": [ + 824, + 967, + 1098, + 1003 + ] + }, + { + "text": "控制专利，提高稳定性和安全", + "boxes": [ + 1136, + 967, + 1410, + 1003 + ] + }, + { + "text": "系伤害都是十万级别的", + "boxes": [ + 200, + 979, + 508, + 1026 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "界的狠人", + "boxes": [ + 826, + 997, + 914, + 1028 + ] + }, + { + "text": "性", + "boxes": [ + 1138, + 1001, + 1170, + 1022 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "△1", + "boxes": [ + 830, + 1028, + 862, + 1042 + ] + } + ] + }, + { + "frame_id": "60_4_8", + "episode_id": 60, + "env_id": 4, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.385417938232422, + "advantages": -0.2128928303718567, + "generated_text": "{\n \"intent\": \"To continue interacting with the desktop environment by exploring a web page or performing a search, I will open a new browser tab.\",\n \"action\": \"Click(243, 43)\"\n}", + "action_string": "Click(243, 43)", + "intent_string": "To continue interacting with the desktop environment by exploring a web page or performing a search, I will open a new browser tab.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1384, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 836, + 582, + 936, + 606 + ] + }, + { + "text": "Privacy Notice", + "boxes": [ + 944, + 582, + 1046, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 258, + 32, + 312, + 52 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 250, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_5_8", + "episode_id": 60, + "env_id": 5, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.317808151245117, + "advantages": -0.2626684904098511, + "generated_text": "{\n \"intent\": \"To scroll through the webpage content to explore more details about the article.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To scroll through the webpage content to explore more details about the article.", + "before_screen_ocr_results": [ + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 252, + 32, + 412, + 50 + ] + }, + { + "text": "C", + "boxes": [ + 74, + 68, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1872, + 89 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 568, + 121, + 636, + 147 + ] + }, + { + "text": "长春市", + "boxes": [ + 1636, + 125, + 1684, + 143 + ] + }, + { + "text": "本文转自：经济日报", + "boxes": [ + 364, + 173, + 524, + 202 + ] + }, + { + "text": "豆包AI", + "boxes": [ + 1568, + 199, + 1708, + 256 + ] + }, + { + "text": "全能高效", + "boxes": [ + 1568, + 246, + 1706, + 296 + ] + }, + { + "text": "小", + "boxes": [ + 196, + 264, + 210, + 280 + ] + }, + { + "text": "写作助手", + "boxes": [ + 1564, + 282, + 1706, + 332 + ] + }, + { + "text": "", + "boxes": [ + 192, + 318, + 214, + 341 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1608, + 326, + 1668, + 351 + ] + }, + { + "text": "豆包PC版_操作便捷_免费下载安", + "boxes": [ + 1422, + 353, + 1688, + 389 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 369, + 218, + 401 + ] + }, + { + "text": "装", + "boxes": [ + 1428, + 383, + 1452, + 405 + ] + }, + { + "text": "橘", + "boxes": [ + 852, + 411, + 904, + 445 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1430, + 411, + 1528, + 435 + ] + }, + { + "text": "?", + "boxes": [ + 194, + 433, + 210, + 451 + ] + }, + { + "text": "方问智作说车", + "boxes": [ + 1464, + 498, + 1548, + 522 + ] + }, + { + "text": "长安汽车首个海外整车工厂在泰国", + "boxes": [ + 1444, + 542, + 1662, + 572 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 382, + 558, + 536, + 586 + ] + }, + { + "text": "投产", + "boxes": [ + 1450, + 568, + 1476, + 588 + ] + }, + { + "text": "最近，一个冷门技术参数上了热搜。起因是有汽车博主自费测试阿维塔12风阻系数，得出的数据与企业公布的数据差", + "boxes": [ + 360, + 590, + 1238, + 625 + ] + }, + { + "text": "距不小；而企业在相关公证机构见证下，通过中国汽车工程研究院在风洞实验室全程直播测试，得出的数据并不存在", + "boxes": [ + 364, + 621, + 1238, + 651 + ] + }, + { + "text": "中国东部小县城撬动全球轮胎市场", + "boxes": [ + 1448, + 623, + 1662, + 647 + ] + }, + { + "text": "虚标，从而引发舆论关注。", + "boxes": [ + 366, + 651, + 560, + 675 + ] + }, + { + "text": "汽车风阻系数是衡量车辆行驶时空气阻力大小的关键参数，直接影响燃油经济性、续航能力、高速稳定性和噪声水", + "boxes": [ + 364, + 689, + 1222, + 719 + ] + }, + { + "text": "长安汽车泰国工厂峻工投产", + "boxes": [ + 1450, + 683, + 1610, + 701 + ] + }, + { + "text": "平。数值越低，说明空气阻力越小。新能源汽车的加速普及和补能焦虑，使得低风阻成为新能源车企竞争中少有的\"有", + "boxes": [ + 364, + 717, + 1244, + 746 + ] + }, + { + "text": "感数据”：不仅能影响消费者认知，还能提升续航里程。", + "boxes": [ + 364, + 742, + 774, + 772 + ] + }, + { + "text": "电", + "boxes": [ + 1422, + 774, + 1442, + 794 + ] + }, + { + "text": "要了解风阻系数在新能源赛道是如何\"吹\"起来的，首先还得回溯到2012年，Model S横空出世，特斯拉便将风阻系数", + "boxes": [ + 362, + 784, + 1226, + 814 + ] + }, + { + "text": "打工人", + "boxes": [ + 1426, + 800, + 1502, + 832 + ] + }, + { + "text": "与科技含量进行了大量捆绑营销，不仅重塑了行业以及消费者对\"低风阻\"的认知，也强化了特斯拉富有科技感的品牌", + "boxes": [ + 364, + 810, + 1232, + 840 + ] + }, + { + "text": "高效工作的", + "boxes": [ + 1430, + 828, + 1568, + 860 + ] + }, + { + "text": "形象。此后，诸多新能源车企开始争相效仿，风阻系数纪录也不断被刷新。", + "boxes": [ + 362, + 838, + 918, + 862 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 28, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 836, + 879, + 1090, + 903 + ] + }, + { + "text": "我接受", + "boxes": [ + 1670, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 798, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1526, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置“改变您的选择。欲了解", + "boxes": [ + 28, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "理性看待风阻··", + "boxes": [ + 216, + 1048, + 306, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 74, + 68, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 570, + 121, + 636, + 147 + ] + }, + { + "text": "长春市", + "boxes": [ + 1636, + 125, + 1684, + 143 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "豆包AI", + "boxes": [ + 1568, + 199, + 1708, + 256 + ] + }, + { + "text": "全能高效", + "boxes": [ + 1568, + 246, + 1706, + 296 + ] + }, + { + "text": "", + "boxes": [ + 194, + 264, + 210, + 282 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 382, + 272, + 536, + 300 + ] + }, + { + "text": "写作助手", + "boxes": [ + 1564, + 280, + 1706, + 330 + ] + }, + { + "text": "", + "boxes": [ + 192, + 316, + 216, + 343 + ] + }, + { + "text": "最近，一个冷门技术参数上了热搜。起因是有汽车博主自费测试阿维塔12风阻系数，得出的数据与企业公布的数据差", + "boxes": [ + 364, + 310, + 1238, + 339 + ] + }, + { + "text": "距不小；而企业在相关公证机构见证下，通过中国汽车工程研究院在风洞实验室全程直播测试，得出的数据并不存在", + "boxes": [ + 366, + 339, + 1236, + 363 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1610, + 330, + 1666, + 349 + ] + }, + { + "text": "虚标，从而引发舆论关注。", + "boxes": [ + 362, + 361, + 560, + 391 + ] + }, + { + "text": "豆包PC版_操作便捷_免费下载安", + "boxes": [ + 1422, + 353, + 1688, + 389 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 369, + 218, + 403 + ] + }, + { + "text": "装", + "boxes": [ + 1422, + 379, + 1455, + 409 + ] + }, + { + "text": "汽车风阻系数是衡量车辆行驶时空气阻力大小的关键参数，直接影响燃油经济性、续航能力、高速稳定性和噪声水", + "boxes": [ + 364, + 403, + 1220, + 433 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 411, + 1528, + 435 + ] + }, + { + "text": "#", + "boxes": [ + 190, + 429, + 212, + 455 + ] + }, + { + "text": "平。数值越低，说明空气阻力越小。新能源汽车的加速普及和补能焦虑，使得低风阻成为新能源车企竞争中少有的\"有", + "boxes": [ + 362, + 429, + 1244, + 461 + ] + }, + { + "text": "感数据”：不仅能影响消费者认知，还能提升续航里程。", + "boxes": [ + 366, + 461, + 772, + 484 + ] + }, + { + "text": "要了解风阻系数在新能源赛道是如何\"吹\"起来的，首先还得回溯到2012年，Model S横空出世，特斯拉便将风阻系数", + "boxes": [ + 364, + 498, + 1226, + 528 + ] + }, + { + "text": "方问智作说车", + "boxes": [ + 1464, + 498, + 1548, + 522 + ] + }, + { + "text": "L", + "boxes": [ + 1682, + 502, + 1694, + 514 + ] + }, + { + "text": "与科技含量进行了大量捆绑营销，不仅重塑了行业以及消费者对\"低风阻\"的认知也强化了特斯拉富有科技感的品牌", + "boxes": [ + 362, + 520, + 1234, + 558 + ] + }, + { + "text": "形象。此后，诸多新能源车企开始争相效仿，风阻系数纪录也不断被刷新。", + "boxes": [ + 364, + 552, + 920, + 582 + ] + }, + { + "text": "长安汽车首个海外整车工厂在泰国", + "boxes": [ + 1446, + 544, + 1664, + 574 + ] + }, + { + "text": "投产", + "boxes": [ + 1450, + 568, + 1476, + 588 + ] + }, + { + "text": "散人打金服", + "boxes": [ + 618, + 621, + 686, + 647 + ] + }, + { + "text": "中国东部小县城撬动全球轮胎市场", + "boxes": [ + 1448, + 623, + 1662, + 647 + ] + }, + { + "text": "装备靠打，刀刀爆装高回收，散人打金天堂！", + "boxes": [ + 618, + 649, + 1020, + 679 + ] + }, + { + "text": "长安汽车泰国工厂峻工投产", + "boxes": [ + 1450, + 683, + 1610, + 701 + ] + }, + { + "text": "广告", + "boxes": [ + 620, + 725, + 642, + 739 + ] + }, + { + "text": "此次风阻系数测试之争表面是数值大小，背后折射出的是行业过度内卷。这两年，车市价格战愈演愈烈，但产品成本", + "boxes": [ + 364, + 780, + 1238, + 810 + ] + }, + { + "text": "电", + "boxes": [ + 1422, + 774, + 1442, + 794 + ] + }, + { + "text": "的下降毕竟没有售价下降得那么快，车市竞争已从价格卷到技术参数。每次新车发布会，车企都在穷尽办法，让各项", + "boxes": [ + 364, + 808, + 1238, + 838 + ] + }, + { + "text": "打工人", + "boxes": [ + 1426, + 800, + 1502, + 832 + ] + }, + { + "text": "数据变得极致一点，至少要比友商领先。在如此内卷的氛围裹挟下，每一个小数点都成了车企比拼的战场，风阻系数", + "boxes": [ + 362, + 836, + 1236, + 860 + ] + }, + { + "text": "高效工作的", + "boxes": [ + 1430, + 828, + 1568, + 860 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 28, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 836, + 879, + 1088, + 903 + ] + }, + { + "text": "我接受", + "boxes": [ + 1670, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 796, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1526, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击\"管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置“改变您的选择。欲了解", + "boxes": [ + 28, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "理性看待风阻··", + "boxes": [ + 218, + 1048, + 306, + 1072 + ] + } + ] + }, + { + "frame_id": "60_6_8", + "episode_id": 60, + "env_id": 6, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.686917304992676, + "advantages": 0.009076926857233047, + "generated_text": "{\n \"intent\": \"To scroll down and view more content on the webpage.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To scroll down and view more content on the webpage.", + "before_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 262, + 73, + 274, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Visior", + "boxes": [ + 838, + 113, + 868, + 127 + ] + }, + { + "text": "AirPods I E TV&Home", + "boxes": [ + 904, + 109, + 1046, + 133 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1396, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1438, + 129 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 141, + 1060, + 177 + ] + }, + { + "text": "Learn more", + "boxes": [ + 864, + 204, + 954, + 230 + ] + }, + { + "text": "Buy", + "boxes": [ + 1010, + 206, + 1044, + 228 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 796, + 752, + 1116, + 808 + ] + }, + { + "text": " Sky blue color.", + "boxes": [ + 860, + 810, + 1048, + 858 + ] + }, + { + "text": "Sky high performance with M4.", + "boxes": [ + 764, + 846, + 1144, + 887 + ] + }, + { + "text": "Learn more ", + "boxes": [ + 852, + 907, + 960, + 939 + ] + }, + { + "text": "Buy", + "boxes": [ + 1006, + 909, + 1048, + 935 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 794, + 469, + 1116, + 524 + ] + }, + { + "text": "Sky bluecolor.", + "boxes": [ + 864, + 530, + 1044, + 568 + ] + }, + { + "text": "Sky high performance with M4.", + "boxes": [ + 762, + 558, + 1146, + 606 + ] + }, + { + "text": "Learn more ", + "boxes": [ + 847, + 617, + 963, + 660 + ] + }, + { + "text": "Buy", + "boxes": [ + 1004, + 623, + 1050, + 649 + ] + }, + { + "text": "Built for Aople Intelliaence.", + "boxes": [ + 830, + 1016, + 1074, + 1054 + ] + } + ] + }, + { + "frame_id": "60_7_8", + "episode_id": 60, + "env_id": 7, + "step": 8, + "format_rewards": 1.0, + "rewards": 8.639925956726074, + "advantages": -0.761738657951355, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment by clicking on various components. This will provide insights into the layout and functionality of the software.\",\n \"action\": \"DoubleClick(900, 550)\"\n}", + "action_string": "DoubleClick(900, 550)", + "intent_string": "To continue exploring the desktop environment by clicking on various components. This will provide insights into the layout and functionality of the software.", + "before_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 10, + 28, + 34, + 42 + ] + }, + { + "text": "Window Help", + "boxes": [ + 248, + 24, + 346, + 48 + ] + }, + { + "text": "----.", + "boxes": [ + 6, + 52, + 458, + 87 + ] + }, + { + "text": "SI④口回", + "boxes": [ + 456, + 49, + 642, + 84 + ] + }, + { + "text": " Unary/Binary Operators", + "boxes": [ + 16, + 103, + 174, + 127 + ] + }, + { + "text": "+口", + "boxes": [ + 30, + 153, + 70, + 179 + ] + }, + { + "text": "-□", + "boxes": [ + 96, + 153, + 142, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 168, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 153, + 282, + 179 + ] + }, + { + "text": "□+□□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 96, + 282 + ] + }, + { + "text": "廿", + "boxes": [ + 102, + 252, + 138, + 288 + ] + }, + { + "text": "□÷口口", + "boxes": [ + 156, + 252, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": "0 %品", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 409, + 72, + 435 + ] + }, + { + "text": ">", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 10, + 28, + 34, + 42 + ] + }, + { + "text": "Window Help", + "boxes": [ + 242, + 24, + 346, + 48 + ] + }, + { + "text": "-----", + "boxes": [ + 6, + 52, + 470, + 87 + ] + }, + { + "text": "SID口D", + "boxes": [ + 454, + 50, + 644, + 85 + ] + }, + { + "text": "A", + "boxes": [ + 724, + 62, + 740, + 77 + ] + }, + { + "text": " Unary/Binary Operators", + "boxes": [ + 16, + 103, + 174, + 127 + ] + }, + { + "text": "+口", + "boxes": [ + 30, + 153, + 70, + 179 + ] + }, + { + "text": "-□", + "boxes": [ + 96, + 153, + 142, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 168, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 151, + 282, + 179 + ] + }, + { + "text": "□+□□-□□·×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 96, + 282 + ] + }, + { + "text": "廿", + "boxes": [ + 102, + 252, + 138, + 288 + ] + }, + { + "text": "□÷口口", + "boxes": [ + 156, + 252, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": "000%品", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 409, + 72, + 435 + ] + }, + { + "text": ">□v□", + "boxes": [ + 70, + 405, + 218, + 437 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ] + }, + { + "frame_id": "60_0_9", + "episode_id": 60, + "env_id": 0, + "step": 9, + "format_rewards": 0.0, + "rewards": 0.0, + "advantages": -7.122622013092041, + "generated_text": "{\n \"intent\": \" Opening the settings by clicking on the wrench icon allows me to explore various options and features of LibreOffice Writer, enriching my understanding of the desktop environment.\",\n \"action\": \"Click(1939, 174)\"\n}", + "action_string": "None", + "intent_string": null, + "before_screen_ocr_results": [ + { + "text": "·目X自", + "boxes": [ + 12, + 52, + 364, + 87 + ] + }, + { + "text": " Abc", + "boxes": [ + 514, + 56, + 582, + 81 + ] + }, + { + "text": "Noto sans", + "boxes": [ + 144, + 93, + 296, + 123 + ] + }, + { + "text": "aaaaabab", + "boxes": [ + 448, + 93, + 650, + 123 + ] + }, + { + "text": "10", + "boxes": [ + 374, + 103, + 386, + 115 + ] + }, + { + "text": "X", + "boxes": [ + 666, + 99, + 692, + 119 + ] + }, + { + "text": "=", + "boxes": [ + 870, + 101, + 890, + 115 + ] + }, + { + "text": "正正E·手", + "boxes": [ + 1034, + 95, + 1194, + 119 + ] + }, + { + "text": "E", + "boxes": [ + 1186, + 101, + 1204, + 115 + ] + }, + { + "text": "SENDER'S NAME", + "boxes": [ + 1036, + 201, + 1128, + 218 + ] + }, + { + "text": " Put ", + "boxes": [ + 1174, + 202, + 1224, + 236 + ] + }, + { + "text": "D", + "boxes": [ + 1890, + 208, + 1912, + 234 + ] + }, + { + "text": " your", + "boxes": [ + 1170, + 232, + 1230, + 266 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": "LOGO", + "boxes": [ + 1164, + 256, + 1242, + 294 + ] + }, + { + "text": "here", + "boxes": [ + 1167, + 287, + 1231, + 325 + ] + }, + { + "text": "V", + "boxes": [ + 1890, + 282, + 1906, + 300 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 310, + 1914, + 343 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 437, + 698, + 463 + ] + }, + { + "text": "Dear <TITLE><RECIPIENT's NAME>", + "boxes": [ + 608, + 478, + 802, + 502 + ] + }, + { + "text": "Leteesedddrt", + "boxes": [ + 604, + 514, + 1258, + 550 + ] + }, + { + "text": "dlore magna aliaua Ut enim ad minim yenim. guis nostrud eeritatin yamco laboris nisi yt", + "boxes": [ + 606, + 538, + 1256, + 568 + ] + }, + { + "text": "sluip exeasomudnseuatuisuteiuredolrineprehendertinyoluteteeit esseilm", + "boxes": [ + 602, + 556, + 1256, + 592 + ] + }, + { + "text": "dQlre ufugiat nu ariatu ceter sintcaecat supidatat non proident, sunt in sua guigffcia", + "boxes": [ + 606, + 582, + 1256, + 611 + ] + }, + { + "text": "deserunt molit anim id est laborum.", + "boxes": [ + 608, + 607, + 838, + 631 + ] + }, + { + "text": "Loreujumdorsit met sonectetr aciniscieltseddogiumodteurniduntytloeet", + "boxes": [ + 604, + 641, + 1258, + 677 + ] + }, + { + "text": "dlore mana aliaua. Ut enim ad minim yenim. auis nostrud exercitation wamco Jaboris nisi ut", + "boxes": [ + 606, + 667, + 1256, + 697 + ] + }, + { + "text": " aliquip ex ea sommodo sonsequat.", + "boxes": [ + 604, + 687, + 832, + 721 + ] + }, + { + "text": "urs sincerely,", + "boxes": [ + 626, + 772, + 708, + 796 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "·目X自", + "boxes": [ + 12, + 52, + 364, + 87 + ] + }, + { + "text": " Abc", + "boxes": [ + 514, + 56, + 582, + 81 + ] + }, + { + "text": "Noto sans", + "boxes": [ + 144, + 93, + 296, + 123 + ] + }, + { + "text": "aaaaabab", + "boxes": [ + 448, + 93, + 650, + 123 + ] + }, + { + "text": "10", + "boxes": [ + 374, + 103, + 386, + 115 + ] + }, + { + "text": "X", + "boxes": [ + 666, + 99, + 692, + 119 + ] + }, + { + "text": "=", + "boxes": [ + 870, + 101, + 890, + 115 + ] + }, + { + "text": "正正E·手", + "boxes": [ + 1034, + 95, + 1194, + 119 + ] + }, + { + "text": "E", + "boxes": [ + 1186, + 101, + 1204, + 115 + ] + }, + { + "text": "SENDER'S NAME", + "boxes": [ + 1036, + 201, + 1128, + 218 + ] + }, + { + "text": " Put ", + "boxes": [ + 1174, + 202, + 1224, + 236 + ] + }, + { + "text": "D", + "boxes": [ + 1890, + 208, + 1912, + 234 + ] + }, + { + "text": " your", + "boxes": [ + 1170, + 232, + 1230, + 266 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 240, + 1908, + 276 + ] + }, + { + "text": "LOGO", + "boxes": [ + 1164, + 256, + 1242, + 294 + ] + }, + { + "text": "here", + "boxes": [ + 1167, + 287, + 1231, + 325 + ] + }, + { + "text": "V", + "boxes": [ + 1890, + 282, + 1906, + 300 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 310, + 1914, + 343 + ] + }, + { + "text": "2025/ May17/", + "boxes": [ + 608, + 437, + 698, + 463 + ] + }, + { + "text": "Dear <TITLE><RECIPIENT's NAME>", + "boxes": [ + 608, + 478, + 802, + 502 + ] + }, + { + "text": "Leteesedddrt", + "boxes": [ + 604, + 514, + 1258, + 550 + ] + }, + { + "text": "dlore magna aliaua Ut enim ad minim yenim. guis nostrud eeritatin yamco laboris nisi yt", + "boxes": [ + 606, + 538, + 1256, + 568 + ] + }, + { + "text": "sluip exeasomudnseuatuisuteiuredolrineprehendertinyoluteteeit esseilm", + "boxes": [ + 602, + 556, + 1256, + 592 + ] + }, + { + "text": "dQlre ufugiat nu ariatu ceter sintcaecat supidatat non proident, sunt in sua guigffcia", + "boxes": [ + 606, + 582, + 1256, + 611 + ] + }, + { + "text": "deserunt molit anim id est laborum.", + "boxes": [ + 608, + 607, + 838, + 631 + ] + }, + { + "text": "Loreujumdorsit met sonectetr aciniscieltseddogiumodteurniduntytloeet", + "boxes": [ + 604, + 641, + 1258, + 677 + ] + }, + { + "text": "dlore mana aliaua. Ut enim ad minim yenim. auis nostrud exercitation wamco Jaboris nisi ut", + "boxes": [ + 606, + 667, + 1256, + 697 + ] + }, + { + "text": " aliquip ex ea sommodo sonsequat.", + "boxes": [ + 604, + 687, + 832, + 721 + ] + }, + { + "text": "urs sincerely,", + "boxes": [ + 626, + 772, + 708, + 796 + ] + } + ] + }, + { + "frame_id": "60_1_9", + "episode_id": 60, + "env_id": 1, + "step": 9, + "format_rewards": 1.0, + "rewards": 8.90507698059082, + "advantages": -0.5665292739868164, + "generated_text": "{\n \"intent\": \"To explore different applications or open a new tab in Firefox.\",\n \"action\": \"DoubleClick(178, 713)\"\n}", + "action_string": "DoubleClick(178, 713)", + "intent_string": "To explore different applications or open a new tab in Firefox.", + "before_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 284, + 34 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 298, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→C", + "boxes": [ + 582, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 904, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 806, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "Apple", + "boxes": [ + 754, + 492, + 796, + 518 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 90, + 528 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 820, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search - Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "2", + "boxes": [ + 30, + 772, + 82, + 832 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "galculator", + "boxes": [ + 360, + 10, + 450, + 34 + ] + }, + { + "text": "□", + "boxes": [ + 482, + 10, + 550, + 36 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1140, + 4, + 1252, + 22 + ] + }, + { + "text": "口口", + "boxes": [ + 1765, + 0, + 1837, + 33 + ] + }, + { + "text": "国", + "boxes": [ + 260, + 12, + 284, + 34 + ] + }, + { + "text": "Edit View Calculator Help", + "boxes": [ + 298, + 36, + 510, + 60 + ] + }, + { + "text": " New Tab", + "boxes": [ + 580, + 34, + 640, + 52 + ] + }, + { + "text": "New Tab", + "boxes": [ + 802, + 34, + 866, + 52 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "→", + "boxes": [ + 584, + 68, + 684, + 91 + ] + }, + { + "text": "三目◎", + "boxes": [ + 1700, + 66, + 1790, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 512, + 79, + 534, + 103 + ] + }, + { + "text": "Getting Startec", + "boxes": [ + 764, + 103, + 862, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1798, + 151, + 1812, + 167 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 904, + 250, + 1000, + 274 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 284, + 102, + 308 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 724, + 361, + 808, + 385 + ] + }, + { + "text": "Calculator", + "boxes": [ + 10, + 393, + 114, + 419 + ] + }, + { + "text": "知", + "boxes": [ + 1212, + 403, + 1296, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1094, + 415, + 1172, + 472 + ] + }, + { + "text": "B", + "boxes": [ + 758, + 443, + 802, + 494 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 16, + 500, + 92, + 532 + ] + }, + { + "text": "Apple", + "boxes": [ + 754, + 492, + 796, + 518 + ] + }, + { + "text": "Amazon", + "boxes": [ + 982, + 494, + 1042, + 514 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 1120, + 496, + 1150, + 516 + ] + }, + { + "text": "weibo", + "boxes": [ + 1474, + 494, + 1516, + 514 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1598, + 496, + 1632, + 516 + ] + }, + { + "text": " Highlights ", + "boxes": [ + 724, + 576, + 820, + 600 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "amazon", + "boxes": [ + 1222, + 631, + 1408, + 681 + ] + }, + { + "text": "moz:l/a", + "boxes": [ + 1468, + 643, + 1644, + 687 + ] + }, + { + "text": "雷", + "boxes": [ + 29, + 671, + 88, + 726 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 700, + 1413, + 735 + ] + }, + { + "text": "?", + "boxes": [ + 1628, + 709, + 1644, + 727 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "apple", + "boxes": [ + 737, + 724, + 783, + 758 + ] + }, + { + "text": " cn.bing", + "boxes": [ + 978, + 729, + 1030, + 754 + ] + }, + { + "text": "amazon", + "boxes": [ + 1224, + 733, + 1268, + 752 + ] + }, + { + "text": "mozilla", + "boxes": [ + 1462, + 733, + 1506, + 752 + ] + }, + { + "text": "Apple", + "boxes": [ + 740, + 750, + 786, + 778 + ] + }, + { + "text": " Search - Microsoft Bing", + "boxes": [ + 976, + 746, + 1142, + 781 + ] + }, + { + "text": "Amazon.com. Spend less...", + "boxes": [ + 1222, + 752, + 1402, + 776 + ] + }, + { + "text": "Firefox Privacy Notice -", + "boxes": [ + 1460, + 752, + 1620, + 776 + ] + }, + { + "text": "", + "boxes": [ + 30, + 769, + 82, + 831 + ] + }, + { + "text": "Draw", + "boxes": [ + 29, + 825, + 89, + 867 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "Mozilla Firefox", + "boxes": [ + 520, + 1048, + 614, + 1072 + ] + } + ] + }, + { + "frame_id": "60_2_9", + "episode_id": 60, + "env_id": 2, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.025543212890625, + "advantages": 0.2583799660205841, + "generated_text": "{\n \"intent\": \"To continue exploring product options on this page, showing more items by scrolling down.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring product options on this page, showing more items by scrolling down.", + "before_screen_ocr_results": [ + { + "text": "（", + "boxes": [ + 16, + 68, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 30, + 61, + 126, + 92 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1878, + 91 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 197, + 94, + 220 + ] + }, + { + "text": "Today's Deal", + "boxes": [ + 18, + 220, + 94, + 238 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 272, + 144, + 302 + ] + }, + { + "text": "Sponsored ", + "boxes": [ + 318, + 272, + 388, + 294 + ] + }, + { + "text": "Sponsored", + "boxes": [ + 906, + 272, + 972, + 292 + ] + }, + { + "text": "Sponsored ", + "boxes": [ + 1196, + 270, + 1270, + 294 + ] + }, + { + "text": "ASUS ROG Strix G16 Gaming ", + "boxes": [ + 1488, + 268, + 1706, + 298 + ] + }, + { + "text": "Magnetic Dart Board - 12pcs", + "boxes": [ + 310, + 290, + 532, + 324 + ] + }, + { + "text": "Amazon Basics IPS Monitor 75 Hz ", + "boxes": [ + 608, + 292, + 860, + 322 + ] + }, + { + "text": "Audioengine A2+ Desktop 24 Bit", + "boxes": [ + 898, + 288, + 1144, + 326 + ] + }, + { + "text": "Mini projector portable projector", + "boxes": [ + 1190, + 290, + 1442, + 326 + ] + }, + { + "text": "Laptop, 165Hz Display, NVIDIA", + "boxes": [ + 1486, + 288, + 1722, + 324 + ] + }, + { + "text": "Department", + "boxes": [ + 12, + 310, + 98, + 336 + ] + }, + { + "text": "Magnetic Darts - Excellent Indoor", + "boxes": [ + 314, + 318, + 564, + 347 + ] + }, + { + "text": "Powered with AOC Technology FHD", + "boxes": [ + 608, + 318, + 872, + 347 + ] + }, + { + "text": "Wireless Bluetooth Desktop", + "boxes": [ + 900, + 316, + 1112, + 350 + ] + }, + { + "text": " movie home cinema bedroom", + "boxes": [ + 1192, + 315, + 1420, + 350 + ] + }, + { + "text": "GeForce RTXTM 4060, Intel Core", + "boxes": [ + 1488, + 316, + 1720, + 345 + ] + }, + { + "text": "Computers&Accessories", + "boxes": [ + 18, + 338, + 172, + 355 + ] + }, + { + "text": "Game and Party Games - Gifts for 5", + "boxes": [ + 312, + 339, + 578, + 369 + ] + }, + { + "text": "1080P HDMI, Display Port and VGA", + "boxes": [ + 610, + 341, + 870, + 371 + ] + }, + { + "text": "Speakers - 60w Comput", + "boxes": [ + 902, + 341, + 1076, + 371 + ] + }, + { + "text": "student dorm projector watch soccer", + "boxes": [ + 1194, + 341, + 1464, + 371 + ] + }, + { + "text": "i7-13650HX, 16GB DDR5, 1TB PCI..", + "boxes": [ + 1486, + 339, + 1754, + 369 + ] + }, + { + "text": " Computers & Tablets", + "boxes": [ + 14, + 351, + 162, + 381 + ] + }, + { + "text": "立", + "boxes": [ + 1794, + 347, + 1820, + 369 + ] + }, + { + "text": "678 9 10 11 12 Year Old Boy Kids", + "boxes": [ + 314, + 363, + 570, + 395 + ] + }, + { + "text": "Input VESA Compatible Built-i...", + "boxes": [ + 608, + 365, + 852, + 395 + ] + }, + { + "text": "for Music and Gaming", + "boxes": [ + 902, + 365, + 1066, + 395 + ] + }, + { + "text": "competition basketball match for...", + "boxes": [ + 1194, + 365, + 1456, + 395 + ] + }, + { + "text": "★★★★1.895", + "boxes": [ + 1488, + 367, + 1632, + 397 + ] + }, + { + "text": "★★★★☆5,961", + "boxes": [ + 316, + 395, + 452, + 419 + ] + }, + { + "text": "Options: 2 sizes", + "boxes": [ + 612, + 399, + 696, + 417 + ] + }, + { + "text": "★★★★☆3.323", + "boxes": [ + 904, + 391, + 1044, + 421 + ] + }, + { + "text": "★★★★☆127", + "boxes": [ + 1194, + 393, + 1324, + 423 + ] + }, + { + "text": "in past month", + "boxes": [ + 1566, + 393, + 1656, + 411 + ] + }, + { + "text": "Brands", + "boxes": [ + 10, + 405, + 62, + 431 + ] + }, + { + "text": "ASUS", + "boxes": [ + 7, + 424, + 77, + 457 + ] + }, + { + "text": "★★★★★1.028", + "boxes": [ + 608, + 419, + 750, + 449 + ] + }, + { + "text": "$8999", + "boxes": [ + 1191, + 422, + 1261, + 459 + ] + }, + { + "text": "$1,24478", + "boxes": [ + 1486, + 419, + 1592, + 457 + ] + }, + { + "text": "VISOONE", + "boxes": [ + 5, + 442, + 99, + 479 + ] + }, + { + "text": "$2279 List: $29.99", + "boxes": [ + 311, + 442, + 453, + 483 + ] + }, + { + "text": "3K+ bought in past month", + "boxes": [ + 606, + 439, + 776, + 469 + ] + }, + { + "text": "$27900", + "boxes": [ + 897, + 440, + 983, + 483 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 455, + 1628, + 484 + ] + }, + { + "text": "$9999", + "boxes": [ + 606, + 472, + 672, + 506 + ] + }, + { + "text": "口A", + "boxes": [ + 14, + 488, + 40, + 510 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 316, + 484, + 452, + 508 + ] + }, + { + "text": "Delivery Wed, May 21", + "boxes": [ + 900, + 480, + 1042, + 510 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 1194, + 480, + 1336, + 510 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1492, + 480, + 1578, + 504 + ] + }, + { + "text": "Amazon Basic", + "boxes": [ + 34, + 494, + 124, + 512 + ] + }, + { + "text": "口R", + "boxes": [ + 12, + 510, + 42, + 530 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 606, + 506, + 750, + 536 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 504, + 990, + 530 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 504, + 1282, + 530 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 506, + 1580, + 538 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1808, + 502, + 1864, + 528 + ] + }, + { + "text": "Razer", + "boxes": [ + 34, + 512, + 68, + 532 + ] + }, + { + "text": "FIFINE", + "boxes": [ + 7, + 525, + 79, + 559 + ] + }, + { + "text": "Ssee optlons", + "boxes": [ + 315, + 531, + 403, + 568 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 608, + 532, + 700, + 556 + ] + }, + { + "text": "Add to t", + "boxes": [ + 903, + 529, + 993, + 566 + ] + }, + { + "text": "Ad t cat", + "boxes": [ + 1198, + 530, + 1288, + 568 + ] + }, + { + "text": " Redragon", + "boxes": [ + 5, + 541, + 97, + 581 + ] + }, + { + "text": "More Buying Choices", + "boxes": [ + 1486, + 538, + 1624, + 568 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 607, + 557, + 703, + 592 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1792, + 548, + 1862, + 574 + ] + }, + { + "text": "$1,219.99 (7 used & ne", + "boxes": [ + 1488, + 562, + 1638, + 586 + ] + }, + { + "text": "V See more", + "boxes": [ + 14, + 572, + 90, + 596 + ] + }, + { + "text": "Condition", + "boxes": [ + 12, + 606, + 82, + 627 + ] + }, + { + "text": "New", + "boxes": [ + 10, + 623, + 68, + 649 + ] + }, + { + "text": "used ", + "boxes": [ + 8, + 641, + 72, + 673 + ] + }, + { + "text": "$10", + "boxes": [ + 1080, + 641, + 1118, + 671 + ] + }, + { + "text": "Seller", + "boxes": [ + 10, + 675, + 54, + 703 + ] + }, + { + "text": "ROBLOX", + "boxes": [ + 964, + 677, + 1114, + 715 + ] + }, + { + "text": " The Wasteland G", + "boxes": [ + 8, + 717, + 144, + 746 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 719, + 1860, + 739 + ] + }, + { + "text": "5070", + "boxes": [ + 692, + 758, + 726, + 772 + ] + }, + { + "text": "New Releases", + "boxes": [ + 12, + 774, + 100, + 798 + ] + }, + { + "text": "口", + "boxes": [ + 1796, + 770, + 1810, + 786 + ] + }, + { + "text": "Last 30 days ", + "boxes": [ + 7, + 794, + 101, + 830 + ] + }, + { + "text": "Last 90 days ", + "boxes": [ + 12, + 818, + 98, + 844 + ] + }, + { + "text": "", + "boxes": [ + 986, + 816, + 1034, + 836 + ] + }, + { + "text": "Coming Soon", + "boxes": [ + 12, + 840, + 102, + 864 + ] + }, + { + "text": "1,000 Robux", + "boxes": [ + 974, + 844, + 1096, + 874 + ] + }, + { + "text": "Video Game DRM", + "boxes": [ + 8, + 868, + 132, + 897 + ] + }, + { + "text": "800+200Bonus Robux", + "boxes": [ + 970, + 883, + 1100, + 901 + ] + }, + { + "text": " DRM-Free", + "boxes": [ + 10, + 893, + 102, + 919 + ] + }, + { + "text": "□orgin", + "boxes": [ + 10, + 911, + 80, + 943 + ] + }, + { + "text": "●●+2", + "boxes": [ + 426, + 917, + 494, + 935 + ] + }, + { + "text": "ASUS TUF Gaming GeForce RTX TM", + "boxes": [ + 606, + 911, + 860, + 941 + ] + }, + { + "text": "Roblox Digital Gift Card - 1,000", + "boxes": [ + 900, + 911, + 1136, + 941 + ] + }, + { + "text": "SAMSUNG 990 PRO SSD 4TB PCle", + "boxes": [ + 1488, + 913, + 1738, + 937 + ] + }, + { + "text": " steam", + "boxes": [ + 8, + 927, + 78, + 961 + ] + }, + { + "text": "5070 12GB GDDR7 OC Edition", + "boxes": [ + 606, + 931, + 840, + 965 + ] + }, + { + "text": "Robux [Includes Exclusive Virtual", + "boxes": [ + 900, + 933, + 1144, + 965 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1814, + 925, + 1860, + 945 + ] + }, + { + "text": "BENGOO G9000 Stereo Gaming ", + "boxes": [ + 312, + 941, + 556, + 975 + ] + }, + { + "text": "Razer BlackShark V2 X Gaming ", + "boxes": [ + 1192, + 941, + 1428, + 977 + ] + }, + { + "text": "4.0 M.2 2280 Internal Solid State", + "boxes": [ + 1490, + 937, + 1736, + 961 + ] + }, + { + "text": "uplay", + "boxes": [ + 7, + 950, + 75, + 981 + ] + }, + { + "text": "Gaming Graphics Card (PCle@ 5.0,", + "boxes": [ + 608, + 959, + 856, + 989 + ] + }, + { + "text": "Item] [Digital Code] ", + "boxes": [ + 902, + 959, + 1052, + 989 + ] + }, + { + "text": "Hard Drive, Seq. Read Speeds Upto", + "boxes": [ + 1486, + 959, + 1752, + 989 + ] + }, + { + "text": "Headset for PS4 PC Xbox One PS5 ", + "boxes": [ + 316, + 971, + 570, + 995 + ] + }, + { + "text": "Headset: 7.1 Surround Sound -", + "boxes": [ + 1194, + 969, + 1424, + 999 + ] + }, + { + "text": "廿1+", + "boxes": [ + 1790, + 971, + 1886, + 997 + ] + }, + { + "text": "HDMI/DP 2.1, 3.125-slot, Military..", + "boxes": [ + 608, + 983, + 874, + 1012 + ] + }, + { + "text": "May 15, 2019 ", + "boxes": [ + 904, + 985, + 996, + 1009 + ] + }, + { + "text": "7,450 MB/s for High End Computi.", + "boxes": [ + 1488, + 983, + 1750, + 1012 + ] + }, + { + "text": "Controlle, Noise Cancelling Over Ear", + "boxes": [ + 314, + 993, + 588, + 1022 + ] + }, + { + "text": "50mm Drivers - Memory Foam", + "boxes": [ + 1194, + 993, + 1426, + 1022 + ] + }, + { + "text": "s with Mic, LED Light, 7...", + "boxes": [ + 404, + 1012, + 584, + 1047 + ] + }, + { + "text": "★★★★★683", + "boxes": [ + 606, + 1006, + 740, + 1040 + ] + }, + { + "text": "★★★★☆171,494", + "boxes": [ + 902, + 1007, + 1062, + 1036 + ] + }, + { + "text": "Cushion - For PC, PS4, PS5, Switch", + "boxes": [ + 1194, + 1016, + 1446, + 1040 + ] + }, + { + "text": "★★★★★12,593", + "boxes": [ + 1486, + 1007, + 1640, + 1040 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→", + "boxes": [ + 38, + 66, + 126, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 254, + 68, + 312, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1589, + 63, + 1661, + 88 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": "★★★★5,961", + "boxes": [ + 318, + 109, + 454, + 133 + ] + }, + { + "text": "Options: 2 sizes", + "boxes": [ + 610, + 111, + 698, + 135 + ] + }, + { + "text": "★★★★☆3,323", + "boxes": [ + 906, + 109, + 1042, + 133 + ] + }, + { + "text": "★★★★★127", + "boxes": [ + 1196, + 107, + 1324, + 137 + ] + }, + { + "text": "3K+ bought in past month", + "boxes": [ + 1490, + 103, + 1656, + 127 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1814, + 105, + 1872, + 131 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 123, + 60, + 143 + ] + }, + { + "text": "★★★★★1.028", + "boxes": [ + 610, + 133, + 750, + 163 + ] + }, + { + "text": "$1,24478", + "boxes": [ + 1484, + 133, + 1594, + 171 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1806, + 127, + 1882, + 151 + ] + }, + { + "text": "ASUS", + "boxes": [ + 10, + 143, + 74, + 169 + ] + }, + { + "text": "$8999", + "boxes": [ + 1194, + 139, + 1258, + 173 + ] + }, + { + "text": "VISOONE", + "boxes": [ + 7, + 158, + 101, + 193 + ] + }, + { + "text": "$2279 List: $29.99", + "boxes": [ + 311, + 158, + 451, + 199 + ] + }, + { + "text": "$27900", + "boxes": [ + 895, + 154, + 983, + 197 + ] + }, + { + "text": "Logitech G", + "boxes": [ + 7, + 178, + 109, + 215 + ] + }, + { + "text": " Save 10%v", + "boxes": [ + 1198, + 173, + 1280, + 199 + ] + }, + { + "text": "$9999", + "boxes": [ + 606, + 189, + 672, + 220 + ] + }, + { + "text": " Amazon Basics", + "boxes": [ + 8, + 202, + 130, + 231 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 316, + 199, + 454, + 222 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 900, + 199, + 1046, + 222 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 1196, + 199, + 1338, + 222 + ] + }, + { + "text": "Razer", + "boxes": [ + 31, + 223, + 71, + 249 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 604, + 218, + 748, + 254 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 218, + 992, + 244 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 218, + 1284, + 244 + ] + }, + { + "text": "Addto cart", + "boxes": [ + 1487, + 216, + 1583, + 259 + ] + }, + { + "text": "F", + "boxes": [ + 16, + 228, + 40, + 242 + ] + }, + { + "text": "Ships to Car", + "boxes": [ + 612, + 248, + 674, + 266 + ] + }, + { + "text": "Addto cart", + "boxes": [ + 906, + 244, + 994, + 282 + ] + }, + { + "text": "Ad t cat", + "boxes": [ + 1198, + 244, + 1288, + 282 + ] + }, + { + "text": "Redragon", + "boxes": [ + 7, + 259, + 95, + 293 + ] + }, + { + "text": "See optio", + "boxes": [ + 328, + 254, + 386, + 272 + ] + }, + { + "text": "More Buying Choice", + "boxes": [ + 1490, + 256, + 1616, + 280 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 610, + 274, + 700, + 306 + ] + }, + { + "text": "$1,219.99 (7 used & new offers)", + "boxes": [ + 1488, + 278, + 1698, + 302 + ] + }, + { + "text": "Condition", + "boxes": [ + 14, + 320, + 76, + 339 + ] + }, + { + "text": "New", + "boxes": [ + 8, + 338, + 64, + 363 + ] + }, + { + "text": "used", + "boxes": [ + 3, + 353, + 73, + 390 + ] + }, + { + "text": "立", + "boxes": [ + 1792, + 345, + 1820, + 371 + ] + }, + { + "text": "Digital Git Card", + "boxes": [ + 960, + 357, + 1038, + 381 + ] + }, + { + "text": "$10", + "boxes": [ + 1080, + 355, + 1118, + 385 + ] + }, + { + "text": "Seller", + "boxes": [ + 10, + 391, + 54, + 419 + ] + }, + { + "text": "ROBLOX", + "boxes": [ + 964, + 391, + 1114, + 429 + ] + }, + { + "text": " The Wasteland Gaming", + "boxes": [ + 8, + 435, + 182, + 463 + ] + }, + { + "text": "VSeemore", + "boxes": [ + 14, + 463, + 88, + 480 + ] + }, + { + "text": "New Releases", + "boxes": [ + 12, + 490, + 102, + 514 + ] + }, + { + "text": "Last 30 days ", + "boxes": [ + 7, + 508, + 101, + 545 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1808, + 502, + 1864, + 528 + ] + }, + { + "text": "Last 90 days ", + "boxes": [ + 12, + 532, + 98, + 558 + ] + }, + { + "text": "Coming Soon", + "boxes": [ + 9, + 551, + 101, + 582 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1796, + 550, + 1860, + 570 + ] + }, + { + "text": "1,000 Robux", + "boxes": [ + 974, + 560, + 1096, + 590 + ] + }, + { + "text": "VideoGameDRM", + "boxes": [ + 14, + 588, + 126, + 606 + ] + }, + { + "text": "800+200BonusRobux", + "boxes": [ + 970, + 598, + 1100, + 615 + ] + }, + { + "text": "DRM-Free", + "boxes": [ + 10, + 608, + 98, + 633 + ] + }, + { + "text": "origin", + "boxes": [ + 4, + 617, + 80, + 663 + ] + }, + { + "text": "○·····", + "boxes": [ + 312, + 623, + 500, + 659 + ] + }, + { + "text": "ASUS TUF Gaming GeForce RTX TM", + "boxes": [ + 608, + 625, + 858, + 655 + ] + }, + { + "text": "Roblox Digital Gift Card - 1,000", + "boxes": [ + 900, + 625, + 1136, + 655 + ] + }, + { + "text": ".", + "boxes": [ + 1194, + 625, + 1304, + 657 + ] + }, + { + "text": "SAMSUNG 990 PRO SSD 4TB PCle", + "boxes": [ + 1488, + 625, + 1740, + 655 + ] + }, + { + "text": " steam", + "boxes": [ + 10, + 647, + 76, + 673 + ] + }, + { + "text": "BENGOO G9000 Stereo Gaming", + "boxes": [ + 312, + 653, + 554, + 693 + ] + }, + { + "text": "5070 12GB GDDR7 OC Edition", + "boxes": [ + 610, + 653, + 838, + 677 + ] + }, + { + "text": "Robux [Includes Exclusive Virtual", + "boxes": [ + 902, + 649, + 1144, + 679 + ] + }, + { + "text": "Razer BlackShark V2 X Gaming", + "boxes": [ + 1192, + 655, + 1424, + 691 + ] + }, + { + "text": "4.0 M.2 2280 Internal Solid State", + "boxes": [ + 1486, + 649, + 1736, + 679 + ] + }, + { + "text": "uplay", + "boxes": [ + 7, + 664, + 73, + 698 + ] + }, + { + "text": "Gaming Graphics Card (PCle@ 5.0,", + "boxes": [ + 610, + 673, + 856, + 703 + ] + }, + { + "text": "Item] [Digital Code] ", + "boxes": [ + 902, + 675, + 1052, + 705 + ] + }, + { + "text": "Hard Drive, Seq. Read Speeds Upto", + "boxes": [ + 1484, + 671, + 1752, + 705 + ] + }, + { + "text": "Headset for PS4 PC Xbox One PS5 ", + "boxes": [ + 316, + 685, + 568, + 709 + ] + }, + { + "text": "Headset: 7.1 Surround Sound -", + "boxes": [ + 1196, + 683, + 1424, + 713 + ] + }, + { + "text": "HDMI@/DP 2.1, 3.125-slot, Military...", + "boxes": [ + 610, + 699, + 874, + 723 + ] + }, + { + "text": "May 15, 2019", + "boxes": [ + 904, + 699, + 994, + 723 + ] + }, + { + "text": "7,450 MB/s for High End Computi...", + "boxes": [ + 1486, + 695, + 1752, + 729 + ] + }, + { + "text": "Controlle, Noise Cancelling Over Ear ", + "boxes": [ + 314, + 705, + 588, + 737 + ] + }, + { + "text": "50mm Drivers - Memory Foam", + "boxes": [ + 1196, + 707, + 1422, + 735 + ] + }, + { + "text": "★★★★★683", + "boxes": [ + 608, + 725, + 736, + 754 + ] + }, + { + "text": "★★★★☆171,494", + "boxes": [ + 902, + 721, + 1058, + 750 + ] + }, + { + "text": "★★★★12.593", + "boxes": [ + 1488, + 720, + 1640, + 755 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 719, + 1860, + 739 + ] + }, + { + "text": "Headphones with Mic, LED Light, 7..", + "boxes": [ + 314, + 730, + 586, + 762 + ] + }, + { + "text": "Cushion - For PC, PS4, PS5, Switch..", + "boxes": [ + 1198, + 735, + 1454, + 758 + ] + }, + { + "text": "2K+ bought in past month", + "boxes": [ + 610, + 750, + 776, + 768 + ] + }, + { + "text": "Windows", + "boxes": [ + 901, + 746, + 967, + 777 + ] + }, + { + "text": "★★★★109.456", + "boxes": [ + 316, + 756, + 470, + 786 + ] + }, + { + "text": "★★★★21.,896", + "boxes": [ + 1194, + 756, + 1344, + 786 + ] + }, + { + "text": "6K+ bought in past month", + "boxes": [ + 1490, + 748, + 1662, + 772 + ] + }, + { + "text": "$73999", + "boxes": [ + 606, + 778, + 686, + 810 + ] + }, + { + "text": "$10%", + "boxes": [ + 900, + 774, + 958, + 806 + ] + }, + { + "text": "6K+ bought in past month", + "boxes": [ + 1198, + 780, + 1366, + 804 + ] + }, + { + "text": "$3029 Lst:$5464.99", + "boxes": [ + 1483, + 774, + 1651, + 817 + ] + }, + { + "text": "$1799 List Price: $35.9", + "boxes": [ + 311, + 807, + 487, + 850 + ] + }, + { + "text": " Delivery May 2 - 25", + "boxes": [ + 604, + 810, + 744, + 846 + ] + }, + { + "text": "$3849 List: 59.99", + "boxes": [ + 1189, + 805, + 1335, + 850 + ] + }, + { + "text": " Delivery Tue, May 20 ", + "boxes": [ + 1486, + 814, + 1632, + 844 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1490, + 838, + 1580, + 862 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 316, + 850, + 454, + 874 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 1196, + 850, + 1334, + 874 + ] + }, + { + "text": "Ad to cart", + "boxes": [ + 1489, + 859, + 1579, + 900 + ] + }, + { + "text": "See option", + "boxes": [ + 614, + 870, + 688, + 893 + ] + }, + { + "text": " 1 sustainablity feature", + "boxes": [ + 1200, + 889, + 1368, + 919 + ] + }, + { + "text": "More Buying Choices", + "boxes": [ + 1492, + 899, + 1622, + 923 + ] + }, + { + "text": "Add tocat", + "boxes": [ + 1197, + 914, + 1289, + 956 + ] + }, + { + "text": "$270.84 (7+ used & new offers)", + "boxes": [ + 1492, + 923, + 1692, + 941 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1818, + 929, + 1858, + 945 + ] + }, + { + "text": "$23.72(10+used & nev", + "boxes": [ + 1200, + 979, + 1348, + 997 + ] + }, + { + "text": "voffers", + "boxes": [ + 1348, + 979, + 1396, + 997 + ] + }, + { + "text": "云", + "boxes": [ + 1796, + 977, + 1814, + 993 + ] + }, + { + "text": "1", + "boxes": [ + 1820, + 977, + 1852, + 991 + ] + } + ] + }, + { + "frame_id": "60_3_9", + "episode_id": 60, + "env_id": 3, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.884627342224121, + "advantages": 0.15463493764400482, + "generated_text": "{\n \"intent\": \"To scroll through the webpage content to explore additional information.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To scroll through the webpage content to explore additional information.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 10, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 34, + 66, + 130, + 91 + ] + }, + { + "text": "① https://cn.bing.cor", + "boxes": [ + 254, + 66, + 436, + 89 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 558, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 486, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "爆料正新鸡排生姐", + "boxes": [ + 492, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 796, + 453, + 970, + 476 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 748 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.0996 ", + "boxes": [ + 1636, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "15087%", + "boxes": [ + 1623, + 888, + 1702, + 944 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 925, + 848, + 941 + ] + }, + { + "text": "一点资讯·1天", + "boxes": [ + 854, + 921, + 942, + 945 + ] + }, + { + "text": "金融界汽车·9小时", + "boxes": [ + 1138, + 921, + 1278, + 945 + ] + }, + { + "text": "梦幻西游：万元赐福，数亿成", + "boxes": [ + 826, + 941, + 1098, + 977 + ] + }, + { + "text": "赛力斯申请高速过弯转向不足", + "boxes": [ + 1138, + 941, + 1408, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "本改书，号主被称为梦幻玩家", + "boxes": [ + 826, + 967, + 1098, + 1003 + ] + }, + { + "text": "控制专利，提高稳定性和安全", + "boxes": [ + 1136, + 967, + 1410, + 1003 + ] + }, + { + "text": "汽车轻量化角逐", + "boxes": [ + 206, + 985, + 408, + 1014 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "界的狠人", + "boxes": [ + 824, + 993, + 916, + 1030 + ] + }, + { + "text": "性", + "boxes": [ + 1138, + 1001, + 1170, + 1022 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "1", + "boxes": [ + 832, + 1028, + 858, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": " Search -Microsoft Bing×", + "boxes": [ + 244, + 32, + 444, + 50 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "mages", + "boxes": [ + 254, + 123, + 292, + 143 + ] + }, + { + "text": "Video", + "boxes": [ + 332, + 125, + 362, + 139 + ] + }, + { + "text": "Translat", + "boxes": [ + 406, + 121, + 452, + 141 + ] + }, + { + "text": "国内版（国际版", + "boxes": [ + 490, + 189, + 638, + 218 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 236, + 614, + 266 + ] + }, + { + "text": "虹中", + "boxes": [ + 1538, + 359, + 1574, + 387 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 389, + 1634, + 421 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 185, + 436, + 273, + 467 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1458, + 480, + 1552, + 512 + ] + }, + { + "text": "特断拉", + "boxes": [ + 1460, + 518, + 1504, + 556 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1636, + 518, + 1698, + 558 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1458, + 564, + 1514, + 606 + ] + }, + { + "text": "", + "boxes": [ + 1589, + 569, + 1639, + 600 + ] + }, + { + "text": ",1024 ", + "boxes": [ + 1628, + 560, + 1702, + 609 + ] + }, + { + "text": "1-087 ", + "boxes": [ + 1619, + 595, + 1719, + 661 + ] + }, + { + "text": "4", + "boxes": [ + 1596, + 623, + 1610, + 641 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 639, + 848, + 655 + ] + }, + { + "text": "点资讯+关注·1天", + "boxes": [ + 864, + 635, + 984, + 659 + ] + }, + { + "text": "金融界汽车·9小时", + "boxes": [ + 1136, + 633, + 1280, + 663 + ] + }, + { + "text": "梦幻西游：万元赐福，数亿成", + "boxes": [ + 826, + 655, + 1098, + 691 + ] + }, + { + "text": "赛力斯申请高速过弯转向不足", + "boxes": [ + 1136, + 655, + 1406, + 691 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 655, + 1510, + 701 + ] + }, + { + "text": "", + "boxes": [ + 1594, + 661, + 1626, + 691 + ] + }, + { + "text": "+3.28%", + "boxes": [ + 1626, + 653, + 1700, + 701 + ] + }, + { + "text": "本改书，号主被称为梦幻玩家", + "boxes": [ + 824, + 681, + 1098, + 717 + ] + }, + { + "text": "控制专利，提高稳定性和安全", + "boxes": [ + 1138, + 685, + 1406, + 715 + ] + }, + { + "text": "界的狠人", + "boxes": [ + 821, + 704, + 917, + 747 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 699, + 1522, + 748 + ] + }, + { + "text": "-1.18%0 ", + "boxes": [ + 1634, + 703, + 1698, + 744 + ] + }, + { + "text": "性", + "boxes": [ + 1140, + 715, + 1164, + 739 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 717, + 1628, + 739 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1622, + 744, + 1726, + 770 + ] + }, + { + "text": "长春市", + "boxes": [ + 1466, + 800, + 1524, + 820 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 202, + 951, + 332, + 975 + ] + }, + { + "text": "ITBEAR科技资讯·2天", + "boxes": [ + 826, + 951, + 986, + 975 + ] + }, + { + "text": "游戏天地·1周", + "boxes": [ + 1142, + 951, + 1254, + 975 + ] + }, + { + "text": "1", + "boxes": [ + 518, + 967, + 538, + 983 + ] + }, + { + "text": "一点视频/南哥说异·1天", + "boxes": [ + 534, + 961, + 684, + 991 + ] + }, + { + "text": "多彩的天涯之国", + "boxes": [ + 202, + 975, + 352, + 1005 + ] + }, + { + "text": "自然吸气渐行渐远，涡轮增压", + "boxes": [ + 826, + 975, + 1096, + 1005 + ] + }, + { + "text": "英雄联盟：打野总被反？这3", + "boxes": [ + 1134, + 969, + 1400, + 1009 + ] + }, + { + "text": "娥被妖怪逮走，下秒一招，", + "boxes": [ + 510, + 985, + 780, + 1020 + ] + }, + { + "text": "车为何成主流？", + "boxes": [ + 826, + 1001, + 970, + 1032 + ] + }, + { + "text": "招轻松化解！", + "boxes": [ + 1138, + 1001, + 1262, + 1032 + ] + } + ] + }, + { + "frame_id": "60_4_9", + "episode_id": 60, + "env_id": 4, + "step": 9, + "format_rewards": 1.0, + "rewards": 8.932393074035645, + "advantages": -0.5464186072349548, + "generated_text": "{\n \"intent\": \"Opening a new browser tab might help explore more web resources or configurations.\",\n \"action\": \"DoubleClick(468, 40)\"\n}", + "action_string": "DoubleClick(468, 40)", + "intent_string": "Opening a new browser tab might help explore more web resources or configurations.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 480, + 32, + 534, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 40, + 66, + 128, + 89 + ] + }, + { + "text": "Qearch with Bing or enter address", + "boxes": [ + 244, + 57, + 524, + 97 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 996, + 534, + 1036, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "60_5_9", + "episode_id": 60, + "env_id": 5, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.222837448120117, + "advantages": -0.3325877785682678, + "generated_text": "{\n \"intent\": \"Scrolling down will allow me to view more content from the webpage, potentially discovering additional interesting elements or information.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "Scrolling down will allow me to view more content from the webpage, potentially discovering additional interesting elements or information.", + "before_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 74, + 68, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 570, + 121, + 636, + 147 + ] + }, + { + "text": "长春市", + "boxes": [ + 1636, + 125, + 1684, + 143 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "豆包AI", + "boxes": [ + 1568, + 199, + 1708, + 256 + ] + }, + { + "text": "全能高效", + "boxes": [ + 1568, + 246, + 1706, + 296 + ] + }, + { + "text": "", + "boxes": [ + 194, + 264, + 210, + 282 + ] + }, + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 382, + 272, + 536, + 300 + ] + }, + { + "text": "写作助手", + "boxes": [ + 1564, + 280, + 1706, + 330 + ] + }, + { + "text": "", + "boxes": [ + 192, + 316, + 216, + 343 + ] + }, + { + "text": "最近，一个冷门技术参数上了热搜。起因是有汽车博主自费测试阿维塔12风阻系数，得出的数据与企业公布的数据差", + "boxes": [ + 364, + 310, + 1238, + 339 + ] + }, + { + "text": "距不小；而企业在相关公证机构见证下，通过中国汽车工程研究院在风洞实验室全程直播测试，得出的数据并不存在", + "boxes": [ + 366, + 339, + 1236, + 363 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1610, + 330, + 1666, + 349 + ] + }, + { + "text": "虚标，从而引发舆论关注。", + "boxes": [ + 362, + 361, + 560, + 391 + ] + }, + { + "text": "豆包PC版_操作便捷_免费下载安", + "boxes": [ + 1422, + 353, + 1688, + 389 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 369, + 218, + 403 + ] + }, + { + "text": "装", + "boxes": [ + 1422, + 379, + 1455, + 409 + ] + }, + { + "text": "汽车风阻系数是衡量车辆行驶时空气阻力大小的关键参数，直接影响燃油经济性、续航能力、高速稳定性和噪声水", + "boxes": [ + 364, + 403, + 1220, + 433 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 411, + 1528, + 435 + ] + }, + { + "text": "#", + "boxes": [ + 190, + 429, + 212, + 455 + ] + }, + { + "text": "平。数值越低，说明空气阻力越小。新能源汽车的加速普及和补能焦虑，使得低风阻成为新能源车企竞争中少有的\"有", + "boxes": [ + 362, + 429, + 1244, + 461 + ] + }, + { + "text": "感数据”：不仅能影响消费者认知，还能提升续航里程。", + "boxes": [ + 366, + 461, + 772, + 484 + ] + }, + { + "text": "要了解风阻系数在新能源赛道是如何\"吹\"起来的，首先还得回溯到2012年，Model S横空出世，特斯拉便将风阻系数", + "boxes": [ + 364, + 498, + 1226, + 528 + ] + }, + { + "text": "方问智作说车", + "boxes": [ + 1464, + 498, + 1548, + 522 + ] + }, + { + "text": "L", + "boxes": [ + 1682, + 502, + 1694, + 514 + ] + }, + { + "text": "与科技含量进行了大量捆绑营销，不仅重塑了行业以及消费者对\"低风阻\"的认知也强化了特斯拉富有科技感的品牌", + "boxes": [ + 362, + 520, + 1234, + 558 + ] + }, + { + "text": "形象。此后，诸多新能源车企开始争相效仿，风阻系数纪录也不断被刷新。", + "boxes": [ + 364, + 552, + 920, + 582 + ] + }, + { + "text": "长安汽车首个海外整车工厂在泰国", + "boxes": [ + 1446, + 544, + 1664, + 574 + ] + }, + { + "text": "投产", + "boxes": [ + 1450, + 568, + 1476, + 588 + ] + }, + { + "text": "散人打金服", + "boxes": [ + 618, + 621, + 686, + 647 + ] + }, + { + "text": "中国东部小县城撬动全球轮胎市场", + "boxes": [ + 1448, + 623, + 1662, + 647 + ] + }, + { + "text": "装备靠打，刀刀爆装高回收，散人打金天堂！", + "boxes": [ + 618, + 649, + 1020, + 679 + ] + }, + { + "text": "长安汽车泰国工厂峻工投产", + "boxes": [ + 1450, + 683, + 1610, + 701 + ] + }, + { + "text": "广告", + "boxes": [ + 620, + 725, + 642, + 739 + ] + }, + { + "text": "此次风阻系数测试之争表面是数值大小，背后折射出的是行业过度内卷。这两年，车市价格战愈演愈烈，但产品成本", + "boxes": [ + 364, + 780, + 1238, + 810 + ] + }, + { + "text": "电", + "boxes": [ + 1422, + 774, + 1442, + 794 + ] + }, + { + "text": "的下降毕竟没有售价下降得那么快，车市竞争已从价格卷到技术参数。每次新车发布会，车企都在穷尽办法，让各项", + "boxes": [ + 364, + 808, + 1238, + 838 + ] + }, + { + "text": "打工人", + "boxes": [ + 1426, + 800, + 1502, + 832 + ] + }, + { + "text": "数据变得极致一点，至少要比友商领先。在如此内卷的氛围裹挟下，每一个小数点都成了车企比拼的战场，风阻系数", + "boxes": [ + 362, + 836, + 1236, + 860 + ] + }, + { + "text": "高效工作的", + "boxes": [ + 1430, + 828, + 1568, + 860 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 28, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 836, + 879, + 1088, + 903 + ] + }, + { + "text": "我接受", + "boxes": [ + 1670, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 796, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1526, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击\"管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置“改变您的选择。欲了解", + "boxes": [ + 28, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "理性看待风阻··", + "boxes": [ + 218, + 1048, + 306, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "理性看待风阻系数测试争议", + "boxes": [ + 252, + 32, + 412, + 50 + ] + }, + { + "text": "C", + "boxes": [ + 74, + 68, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": "?", + "boxes": [ + 1850, + 69, + 1868, + 83 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 566, + 121, + 636, + 147 + ] + }, + { + "text": "长春市", + "boxes": [ + 1636, + 123, + 1684, + 143 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "感数据”：不仅能影响消费者认知，还能提升续航里程", + "boxes": [ + 368, + 175, + 766, + 199 + ] + }, + { + "text": "豆包AI", + "boxes": [ + 1568, + 199, + 1708, + 256 + ] + }, + { + "text": "要了解风阻系数在新能源赛道是如何“吹\"起来的，首先还得回溯到2012年，Model S横空出世，特斯拉便将风阻系数", + "boxes": [ + 364, + 212, + 1226, + 242 + ] + }, + { + "text": "与科技含量进行了大量捆绑营销，不仅重塑了行业以及消费者对\"低风阻\"的认知，也强化了特斯拉富有科技感的品牌", + "boxes": [ + 368, + 242, + 1230, + 266 + ] + }, + { + "text": "全能高效", + "boxes": [ + 1568, + 246, + 1706, + 296 + ] + }, + { + "text": "", + "boxes": [ + 194, + 262, + 210, + 282 + ] + }, + { + "text": "形象。此后，诸多新能源车企开始争相效仿，风阻系数纪录也不断被刷新。", + "boxes": [ + 364, + 266, + 920, + 296 + ] + }, + { + "text": "写作助手", + "boxes": [ + 1565, + 279, + 1705, + 330 + ] + }, + { + "text": "", + "boxes": [ + 192, + 316, + 216, + 343 + ] + }, + { + "text": "散人打金服", + "boxes": [ + 618, + 336, + 688, + 361 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1610, + 330, + 1664, + 349 + ] + }, + { + "text": "#", + "boxes": [ + 180, + 361, + 226, + 407 + ] + }, + { + "text": "装备靠打，刀刀爆装高回收，散人打金天堂！", + "boxes": [ + 616, + 361, + 1022, + 397 + ] + }, + { + "text": "豆包PC版_操作便捷_免费下载安", + "boxes": [ + 1422, + 353, + 1688, + 389 + ] + }, + { + "text": "装", + "boxes": [ + 1428, + 383, + 1452, + 405 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 411, + 1526, + 435 + ] + }, + { + "text": "#", + "boxes": [ + 190, + 429, + 212, + 455 + ] + }, + { + "text": "广告", + "boxes": [ + 620, + 441, + 642, + 455 + ] + }, + { + "text": "此次风阻系数测试之争表面是数值大小，背后折射出的是行业过度内卷。这两年，车市价格战愈演愈烈，但产品成本", + "boxes": [ + 362, + 492, + 1240, + 528 + ] + }, + { + "text": "方问智作说车", + "boxes": [ + 1464, + 498, + 1548, + 522 + ] + }, + { + "text": "L", + "boxes": [ + 1682, + 502, + 1694, + 514 + ] + }, + { + "text": "的下降毕竟没有售价下降得那么快，车市竞争已从价格卷到技术参数。每次新车发布会，车企都在穷尽办法，让各项", + "boxes": [ + 362, + 520, + 1236, + 550 + ] + }, + { + "text": "数据变得极致一点，至少要比友商领先。在如此内卷的氛围裹挟下，每一个小数点都成了车企比拼的战场，风阻系数", + "boxes": [ + 364, + 548, + 1238, + 578 + ] + }, + { + "text": "长安汽车首个海外整车工厂在泰国", + "boxes": [ + 1444, + 542, + 1662, + 572 + ] + }, + { + "text": "的锚铢必较就成为一种必然。", + "boxes": [ + 360, + 572, + 578, + 604 + ] + }, + { + "text": "投产", + "boxes": [ + 1452, + 570, + 1472, + 586 + ] + }, + { + "text": "同时，流量至上思维也在深刻影响营销传播生态。随着数字技术和移动互联网的快速发展，没有门槛的自媒体从业者", + "boxes": [ + 362, + 615, + 1236, + 645 + ] + }, + { + "text": "中国东部小县城撬动全球轮胎市场", + "boxes": [ + 1446, + 623, + 1662, + 647 + ] + }, + { + "text": "越来越多，而流量越来越稀缺，且不可能平均分配。为了博取眼球，部分自媒体便通过碰瓷名企或名人，制造对立", + "boxes": [ + 364, + 639, + 1224, + 669 + ] + }, + { + "text": "从而实现流量的快速积累和商业价值变现。", + "boxes": [ + 366, + 671, + 678, + 695 + ] + }, + { + "text": "长安汽车泰国工厂峻工投产", + "boxes": [ + 1450, + 683, + 1612, + 701 + ] + }, + { + "text": "汽车风阻系数测试是一项非常严谨的工作，不仅要遵循科学的测试标准和方法，还要有强大的技术和实力支撑。比", + "boxes": [ + 360, + 707, + 1224, + 741 + ] + }, + { + "text": "如，中国汽研风洞实验室造价高达5.5亿元，且在国内极其稀缺；测试车辆在进风洞之前，应当遵循国家或行业认可的", + "boxes": [ + 364, + 737, + 1242, + 766 + ] + }, + { + "text": "标准，在调校车间进行详细检查，确保整体测试的规范性和结果的可靠性。任何测试条件的改变、人员操作的不专", + "boxes": [ + 364, + 762, + 1226, + 792 + ] + }, + { + "text": "电", + "boxes": [ + 1422, + 774, + 1442, + 794 + ] + }, + { + "text": "业，都会影响测试结果，透支公信力。然而，我国在整车空气动力学风洞测试方面缺乏统一的国家标准，这为各种民", + "boxes": [ + 364, + 788, + 1236, + 818 + ] + }, + { + "text": "打工人", + "boxes": [ + 1426, + 800, + 1502, + 832 + ] + }, + { + "text": "间野生测试留下了自由发挥的空间。", + "boxes": [ + 360, + 812, + 632, + 844 + ] + }, + { + "text": "高效工作的", + "boxes": [ + 1430, + 828, + 1568, + 860 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 28, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 834, + 879, + 1090, + 903 + ] + }, + { + "text": "我接受", + "boxes": [ + 1670, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 798, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 834, + 911, + 1526, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击\"管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置“改变您的选择。欲了解", + "boxes": [ + 28, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 126, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "理性看待风阻··", + "boxes": [ + 218, + 1048, + 306, + 1072 + ] + } + ] + }, + { + "frame_id": "60_6_9", + "episode_id": 60, + "env_id": 6, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.05478572845459, + "advantages": -0.4563107490539551, + "generated_text": "{\n \"intent\": \"To scroll down and explore more content on the webpage.\",\n \"action\": \"ScrollDown(100, 50)\"\n}", + "action_string": "ScrollDown(100, 50)", + "intent_string": "To scroll down and explore more content on the webpage.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口9", + "boxes": [ + 1790, + 69, + 1864, + 87 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 794, + 469, + 1116, + 524 + ] + }, + { + "text": "Sky bluecolor.", + "boxes": [ + 864, + 530, + 1044, + 568 + ] + }, + { + "text": "Sky high performance with M4.", + "boxes": [ + 762, + 558, + 1146, + 606 + ] + }, + { + "text": "Learn more ", + "boxes": [ + 847, + 617, + 963, + 660 + ] + }, + { + "text": "Buy", + "boxes": [ + 1004, + 623, + 1050, + 649 + ] + }, + { + "text": "Built for Aople Intelliaence.", + "boxes": [ + 830, + 1016, + 1074, + 1054 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三目（", + "boxes": [ + 1788, + 69, + 1856, + 87 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 794, + 469, + 1116, + 524 + ] + }, + { + "text": "Sky blue color.", + "boxes": [ + 864, + 530, + 1044, + 568 + ] + }, + { + "text": "Sky high performance with M4.", + "boxes": [ + 762, + 558, + 1146, + 606 + ] + }, + { + "text": "Learn more ", + "boxes": [ + 847, + 617, + 963, + 660 + ] + }, + { + "text": "Built for Aople Intelliaence.", + "boxes": [ + 830, + 1016, + 1074, + 1054 + ] + } + ] + }, + { + "frame_id": "60_7_9", + "episode_id": 60, + "env_id": 7, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.305046081542969, + "advantages": -0.2720641791820526, + "generated_text": "{\n \"intent\": \"To understand the functionality of the operators available in the Unary/Binary Operators panel, I will double-click an operator, which usually opens a dialog or provides further options.\",\n \"action\": \"DoubleClick(50, 165)\"\n}", + "action_string": "DoubleClick(50, 165)", + "intent_string": "To understand the functionality of the operators available in the Unary/Binary Operators panel, I will double-click an operator, which usually opens a dialog or provides further options.", + "before_screen_ocr_results": [ + { + "text": "File", + "boxes": [ + 10, + 28, + 34, + 42 + ] + }, + { + "text": "Window Help", + "boxes": [ + 242, + 24, + 346, + 48 + ] + }, + { + "text": "-----", + "boxes": [ + 6, + 52, + 470, + 87 + ] + }, + { + "text": "SID口D", + "boxes": [ + 454, + 50, + 644, + 85 + ] + }, + { + "text": "A", + "boxes": [ + 724, + 62, + 740, + 77 + ] + }, + { + "text": " Unary/Binary Operators", + "boxes": [ + 16, + 103, + 174, + 127 + ] + }, + { + "text": "+口", + "boxes": [ + 30, + 153, + 70, + 179 + ] + }, + { + "text": "-□", + "boxes": [ + 96, + 153, + 142, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 168, + 151, + 214, + 179 + ] + }, + { + "text": "+口", + "boxes": [ + 240, + 151, + 282, + 179 + ] + }, + { + "text": "□+□□-□□·×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 96, + 282 + ] + }, + { + "text": "廿", + "boxes": [ + 102, + 252, + 138, + 288 + ] + }, + { + "text": "□÷口口", + "boxes": [ + 156, + 252, + 288, + 284 + ] + }, + { + "text": "", + "boxes": [ + 16, + 298, + 292, + 334 + ] + }, + { + "text": "000%品", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 32, + 409, + 72, + 435 + ] + }, + { + "text": "□>□v□", + "boxes": [ + 62, + 403, + 220, + 439 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Help", + "boxes": [ + 300, + 26, + 344, + 46 + ] + }, + { + "text": "----", + "boxes": [ + 6, + 52, + 468, + 87 + ] + }, + { + "text": "SID@D", + "boxes": [ + 454, + 50, + 644, + 85 + ] + }, + { + "text": "A", + "boxes": [ + 724, + 62, + 740, + 77 + ] + }, + { + "text": "±口", + "boxes": [ + 174, + 155, + 208, + 175 + ] + }, + { + "text": "口", + "boxes": [ + 242, + 153, + 282, + 179 + ] + }, + { + "text": "□+□□-□冏□·□冏□×□", + "boxes": [ + 18, + 206, + 292, + 236 + ] + }, + { + "text": "□*□", + "boxes": [ + 22, + 256, + 88, + 282 + ] + }, + { + "text": "号", + "boxes": [ + 106, + 254, + 134, + 282 + ] + }, + { + "text": "□÷□□口", + "boxes": [ + 156, + 252, + 286, + 282 + ] + }, + { + "text": "Φ", + "boxes": [ + 14, + 296, + 290, + 334 + ] + }, + { + "text": "00%品", + "boxes": [ + 10, + 336, + 290, + 391 + ] + }, + { + "text": "J口", + "boxes": [ + 34, + 411, + 68, + 433 + ] + }, + { + "text": ">", + "boxes": [ + 70, + 405, + 218, + 437 + ] + }, + { + "text": "ntitled 2 - Li.", + "boxes": [ + 376, + 1048, + 466, + 1072 + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_60/scores_v3.json b/ScreenExplorer-7B-E1/episode_60/scores_v3.json new file mode 100644 index 0000000000000000000000000000000000000000..26ef8fe295f199025ac6a0f00eaef7ce910fd645 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_60/scores_v3.json @@ -0,0 +1,19 @@ +{ + "directory_path": "logs/ScreenExplorer_2025-05-17_22-11-27ab2072/episode_60", + "single_trajectory_image_diversity": 0.40707466565072536, + "single_trajectory_text_diversity": 0.2743259072303772, + "single_trajectory_image_diversity_max": 0.96875, + "single_trajectory_image_diversity_min": 0.0009765625, + "single_trajectory_image_diversity_std": 0.33233642578125, + "single_trajectory_text_diversity_max": 0.50738525390625, + "single_trajectory_text_diversity_min": 0.00787353515625, + "single_trajectory_text_diversity_std": 0.1514434814453125, + "group_trajectories_image_diversity": 0.5073378086090088, + "group_trajectories_text_diversity": 0.4063767194747925, + "group_trajectories_image_diversity_max": 1.171875, + "group_trajectories_image_diversity_min": 0.0, + "group_trajectories_image_diversity_std": 0.341796875, + "group_trajectories_text_diversity_max": 0.6630859375, + "group_trajectories_text_diversity_min": 0.0, + "group_trajectories_text_diversity_std": 0.151611328125 +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_70/game_record_20250518_061557.json b/ScreenExplorer-7B-E1/episode_70/game_record_20250518_061557.json new file mode 100644 index 0000000000000000000000000000000000000000..722fc3baa111bb65bba9bef69b0b5b424f558aa8 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_70/game_record_20250518_061557.json @@ -0,0 +1,42345 @@ +{ + "start_time": "2025-05-18T06:15:57.850913", + "end_time": "2025-05-18T06:23:07.041245", + "buffer_size": 80, + "all_records": [ + { + "frame_id": "70_0_0", + "episode_id": 70, + "env_id": 0, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.181640625, + "advantages": 0.8495160937309265, + "generated_text": "{\n \"intent\": \"To begin exploring available applications on the desktop, I will open Firefox by clicking on its icon.\",\n \"action\": \"DoubleClick(73, 53)\"\n}", + "action_string": "DoubleClick(73, 53)", + "intent_string": "To begin exploring available applications on the desktop, I will open Firefox by clicking on its icon.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Qbearch with Bing orenteraddress", + "boxes": [ + 244, + 57, + 522, + 100 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 516, + 479, + 567, + 535 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_1_0", + "episode_id": 70, + "env_id": 1, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.899724960327148, + "advantages": 1.2640211582183838, + "generated_text": "{\n \"intent\": \"Clicking on one of the icons will allow me to open a program or folder, which is necessary for further exploration.\",\n \"action\": \"DoubleClick(55, 707)\"\n}", + "action_string": "DoubleClick(55, 707)", + "intent_string": "Clicking on one of the icons will allow me to open a program or folder, which is necessary for further exploration.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled1-LibreOfficeCalc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "·口·目·目@自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "G-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "x =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "21", + "boxes": [ + 16, + 512, + 30, + 528 + ] + }, + { + "text": "25", + "boxes": [ + 16, + 584, + 30, + 602 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "70_2_0", + "episode_id": 70, + "env_id": 2, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.38414478302002, + "advantages": 0.9664090275764465, + "generated_text": "{\n \"intent\": \"To start exploring the available applications, I will open the Firefox browser first.\",\n \"action\": \"DoubleClick(67, 33)\"\n}", + "action_string": "DoubleClick(67, 33)", + "intent_string": "To start exploring the available applications, I will open the Firefox browser first.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 56, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_3_0", + "episode_id": 70, + "env_id": 3, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.278422355651855, + "advantages": 0.9053821563720703, + "generated_text": "{\n \"intent\": \"The first step should be to open an application or file that could provide more information about the system's capabilities. Since each图标 on the desktop represents a separate program or file, clicking on one will immediately provide interaction with that specific location.\",\n \"action\": \"DoubleClick(43, 37)\"\n}", + "action_string": "DoubleClick(43, 37)", + "intent_string": "The first step should be to open an application or file that could provide more information about the system's capabilities. Since each图标 on the desktop represents a separate program or file, clicking on one will immediately provide interaction with that specific location.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_4_0", + "episode_id": 70, + "env_id": 4, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.395389556884766, + "advantages": 0.9728999733924866, + "generated_text": "{\n \"intent\": \"To start exploring by opening an application, let's begin with Firefox.\",\n \"action\": \"DoubleClick(61, 52)\"\n}", + "action_string": "DoubleClick(61, 52)", + "intent_string": "To start exploring by opening an application, let's begin with Firefox.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_5_0", + "episode_id": 70, + "env_id": 5, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.309697151184082, + "advantages": 0.9234350919723511, + "generated_text": "{\n \"intent\": \"The first action will be to open the Firefox browser icon to possibly open new tabs or pages, which allows us to explore more content online.\",\n \"action\": \"DoubleClick(56, 46)\"\n}", + "action_string": "DoubleClick(56, 46)", + "intent_string": "The first action will be to open the Firefox browser icon to possibly open new tabs or pages, which allows us to explore more content online.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C仓", + "boxes": [ + 74, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_6_0", + "episode_id": 70, + "env_id": 6, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.286810874938965, + "advantages": 0.9102243185043335, + "generated_text": "{\n \"intent\": \"To begin exploring the available applications by trying one of them. This will help in discovering various system functions and capabilities.\",\n \"action\": \"DoubleClick(37, 42)\"\n}", + "action_string": "DoubleClick(37, 42)", + "intent_string": "To begin exploring the available applications by trying one of them. This will help in discovering various system functions and capabilities.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 516, + 479, + 567, + 535 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_7_0", + "episode_id": 70, + "env_id": 7, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.109670639038086, + "advantages": 0.8079723715782166, + "generated_text": "{\n \"intent\": \"To explore files or applications further I will open an application like Firefox or Chrome.\",\n \"action\": \"DoubleClick(41, 37)\"\n}", + "action_string": "DoubleClick(41, 37)", + "intent_string": "To explore files or applications further I will open an application like Firefox or Chrome.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_0_1", + "episode_id": 70, + "env_id": 0, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.496247291564941, + "advantages": -0.12335577607154846, + "generated_text": "{\n \"intent\": \"To enhance the exploration experience by clicking on different sites for content discovery and information gathering.\",\n \"action\": \"DoubleClick(722, 504)\"\n}", + "action_string": "DoubleClick(722, 504)", + "intent_string": "To enhance the exploration experience by clicking on different sites for content discovery and information gathering.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 742, + 534, + 808, + 554 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_1_1", + "episode_id": 70, + "env_id": 1, + "step": 1, + "format_rewards": 0.0, + "rewards": 0.0, + "advantages": -5.604943752288818, + "generated_text": "{\n \"intent\": \"To start exploring how the spreadsheet works, I will enter a sample text directly into cell A1.\",\n \"action\": \"Text(100, 150, \" \n \n \n \t\t\t\t\t\t\t\t \t\t\t\t\t\t \t\t\t\t\t\t\t\t\t\n \n \t\t\t\t \n \n \n \n \t\t\t\t\t\t\t\t \t\t\t\t\t\t \t\t\t\t\t\t\t\t \t\t \t\t\t\t\t\t\t\t \t\t\t\t\t\t \t\t\t\t\t\t\t\t \t\t\t\t\t\t \t\t\t\t\t\t\t\t \t\t\t\t\t\t \t\t\t\t\t\t\t\t \t\t\t \t\t\t\t\t\t\t\t \t\t\t\t\t\t\t \n \n \t\t\t\t \n \n \n \n \t\t\t\t\t\t\t\t \t\t\t\t\t\t \t\t\t\t\t\n \t\t\t \t\t\t\t\t\t\t\t\t\t \t\t\t\t\t \t\t\t\t\t\t \t\t\t\t\t\t\t\t\t \t \t\t\t\t\t\t\t \n \t\t\t\t \t\t \t\t\t\t\t \t\t\t\t\t\t", + "action_string": "None", + "intent_string": null, + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 842, + 0, + 1078, + 24 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "·口·目·目@自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "上&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "←-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": " =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "25", + "boxes": [ + 16, + 584, + 30, + 602 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→=+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "--目- -", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "上&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "勺-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": " =[", + "boxes": [ + 126, + 125, + 260, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "25", + "boxes": [ + 16, + 584, + 30, + 602 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "70_2_1", + "episode_id": 70, + "env_id": 2, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.41934585571289, + "advantages": -0.1677461564540863, + "generated_text": "{\n \"intent\": \"To continue exploring the browser interface, I will scroll through the page to view more content.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring the browser interface, I will scroll through the page to view more content.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 56, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1270, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_3_1", + "episode_id": 70, + "env_id": 3, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.571406364440918, + "advantages": 0.49726614356040955, + "generated_text": "{\n \"intent\": \"To explore websites, starting with a popular one like Amazon, which often exposes diverse content including reviews, news, and advertisements.\",\n \"action\": \"DoubleClick(1073, 722)\"\n}", + "action_string": "DoubleClick(1073, 722)", + "intent_string": "To explore websites, starting with a popular one like Amazon, which often exposes diverse content including reviews, news, and advertisements.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 412, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 121, + 1474, + 135 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 918, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 660, + 294, + 942, + 349 + ] + }, + { + "text": "You are on ama", + "boxes": [ + 510, + 461, + 606, + 478 + ] + }, + { + "text": "1", + "boxes": [ + 1344, + 488, + 1386, + 508 + ] + }, + { + "text": "", + "boxes": [ + 1416, + 486, + 1550, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 916, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 982, + 530, + 1164, + 560 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 1350, + 530, + 1608, + 560 + ] + }, + { + "text": "appliances", + "boxes": [ + 1350, + 558, + 1464, + 588 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 980, + 889, + 1066, + 915 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 242, + 969, + 464, + 999 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1220, + 999 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1352, + 969, + 1508, + 999 + ] + }, + { + "text": "g=UTF8&k", + "boxes": [ + 256, + 1016, + 330, + 1042 + ] + } + ] + }, + { + "frame_id": "70_4_1", + "episode_id": 70, + "env_id": 4, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.447333335876465, + "advantages": -0.1515907347202301, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment by potentially discovering new websites or navigating existing ones, I'll click on the 'bibt' icon, which seems to be an abbreviation for Bing search.\",\n \"action\": \"DoubleClick(514, 307)\"\n}", + "action_string": "DoubleClick(514, 307)", + "intent_string": "To continue exploring the desktop environment by potentially discovering new websites or navigating existing ones, I'll click on the 'bibt' icon, which seems to be an abbreviation for Bing search.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFiref", + "boxes": [ + 904, + 4, + 1004, + 22 + ] + }, + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 250, + 61, + 518, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_5_1", + "episode_id": 70, + "env_id": 5, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.83297348022461, + "advantages": 0.0710151344537735, + "generated_text": "{\n \"intent\": \"To start exploring by navigating to another webpage or saving the current tab for future reference.\",\n \"action\": \"DoubleClick(724, 87)\"\n}", + "action_string": "DoubleClick(724, 87)", + "intent_string": "To start exploring by navigating to another webpage or saving the current tab for future reference.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 60, + 524, + 95 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 14, + 103, + 156, + 121 + ] + }, + { + "text": " Apple --htt://ww.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 252, + 210, + 592, + 242 + ] + }, + { + "text": "知zhihu -", + "boxes": [ + 254, + 250, + 328, + 274 + ] + }, + { + "text": "G ifeng - ifeng.com", + "boxes": [ + 254, + 284, + 402, + 314 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1223, + 668, + 1419, + 737 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ] + }, + { + "frame_id": "70_6_1", + "episode_id": 70, + "env_id": 6, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.359089851379395, + "advantages": -0.20252816379070282, + "generated_text": "{\n \"intent\": \"To begin exploring other applications or functionalities on the computer by opening a new browser tab.\",\n \"action\": \"DoubleClick(170, 42)\"\n}", + "action_string": "DoubleClick(170, 42)", + "intent_string": "To begin exploring other applications or functionalities on the computer by opening a new browser tab.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": "C", + "boxes": [ + 72, + 69, + 124, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_7_1", + "episode_id": 70, + "env_id": 7, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.8668851852417, + "advantages": 0.0905902311205864, + "generated_text": "{\n \"intent\": \"Since we have different browser sites available, clicking on one could help me explore different websites or functionalities. I'll start by visiting Amazon to possibly look around, learn more about the website design, or see if the interface is robust.\",\n \"action\": \"DoubleClick(1747, 8)\" }", + "action_string": "DoubleClick(1747, 8)", + "intent_string": "Since we have different browser sites available, clicking on one could help me explore different websites or functionalities. I'll start by visiting Amazon to possibly look around, learn more about the website design, or see if the interface is robust.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Mozilla Firefox", + "boxes": [ + 898, + 40, + 1024, + 64 + ] + }, + { + "text": "Firefox", + "boxes": [ + 19, + 59, + 97, + 98 + ] + }, + { + "text": " New Tab", + "boxes": [ + 328, + 68, + 412, + 93 + ] + }, + { + "text": "→白", + "boxes": [ + 350, + 101, + 454, + 133 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 548, + 99, + 818, + 133 + ] + }, + { + "text": "I四?", + "boxes": [ + 1468, + 103, + 1558, + 129 + ] + }, + { + "text": " Import bookmarks...", + "boxes": [ + 328, + 137, + 494, + 165 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 504, + 139, + 634, + 163 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 668, + 288, + 766, + 312 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 490, + 399, + 580, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 443, + 1066, + 516 + ] + }, + { + "text": "QIYI", + "boxes": [ + 862, + 455, + 934, + 502 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 482, + 1278, + 502 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 90, + 528 + ] + }, + { + "text": "“", + "boxes": [ + 564, + 504, + 584, + 524 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 530, + 562, + 556 + ] + }, + { + "text": "bing", + "boxes": [ + 635, + 525, + 679, + 557 + ] + }, + { + "text": "Amazon", + "boxes": [ + 742, + 532, + 812, + 554 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 532, + 916, + 554 + ] + }, + { + "text": " zhihu", + "boxes": [ + 998, + 532, + 1040, + 552 + ] + }, + { + "text": "ifeng", + "boxes": [ + 1112, + 524, + 1164, + 559 + ] + }, + { + "text": "weibo", + "boxes": [ + 1236, + 532, + 1284, + 552 + ] + }, + { + "text": " ctrip", + "boxes": [ + 1360, + 530, + 1400, + 556 + ] + }, + { + "text": "", + "boxes": [ + 20, + 543, + 90, + 621 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "雷", + "boxes": [ + 27, + 670, + 88, + 727 + ] + }, + { + "text": "amazon", + "boxes": [ + 988, + 669, + 1174, + 719 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 675, + 1418, + 731 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 742, + 1172, + 768 + ] + }, + { + "text": "", + "boxes": [ + 1392, + 746, + 1412, + 766 + ] + }, + { + "text": "", + "boxes": [ + 30, + 771, + 82, + 832 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 768, + 544, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 506, + 790, + 552, + 816 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 900, + 814 + ] + }, + { + "text": "Firefox Privac", + "boxes": [ + 1228, + 790, + 1320, + 814 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "lozilla Firefox", + "boxes": [ + 224, + 1048, + 312, + 1072 + ] + } + ] + }, + { + "frame_id": "70_0_2", + "episode_id": 70, + "env_id": 0, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.7470121383667, + "advantages": 0.5986323356628418, + "generated_text": "{\n \"intent\": \"To explore more web pages or applications, I'll start by clicking on one of the browser highlights. This will help me understand the available resources.\",\n \"action\": \"DoubleClick(844, 742)\"\n}", + "action_string": "DoubleClick(844, 742)", + "intent_string": "To explore more web pages or applications, I'll start by clicking on one of the browser highlights. This will help me understand the available resources.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 742, + 534, + 808, + 554 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " - Microsoft Bing", + "boxes": [ + 76, + 32, + 188, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 406, + 121, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "O", + "boxes": [ + 1342, + 302, + 1356, + 324 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1354, + 403 + ] + }, + { + "text": "CNext", + "boxes": [ + 1356, + 373, + 1418, + 405 + ] + }, + { + "text": "", + "boxes": [ + 1546, + 367, + 1562, + 381 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 274, + 750 + ] + }, + { + "text": "点击关注", + "boxes": [ + 687, + 742, + 797, + 797 + ] + }, + { + "text": "网友关注", + "boxes": [ + 832, + 768, + 914, + 794 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 764, + 1554, + 796 + ] + }, + { + "text": "TREILEGIAIE", + "boxes": [ + 706, + 788, + 774, + 806 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "+2,49 26 ", + "boxes": [ + 1634, + 802, + 1698, + 844 + ] + }, + { + "text": "做打事", + "boxes": [ + 1226, + 818, + 1326, + 856 + ] + }, + { + "text": "新腰：74岁了还敢演女将军，女主若", + "boxes": [ + 826, + 846, + 1094, + 876 + ] + }, + { + "text": "被抓包的猫咪", + "boxes": [ + 1194, + 843, + 1360, + 886 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1516, + 891 + ] + }, + { + "text": "261029 ", + "boxes": [ + 1632, + 846, + 1700, + 893 + ] + }, + { + "text": "双提基租成本，真的规救降继87分", + "boxes": [ + 828, + 885, + 1082, + 915 + ] + }, + { + "text": "高附时中国A50", + "boxes": [ + 1452, + 889, + 1550, + 941 + ] + }, + { + "text": "13,087%6", + "boxes": [ + 1626, + 891, + 1698, + 941 + ] + }, + { + "text": "链破宏冠氨业或巨额亏损", + "boxes": [ + 826, + 919, + 1010, + 953 + ] + }, + { + "text": "D", + "boxes": [ + 1138, + 935, + 1164, + 955 + ] + }, + { + "text": "一点视频/萌宠旅行者·3周", + "boxes": [ + 1158, + 931, + 1322, + 961 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "*322", + "boxes": [ + 1628, + 935, + 1702, + 991 + ] + }, + { + "text": "很显然，猫咪也知道自己在使", + "boxes": [ + 1134, + 953, + 1410, + 989 + ] + }, + { + "text": "致假通资讯", + "boxes": [ + 828, + 963, + 914, + 989 + ] + }, + { + "text": "猫咪打针名场面，哪只反应最夸张？", + "boxes": [ + 198, + 981, + 668, + 1022 + ] + }, + { + "text": "坏！", + "boxes": [ + 1138, + 983, + 1182, + 1011 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "妞强胞网看斑技挑军要戏负罗", + "boxes": [ + 828, + 999, + 1094, + 1028 + ] + }, + { + "text": "_101", + "boxes": [ + 1146, + 1028, + 1178, + 1042 + ] + } + ] + }, + { + "frame_id": "70_1_2", + "episode_id": 70, + "env_id": 1, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.322162628173828, + "advantages": -0.22384393215179443, + "generated_text": "{\n \"intent\": \"This action is aimed at familiarizing myself more deeply into the application interface. Right-clicking will likely bring up additional menu options which could offer more insights or functionalities about this spreadsheet software.\",\n \"action\": \"RightClick(100, 200)\"\n}", + "action_string": "RightClick(100, 200)", + "intent_string": "This action is aimed at familiarizing myself more deeply into the application interface. Right-clicking will likely bring up additional menu options which could offer more insights or functionalities about this spreadsheet software.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "-·目- 自. ", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 51, + 445, + 86 + ] + }, + { + "text": "勺-→、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa=要目目% 00t三图-口目", + "boxes": [ + 12, + 93, + 1208, + 123 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 260, + 157 + ] + }, + { + "text": "A1", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "C", + "boxes": [ + 236, + 159, + 254, + 173 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 480 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "--目·目@X自-", + "boxes": [ + 12, + 54, + 372, + 83 + ] + }, + { + "text": "血&", + "boxes": [ + 376, + 54, + 444, + 85 + ] + }, + { + "text": "G-.", + "boxes": [ + 452, + 56, + 550, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": " Liberation San ", + "boxes": [ + 14, + 95, + 118, + 119 + ] + }, + { + "text": "10", + "boxes": [ + 154, + 97, + 188, + 117 + ] + }, + { + "text": "aaa.", + "boxes": [ + 230, + 93, + 348, + 123 + ] + }, + { + "text": "a·旦·三三", + "boxes": [ + 342, + 93, + 560, + 123 + ] + }, + { + "text": "要回日日-%00+正至田二口-目", + "boxes": [ + 544, + 93, + 1206, + 123 + ] + }, + { + "text": "A2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 152, + 125, + 254, + 157 + ] + }, + { + "text": "Cut", + "boxes": [ + 122, + 206, + 154, + 228 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "Copy", + "boxes": [ + 124, + 230, + 166, + 256 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Paste Special ", + "boxes": [ + 126, + 258, + 220, + 282 + ] + }, + { + "text": "Insert..", + "boxes": [ + 128, + 288, + 174, + 314 + ] + }, + { + "text": "Delete", + "boxes": [ + 125, + 311, + 173, + 343 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 12, + 330, + 34, + 343 + ] + }, + { + "text": "Clear Contents", + "boxes": [ + 130, + 347, + 224, + 365 + ] + }, + { + "text": "12", + "boxes": [ + 14, + 361, + 34, + 375 + ] + }, + { + "text": "14", + "boxes": [ + 14, + 393, + 32, + 409 + ] + }, + { + "text": "Styles", + "boxes": [ + 128, + 429, + 170, + 455 + ] + }, + { + "text": "17", + "boxes": [ + 14, + 449, + 32, + 463 + ] + }, + { + "text": "18", + "boxes": [ + 12, + 465, + 32, + 480 + ] + }, + { + "text": "19", + "boxes": [ + 14, + 480, + 30, + 494 + ] + }, + { + "text": "21", + "boxes": [ + 14, + 512, + 30, + 528 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "1→二+", + "boxes": [ + 18, + 994, + 132, + 1017 + ] + } + ] + }, + { + "frame_id": "70_2_2", + "episode_id": 70, + "env_id": 2, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.742648124694824, + "advantages": 0.0188759695738554, + "generated_text": "{\n \"intent\": \"To continue exploring the document, I will open a new tab. This will allow me to visit different websites or explore other content.\",\n \"action\": \"DoubleClick(241, 43)\"\n}", + "action_string": "DoubleClick(241, 43)", + "intent_string": "To continue exploring the document, I will open a new tab. This will allow me to visit different websites or explore other content.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 443, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "6", + "boxes": [ + 1233, + 479, + 1271, + 506 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": "zhihu", + "boxes": [ + 1002, + 536, + 1034, + 550 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1114, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 258, + 32, + 312, + 52 + ] + }, + { + "text": "C", + "boxes": [ + 54, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 250, + 64, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 202, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_3_2", + "episode_id": 70, + "env_id": 3, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.276390075683594, + "advantages": 0.32697173953056335, + "generated_text": "{\n \"intent\": \"Scrolling down will help me explore more of the Amazon homepage content.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "Scrolling down will help me explore more of the Amazon homepage content.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "MEN+", + "boxes": [ + 1517, + 116, + 1583, + 154 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1860, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": " Signin ", + "boxes": [ + 1622, + 175, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "", + "boxes": [ + 1100, + 490, + 1156, + 508 + ] + }, + { + "text": "1", + "boxes": [ + 1434, + 488, + 1472, + 508 + ] + }, + { + "text": "", + "boxes": [ + 1522, + 488, + 1564, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 982, + 530, + 1164, + 560 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 1350, + 530, + 1608, + 560 + ] + }, + { + "text": "appliances", + "boxes": [ + 1352, + 558, + 1464, + 588 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 980, + 889, + 1066, + 915 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 240, + 969, + 464, + 999 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1220, + 999 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1352, + 969, + 1508, + 999 + ] + }, + { + "text": "！.", + "boxes": [ + 358, + 1001, + 398, + 1026 + ] + }, + { + "text": "UTF8&k", + "boxes": [ + 266, + 1018, + 326, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1660, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1872, + 89 + ] + }, + { + "text": "Gamingaccessories", + "boxes": [ + 246, + 191, + 436, + 208 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 185, + 916, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 982, + 185, + 1164, + 214 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 1350, + 185, + 1608, + 214 + ] + }, + { + "text": "appliances", + "boxes": [ + 1350, + 212, + 1462, + 242 + ] + }, + { + "text": "M", + "boxes": [ + 1478, + 465, + 1544, + 482 + ] + }, + { + "text": "Pots and Pans Kettles", + "boxes": [ + 1458, + 498, + 1614, + 522 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 982, + 544, + 1064, + 568 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 238, + 619, + 464, + 655 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 621, + 1508, + 655 + ] + }, + { + "text": " See all deals", + "boxes": [ + 240, + 985, + 318, + 1009 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Shopnow", + "boxes": [ + 1354, + 987, + 1412, + 1007 + ] + } + ] + }, + { + "frame_id": "70_4_2", + "episode_id": 70, + "env_id": 4, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.671089172363281, + "advantages": -0.022430526092648506, + "generated_text": "{\n \"intent\": \"To begin exploring further into the environment by interacting with the browser's new tab. Double-clicking the 'New Tab' option will allow me to open a new browsing session.\",\n \"action\": \"DoubleClick(47, 37)\"\n}", + "action_string": "DoubleClick(47, 37)", + "intent_string": "To begin exploring further into the environment by interacting with the browser's new tab. Double-clicking the 'New Tab' option will allow me to open a new browsing session.", + "before_screen_ocr_results": [ + { + "text": "MozillaFiref", + "boxes": [ + 904, + 4, + 1004, + 22 + ] + }, + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 250, + 61, + 518, + 93 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "e仓", + "boxes": [ + 58, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_5_2", + "episode_id": 70, + "env_id": 5, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.143619537353516, + "advantages": 0.2503316104412079, + "generated_text": "{\n \"intent\": \"To explore more of the web by opening new tabs or accessing various websites, opening each could provide a better understanding of the environment.\",\n \"action\": \"DoubleClick(636, 726)\"\n}", + "action_string": "DoubleClick(636, 726)", + "intent_string": "To explore more of the web by opening new tabs or accessing various websites, opening each could provide a better understanding of the environment.", + "before_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 36, + 68, + 128, + 91 + ] + }, + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 62, + 522, + 91 + ] + }, + { + "text": " Apple --htt://ww.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 252, + 210, + 592, + 242 + ] + }, + { + "text": "iqiyi.com", + "boxes": [ + 602, + 220, + 662, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1223, + 668, + 1419, + 737 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 189, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 822, + 318, + 932, + 361 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "70_6_2", + "episode_id": 70, + "env_id": 6, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.935757637023926, + "advantages": 0.13034598529338837, + "generated_text": "{\n \"intent\": \"Exploring the web browser is a logical next step after navigating through the desktop to gain access to information or applications by clicking on available shortcuts directly.\",\n \"action\": \"DoubleClick(510, 691)\"\n}", + "action_string": "DoubleClick(510, 691)", + "intent_string": "Exploring the web browser is a logical next step after navigating through the desktop to gain access to information or applications by clicking on available shortcuts directly.", + "before_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 70, + 69, + 124, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 824, + 320, + 930, + 357 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 994, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "70_7_2", + "episode_id": 70, + "env_id": 7, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.793655395507812, + "advantages": 0.048319268971681595, + "generated_text": "{\n \"intent\": \"To explore available applications or settings by interacting with the Firefox browser directly.\",\n \"action\": \"Click(510, 476)\"\n}", + "action_string": "Click(510, 476)", + "intent_string": "To explore available applications or settings by interacting with the Firefox browser directly.", + "before_screen_ocr_results": [ + { + "text": "Mozilla Firefox", + "boxes": [ + 898, + 40, + 1024, + 64 + ] + }, + { + "text": "Firefox", + "boxes": [ + 19, + 59, + 97, + 98 + ] + }, + { + "text": " New Tab", + "boxes": [ + 328, + 68, + 412, + 93 + ] + }, + { + "text": "→白", + "boxes": [ + 350, + 101, + 454, + 133 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 548, + 99, + 818, + 133 + ] + }, + { + "text": "I四?", + "boxes": [ + 1468, + 103, + 1558, + 129 + ] + }, + { + "text": " Import bookmarks...", + "boxes": [ + 328, + 137, + 494, + 165 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 504, + 139, + 634, + 163 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "earch the Web", + "boxes": [ + 668, + 288, + 766, + 312 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": " Top Sites", + "boxes": [ + 490, + 399, + 580, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 443, + 1066, + 516 + ] + }, + { + "text": "QIYI", + "boxes": [ + 862, + 455, + 934, + 502 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 482, + 1278, + 502 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 18, + 502, + 90, + 528 + ] + }, + { + "text": "“", + "boxes": [ + 564, + 504, + 584, + 524 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 530, + 562, + 556 + ] + }, + { + "text": "bing", + "boxes": [ + 635, + 525, + 679, + 557 + ] + }, + { + "text": "Amazon", + "boxes": [ + 742, + 532, + 812, + 554 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 532, + 916, + 554 + ] + }, + { + "text": " zhihu", + "boxes": [ + 998, + 532, + 1040, + 552 + ] + }, + { + "text": "ifeng", + "boxes": [ + 1112, + 524, + 1164, + 559 + ] + }, + { + "text": "weibo", + "boxes": [ + 1236, + 532, + 1284, + 552 + ] + }, + { + "text": " ctrip", + "boxes": [ + 1360, + 530, + 1400, + 556 + ] + }, + { + "text": "", + "boxes": [ + 20, + 543, + 90, + 621 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 94, + 641 + ] + }, + { + "text": "雷", + "boxes": [ + 27, + 670, + 88, + 727 + ] + }, + { + "text": "amazon", + "boxes": [ + 988, + 669, + 1174, + 719 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 675, + 1418, + 731 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 742, + 1172, + 768 + ] + }, + { + "text": "", + "boxes": [ + 1392, + 746, + 1412, + 766 + ] + }, + { + "text": "", + "boxes": [ + 30, + 771, + 82, + 832 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 768, + 544, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 506, + 790, + 552, + 816 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 900, + 814 + ] + }, + { + "text": "Firefox Privac", + "boxes": [ + 1228, + 790, + 1320, + 814 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "lozilla Firefox", + "boxes": [ + 224, + 1048, + 312, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 19, + 59, + 97, + 98 + ] + }, + { + "text": "Apple", + "boxes": [ + 332, + 71, + 390, + 91 + ] + }, + { + "text": "×+", + "boxes": [ + 520, + 69, + 570, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 348, + 103, + 450, + 129 + ] + }, + { + "text": "| https://www.apple.com", + "boxes": [ + 548, + 101, + 766, + 131 + ] + }, + { + "text": "·☆", + "boxes": [ + 1284, + 101, + 1368, + 133 + ] + }, + { + "text": "三四@", + "boxes": [ + 1464, + 99, + 1564, + 131 + ] + }, + { + "text": "iPhone", + "boxes": [ + 694, + 153, + 730, + 167 + ] + }, + { + "text": "Watch", + "boxes": [ + 766, + 153, + 802, + 167 + ] + }, + { + "text": "Vision", + "boxes": [ + 836, + 153, + 876, + 169 + ] + }, + { + "text": "AirPods", + "boxes": [ + 898, + 149, + 952, + 169 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 976, + 147, + 1052, + 171 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1076, + 147, + 1164, + 171 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 151, + 1266, + 169 + ] + }, + { + "text": "Support", + "boxes": [ + 1294, + 151, + 1348, + 169 + ] + }, + { + "text": "Q", + "boxes": [ + 1364, + 149, + 1394, + 169 + ] + }, + { + "text": "2", + "boxes": [ + 1421, + 148, + 1444, + 171 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "iPhone", + "boxes": [ + 859, + 231, + 1053, + 305 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "Meet the iPhone 16 family.", + "boxes": [ + 794, + 300, + 1116, + 341 + ] + }, + { + "text": " Learn more", + "boxes": [ + 826, + 363, + 918, + 389 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 967, + 355, + 1085, + 398 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": " Built for Apple Intelligence.", + "boxes": [ + 828, + 419, + 1078, + 459 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 20, + 502, + 90, + 528 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 719, + 90, + 752 + ] + }, + { + "text": "D", + "boxes": [ + 31, + 775, + 82, + 831 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 828, + 88, + 862 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 856, + 941, + 1072, + 1001 + ] + } + ] + }, + { + "frame_id": "70_0_3", + "episode_id": 70, + "env_id": 0, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.091133117675781, + "advantages": 0.22003450989723206, + "generated_text": "{\n \"intent\": \"To continue exploring the webpage, I will scroll down to view more content.\",\n \"action\": \"ScrollDown(950, 540)\"\n}", + "action_string": "ScrollDown(950, 540)", + "intent_string": "To continue exploring the webpage, I will scroll down to view more content.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 488, + 451, + 634, + 474 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "网友关注", + "boxes": [ + 832, + 768, + 914, + 794 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 768, + 1552, + 794 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "Ww", + "boxes": [ + 1594, + 810, + 1628, + 830 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1632, + 802, + 1698, + 842 + ] + }, + { + "text": "做打事", + "boxes": [ + 1226, + 818, + 1326, + 856 + ] + }, + { + "text": "新腰：74岁了还敢演女将军，女主若", + "boxes": [ + 828, + 846, + 1094, + 876 + ] + }, + { + "text": "被抓包的猫咪", + "boxes": [ + 1194, + 843, + 1360, + 886 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1516, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "双是基租成本，真我城降继制87分", + "boxes": [ + 830, + 887, + 1080, + 911 + ] + }, + { + "text": "高附时中国A50", + "boxes": [ + 1452, + 889, + 1550, + 941 + ] + }, + { + "text": "15087%", + "boxes": [ + 1623, + 888, + 1702, + 944 + ] + }, + { + "text": "破宏业威巨额亏损", + "boxes": [ + 826, + 919, + 1010, + 953 + ] + }, + { + "text": "D", + "boxes": [ + 1138, + 935, + 1164, + 955 + ] + }, + { + "text": "一点视频/萌宠旅行者·3周", + "boxes": [ + 1158, + 931, + 1322, + 961 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "你还不知道怎么选梦幻西游命魂之玉吗？一", + "boxes": [ + 200, + 945, + 740, + 985 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "一篇", + "boxes": [ + 740, + 953, + 788, + 981 + ] + }, + { + "text": "很显然，猫咪也知道自己在使", + "boxes": [ + 1134, + 953, + 1410, + 989 + ] + }, + { + "text": "放假通资讯", + "boxes": [ + 828, + 963, + 914, + 989 + ] + }, + { + "text": "带你看懂", + "boxes": [ + 195, + 976, + 333, + 1025 + ] + }, + { + "text": "坏！", + "boxes": [ + 1138, + 983, + 1182, + 1011 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "刻强胞网看斑技撤军要戏负罗", + "boxes": [ + 828, + 999, + 1094, + 1028 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "_101", + "boxes": [ + 1146, + 1028, + 1180, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 68, + 1658, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 406, + 121, + 452, + 141 + ] + }, + { + "text": "国内版（国际版", + "boxes": [ + 490, + 189, + 638, + 218 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 236, + 614, + 266 + ] + }, + { + "text": "中忠空", + "boxes": [ + 1538, + 357, + 1574, + 387 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 389, + 1634, + 421 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 439, + 272, + 465 + ] + }, + { + "text": "网友关注", + "boxes": [ + 832, + 482, + 914, + 508 + ] + }, + { + "text": "M自选股建议", + "boxes": [ + 1458, + 482, + 1550, + 508 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1456, + 516, + 1506, + 560 + ] + }, + { + "text": "+ 4998", + "boxes": [ + 1634, + 518, + 1698, + 560 + ] + }, + { + "text": "司", + "boxes": [ + 948, + 540, + 966, + 556 + ] + }, + { + "text": "做打事", + "boxes": [ + 1230, + 536, + 1324, + 568 + ] + }, + { + "text": "新腰：74岁了还敢演女将军，女主若", + "boxes": [ + 826, + 562, + 1094, + 592 + ] + }, + { + "text": "被抓包的猫咪", + "boxes": [ + 1192, + 557, + 1362, + 606 + ] + }, + { + "text": "贵州台", + "boxes": [ + 1458, + 564, + 1516, + 608 + ] + }, + { + "text": "", + "boxes": [ + 1589, + 569, + 1639, + 600 + ] + }, + { + "text": ",1024 ", + "boxes": [ + 1628, + 560, + 1702, + 609 + ] + }, + { + "text": "提基租成本，真被隆细密87分", + "boxes": [ + 822, + 602, + 1084, + 631 + ] + }, + { + "text": "时中国A50", + "boxes": [ + 1449, + 601, + 1551, + 658 + ] + }, + { + "text": "13 87%", + "boxes": [ + 1624, + 604, + 1704, + 657 + ] + }, + { + "text": "4", + "boxes": [ + 1596, + 623, + 1610, + 641 + ] + }, + { + "text": "破业威巨额亏损", + "boxes": [ + 824, + 633, + 1010, + 669 + ] + }, + { + "text": "D", + "boxes": [ + 1138, + 649, + 1164, + 671 + ] + }, + { + "text": "一点视频/萌宠旅行者·3周", + "boxes": [ + 1158, + 645, + 1320, + 675 + ] + }, + { + "text": "很显然，猫咪也知道自己在使", + "boxes": [ + 1130, + 661, + 1412, + 707 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 655, + 1510, + 701 + ] + }, + { + "text": "", + "boxes": [ + 1594, + 661, + 1628, + 691 + ] + }, + { + "text": "+3.28%", + "boxes": [ + 1626, + 653, + 1700, + 701 + ] + }, + { + "text": "3周", + "boxes": [ + 292, + 677, + 322, + 691 + ] + }, + { + "text": "放假通资讯", + "boxes": [ + 828, + 679, + 914, + 705 + ] + }, + { + "text": "被打翻的自然调色盘", + "boxes": [ + 202, + 695, + 470, + 737 + ] + }, + { + "text": "坏！", + "boxes": [ + 1136, + 695, + 1182, + 729 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 699, + 1522, + 748 + ] + }, + { + "text": "刘强舰网性看威搬军要求负罗", + "boxes": [ + 828, + 711, + 1094, + 741 + ] + }, + { + "text": "-1. 163%0", + "boxes": [ + 1636, + 703, + 1698, + 744 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 717, + 1628, + 739 + ] + }, + { + "text": "101", + "boxes": [ + 1138, + 741, + 1190, + 766 + ] + }, + { + "text": "", + "boxes": [ + 1188, + 746, + 1210, + 760 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1622, + 744, + 1726, + 770 + ] + }, + { + "text": "长春市", + "boxes": [ + 1460, + 800, + 1524, + 820 + ] + }, + { + "text": "下周五气", + "boxes": [ + 1618, + 854, + 1672, + 872 + ] + }, + { + "text": "KILOAD", + "boxes": [ + 1330, + 868, + 1364, + 887 + ] + }, + { + "text": "魔法与神秘学（一", + "boxes": [ + 1304, + 889, + 1414, + 913 + ] + }, + { + "text": "ITBEAR科技资讯·15小时", + "boxes": [ + 198, + 947, + 382, + 979 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 512, + 949, + 644, + 979 + ] + }, + { + "text": "证券日报·18小时", + "boxes": [ + 828, + 949, + 966, + 979 + ] + }, + { + "text": "校机核·2周", + "boxes": [ + 1140, + 951, + 1232, + 977 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，", + "boxes": [ + 202, + 975, + 454, + 1005 + ] + }, + { + "text": "探秘荒野之魅", + "boxes": [ + 514, + 975, + 642, + 1007 + ] + }, + { + "text": "从“底层重构”到“文明融合”", + "boxes": [ + 826, + 971, + 1090, + 1007 + ] + }, + { + "text": "《女神异闻录3》中的魔法与", + "boxes": [ + 1142, + 969, + 1402, + 1009 + ] + }, + { + "text": "能否引领新能源市场新潮流？", + "boxes": [ + 202, + 999, + 468, + 1028 + ] + }, + { + "text": "光启技术加速推进超材料产业", + "boxes": [ + 822, + 995, + 1098, + 1034 + ] + }, + { + "text": "神秘学（一)", + "boxes": [ + 1135, + 994, + 1259, + 1037 + ] + }, + { + "text": "生太升级", + "boxes": [ + 834, + 1030, + 908, + 1048 + ] + } + ] + }, + { + "frame_id": "70_1_3", + "episode_id": 70, + "env_id": 1, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.082418441772461, + "advantages": -0.3622332215309143, + "generated_text": "{\n \"intent\": \"The user wants to explore the spreadsheet environment by interacting with one of its cells. The mouse over the cell A2 suggests an intention to open context menu options typically used for cell actions such as formatting or editing.\",\n \"action\": \"DoubleClick(72, 264)\"\n}", + "action_string": "DoubleClick(72, 264)", + "intent_string": "The user wants to explore the spreadsheet environment by interacting with one of its cells. The mouse over the cell A2 suggests an intention to open context menu options typically used for cell actions such as formatting or editing.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "--目·目@X自-", + "boxes": [ + 12, + 54, + 372, + 83 + ] + }, + { + "text": "血&", + "boxes": [ + 376, + 54, + 444, + 85 + ] + }, + { + "text": "G-.", + "boxes": [ + 452, + 56, + 550, + 79 + ] + }, + { + "text": "田-", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": " Liberation San ", + "boxes": [ + 14, + 95, + 118, + 119 + ] + }, + { + "text": "10", + "boxes": [ + 154, + 97, + 188, + 117 + ] + }, + { + "text": "aaa.", + "boxes": [ + 230, + 93, + 348, + 123 + ] + }, + { + "text": "a·旦·三三", + "boxes": [ + 342, + 93, + 560, + 123 + ] + }, + { + "text": "要回日日-%00+正至田二口-目", + "boxes": [ + 544, + 93, + 1206, + 123 + ] + }, + { + "text": "A2", + "boxes": [ + 8, + 133, + 26, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 152, + 125, + 254, + 157 + ] + }, + { + "text": "Cut", + "boxes": [ + 122, + 206, + 154, + 228 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "Copy", + "boxes": [ + 124, + 230, + 166, + 256 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Paste Special ", + "boxes": [ + 126, + 258, + 220, + 282 + ] + }, + { + "text": "Insert..", + "boxes": [ + 128, + 288, + 174, + 314 + ] + }, + { + "text": "Delete", + "boxes": [ + 125, + 311, + 173, + 343 + ] + }, + { + "text": "发", + "boxes": [ + 1888, + 312, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 12, + 330, + 34, + 343 + ] + }, + { + "text": "Clear Contents", + "boxes": [ + 130, + 347, + 224, + 365 + ] + }, + { + "text": "12", + "boxes": [ + 14, + 361, + 34, + 375 + ] + }, + { + "text": "14", + "boxes": [ + 14, + 393, + 32, + 409 + ] + }, + { + "text": "Styles", + "boxes": [ + 128, + 429, + 170, + 455 + ] + }, + { + "text": "17", + "boxes": [ + 14, + 449, + 32, + 463 + ] + }, + { + "text": "18", + "boxes": [ + 12, + 465, + 32, + 480 + ] + }, + { + "text": "19", + "boxes": [ + 14, + 480, + 30, + 494 + ] + }, + { + "text": "21", + "boxes": [ + 14, + 512, + 30, + 528 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "1→二+", + "boxes": [ + 18, + 994, + 132, + 1017 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 28, + 42, + 42 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "·口·目·目@自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "G-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa要目目-% 00t三图-口目", + "boxes": [ + 12, + 93, + 1206, + 123 + ] + }, + { + "text": "、", + "boxes": [ + 130, + 129, + 160, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 150, + 125, + 258, + 157 + ] + }, + { + "text": "A6", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "21", + "boxes": [ + 16, + 512, + 30, + 528 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "K\\→→+", + "boxes": [ + 14, + 995, + 132, + 1017 + ] + } + ] + }, + { + "frame_id": "70_2_3", + "episode_id": 70, + "env_id": 2, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.985912322998047, + "advantages": 0.1592971384525299, + "generated_text": "{\n \"intent\": \"To explore more of the browser ecosystem, I will click on one of the available website icons or search for a new webpage.\",\n \"action\": \"DoubleClick(661, 794)\"\n}", + "action_string": "DoubleClick(661, 794)", + "intent_string": "To explore more of the browser ecosystem, I will click on one of the available website icons or search for a new webpage.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 258, + 32, + 312, + 52 + ] + }, + { + "text": "C", + "boxes": [ + 54, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 250, + 64, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 202, + 103, + 310, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 822, + 318, + 932, + 361 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "70_3_3", + "episode_id": 70, + "env_id": 3, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.39560317993164, + "advantages": -0.18145132064819336, + "generated_text": "{\n \"intent\": \"To expand my exploration by moving the mouse to a familiar area where further actions can be performed. This will help me interact with more sections of the page.\",\n \"action\": \"Move(1000, 500)\"\n}", + "action_string": "Move(1000, 500)", + "intent_string": "To expand my exploration by moving the mouse to a familiar area where further actions can be performed. This will help me interact with more sections of the page.", + "before_screen_ocr_results": [ + { + "text": "Gaming accessories", + "boxes": [ + 244, + 189, + 438, + 212 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 185, + 918, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 982, + 185, + 1164, + 214 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 1350, + 185, + 1608, + 214 + ] + }, + { + "text": "appliances", + "boxes": [ + 1352, + 212, + 1464, + 242 + ] + }, + { + "text": "M", + "boxes": [ + 1480, + 465, + 1540, + 482 + ] + }, + { + "text": "Pots and Pans ", + "boxes": [ + 1458, + 498, + 1568, + 522 + ] + }, + { + "text": "Kettles", + "boxes": [ + 1555, + 498, + 1613, + 523 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 982, + 544, + 1064, + 568 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 238, + 619, + 464, + 655 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 619, + 1508, + 655 + ] + }, + { + "text": " See all deals", + "boxes": [ + 240, + 985, + 318, + 1009 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Shopnow", + "boxes": [ + 1354, + 987, + 1412, + 1007 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1598, + 68, + 1660, + 87 + ] + }, + { + "text": "S", + "boxes": [ + 1346, + 141, + 1534, + 165 + ] + }, + { + "text": "Gamingacce", + "boxes": [ + 248, + 191, + 366, + 208 + ] + }, + { + "text": "Shopfor your home essentials", + "boxes": [ + 614, + 187, + 914, + 210 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 982, + 185, + 1164, + 214 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 1350, + 185, + 1608, + 214 + ] + }, + { + "text": "appliances", + "boxes": [ + 1350, + 212, + 1462, + 242 + ] + }, + { + "text": "NU", + "boxes": [ + 1484, + 465, + 1540, + 482 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 981, + 541, + 1065, + 570 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 238, + 621, + 464, + 657 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 623, + 1220, + 653 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 619, + 1508, + 655 + ] + }, + { + "text": "Seealldeals", + "boxes": [ + 246, + 987, + 316, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 985, + 1058, + 1009 + ] + }, + { + "text": "Shopnov", + "boxes": [ + 1356, + 987, + 1406, + 1005 + ] + } + ] + }, + { + "frame_id": "70_4_3", + "episode_id": 70, + "env_id": 4, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.134286880493164, + "advantages": -0.3322928249835968, + "generated_text": "{\n \"intent\": \"To begin interacting with the browser by opening a new tab or clicking on a bookmark, which will allow for further exploration of web content.\",\n \"action\": \"DoubleClick(224, 88)\"\n}", + "action_string": "DoubleClick(224, 88)", + "intent_string": "To begin interacting with the browser by opening a new tab or clicking on a bookmark, which will allow for further exploration of web content.", + "before_screen_ocr_results": [ + { + "text": "C仓", + "boxes": [ + 56, + 69, + 126, + 87 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 61, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 103, + 308, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 182, + 99, + 310, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_5_3", + "episode_id": 70, + "env_id": 5, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.211172103881836, + "advantages": 0.289325475692749, + "generated_text": "{\n \"intent\": \"I am planning to explore more products by scrolling down the page. This will help me view additional sections and information.\",\n \"action\": \"ScrollDown(1000, 540)\"\n}", + "action_string": "ScrollDown(1000, 540)", + "intent_string": "I am planning to explore more products by scrolling down the page. This will help me view additional sections and information.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 189, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 822, + 318, + 932, + 361 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Apple", + "boxes": [ + 28, + 34, + 64, + 48 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→", + "boxes": [ + 32, + 66, + 126, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Visint for Airpodse Intw& Homece.", + "boxes": [ + 834, + 101, + 1082, + 131 + ] + }, + { + "text": "Entertainmer", + "boxes": [ + 1070, + 111, + 1150, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1370, + 113, + 1392, + 127 + ] + }, + { + "text": "②", + "boxes": [ + 1422, + 115, + 1440, + 131 + ] + }, + { + "text": "WATCH", + "boxes": [ + 874, + 621, + 1072, + 677 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 904, + 677, + 1030, + 709 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 846, + 710, + 1062, + 753 + ] + }, + { + "text": " Learn more", + "boxes": [ + 858, + 774, + 958, + 800 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 774, + 1048, + 800 + ] + } + ] + }, + { + "frame_id": "70_6_3", + "episode_id": 70, + "env_id": 6, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.012502670288086, + "advantages": 0.17464607954025269, + "generated_text": "{\n \"intent\": \"To continue exploring the Apple website by clicking 'Shop iPhone' which might offer detailed product options or further exploration into iPhone series such as the one mentioned, aiming to learn more about available models.\",\n \"action\": \"DoubleClick(1032, 337)\"\n}", + "action_string": "DoubleClick(1032, 337)", + "intent_string": "To continue exploring the Apple website by clicking 'Shop iPhone' which might offer detailed product options or further exploration into iPhone series such as the one mentioned, aiming to learn more about available models.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 824, + 320, + 930, + 357 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 994, + 1060, + 1035 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→×", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1370, + 113, + 1392, + 127 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1438, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 824, + 320, + 930, + 357 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 322, + 1082, + 359 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 828, + 379, + 1078, + 421 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 994, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "70_7_3", + "episode_id": 70, + "env_id": 7, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.215646743774414, + "advantages": -0.28532886505126953, + "generated_text": "{\n \"intent\": \"The goal is to explore the system thoroughly by interacting with various applications and tasks available on the desktop. Clicking on 'Files' will open it, allowing access to files and folders stored within.\",\n \"action\": \"Click(98, 937)\"\n}", + "action_string": "Click(98, 937)", + "intent_string": "The goal is to explore the system thoroughly by interacting with various applications and tasks available on the desktop. Clicking on 'Files' will open it, allowing access to files and folders stored within.", + "before_screen_ocr_results": [ + { + "text": "Firefox", + "boxes": [ + 19, + 59, + 97, + 98 + ] + }, + { + "text": "Apple", + "boxes": [ + 332, + 71, + 390, + 91 + ] + }, + { + "text": "×+", + "boxes": [ + 520, + 69, + 570, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 348, + 103, + 450, + 129 + ] + }, + { + "text": "| https://www.apple.com", + "boxes": [ + 548, + 101, + 766, + 131 + ] + }, + { + "text": "·☆", + "boxes": [ + 1284, + 101, + 1368, + 133 + ] + }, + { + "text": "三四@", + "boxes": [ + 1464, + 99, + 1562, + 131 + ] + }, + { + "text": "iPhone", + "boxes": [ + 694, + 153, + 730, + 167 + ] + }, + { + "text": "Watch", + "boxes": [ + 766, + 153, + 802, + 167 + ] + }, + { + "text": "Vision", + "boxes": [ + 836, + 153, + 876, + 169 + ] + }, + { + "text": "AirPods", + "boxes": [ + 898, + 149, + 952, + 169 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 976, + 147, + 1052, + 171 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1076, + 147, + 1164, + 171 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 151, + 1266, + 169 + ] + }, + { + "text": "Support", + "boxes": [ + 1294, + 151, + 1348, + 169 + ] + }, + { + "text": "Q", + "boxes": [ + 1364, + 149, + 1394, + 169 + ] + }, + { + "text": "2", + "boxes": [ + 1421, + 148, + 1444, + 171 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "iPhone", + "boxes": [ + 859, + 231, + 1053, + 305 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "Meet the iPhone 16 family.", + "boxes": [ + 794, + 300, + 1116, + 341 + ] + }, + { + "text": " Learn more", + "boxes": [ + 826, + 363, + 918, + 389 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 967, + 355, + 1085, + 398 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": " Built for Apple Intelligence.", + "boxes": [ + 828, + 419, + 1078, + 459 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 20, + 502, + 90, + 528 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 719, + 90, + 752 + ] + }, + { + "text": "D", + "boxes": [ + 31, + 775, + 82, + 831 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 828, + 88, + 862 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 856, + 941, + 1072, + 1001 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 320, + 40, + 347, + 66 + ] + }, + { + "text": "□", + "boxes": [ + 1536, + 40, + 1598, + 66 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": " Apple", + "boxes": [ + 330, + 71, + 392, + 91 + ] + }, + { + "text": "×+", + "boxes": [ + 518, + 69, + 570, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 343, + 97, + 453, + 134 + ] + }, + { + "text": "① https://ww.apple.com", + "boxes": [ + 550, + 101, + 768, + 131 + ] + }, + { + "text": "·☆", + "boxes": [ + 1284, + 101, + 1368, + 133 + ] + }, + { + "text": "三四?", + "boxes": [ + 1466, + 103, + 1564, + 129 + ] + }, + { + "text": "三", + "boxes": [ + 1572, + 107, + 1588, + 123 + ] + }, + { + "text": "iPhone", + "boxes": [ + 694, + 153, + 730, + 167 + ] + }, + { + "text": "Watch", + "boxes": [ + 768, + 153, + 802, + 167 + ] + }, + { + "text": "Vision", + "boxes": [ + 836, + 153, + 876, + 169 + ] + }, + { + "text": "AirPods", + "boxes": [ + 898, + 149, + 952, + 169 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 978, + 147, + 1052, + 171 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1076, + 147, + 1164, + 171 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 151, + 1266, + 169 + ] + }, + { + "text": "Support", + "boxes": [ + 1294, + 151, + 1348, + 169 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 149, + 1394, + 169 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "iPhone", + "boxes": [ + 859, + 231, + 1053, + 305 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "Meet the iPhone 16 family.", + "boxes": [ + 792, + 300, + 1116, + 341 + ] + }, + { + "text": " Learn more", + "boxes": [ + 826, + 363, + 918, + 389 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 967, + 355, + 1085, + 398 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": "Built for Apple Intligence.", + "boxes": [ + 830, + 423, + 1078, + 459 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 20, + 502, + 90, + 528 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 719, + 90, + 752 + ] + }, + { + "text": "D", + "boxes": [ + 31, + 775, + 82, + 831 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 828, + 88, + 862 + ] + }, + { + "text": "Files", + "boxes": [ + 33, + 938, + 87, + 972 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 856, + 941, + 1072, + 1001 + ] + } + ] + }, + { + "frame_id": "70_0_4", + "episode_id": 70, + "env_id": 0, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.280988693237305, + "advantages": 0.3296262323856354, + "generated_text": "{\n \"intent\": \"The initial page displayed is a search engine website, likely Microsoft Bing. To explore more about the content on the page, a reasonable action is to scroll down slightly to view additional articles or information.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "The initial page displayed is a search engine website, likely Microsoft Bing. To explore more about the content on the page, a reasonable action is to scroll down slightly to view additional articles or information.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 28, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1608, + 68, + 1658, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 406, + 121, + 452, + 141 + ] + }, + { + "text": "国内版（国际版", + "boxes": [ + 490, + 189, + 638, + 218 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 236, + 614, + 266 + ] + }, + { + "text": "中忠空", + "boxes": [ + 1538, + 357, + 1574, + 387 + ] + }, + { + "text": "图卢兹的暮色", + "boxes": [ + 1516, + 389, + 1634, + 421 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 441, + 272, + 465 + ] + }, + { + "text": "网友关注", + "boxes": [ + 832, + 482, + 914, + 508 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1456, + 480, + 1552, + 512 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1456, + 516, + 1506, + 560 + ] + }, + { + "text": "Mw", + "boxes": [ + 1588, + 519, + 1632, + 552 + ] + }, + { + "text": "+ 4998", + "boxes": [ + 1634, + 518, + 1698, + 560 + ] + }, + { + "text": "司", + "boxes": [ + 948, + 540, + 966, + 556 + ] + }, + { + "text": "做打事", + "boxes": [ + 1230, + 536, + 1324, + 568 + ] + }, + { + "text": "新腰：74岁了还敢演女将军，女主若", + "boxes": [ + 826, + 562, + 1094, + 592 + ] + }, + { + "text": "被抓包的猫咪", + "boxes": [ + 1192, + 557, + 1362, + 606 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 560, + 1518, + 609 + ] + }, + { + "text": ",1024 ", + "boxes": [ + 1628, + 560, + 1702, + 609 + ] + }, + { + "text": "", + "boxes": [ + 1592, + 572, + 1638, + 600 + ] + }, + { + "text": "坚提基租成本，真琳被降继性密87分", + "boxes": [ + 820, + 602, + 1084, + 631 + ] + }, + { + "text": "时中国A50", + "boxes": [ + 1449, + 601, + 1551, + 658 + ] + }, + { + "text": "L", + "boxes": [ + 1594, + 623, + 1610, + 641 + ] + }, + { + "text": "破室业威巨额亏损", + "boxes": [ + 824, + 633, + 1010, + 669 + ] + }, + { + "text": "D", + "boxes": [ + 1138, + 649, + 1164, + 671 + ] + }, + { + "text": "一点视频/萌宠旅行者·3周", + "boxes": [ + 1158, + 645, + 1320, + 675 + ] + }, + { + "text": "很显然，猫咪也知道自己在使", + "boxes": [ + 1130, + 661, + 1412, + 707 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 655, + 1510, + 701 + ] + }, + { + "text": "+3.28%", + "boxes": [ + 1626, + 653, + 1700, + 701 + ] + }, + { + "text": "热血传奇新百区·四枚高魔戒指个个稀罕", + "boxes": [ + 200, + 663, + 734, + 699 + ] + }, + { + "text": "[~.", + "boxes": [ + 1588, + 655, + 1630, + 695 + ] + }, + { + "text": "放假通凝讯", + "boxes": [ + 826, + 675, + 916, + 707 + ] + }, + { + "text": "们却不是952服的巅峰", + "boxes": [ + 200, + 699, + 496, + 733 + ] + }, + { + "text": "坏！", + "boxes": [ + 1134, + 695, + 1182, + 729 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 699, + 1522, + 748 + ] + }, + { + "text": "刘强舰网性看威鼓搬军要求负罗", + "boxes": [ + 828, + 711, + 1094, + 741 + ] + }, + { + "text": "-1.18%0 ", + "boxes": [ + 1634, + 703, + 1698, + 744 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 717, + 1628, + 739 + ] + }, + { + "text": "+", + "boxes": [ + 1688, + 717, + 1712, + 731 + ] + }, + { + "text": "101", + "boxes": [ + 1138, + 741, + 1190, + 766 + ] + }, + { + "text": "", + "boxes": [ + 1188, + 746, + 1210, + 760 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1622, + 744, + 1726, + 770 + ] + }, + { + "text": "长春市", + "boxes": [ + 1458, + 800, + 1524, + 820 + ] + }, + { + "text": "下周五", + "boxes": [ + 1618, + 854, + 1668, + 872 + ] + }, + { + "text": "魔法与神之学（一）", + "boxes": [ + 1302, + 885, + 1416, + 915 + ] + }, + { + "text": "ITBEAR科技资讯·15小时", + "boxes": [ + 198, + 947, + 382, + 979 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 512, + 949, + 644, + 979 + ] + }, + { + "text": "证券日报·18小时", + "boxes": [ + 828, + 949, + 966, + 979 + ] + }, + { + "text": "机核·2周", + "boxes": [ + 1138, + 951, + 1232, + 977 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，", + "boxes": [ + 202, + 975, + 454, + 1005 + ] + }, + { + "text": "探秘荒野之魅", + "boxes": [ + 514, + 975, + 642, + 1007 + ] + }, + { + "text": "从“底层重构”到“文明融合”", + "boxes": [ + 826, + 971, + 1090, + 1007 + ] + }, + { + "text": "《女神异闻录3》中的魔法与", + "boxes": [ + 1142, + 969, + 1402, + 1009 + ] + }, + { + "text": "能否引领新能源市场新潮流？", + "boxes": [ + 202, + 999, + 468, + 1028 + ] + }, + { + "text": "光启技术加速推进超材料产业", + "boxes": [ + 822, + 995, + 1098, + 1034 + ] + }, + { + "text": "神秘学（一)", + "boxes": [ + 1135, + 994, + 1259, + 1037 + ] + }, + { + "text": "生太升级", + "boxes": [ + 834, + 1030, + 908, + 1048 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 40, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 28, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 62, + 446, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1597, + 61, + 1663, + 92 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 22, + 109, + 184, + 151 + ] + }, + { + "text": "Translate", + "boxes": [ + 408, + 121, + 454, + 141 + ] + }, + { + "text": "ign", + "boxes": [ + 1648, + 125, + 1666, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 486, + 153, + 560, + 185 + ] + }, + { + "text": "国际版", + "boxes": [ + 546, + 151, + 638, + 183 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 470, + 192, + 616, + 235 + ] + }, + { + "text": "0", + "boxes": [ + 1342, + 204, + 1356, + 224 + ] + }, + { + "text": "的猫味", + "boxes": [ + 1266, + 288, + 1354, + 312 + ] + }, + { + "text": "600519", + "boxes": [ + 1460, + 290, + 1510, + 316 + ] + }, + { + "text": " 1,614.13", + "boxes": [ + 1618, + 288, + 1696, + 320 + ] + }, + { + "text": "不基租成本，真的被隆细87分", + "boxes": [ + 830, + 316, + 1080, + 339 + ] + }, + { + "text": "13027%", + "boxes": [ + 1623, + 318, + 1701, + 373 + ] + }, + { + "text": "破宏业威巨额亏损", + "boxes": [ + 824, + 349, + 1008, + 383 + ] + }, + { + "text": "", + "boxes": [ + 1802, + 349, + 1830, + 373 + ] + }, + { + "text": "1", + "boxes": [ + 1142, + 367, + 1162, + 383 + ] + }, + { + "text": "一点视频/萌宠旅行者·3周", + "boxes": [ + 1154, + 359, + 1322, + 389 + ] + }, + { + "text": "山东博物馆里“涨知识”感受历史文化的独特", + "boxes": [ + 200, + 375, + 780, + 417 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 369, + 1510, + 415 + ] + }, + { + "text": "r", + "boxes": [ + 1594, + 375, + 1630, + 407 + ] + }, + { + "text": "+3.22", + "boxes": [ + 1628, + 367, + 1700, + 415 + ] + }, + { + "text": "很显然，猫咪也知道自己在使", + "boxes": [ + 1132, + 377, + 1410, + 421 + ] + }, + { + "text": "放假通资讯", + "boxes": [ + 828, + 393, + 914, + 419 + ] + }, + { + "text": "魅力", + "boxes": [ + 197, + 408, + 271, + 454 + ] + }, + { + "text": "坏！", + "boxes": [ + 1138, + 413, + 1180, + 441 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 413, + 1524, + 467 + ] + }, + { + "text": "-426 ", + "boxes": [ + 1632, + 415, + 1700, + 465 + ] + }, + { + "text": "刻魏测性着斑戚病技撕军要戏寅罗", + "boxes": [ + 826, + 429, + 1094, + 459 + ] + }, + { + "text": "VLM", + "boxes": [ + 1596, + 431, + 1628, + 453 + ] + }, + { + "text": "101", + "boxes": [ + 1138, + 455, + 1190, + 480 + ] + }, + { + "text": "", + "boxes": [ + 1188, + 461, + 1210, + 474 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1622, + 459, + 1726, + 484 + ] + }, + { + "text": "长春市", + "boxes": [ + 1466, + 516, + 1524, + 536 + ] + }, + { + "text": "魔法与神之学（一）", + "boxes": [ + 1304, + 602, + 1418, + 631 + ] + }, + { + "text": "", + "boxes": [ + 598, + 637, + 682, + 661 + ] + }, + { + "text": "ITBEAR科技资讯·15小时", + "boxes": [ + 198, + 663, + 384, + 693 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 514, + 667, + 642, + 691 + ] + }, + { + "text": "证券日报+关注·18小时", + "boxes": [ + 828, + 667, + 1006, + 691 + ] + }, + { + "text": "机核·2周", + "boxes": [ + 1138, + 667, + 1232, + 693 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，", + "boxes": [ + 202, + 689, + 452, + 719 + ] + }, + { + "text": "探秘荒野之魅", + "boxes": [ + 514, + 689, + 644, + 721 + ] + }, + { + "text": "从“底层重构\"到“文明融合”", + "boxes": [ + 824, + 685, + 1090, + 721 + ] + }, + { + "text": "《女神异闻录3》中的魔法与", + "boxes": [ + 1140, + 685, + 1404, + 727 + ] + }, + { + "text": "能否引领新能源市场新潮流？", + "boxes": [ + 204, + 719, + 464, + 742 + ] + }, + { + "text": "光启技术加速推进超材料产业", + "boxes": [ + 824, + 713, + 1098, + 748 + ] + }, + { + "text": "神秘学（一)", + "boxes": [ + 1136, + 712, + 1256, + 751 + ] + }, + { + "text": "生态升级", + "boxes": [ + 826, + 741, + 916, + 772 + ] + }, + { + "text": "10", + "boxes": [ + 514, + 768, + 584, + 800 + ] + }, + { + "text": "", + "boxes": [ + 824, + 772, + 878, + 798 + ] + }, + { + "text": "4", + "boxes": [ + 1136, + 768, + 1208, + 800 + ] + }, + { + "text": "交通事件", + "boxes": [ + 1446, + 826, + 1542, + 858 + ] + }, + { + "text": "尚未升灵", + "boxes": [ + 666, + 858, + 712, + 878 + ] + }, + { + "text": "硅谷大街 禁止通行 原因：路面施工", + "boxes": [ + 1448, + 858, + 1708, + 895 + ] + }, + { + "text": "手套", + "boxes": [ + 554, + 925, + 586, + 947 + ] + }, + { + "text": "已绑定", + "boxes": [ + 532, + 951, + 582, + 977 + ] + }, + { + "text": "网上车市·1周", + "boxes": [ + 202, + 983, + 318, + 1007 + ] + }, + { + "text": "重游戏天地·13小时", + "boxes": [ + 514, + 981, + 652, + 1011 + ] + }, + { + "text": "江湖上流传着一", + "boxes": [ + 820, + 1002, + 1036, + 1047 + ] + }, + { + "text": "句话，能被住的猫都不是善", + "boxes": [ + 1030, + 1007, + 1416, + 1042 + ] + } + ] + }, + { + "frame_id": "70_1_4", + "episode_id": 70, + "env_id": 1, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.010412216186523, + "advantages": -0.4037978947162628, + "generated_text": "{\n \"intent\": \"The current screen shows an open spreadsheet software which allows data entry and manipulation. To further explore the environment, it's logical to interact directly with the application interface by clicking into a cell to start entering or modifying data.\",\n \"action\": \"DoubleClick(150, 300)\"\n}", + "action_string": "DoubleClick(150, 300)", + "intent_string": "The current screen shows an open spreadsheet software which allows data entry and manipulation. To further explore the environment, it's logical to interact directly with the application interface by clicking into a cell to start entering or modifying data.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 28, + 42, + 42 + ] + }, + { + "text": "Edit View", + "boxes": [ + 38, + 26, + 116, + 44 + ] + }, + { + "text": "Help", + "boxes": [ + 526, + 30, + 550, + 44 + ] + }, + { + "text": "·口·目·目@自、", + "boxes": [ + 8, + 50, + 378, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 373, + 51, + 443, + 86 + ] + }, + { + "text": "G-、", + "boxes": [ + 450, + 56, + 552, + 79 + ] + }, + { + "text": "αv田·I·子D目I·口田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "Liberationsan10aαaa要目目-% 00t三图-口目", + "boxes": [ + 12, + 93, + 1206, + 123 + ] + }, + { + "text": "、", + "boxes": [ + 130, + 129, + 160, + 149 + ] + }, + { + "text": "=", + "boxes": [ + 148, + 125, + 258, + 157 + ] + }, + { + "text": "A6", + "boxes": [ + 8, + 135, + 26, + 149 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 199 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "21", + "boxes": [ + 16, + 512, + 30, + 528 + ] + }, + { + "text": "25", + "boxes": [ + 16, + 584, + 30, + 602 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "→→+", + "boxes": [ + 14, + 995, + 132, + 1017 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "画··目·目@自、", + "boxes": [ + 12, + 54, + 390, + 83 + ] + }, + { + "text": "&", + "boxes": [ + 371, + 53, + 445, + 84 + ] + }, + { + "text": "←-、", + "boxes": [ + 444, + 56, + 552, + 79 + ] + }, + { + "text": "v田·I·子DI田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "10", + "boxes": [ + 140, + 95, + 230, + 119 + ] + }, + { + "text": "aaaa旦目日日%00三三图目", + "boxes": [ + 230, + 93, + 1206, + 123 + ] + }, + { + "text": "游×化", + "boxes": [ + 152, + 129, + 242, + 155 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 197 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 484 + ] + }, + { + "text": ".9", + "boxes": [ + 18, + 482, + 30, + 494 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "70_2_4", + "episode_id": 70, + "env_id": 2, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.430975914001465, + "advantages": 0.41620445251464844, + "generated_text": "{\n \"intent\": \"To continue exploring different sections of the Apple website, clicking on 'Shop iPhone' will allow access to more details about available iPhone models.\",\n \"action\": \"DoubleClick(1250, 339)\"\n}", + "action_string": "DoubleClick(1250, 339)", + "intent_string": "To continue exploring different sections of the Apple website, clicking on 'Shop iPhone' will allow access to more details about available iPhone models.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 822, + 318, + 932, + 361 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1603, + 65, + 1659, + 90 + ] + }, + { + "text": "Watch", + "boxes": [ + 768, + 113, + 798, + 127 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 986, + 109, + 1046, + 129 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1082, + 111, + 1160, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1296, + 111, + 1342, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1370, + 109, + 1396, + 129 + ] + }, + { + "text": "?", + "boxes": [ + 1414, + 105, + 1446, + 137 + ] + }, + { + "text": "B", + "boxes": [ + 454, + 157, + 522, + 218 + ] + }, + { + "text": "B", + "boxes": [ + 562, + 157, + 630, + 224 + ] + }, + { + "text": "D", + "boxes": [ + 754, + 157, + 820, + 218 + ] + }, + { + "text": "18", + "boxes": [ + 1106, + 155, + 1170, + 222 + ] + }, + { + "text": "", + "boxes": [ + 1378, + 153, + 1464, + 226 + ] + }, + { + "text": "0", + "boxes": [ + 670, + 163, + 716, + 214 + ] + }, + { + "text": "00", + "boxes": [ + 846, + 165, + 922, + 222 + ] + }, + { + "text": "·", + "boxes": [ + 1026, + 179, + 1068, + 218 + ] + }, + { + "text": "口", + "boxes": [ + 1210, + 173, + 1276, + 220 + ] + }, + { + "text": "OS", + "boxes": [ + 1306, + 171, + 1354, + 218 + ] + }, + { + "text": "9?", + "boxes": [ + 944, + 181, + 992, + 218 + ] + }, + { + "text": "ipPhone 16 Pro", + "boxes": [ + 448, + 218, + 530, + 244 + ] + }, + { + "text": "iPhone 16", + "boxes": [ + 564, + 218, + 626, + 244 + ] + }, + { + "text": "iphone 16e", + "boxes": [ + 660, + 218, + 726, + 244 + ] + }, + { + "text": "iPhone 15", + "boxes": [ + 758, + 218, + 820, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 852, + 218, + 908, + 244 + ] + }, + { + "text": "AirPods", + "boxes": [ + 944, + 218, + 994, + 244 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1026, + 218, + 1066, + 246 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1102, + 218, + 1172, + 244 + ] + }, + { + "text": "Apple Card", + "boxes": [ + 1210, + 218, + 1274, + 244 + ] + }, + { + "text": "ios 18", + "boxes": [ + 1308, + 218, + 1350, + 244 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 1386, + 218, + 1458, + 244 + ] + }, + { + "text": "iPhone", + "boxes": [ + 99, + 384, + 369, + 488 + ] + }, + { + "text": "Designed to be loved.", + "boxes": [ + 1520, + 433, + 1800, + 474 + ] + }, + { + "text": "9:41", + "boxes": [ + 836, + 566, + 1044, + 693 + ] + } + ] + }, + { + "frame_id": "70_3_4", + "episode_id": 70, + "env_id": 3, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.061951637268066, + "advantages": 0.20318986475467682, + "generated_text": "{\n \"intent\": \"To continue exploring the Amazon marketplace, we will scroll down to view more product categories and details.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring the Amazon marketplace, we will scroll down to view more product categories and details.", + "before_screen_ocr_results": [ + { + "text": "Shopfor your home essentials", + "boxes": [ + 614, + 187, + 914, + 210 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 982, + 185, + 1164, + 214 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 1350, + 185, + 1606, + 214 + ] + }, + { + "text": "appliances", + "boxes": [ + 1352, + 212, + 1464, + 242 + ] + }, + { + "text": "NU", + "boxes": [ + 1484, + 465, + 1540, + 482 + ] + }, + { + "text": "Pots and Pans", + "boxes": [ + 1460, + 502, + 1566, + 520 + ] + }, + { + "text": "Kettles", + "boxes": [ + 1555, + 498, + 1611, + 523 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 981, + 541, + 1065, + 570 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 238, + 621, + 464, + 657 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 623, + 1220, + 653 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1348, + 619, + 1508, + 655 + ] + }, + { + "text": "See all deals", + "boxes": [ + 244, + 987, + 316, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 985, + 1058, + 1009 + ] + }, + { + "text": "Shopnov", + "boxes": [ + 1356, + 987, + 1406, + 1005 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "", + "boxes": [ + 1468, + 177, + 1554, + 201 + ] + }, + { + "text": "Coffee", + "boxes": [ + 1358, + 218, + 1394, + 232 + ] + }, + { + "text": "Pots and Pans", + "boxes": [ + 1460, + 216, + 1556, + 234 + ] + }, + { + "text": "Kettles", + "boxes": [ + 1562, + 216, + 1610, + 234 + ] + }, + { + "text": "See more", + "boxes": [ + 244, + 262, + 300, + 280 + ] + }, + { + "text": "Shop gaming ", + "boxes": [ + 982, + 260, + 1066, + 284 + ] + }, + { + "text": "Explore all products in Kitchen", + "boxes": [ + 1354, + 262, + 1532, + 280 + ] + }, + { + "text": "Shop deals in Fashion ", + "boxes": [ + 240, + 339, + 464, + 369 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 338, + 906, + 367 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 339, + 1220, + 369 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1350, + 339, + 1508, + 369 + ] + }, + { + "text": "ee all deals", + "boxes": [ + 252, + 701, + 316, + 719 + ] + }, + { + "text": "See all deals", + "boxes": [ + 988, + 701, + 1054, + 719 + ] + }, + { + "text": " Gift ideas inspired by your shopping history", + "boxes": [ + 236, + 776, + 682, + 814 + ] + }, + { + "text": "CD", + "boxes": [ + 794, + 848, + 834, + 885 + ] + } + ] + }, + { + "frame_id": "70_4_4", + "episode_id": 70, + "env_id": 4, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.3412446975708, + "advantages": 0.3644082248210907, + "generated_text": "{\n \"intent\": \"Scanning for useful actions, clicking on 'Getting Started' appears to be informative.\",\n \"action\": \"DoubleClick(258, 110)\"\n}", + "action_string": "DoubleClick(258, 110)", + "intent_string": "Scanning for useful actions, clicking on 'Getting Started' appears to be informative.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 61, + 520, + 93 + ] + }, + { + "text": " Import bookmarks.", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 182, + 99, + 310, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozilla I", + "boxes": [ + 251, + 103, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 111, + 1048, + 143 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "70_5_4", + "episode_id": 70, + "env_id": 5, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.744073867797852, + "advantages": 0.5969362258911133, + "generated_text": "{\n \"intent\": \"Given the goal is to explore this environment by interacting as much as possible, clicking on elements can provide more information or navigate to different parts of the site. The 'Learn more' button on the WATCH SERIES 10 section seems like an interesting option to explore further.\",\n \"action\": \"DoubleClick(918, 791)\"\n}", + "action_string": "DoubleClick(918, 791)", + "intent_string": "Given the goal is to explore this environment by interacting as much as possible, clicking on elements can provide more information or navigate to different parts of the site. The 'Learn more' button on the WATCH SERIES 10 section seems like an interesting option to explore further.", + "before_screen_ocr_results": [ + { + "text": "Appl", + "boxes": [ + 28, + 34, + 62, + 48 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→", + "boxes": [ + 32, + 66, + 126, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Visint for Airpodse Intw& Homece.", + "boxes": [ + 834, + 101, + 1082, + 131 + ] + }, + { + "text": "Entertainmer", + "boxes": [ + 1070, + 111, + 1150, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1370, + 113, + 1392, + 127 + ] + }, + { + "text": "②", + "boxes": [ + 1422, + 115, + 1440, + 131 + ] + }, + { + "text": "WATCH", + "boxes": [ + 874, + 621, + 1072, + 677 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 904, + 677, + 1030, + 709 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 846, + 710, + 1062, + 753 + ] + }, + { + "text": " Learn more", + "boxes": [ + 858, + 774, + 958, + 800 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 774, + 1048, + 800 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1605, + 65, + 1659, + 90 + ] + }, + { + "text": "e", + "boxes": [ + 257, + 67, + 277, + 88 + ] + }, + { + "text": "Visior", + "boxes": [ + 842, + 113, + 870, + 127 + ] + }, + { + "text": "AirPods", + "boxes": [ + 906, + 109, + 950, + 129 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 986, + 109, + 1052, + 129 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1082, + 111, + 1164, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1302, + 111, + 1346, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1382, + 113, + 1394, + 125 + ] + }, + { + "text": "二", + "boxes": [ + 1430, + 113, + 1442, + 125 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 652, + 334, + 1250, + 419 + ] + } + ] + }, + { + "frame_id": "70_6_4", + "episode_id": 70, + "env_id": 6, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.77735710144043, + "advantages": 0.03891128674149513, + "generated_text": "{\n \"intent\": \"The goal is to continue exploring different sections of the webpage by clicking on an item or link that promotes learning more about iPhones.\",\n \"action\": \"Click(368, 992)\"\n}", + "action_string": "Click(368, 992)", + "intent_string": "The goal is to continue exploring different sections of the webpage by clicking on an item or link that promotes learning more about iPhones.", + "before_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1544, + 69, + 1562, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1660, + 89 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 944, + 127 + ] + }, + { + "text": "TV & Hon", + "boxes": [ + 986, + 111, + 1036, + 129 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 134, + 218, + 412, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1600, + 230, + 1742, + 254 + ] + }, + { + "text": "ecialist7", + "boxes": [ + 1714, + 254, + 1766, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1564, + 292, + 1588, + 318 + ] + }, + { + "text": "Visit an Apple Store", + "boxes": [ + 1596, + 284, + 1732, + 312 + ] + }, + { + "text": "Find one near you 7", + "boxes": [ + 1598, + 303, + 1726, + 332 + ] + }, + { + "text": "?", + "boxes": [ + 1028, + 339, + 1042, + 357 + ] + }, + { + "text": "All Models ", + "boxes": [ + 144, + 389, + 230, + 413 + ] + }, + { + "text": " Shopping guides ", + "boxes": [ + 260, + 387, + 400, + 417 + ] + }, + { + "text": "Ways to Save", + "boxes": [ + 430, + 389, + 540, + 413 + ] + }, + { + "text": " Sustainability", + "boxes": [ + 570, + 387, + 682, + 417 + ] + }, + { + "text": "Accessories", + "boxes": [ + 716, + 389, + 814, + 413 + ] + }, + { + "text": " Setup and Support ", + "boxes": [ + 846, + 387, + 1000, + 417 + ] + }, + { + "text": "The iPhone Experience", + "boxes": [ + 1032, + 389, + 1214, + 413 + ] + }, + { + "text": "Special Stores ", + "boxes": [ + 1248, + 387, + 1366, + 417 + ] + }, + { + "text": "All models. Take your pick.", + "boxes": [ + 138, + 472, + 464, + 508 + ] + }, + { + "text": "PPLE INTELLIGENCE", + "boxes": [ + 180, + 560, + 296, + 578 + ] + }, + { + "text": "APPLE INTELLIGENCE", + "boxes": [ + 1012, + 556, + 1140, + 580 + ] + }, + { + "text": "iPhone 16 Pro &", + "boxes": [ + 164, + 580, + 376, + 621 + ] + }, + { + "text": "iPhone 16 &", + "boxes": [ + 586, + 582, + 746, + 619 + ] + }, + { + "text": "iPhone 16e", + "boxes": [ + 1004, + 579, + 1156, + 622 + ] + }, + { + "text": "iPhone 15 &", + "boxes": [ + 1424, + 580, + 1582, + 617 + ] + }, + { + "text": "Expl", + "boxes": [ + 1841, + 579, + 1917, + 624 + ] + }, + { + "text": "iPhone 16 Pro Max", + "boxes": [ + 164, + 611, + 408, + 653 + ] + }, + { + "text": "iPhone 16 Plus", + "boxes": [ + 586, + 613, + 778, + 651 + ] + }, + { + "text": "iPhone 15 Plus", + "boxes": [ + 1424, + 611, + 1618, + 653 + ] + }, + { + "text": "acce", + "boxes": [ + 1846, + 619, + 1914, + 651 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "BuyiPhone16", + "boxes": [ + 674, + 4, + 804, + 22 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1128, + 2, + 1246, + 26 + ] + }, + { + "text": "→白", + "boxes": [ + 8, + 60, + 128, + 91 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1882, + 91 + ] + }, + { + "text": "Store", + "boxes": [ + 512, + 113, + 546, + 127 + ] + }, + { + "text": "Mac", + "boxes": [ + 572, + 113, + 602, + 127 + ] + }, + { + "text": "iPad", + "boxes": [ + 636, + 113, + 660, + 127 + ] + }, + { + "text": "iPhone", + "boxes": [ + 692, + 113, + 732, + 129 + ] + }, + { + "text": "Watch", + "boxes": [ + 764, + 113, + 804, + 129 + ] + }, + { + "text": "Vision", + "boxes": [ + 834, + 111, + 874, + 131 + ] + }, + { + "text": "AirPods", + "boxes": [ + 904, + 111, + 948, + 131 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 982, + 111, + 1048, + 131 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1078, + 109, + 1164, + 133 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 111, + 1264, + 133 + ] + }, + { + "text": "Support", + "boxes": [ + 1294, + 111, + 1344, + 131 + ] + }, + { + "text": "Q", + "boxes": [ + 1368, + 109, + 1396, + 131 + ] + }, + { + "text": "6", + "boxes": [ + 1418, + 111, + 1442, + 135 + ] + }, + { + "text": "Carrer Deals at Apple", + "boxes": [ + 496, + 170, + 654, + 207 + ] + }, + { + "text": "", + "boxes": [ + 672, + 187, + 706, + 214 + ] + }, + { + "text": "S", + "boxes": [ + 864, + 191, + 892, + 210 + ] + }, + { + "text": "T", + "boxes": [ + 1058, + 189, + 1084, + 212 + ] + }, + { + "text": "p to $1000 afte", + "boxes": [ + 1324, + 183, + 1410, + 201 + ] + }, + { + "text": "See all deals ①", + "boxes": [ + 500, + 197, + 606, + 220 + ] + }, + { + "text": "V", + "boxes": [ + 1256, + 193, + 1274, + 206 + ] + } + ] + }, + { + "frame_id": "70_7_4", + "episode_id": 70, + "env_id": 7, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.656171798706055, + "advantages": -0.03104138933122158, + "generated_text": "{\n \"intent\": \"To explore further and possibly find more clickable elements or other useful features by scrolling through the page.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore further and possibly find more clickable elements or other useful features by scrolling through the page.", + "before_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 320, + 40, + 347, + 66 + ] + }, + { + "text": "□", + "boxes": [ + 1536, + 40, + 1598, + 66 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": " Apple", + "boxes": [ + 330, + 71, + 392, + 91 + ] + }, + { + "text": "×+", + "boxes": [ + 518, + 69, + 570, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 343, + 97, + 453, + 134 + ] + }, + { + "text": "① https://ww.apple.com", + "boxes": [ + 550, + 101, + 768, + 131 + ] + }, + { + "text": "·☆", + "boxes": [ + 1284, + 101, + 1368, + 133 + ] + }, + { + "text": "三四?", + "boxes": [ + 1466, + 103, + 1564, + 129 + ] + }, + { + "text": "三", + "boxes": [ + 1572, + 107, + 1588, + 123 + ] + }, + { + "text": "iPhone", + "boxes": [ + 694, + 153, + 730, + 167 + ] + }, + { + "text": "Watch", + "boxes": [ + 768, + 153, + 802, + 167 + ] + }, + { + "text": "Vision", + "boxes": [ + 836, + 153, + 876, + 169 + ] + }, + { + "text": "AirPods", + "boxes": [ + 898, + 149, + 952, + 169 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 978, + 147, + 1052, + 171 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1076, + 147, + 1164, + 171 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 151, + 1266, + 169 + ] + }, + { + "text": "Support", + "boxes": [ + 1294, + 151, + 1348, + 169 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 149, + 1394, + 169 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "iPhone", + "boxes": [ + 859, + 231, + 1053, + 305 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "Meet the iPhone 16 family.", + "boxes": [ + 792, + 300, + 1116, + 341 + ] + }, + { + "text": " Learn more", + "boxes": [ + 826, + 363, + 918, + 389 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 967, + 355, + 1085, + 398 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": " Built for Apple Intelligence.", + "boxes": [ + 828, + 419, + 1078, + 459 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 20, + 502, + 90, + 528 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "國", + "boxes": [ + 36, + 681, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 719, + 90, + 752 + ] + }, + { + "text": "D", + "boxes": [ + 31, + 775, + 82, + 831 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 828, + 88, + 862 + ] + }, + { + "text": "Files", + "boxes": [ + 33, + 938, + 87, + 972 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 856, + 941, + 1072, + 1001 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 320, + 40, + 347, + 66 + ] + }, + { + "text": "ppl", + "boxes": [ + 876, + 46, + 906, + 60 + ] + }, + { + "text": "□", + "boxes": [ + 1536, + 40, + 1598, + 66 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "Apple", + "boxes": [ + 332, + 71, + 392, + 91 + ] + }, + { + "text": "×+", + "boxes": [ + 518, + 68, + 572, + 93 + ] + }, + { + "text": "↑→", + "boxes": [ + 330, + 101, + 452, + 131 + ] + }, + { + "text": "① https:/www.apple.com", + "boxes": [ + 548, + 99, + 768, + 133 + ] + }, + { + "text": "·☆", + "boxes": [ + 1286, + 101, + 1368, + 133 + ] + }, + { + "text": "三四?", + "boxes": [ + 1466, + 103, + 1568, + 129 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1196, + 151, + 1262, + 169 + ] + }, + { + "text": "Support", + "boxes": [ + 1296, + 151, + 1346, + 169 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 149, + 1394, + 169 + ] + }, + { + "text": "?", + "boxes": [ + 1420, + 148, + 1444, + 173 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 20, + 502, + 90, + 528 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "WATCH", + "boxes": [ + 872, + 659, + 1070, + 715 + ] + }, + { + "text": "國", + "boxes": [ + 38, + 681, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 904, + 715, + 1032, + 746 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 850, + 752, + 1064, + 788 + ] + }, + { + "text": "D", + "boxes": [ + 34, + 778, + 80, + 828 + ] + }, + { + "text": "Learn more", + "boxes": [ + 862, + 814, + 958, + 840 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 812, + 1054, + 838 + ] + }, + { + "text": "Draw", + "boxes": [ + 34, + 832, + 88, + 862 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + } + ] + }, + { + "frame_id": "70_0_5", + "episode_id": 70, + "env_id": 0, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.137073516845703, + "advantages": 0.24655301868915558, + "generated_text": "{\n \"intent\": \"To explore more content by scrolling or interacting with elements on the Bing homepage, clicking on any item could give us more detailed information.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore more content by scrolling or interacting with elements on the Bing homepage, clicking on any item could give us more detailed information.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": " https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 22, + 111, + 184, + 153 + ] + }, + { + "text": "Translate", + "boxes": [ + 404, + 119, + 458, + 145 + ] + }, + { + "text": "Maps", + "boxes": [ + 492, + 121, + 526, + 143 + ] + }, + { + "text": "ign", + "boxes": [ + 1648, + 125, + 1666, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 488, + 153, + 560, + 185 + ] + }, + { + "text": "国际版", + "boxes": [ + 546, + 151, + 638, + 183 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 468, + 192, + 616, + 235 + ] + }, + { + "text": "0", + "boxes": [ + 1342, + 204, + 1356, + 224 + ] + }, + { + "text": "的猫味", + "boxes": [ + 1266, + 288, + 1354, + 312 + ] + }, + { + "text": "600519", + "boxes": [ + 1460, + 290, + 1510, + 316 + ] + }, + { + "text": " 1,614.13", + "boxes": [ + 1618, + 288, + 1696, + 320 + ] + }, + { + "text": "亚提基租成本_真救降继87分", + "boxes": [ + 828, + 312, + 1082, + 341 + ] + }, + { + "text": "13,287%", + "boxes": [ + 1628, + 326, + 1698, + 367 + ] + }, + { + "text": "塑破宏氨或巨额亏损", + "boxes": [ + 826, + 349, + 1008, + 383 + ] + }, + { + "text": "", + "boxes": [ + 1802, + 349, + 1830, + 373 + ] + }, + { + "text": "1", + "boxes": [ + 1142, + 367, + 1162, + 383 + ] + }, + { + "text": "一点视频/萌宠旅行者·3周", + "boxes": [ + 1154, + 359, + 1322, + 389 + ] + }, + { + "text": "你还不知道怎么选梦幻西游命魂之玉吗？", + "boxes": [ + 200, + 375, + 734, + 417 + ] + }, + { + "text": "比油", + "boxes": [ + 1456, + 369, + 1510, + 415 + ] + }, + { + "text": "r", + "boxes": [ + 1594, + 375, + 1630, + 407 + ] + }, + { + "text": "+3.22", + "boxes": [ + 1628, + 367, + 1700, + 415 + ] + }, + { + "text": "篇", + "boxes": [ + 754, + 385, + 788, + 407 + ] + }, + { + "text": "很显然，猫咪也知道自己在使", + "boxes": [ + 1134, + 379, + 1410, + 419 + ] + }, + { + "text": "放假通资讯", + "boxes": [ + 828, + 393, + 914, + 419 + ] + }, + { + "text": "带你看懂", + "boxes": [ + 198, + 411, + 330, + 455 + ] + }, + { + "text": "坏！", + "boxes": [ + 1138, + 413, + 1180, + 441 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 413, + 1524, + 467 + ] + }, + { + "text": "-426 ", + "boxes": [ + 1632, + 415, + 1700, + 465 + ] + }, + { + "text": "剑强魏钢性看斑瞰技撇军要求寅罗", + "boxes": [ + 828, + 429, + 1094, + 459 + ] + }, + { + "text": "VLM", + "boxes": [ + 1596, + 431, + 1628, + 453 + ] + }, + { + "text": "△1", + "boxes": [ + 204, + 459, + 246, + 478 + ] + }, + { + "text": "101", + "boxes": [ + 1138, + 455, + 1190, + 480 + ] + }, + { + "text": "", + "boxes": [ + 1188, + 461, + 1210, + 474 + ] + }, + { + "text": "查看自选股建议", + "boxes": [ + 1622, + 459, + 1726, + 484 + ] + }, + { + "text": "长春市", + "boxes": [ + 1466, + 516, + 1524, + 536 + ] + }, + { + "text": "魔法与神之学（一）", + "boxes": [ + 1304, + 602, + 1418, + 631 + ] + }, + { + "text": "ITBEAR科技资讯·15小时", + "boxes": [ + 198, + 663, + 384, + 693 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 514, + 667, + 642, + 691 + ] + }, + { + "text": "证券日报+关注·18小时", + "boxes": [ + 830, + 667, + 1006, + 691 + ] + }, + { + "text": "机核·2周", + "boxes": [ + 1138, + 667, + 1232, + 693 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，", + "boxes": [ + 202, + 689, + 452, + 719 + ] + }, + { + "text": "探秘荒野之魅", + "boxes": [ + 514, + 689, + 644, + 721 + ] + }, + { + "text": "从“底层重构\"到“文明融合”", + "boxes": [ + 826, + 685, + 1090, + 721 + ] + }, + { + "text": "《女神异闻录3》中的魔法与", + "boxes": [ + 1140, + 685, + 1404, + 727 + ] + }, + { + "text": "能否引领新能源市场新潮流？", + "boxes": [ + 204, + 719, + 464, + 742 + ] + }, + { + "text": "光启技术加速推进超材料产业", + "boxes": [ + 824, + 713, + 1098, + 748 + ] + }, + { + "text": "神秘学（一)", + "boxes": [ + 1136, + 712, + 1256, + 751 + ] + }, + { + "text": "生态升级", + "boxes": [ + 826, + 741, + 916, + 772 + ] + }, + { + "text": "10", + "boxes": [ + 514, + 768, + 584, + 800 + ] + }, + { + "text": "", + "boxes": [ + 824, + 772, + 878, + 798 + ] + }, + { + "text": "4", + "boxes": [ + 1136, + 768, + 1208, + 800 + ] + }, + { + "text": "交通事件", + "boxes": [ + 1446, + 826, + 1542, + 858 + ] + }, + { + "text": "尚未升灵", + "boxes": [ + 666, + 858, + 712, + 878 + ] + }, + { + "text": "硅谷大街 禁止通行 原因：路面施工", + "boxes": [ + 1448, + 858, + 1708, + 895 + ] + }, + { + "text": "手套", + "boxes": [ + 554, + 925, + 586, + 947 + ] + }, + { + "text": "已绑定", + "boxes": [ + 532, + 951, + 582, + 977 + ] + }, + { + "text": "网上车市·1周", + "boxes": [ + 202, + 983, + 318, + 1007 + ] + }, + { + "text": "重游戏天地·13小时", + "boxes": [ + 514, + 981, + 652, + 1011 + ] + }, + { + "text": "江湖上流传着一", + "boxes": [ + 820, + 1002, + 1036, + 1047 + ] + }, + { + "text": "句话，能被住的猫都不是善", + "boxes": [ + 1032, + 1007, + 1416, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 40, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 28, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 24, + 109, + 184, + 151 + ] + }, + { + "text": "Translate", + "boxes": [ + 406, + 121, + 454, + 141 + ] + }, + { + "text": "ign", + "boxes": [ + 1646, + 125, + 1666, + 141 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 482, + 145, + 640, + 187 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 468, + 192, + 616, + 235 + ] + }, + { + "text": "9", + "boxes": [ + 1336, + 200, + 1360, + 226 + ] + }, + { + "text": "", + "boxes": [ + 1300, + 286, + 1378, + 326 + ] + }, + { + "text": "魔法与神移学（一）", + "boxes": [ + 1302, + 316, + 1418, + 347 + ] + }, + { + "text": "ITBEAR科技资讯·15小时", + "boxes": [ + 200, + 379, + 384, + 409 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 512, + 379, + 644, + 409 + ] + }, + { + "text": "证券日报·18小时", + "boxes": [ + 830, + 379, + 966, + 409 + ] + }, + { + "text": "机核·2周", + "boxes": [ + 1138, + 381, + 1232, + 407 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，", + "boxes": [ + 200, + 401, + 450, + 437 + ] + }, + { + "text": "探秘荒野之魅", + "boxes": [ + 509, + 397, + 645, + 439 + ] + }, + { + "text": "从“底层重构”到“文明融合”", + "boxes": [ + 824, + 401, + 1090, + 437 + ] + }, + { + "text": "《女神异闻录3》中的魔法与", + "boxes": [ + 1140, + 399, + 1404, + 441 + ] + }, + { + "text": "能否引领新能源市场新潮流？", + "boxes": [ + 202, + 431, + 466, + 461 + ] + }, + { + "text": "光启技术加速推进超材料产业", + "boxes": [ + 824, + 427, + 1098, + 463 + ] + }, + { + "text": "神秘学（一)", + "boxes": [ + 1136, + 426, + 1256, + 463 + ] + }, + { + "text": "生态升级", + "boxes": [ + 824, + 453, + 918, + 490 + ] + }, + { + "text": "1", + "boxes": [ + 206, + 488, + 236, + 508 + ] + }, + { + "text": "", + "boxes": [ + 514, + 486, + 582, + 512 + ] + }, + { + "text": "", + "boxes": [ + 822, + 484, + 880, + 516 + ] + }, + { + "text": "44", + "boxes": [ + 1138, + 486, + 1206, + 512 + ] + }, + { + "text": "X.", + "boxes": [ + 1354, + 542, + 1416, + 568 + ] + }, + { + "text": "交通事件", + "boxes": [ + 1448, + 544, + 1538, + 570 + ] + }, + { + "text": "尚未升灵", + "boxes": [ + 650, + 574, + 712, + 594 + ] + }, + { + "text": "硅谷大街 禁止通行 原因：路面施工", + "boxes": [ + 1448, + 578, + 1710, + 607 + ] + }, + { + "text": "理次数：3/3", + "boxes": [ + 626, + 606, + 680, + 623 + ] + }, + { + "text": "手容", + "boxes": [ + 544, + 637, + 588, + 665 + ] + }, + { + "text": "已绑定", + "boxes": [ + 532, + 667, + 582, + 693 + ] + }, + { + "text": "网上车市·1周", + "boxes": [ + 200, + 695, + 320, + 725 + ] + }, + { + "text": "爱游戏天地·13小时", + "boxes": [ + 516, + 695, + 650, + 725 + ] + }, + { + "text": "L2.9级智能驾驶可还行？天", + "boxes": [ + 202, + 721, + 462, + 750 + ] + }, + { + "text": "天龙网游八卦：15个材料出俩", + "boxes": [ + 512, + 717, + 794, + 752 + ] + }, + { + "text": "江湖上流传着一", + "boxes": [ + 820, + 719, + 1032, + 760 + ] + }, + { + "text": "句话，能被拴住的猫都不是善", + "boxes": [ + 1028, + 723, + 1414, + 758 + ] + }, + { + "text": "工05表现不差，无奈销量太拉", + "boxes": [ + 202, + 746, + 478, + 776 + ] + }, + { + "text": "极品装备，这是脚本开挂？", + "boxes": [ + 512, + 744, + 758, + 774 + ] + }, + { + "text": "茬！", + "boxes": [ + 824, + 758, + 888, + 792 + ] + }, + { + "text": "膀", + "boxes": [ + 204, + 776, + 228, + 798 + ] + }, + { + "text": "林大学校区南4", + "boxes": [ + 1516, + 774, + 1720, + 808 + ] + }, + { + "text": "3", + "boxes": [ + 204, + 804, + 236, + 824 + ] + }, + { + "text": "1", + "boxes": [ + 516, + 802, + 576, + 828 + ] + }, + { + "text": "在地图上查看", + "boxes": [ + 1548, + 810, + 1628, + 834 + ] + }, + { + "text": "美国职业篮球联赛", + "boxes": [ + 1450, + 858, + 1602, + 887 + ] + }, + { + "text": "在你的区域中很受欢迎", + "boxes": [ + 1450, + 887, + 1580, + 924 + ] + }, + { + "text": "完赛·5月17日", + "boxes": [ + 1554, + 931, + 1624, + 949 + ] + }, + { + "text": "88", + "boxes": [ + 1126, + 947, + 1194, + 979 + ] + }, + { + "text": "31-119", + "boxes": [ + 1536, + 949, + 1636, + 975 + ] + }, + { + "text": "下午7:30", + "boxes": [ + 1542, + 999, + 1640, + 1030 + ] + }, + { + "text": "重游戏天地·1周", + "boxes": [ + 1138, + 1011, + 1256, + 1042 + ] + }, + { + "text": "arch - Micr.", + "boxes": [ + 232, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "70_1_5", + "episode_id": 70, + "env_id": 1, + "step": 5, + "format_rewards": 1.0, + "rewards": 8.755910873413086, + "advantages": -0.5507055521011353, + "generated_text": "{\n \"intent\": \"To continue exploring by accessing different applications or settings available on the desktop, the best action would be to click on an icon or area that represents another application or feature.\",\n \"action\": \"DoubleClick(100, 100)\"\n}", + "action_string": "DoubleClick(100, 100)", + "intent_string": "To continue exploring by accessing different applications or settings available on the desktop, the best action would be to click on an icon or area that represents another application or feature.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "画··目·目@自、", + "boxes": [ + 12, + 54, + 390, + 83 + ] + }, + { + "text": "&", + "boxes": [ + 371, + 53, + 445, + 84 + ] + }, + { + "text": "←-、", + "boxes": [ + 444, + 56, + 552, + 79 + ] + }, + { + "text": "v田·I·子DI田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "10", + "boxes": [ + 140, + 95, + 230, + 119 + ] + }, + { + "text": "aaaa旦目日日%00三三图目", + "boxes": [ + 230, + 93, + 1206, + 123 + ] + }, + { + "text": "游×化", + "boxes": [ + 152, + 129, + 242, + 155 + ] + }, + { + "text": "", + "boxes": [ + 1890, + 171, + 1910, + 197 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "18", + "boxes": [ + 16, + 465, + 30, + 484 + ] + }, + { + "text": ".9", + "boxes": [ + 18, + 482, + 30, + 494 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 -LibreOfficeCalc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "-口·目·口%自、", + "boxes": [ + 10, + 48, + 390, + 85 + ] + }, + { + "text": "血&", + "boxes": [ + 371, + 53, + 445, + 84 + ] + }, + { + "text": "-、", + "boxes": [ + 444, + 56, + 552, + 79 + ] + }, + { + "text": "v田·I·子DI田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "aaaa旦目日日%00三三图目", + "boxes": [ + 230, + 93, + 1206, + 123 + ] + }, + { + "text": "浆×", + "boxes": [ + 152, + 129, + 242, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "12", + "boxes": [ + 16, + 361, + 30, + 377 + ] + }, + { + "text": "13", + "boxes": [ + 16, + 377, + 30, + 397 + ] + }, + { + "text": "14", + "boxes": [ + 16, + 393, + 30, + 409 + ] + }, + { + "text": "17", + "boxes": [ + 16, + 449, + 30, + 465 + ] + }, + { + "text": "21", + "boxes": [ + 16, + 512, + 30, + 528 + ] + }, + { + "text": "4", + "boxes": [ + 18, + 570, + 30, + 582 + ] + }, + { + "text": "25", + "boxes": [ + 16, + 584, + 30, + 602 + ] + }, + { + "text": "8", + "boxes": [ + 18, + 806, + 32, + 822 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 30, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→→+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ] + }, + { + "frame_id": "70_2_5", + "episode_id": 70, + "env_id": 2, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.304304122924805, + "advantages": -0.23415254056453705, + "generated_text": "{\n \"intent\": \"To continue exploring various products on Apple's website by interacting with different areas or navigating through links.\",\n \"action\": \"DoubleClick(700, 150)\"\n}", + "action_string": "DoubleClick(700, 150)", + "intent_string": "To continue exploring various products on Apple's website by interacting with different areas or navigating through links.", + "before_screen_ocr_results": [ + { + "text": "Appl", + "boxes": [ + 316, + 34, + 346, + 48 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1600, + 66, + 1662, + 91 + ] + }, + { + "text": "亦口", + "boxes": [ + 1790, + 69, + 1848, + 87 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1082, + 111, + 1158, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1298, + 111, + 1340, + 131 + ] + }, + { + "text": "Q", + "boxes": [ + 1370, + 109, + 1396, + 129 + ] + }, + { + "text": "B", + "boxes": [ + 456, + 157, + 522, + 218 + ] + }, + { + "text": "D", + "boxes": [ + 560, + 157, + 628, + 218 + ] + }, + { + "text": "0", + "boxes": [ + 670, + 163, + 714, + 214 + ] + }, + { + "text": "C", + "boxes": [ + 754, + 159, + 820, + 218 + ] + }, + { + "text": "18", + "boxes": [ + 1106, + 155, + 1170, + 222 + ] + }, + { + "text": "00", + "boxes": [ + 846, + 165, + 920, + 222 + ] + }, + { + "text": "0", + "boxes": [ + 1380, + 157, + 1462, + 220 + ] + }, + { + "text": "·", + "boxes": [ + 1026, + 179, + 1066, + 218 + ] + }, + { + "text": "口", + "boxes": [ + 1210, + 173, + 1276, + 220 + ] + }, + { + "text": "OS", + "boxes": [ + 1306, + 171, + 1354, + 218 + ] + }, + { + "text": "9?", + "boxes": [ + 944, + 181, + 992, + 218 + ] + }, + { + "text": "iphone 16 Pro ", + "boxes": [ + 446, + 216, + 532, + 248 + ] + }, + { + "text": "ipPhone 16 ", + "boxes": [ + 564, + 218, + 626, + 244 + ] + }, + { + "text": "iphone 16e", + "boxes": [ + 660, + 218, + 726, + 244 + ] + }, + { + "text": "iphone 15 ", + "boxes": [ + 756, + 216, + 822, + 248 + ] + }, + { + "text": "Compare", + "boxes": [ + 850, + 216, + 910, + 248 + ] + }, + { + "text": "AirPods", + "boxes": [ + 944, + 218, + 992, + 246 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1026, + 218, + 1066, + 246 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1102, + 218, + 1172, + 244 + ] + }, + { + "text": "Apple Card", + "boxes": [ + 1208, + 216, + 1276, + 248 + ] + }, + { + "text": "ios 18", + "boxes": [ + 1308, + 218, + 1350, + 244 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 1386, + 218, + 1458, + 244 + ] + }, + { + "text": "iPhone", + "boxes": [ + 101, + 388, + 367, + 486 + ] + }, + { + "text": "Designed to be loved.", + "boxes": [ + 1520, + 433, + 1800, + 474 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1600, + 66, + 1662, + 91 + ] + }, + { + "text": "亦口", + "boxes": [ + 1790, + 69, + 1846, + 87 + ] + }, + { + "text": "Entertainmen", + "boxes": [ + 1082, + 111, + 1156, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1372, + 111, + 1390, + 127 + ] + }, + { + "text": "D", + "boxes": [ + 456, + 159, + 522, + 218 + ] + }, + { + "text": "口", + "boxes": [ + 560, + 155, + 626, + 218 + ] + }, + { + "text": "#", + "boxes": [ + 674, + 159, + 712, + 204 + ] + }, + { + "text": "D", + "boxes": [ + 756, + 157, + 820, + 218 + ] + }, + { + "text": "18", + "boxes": [ + 1106, + 155, + 1170, + 222 + ] + }, + { + "text": "00", + "boxes": [ + 846, + 165, + 918, + 222 + ] + }, + { + "text": "80", + "boxes": [ + 1382, + 157, + 1462, + 220 + ] + }, + { + "text": "口", + "boxes": [ + 1210, + 173, + 1276, + 220 + ] + }, + { + "text": "OS", + "boxes": [ + 1306, + 171, + 1354, + 218 + ] + }, + { + "text": "9?", + "boxes": [ + 944, + 181, + 992, + 218 + ] + }, + { + "text": "O", + "boxes": [ + 1019, + 172, + 1070, + 224 + ] + }, + { + "text": "iphone 16 Pro ", + "boxes": [ + 446, + 216, + 532, + 248 + ] + }, + { + "text": "ipPhone 16 ", + "boxes": [ + 564, + 218, + 626, + 244 + ] + }, + { + "text": "iphone 16e", + "boxes": [ + 660, + 218, + 724, + 244 + ] + }, + { + "text": "iphone 15 ", + "boxes": [ + 756, + 216, + 822, + 248 + ] + }, + { + "text": "Compare", + "boxes": [ + 850, + 216, + 910, + 248 + ] + }, + { + "text": "AirPods", + "boxes": [ + 944, + 218, + 992, + 246 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1026, + 218, + 1066, + 246 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1102, + 218, + 1172, + 244 + ] + }, + { + "text": "Apple Card", + "boxes": [ + 1208, + 216, + 1276, + 248 + ] + }, + { + "text": "ios 18", + "boxes": [ + 1308, + 218, + 1350, + 244 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 1386, + 218, + 1458, + 244 + ] + }, + { + "text": "Get $170-$630 in credit t", + "boxes": [ + 592, + 282, + 758, + 306 + ] + }, + { + "text": "iPhone", + "boxes": [ + 101, + 388, + 367, + 486 + ] + }, + { + "text": " Designed to be loved.", + "boxes": [ + 1518, + 433, + 1798, + 474 + ] + }, + { + "text": "1", + "boxes": [ + 1838, + 973, + 1862, + 995 + ] + } + ] + }, + { + "frame_id": "70_3_5", + "episode_id": 70, + "env_id": 3, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.305218696594238, + "advantages": 0.3436126708984375, + "generated_text": "{\n \"intent\": \"The aim here is to explore more products offered by Amazon. Double clicking any product link underatured on the home page will likely lead to detailed product pages, enhancing my exploration.\",\n \"action\": \"DoubleClick(771, 857)\"\n}", + "action_string": "DoubleClick(771, 857)", + "intent_string": "The aim here is to explore more products offered by Amazon. Double clicking any product link underatured on the home page will likely lead to detailed product pages, enhancing my exploration.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "", + "boxes": [ + 1468, + 177, + 1554, + 201 + ] + }, + { + "text": "Coffee", + "boxes": [ + 1358, + 218, + 1394, + 232 + ] + }, + { + "text": "Pots and Pans", + "boxes": [ + 1460, + 216, + 1556, + 234 + ] + }, + { + "text": "Kettles", + "boxes": [ + 1562, + 216, + 1610, + 234 + ] + }, + { + "text": "See more", + "boxes": [ + 244, + 262, + 300, + 280 + ] + }, + { + "text": "Shop gaming ", + "boxes": [ + 982, + 260, + 1066, + 284 + ] + }, + { + "text": "Explore all products in Kitchen", + "boxes": [ + 1354, + 262, + 1532, + 280 + ] + }, + { + "text": "Shop deals in Fashion ", + "boxes": [ + 240, + 339, + 464, + 369 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 608, + 338, + 906, + 367 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 339, + 1220, + 369 + ] + }, + { + "text": "Toys under $25", + "boxes": [ + 1350, + 339, + 1508, + 369 + ] + }, + { + "text": "ee all deals", + "boxes": [ + 252, + 701, + 316, + 719 + ] + }, + { + "text": "See all deals", + "boxes": [ + 988, + 701, + 1054, + 719 + ] + }, + { + "text": " Gift ideas inspired by your shopping history", + "boxes": [ + 236, + 776, + 682, + 814 + ] + }, + { + "text": "CD", + "boxes": [ + 794, + 848, + 834, + 885 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→×白", + "boxes": [ + 6, + 59, + 132, + 96 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 300, + 115, + 414, + 139 + ] + }, + { + "text": "EN", + "boxes": [ + 1394, + 121, + 1442, + 147 + ] + }, + { + "text": "Retums", + "boxes": [ + 1606, + 113, + 1650, + 139 + ] + }, + { + "text": "International Kindle PaperwhiteBuy Now", + "boxes": [ + 754, + 204, + 1072, + 234 + ] + }, + { + "text": " Nintendo SwitchtM with Neon Blue and Neon Red Joy-ConTM ", + "boxes": [ + 640, + 304, + 1298, + 339 + ] + }, + { + "text": "", + "boxes": [ + 586, + 320, + 610, + 343 + ] + }, + { + "text": "Buy used: $279.99", + "boxes": [ + 1394, + 327, + 1518, + 356 + ] + }, + { + "text": "Platform : Nintendo Switch ", + "boxes": [ + 644, + 351, + 828, + 381 + ] + }, + { + "text": "4.7★★★★☆", + "boxes": [ + 642, + 369, + 776, + 403 + ] + }, + { + "text": "8,882 ratings", + "boxes": [ + 784, + 375, + 876, + 399 + ] + }, + { + "text": "#1 Best Seller", + "boxes": [ + 646, + 401, + 732, + 425 + ] + }, + { + "text": "in Nintendo Switch Consoles", + "boxes": [ + 734, + 405, + 890, + 423 + ] + }, + { + "text": "June 5", + "boxes": [ + 1396, + 415, + 1442, + 435 + ] + }, + { + "text": "3K+ bought in past 1", + "boxes": [ + 646, + 435, + 756, + 453 + ] + }, + { + "text": "$27999", + "boxes": [ + 642, + 474, + 724, + 506 + ] + }, + { + "text": "Used: Very Good | Details", + "boxes": [ + 1394, + 474, + 1558, + 498 + ] + }, + { + "text": "SWITCH.", + "boxes": [ + 352, + 498, + 410, + 532 + ] + }, + { + "text": "$192.10 Shipping &", + "boxes": [ + 646, + 512, + 774, + 536 + ] + }, + { + "text": "Store", + "boxes": [ + 1398, + 520, + 1432, + 534 + ] + }, + { + "text": "Up to 15% off", + "boxes": [ + 648, + 538, + 750, + 564 + ] + }, + { + "text": "if you qualify", + "boxes": [ + 746, + 542, + 834, + 560 + ] + }, + { + "text": "Fulfilled by Amazon", + "boxes": [ + 1396, + 534, + 1522, + 558 + ] + }, + { + "text": "！", + "boxes": [ + 418, + 592, + 438, + 608 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1464, + 588, + 1536, + 613 + ] + }, + { + "text": "Color:Black", + "boxes": [ + 648, + 613, + 724, + 631 + ] + }, + { + "text": "About this item", + "boxes": [ + 642, + 657, + 796, + 687 + ] + }, + { + "text": "Add to List", + "boxes": [ + 1399, + 670, + 1479, + 699 + ] + }, + { + "text": " Other sellers on Amazon", + "boxes": [ + 1384, + 742, + 1580, + 774 + ] + }, + { + "text": "New & Used (9) from $17200", + "boxes": [ + 1392, + 792, + 1590, + 820 + ] + }, + { + "text": "曰", + "boxes": [ + 640, + 812, + 676, + 844 + ] + }, + { + "text": "This product islavailable as Renewed.", + "boxes": [ + 642, + 846, + 964, + 876 + ] + }, + { + "text": "$255.99 + $7.41 shipping", + "boxes": [ + 726, + 891, + 890, + 921 + ] + }, + { + "text": "★★★★☆（449)", + "boxes": [ + 726, + 911, + 848, + 941 + ] + }, + { + "text": "Works and looks like new and backed by th", + "boxes": [ + 726, + 935, + 998, + 959 + ] + }, + { + "text": "Unwell HydrationfromAlexCooper", + "boxes": [ + 740, + 991, + 980, + 1017 + ] + }, + { + "text": "Hydrate & focus with every sip Shop nov", + "boxes": [ + 740, + 1011, + 998, + 1040 + ] + } + ] + }, + { + "frame_id": "70_4_5", + "episode_id": 70, + "env_id": 4, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.468633651733398, + "advantages": -0.13929539918899536, + "generated_text": "{\n \"intent\": \"The current page provides information about Firefox browser downloads and settings. To continue exploring various functionalities or settings provided by Isaiah Firefox, scrolling might be necessary to view more options.\",\n \"action\": \"ScrollUp(960, 540)\"\n}", + "action_string": "ScrollUp(960, 540)", + "intent_string": "The current page provides information about Firefox browser downloads and settings. To continue exploring various functionalities or settings provided by Isaiah Firefox, scrolling might be necessary to view more options.", + "before_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozilla I", + "boxes": [ + 251, + 103, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 111, + 1048, + 143 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "70_5_5", + "episode_id": 70, + "env_id": 5, + "step": 5, + "format_rewards": 1.0, + "rewards": 11.041315078735352, + "advantages": 0.7685149908065796, + "generated_text": "{\n \"intent\": \"Choosing the 'Watch' option will likely expand the section related to watches, which might contain more information about Apple watch models and possibly other relevant features.\",\n \"action\": \"DoubleClick(786, 119)\"\n}", + "action_string": "DoubleClick(786, 119)", + "intent_string": "Choosing the 'Watch' option will likely expand the section related to watches, which might contain more information about Apple watch models and possibly other relevant features.", + "before_screen_ocr_results": [ + { + "text": "目☆", + "boxes": [ + 1538, + 62, + 1662, + 91 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 944, + 127 + ] + }, + { + "text": "TV & Hom", + "boxes": [ + 986, + 111, + 1038, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1370, + 109, + 1396, + 129 + ] + }, + { + "text": "?", + "boxes": [ + 1416, + 105, + 1446, + 135 + ] + }, + { + "text": "ness+ free when you buy any Apple Watch.1 Shop Apple Watch", + "boxes": [ + 812, + 161, + 1214, + 179 + ] + }, + { + "text": "WATCH", + "boxes": [ + 867, + 260, + 1033, + 314 + ] + }, + { + "text": "SERIES10", + "boxes": [ + 912, + 308, + 1000, + 334 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 646, + 327, + 1252, + 425 + ] + }, + { + "text": "From $399 or $33.25/mo.for 12 mo.*", + "boxes": [ + 804, + 427, + 1106, + 457 + ] + }, + { + "text": "Buy", + "boxes": [ + 934, + 486, + 974, + 512 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 204, + 34, + 244, + 50 + ] + }, + { + "text": "←", + "boxes": [ + 12, + 68, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 66, + 128, + 91 + ] + }, + { + "text": "", + "boxes": [ + 370, + 159, + 414, + 212 + ] + }, + { + "text": "0", + "boxes": [ + 468, + 155, + 520, + 214 + ] + }, + { + "text": "0", + "boxes": [ + 574, + 163, + 618, + 206 + ] + }, + { + "text": "O", + "boxes": [ + 674, + 161, + 718, + 210 + ] + }, + { + "text": "6", + "boxes": [ + 770, + 161, + 828, + 214 + ] + }, + { + "text": "0", + "boxes": [ + 864, + 159, + 940, + 214 + ] + }, + { + "text": "00", + "boxes": [ + 964, + 163, + 1034, + 216 + ] + }, + { + "text": "", + "boxes": [ + 1050, + 167, + 1078, + 210 + ] + }, + { + "text": "?", + "boxes": [ + 1208, + 169, + 1246, + 210 + ] + }, + { + "text": "00", + "boxes": [ + 1476, + 161, + 1562, + 214 + ] + }, + { + "text": "9?", + "boxes": [ + 1118, + 177, + 1166, + 214 + ] + }, + { + "text": "?", + "boxes": [ + 1302, + 171, + 1358, + 214 + ] + }, + { + "text": "O", + "boxes": [ + 1398, + 171, + 1454, + 212 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 458, + 214, + 532, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 556, + 214, + 636, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 660, + 214, + 740, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 764, + 214, + 838, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 864, + 212, + 940, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 964, + 214, + 1022, + 240 + ] + }, + { + "text": "Bands", + "boxes": [ + 1048, + 216, + 1084, + 236 + ] + }, + { + "text": "AirPods", + "boxes": [ + 1122, + 218, + 1162, + 234 + ] + }, + { + "text": "atchos 11", + "boxes": [ + 1406, + 216, + 1462, + 234 + ] + }, + { + "text": "Shop Watch", + "boxes": [ + 1490, + 216, + 1554, + 234 + ] + }, + { + "text": "Ultra 2", + "boxes": [ + 468, + 232, + 516, + 252 + ] + }, + { + "text": "Nike", + "boxes": [ + 680, + 232, + 710, + 252 + ] + }, + { + "text": "Hermes", + "boxes": [ + 774, + 232, + 824, + 250 + ] + }, + { + "text": "Studio", + "boxes": [ + 878, + 232, + 924, + 252 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 108, + 381, + 574, + 489 + ] + }, + { + "text": "The ultimate device", + "boxes": [ + 1536, + 393, + 1796, + 439 + ] + }, + { + "text": "for a healthy life.", + "boxes": [ + 1538, + 433, + 1752, + 469 + ] + }, + { + "text": "e Watch", + "boxes": [ + 250, + 1050, + 302, + 1070 + ] + } + ] + }, + { + "frame_id": "70_6_5", + "episode_id": 70, + "env_id": 6, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.410236358642578, + "advantages": -0.17300449311733246, + "generated_text": "{\n \"intent\": \"To explore the website by scrolling down to see more content and save browsing steps for other actions.\",\n \"action\": \"ScrollDown(960, 539)\"\n}", + "action_string": "ScrollDown(960, 539)", + "intent_string": "To explore the website by scrolling down to see more content and save browsing steps for other actions.", + "before_screen_ocr_results": [ + { + "text": "BuyiPhone16", + "boxes": [ + 674, + 4, + 804, + 22 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1128, + 2, + 1246, + 26 + ] + }, + { + "text": "→白", + "boxes": [ + 8, + 60, + 128, + 91 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1882, + 91 + ] + }, + { + "text": "Store", + "boxes": [ + 512, + 113, + 546, + 127 + ] + }, + { + "text": "Mac", + "boxes": [ + 572, + 113, + 602, + 127 + ] + }, + { + "text": "iPad", + "boxes": [ + 636, + 113, + 660, + 127 + ] + }, + { + "text": "iPhone", + "boxes": [ + 692, + 113, + 732, + 129 + ] + }, + { + "text": "Watch", + "boxes": [ + 764, + 113, + 804, + 129 + ] + }, + { + "text": "Vision", + "boxes": [ + 834, + 111, + 874, + 131 + ] + }, + { + "text": "AirPods", + "boxes": [ + 904, + 111, + 948, + 131 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 982, + 111, + 1048, + 131 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1078, + 109, + 1164, + 133 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 111, + 1264, + 133 + ] + }, + { + "text": "Support", + "boxes": [ + 1294, + 111, + 1344, + 131 + ] + }, + { + "text": "Q", + "boxes": [ + 1368, + 109, + 1396, + 131 + ] + }, + { + "text": "6", + "boxes": [ + 1418, + 111, + 1442, + 135 + ] + }, + { + "text": "Carrer Deals at Apple", + "boxes": [ + 496, + 170, + 654, + 207 + ] + }, + { + "text": "", + "boxes": [ + 672, + 187, + 706, + 214 + ] + }, + { + "text": "S", + "boxes": [ + 864, + 191, + 892, + 210 + ] + }, + { + "text": "T", + "boxes": [ + 1058, + 189, + 1084, + 212 + ] + }, + { + "text": "p to $1000 afte", + "boxes": [ + 1324, + 183, + 1410, + 201 + ] + }, + { + "text": "See all deals ①", + "boxes": [ + 500, + 197, + 606, + 220 + ] + }, + { + "text": "V", + "boxes": [ + 1256, + 193, + 1274, + 206 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "①| https://www.apple.com/shop/buy-iphone/iphone-16-pro", + "boxes": [ + 250, + 66, + 692, + 89 + ] + }, + { + "text": "目☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "I四三", + "boxes": [ + 1782, + 64, + 1912, + 93 + ] + }, + { + "text": "2", + "boxes": [ + 102, + 1048, + 152, + 1066 + ] + }, + { + "text": " Buy iPhone 1..", + "boxes": [ + 142, + 1044, + 318, + 1074 + ] + } + ] + }, + { + "frame_id": "70_7_5", + "episode_id": 70, + "env_id": 7, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.406832695007324, + "advantages": -0.17496921122074127, + "generated_text": "{\n \"intent\": \"To continue browsing through the webpage and explore further information about Apple products, clicking on different sections or links would be beneficial.\",\n \"action\": \"DoubleClick(471, 85)\"\n}", + "action_string": "DoubleClick(471, 85)", + "intent_string": "To continue browsing through the webpage and explore further information about Apple products, clicking on different sections or links would be beneficial.", + "before_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 320, + 40, + 347, + 66 + ] + }, + { + "text": "ppl", + "boxes": [ + 876, + 46, + 906, + 60 + ] + }, + { + "text": "□", + "boxes": [ + 1536, + 40, + 1598, + 66 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "Apple", + "boxes": [ + 332, + 71, + 392, + 91 + ] + }, + { + "text": "×+", + "boxes": [ + 520, + 69, + 570, + 89 + ] + }, + { + "text": "↑→", + "boxes": [ + 330, + 101, + 452, + 131 + ] + }, + { + "text": "① https:/www.apple.com", + "boxes": [ + 548, + 99, + 768, + 133 + ] + }, + { + "text": "·☆", + "boxes": [ + 1286, + 101, + 1368, + 133 + ] + }, + { + "text": "三四?", + "boxes": [ + 1466, + 103, + 1568, + 129 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1196, + 151, + 1262, + 169 + ] + }, + { + "text": "Support", + "boxes": [ + 1296, + 151, + 1346, + 169 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 149, + 1394, + 169 + ] + }, + { + "text": "?", + "boxes": [ + 1420, + 148, + 1444, + 173 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 20, + 502, + 90, + 528 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "WATCH", + "boxes": [ + 872, + 659, + 1070, + 715 + ] + }, + { + "text": "國", + "boxes": [ + 38, + 681, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 904, + 715, + 1032, + 746 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 850, + 752, + 1064, + 788 + ] + }, + { + "text": "D", + "boxes": [ + 34, + 778, + 80, + 828 + ] + }, + { + "text": "Learn more", + "boxes": [ + 862, + 814, + 958, + 840 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 812, + 1054, + 838 + ] + }, + { + "text": "Draw", + "boxes": [ + 34, + 832, + 88, + 862 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 940, + 42, + 1054, + 60 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "Apple", + "boxes": [ + 334, + 71, + 390, + 91 + ] + }, + { + "text": "x+", + "boxes": [ + 522, + 69, + 570, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 352, + 107, + 396, + 125 + ] + }, + { + "text": "C", + "boxes": [ + 390, + 105, + 422, + 127 + ] + }, + { + "text": "仓", + "boxes": [ + 418, + 107, + 448, + 127 + ] + }, + { + "text": "① https:/www.apple.com", + "boxes": [ + 548, + 99, + 768, + 133 + ] + }, + { + "text": "·☆", + "boxes": [ + 1286, + 101, + 1368, + 133 + ] + }, + { + "text": "三四@", + "boxes": [ + 1464, + 99, + 1568, + 131 + ] + }, + { + "text": "iPhone", + "boxes": [ + 696, + 153, + 730, + 167 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1196, + 151, + 1262, + 169 + ] + }, + { + "text": "Support", + "boxes": [ + 1296, + 151, + 1346, + 169 + ] + }, + { + "text": "Q", + "boxes": [ + 1364, + 149, + 1394, + 169 + ] + }, + { + "text": "?", + "boxes": [ + 1420, + 148, + 1444, + 173 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 20, + 502, + 90, + 528 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "WATCH", + "boxes": [ + 870, + 659, + 1070, + 715 + ] + }, + { + "text": "國", + "boxes": [ + 38, + 681, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 904, + 715, + 1032, + 746 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 850, + 752, + 1064, + 788 + ] + }, + { + "text": "", + "boxes": [ + 30, + 771, + 84, + 832 + ] + }, + { + "text": "Learn more", + "boxes": [ + 862, + 814, + 958, + 840 + ] + }, + { + "text": "Buy", + "boxes": [ + 1008, + 812, + 1054, + 838 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + } + ] + }, + { + "frame_id": "70_0_6", + "episode_id": 70, + "env_id": 0, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.823436737060547, + "advantages": 0.6427474617958069, + "generated_text": "{\n \"intent\": \"Exploring more content and navigating through various sections of the Bing search page.\",\n \"action\": \"DoubleClick(450, 376)\"\n}", + "action_string": "DoubleClick(450, 376)", + "intent_string": "Exploring more content and navigating through various sections of the Bing search page.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 40, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 28, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 252, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 66, + 1662, + 91 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1870, + 89 + ] + }, + { + "text": "Microsoft Bing", + "boxes": [ + 24, + 109, + 184, + 151 + ] + }, + { + "text": "Translate", + "boxes": [ + 406, + 121, + 454, + 141 + ] + }, + { + "text": "ign", + "boxes": [ + 1646, + 125, + 1666, + 141 + ] + }, + { + "text": "国内版国际版", + "boxes": [ + 482, + 145, + 640, + 187 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 468, + 192, + 616, + 235 + ] + }, + { + "text": "9", + "boxes": [ + 1336, + 200, + 1360, + 226 + ] + }, + { + "text": "", + "boxes": [ + 1300, + 286, + 1378, + 326 + ] + }, + { + "text": "魔法与神移学（一）", + "boxes": [ + 1302, + 316, + 1418, + 347 + ] + }, + { + "text": "ITBEAR科技资讯·15小时", + "boxes": [ + 200, + 379, + 384, + 409 + ] + }, + { + "text": "生活研究所·3周", + "boxes": [ + 512, + 379, + 644, + 409 + ] + }, + { + "text": "证券日报·18小时", + "boxes": [ + 830, + 379, + 966, + 409 + ] + }, + { + "text": "机核·2周", + "boxes": [ + 1138, + 381, + 1232, + 407 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，", + "boxes": [ + 200, + 401, + 450, + 437 + ] + }, + { + "text": "探秘荒野之魅", + "boxes": [ + 509, + 397, + 645, + 439 + ] + }, + { + "text": "从“底层重构”到“文明融合”", + "boxes": [ + 824, + 401, + 1090, + 437 + ] + }, + { + "text": "《女神异闻录3》中的魔法与", + "boxes": [ + 1140, + 399, + 1404, + 441 + ] + }, + { + "text": "能否引领新能源市场新潮流？", + "boxes": [ + 202, + 431, + 466, + 461 + ] + }, + { + "text": "光启技术加速推进超材料产业", + "boxes": [ + 824, + 427, + 1098, + 463 + ] + }, + { + "text": "神秘学（一)", + "boxes": [ + 1136, + 426, + 1256, + 463 + ] + }, + { + "text": "生态升级", + "boxes": [ + 824, + 453, + 918, + 490 + ] + }, + { + "text": "1", + "boxes": [ + 206, + 488, + 236, + 508 + ] + }, + { + "text": "", + "boxes": [ + 514, + 486, + 582, + 512 + ] + }, + { + "text": "", + "boxes": [ + 822, + 484, + 880, + 516 + ] + }, + { + "text": "44", + "boxes": [ + 1138, + 486, + 1206, + 512 + ] + }, + { + "text": "X.", + "boxes": [ + 1354, + 542, + 1416, + 568 + ] + }, + { + "text": "交通事件", + "boxes": [ + 1448, + 544, + 1538, + 570 + ] + }, + { + "text": "尚未升灵", + "boxes": [ + 650, + 574, + 712, + 594 + ] + }, + { + "text": "硅谷大街 禁止通行 原因：路面施工", + "boxes": [ + 1448, + 578, + 1710, + 607 + ] + }, + { + "text": "理次数：3/3", + "boxes": [ + 626, + 606, + 680, + 623 + ] + }, + { + "text": "手容", + "boxes": [ + 544, + 637, + 588, + 665 + ] + }, + { + "text": "已绑定", + "boxes": [ + 532, + 667, + 582, + 693 + ] + }, + { + "text": "网上车市·1周", + "boxes": [ + 200, + 695, + 320, + 725 + ] + }, + { + "text": "爱游戏天地·13小时", + "boxes": [ + 516, + 695, + 650, + 725 + ] + }, + { + "text": "L2.9级智能驾驶可还行？天", + "boxes": [ + 202, + 721, + 462, + 750 + ] + }, + { + "text": "天龙网游八卦：15个材料出俩", + "boxes": [ + 512, + 717, + 794, + 752 + ] + }, + { + "text": "江湖上流传着一", + "boxes": [ + 820, + 719, + 1032, + 760 + ] + }, + { + "text": "句话，能被拴住的猫都不是善", + "boxes": [ + 1028, + 723, + 1414, + 758 + ] + }, + { + "text": "工05表现不差，无奈销量太拉", + "boxes": [ + 202, + 746, + 478, + 776 + ] + }, + { + "text": "极品装备，这是脚本开挂？", + "boxes": [ + 512, + 744, + 758, + 774 + ] + }, + { + "text": "茬！", + "boxes": [ + 824, + 758, + 888, + 792 + ] + }, + { + "text": "膀", + "boxes": [ + 204, + 776, + 228, + 798 + ] + }, + { + "text": "林大学校区南4", + "boxes": [ + 1516, + 774, + 1720, + 808 + ] + }, + { + "text": "3", + "boxes": [ + 204, + 804, + 236, + 824 + ] + }, + { + "text": "1", + "boxes": [ + 516, + 802, + 576, + 828 + ] + }, + { + "text": "在地图上查看", + "boxes": [ + 1548, + 810, + 1628, + 834 + ] + }, + { + "text": "美国职业篮球联赛", + "boxes": [ + 1450, + 858, + 1602, + 887 + ] + }, + { + "text": "在你的区域中很受欢迎", + "boxes": [ + 1450, + 887, + 1580, + 924 + ] + }, + { + "text": "完赛·5月17日", + "boxes": [ + 1554, + 931, + 1624, + 949 + ] + }, + { + "text": "88", + "boxes": [ + 1126, + 947, + 1194, + 979 + ] + }, + { + "text": "31-119", + "boxes": [ + 1536, + 949, + 1636, + 975 + ] + }, + { + "text": "下午7:30", + "boxes": [ + 1542, + 999, + 1640, + 1030 + ] + }, + { + "text": "重游戏天地·1周", + "boxes": [ + 1138, + 1011, + 1256, + 1042 + ] + }, + { + "text": "arch - Micr.", + "boxes": [ + 232, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "osoft Bing", + "boxes": [ + 118, + 32, + 188, + 50 + ] + }, + { + "text": "哈弗枭龙MAX携H", + "boxes": [ + 258, + 32, + 380, + 50 + ] + }, + { + "text": "C", + "boxes": [ + 56, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1664, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 564, + 117, + 638, + 149 + ] + }, + { + "text": "msn", + "boxes": [ + 58, + 129, + 104, + 149 + ] + }, + { + "text": "中", + "boxes": [ + 1764, + 121, + 1790, + 145 + ] + }, + { + "text": "登录", + "boxes": [ + 1830, + 123, + 1858, + 145 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "资讯 娱乐　 时尚 教育", + "boxes": [ + 344, + 175, + 542, + 204 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1606, + 175, + 1726, + 206 + ] + }, + { + "text": "精确分析", + "boxes": [ + 1444, + 250, + 1588, + 302 + ] + }, + { + "text": "ITBEAR科技资讯", + "boxes": [ + 380, + 260, + 554, + 308 + ] + }, + { + "text": "专", + "boxes": [ + 1426, + 260, + 1442, + 274 + ] + }, + { + "text": "绘制图表", + "boxes": [ + 1446, + 292, + 1586, + 336 + ] + }, + { + "text": "力您高效完成数据分析", + "boxes": [ + 1456, + 334, + 1576, + 351 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，能否引领新能源市场新潮流？", + "boxes": [ + 366, + 349, + 1178, + 391 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1456, + 351, + 1506, + 369 + ] + }, + { + "text": "#", + "boxes": [ + 180, + 385, + 226, + 431 + ] + }, + { + "text": "k王者荣耀的报道·15小时·① 1 分钟读取", + "boxes": [ + 364, + 413, + 708, + 449 + ] + }, + { + "text": "让你成为 Excel 达人，工作学习效", + "boxes": [ + 1422, + 411, + 1702, + 447 + ] + }, + { + "text": "率倍增", + "boxes": [ + 1426, + 437, + 1490, + 469 + ] + }, + { + "text": "B", + "boxes": [ + 194, + 455, + 214, + 474 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 469, + 1530, + 492 + ] + }, + { + "text": "在汽车市场风起云涌的今天，哈弗汽车，这个长久以来以\"性价比\"著称的品牌，正积极布局新能源领域，意图在新的", + "boxes": [ + 364, + 504, + 1232, + 534 + ] + }, + { + "text": "赛道上续写辉煌。近日，哈弗汽车两款全新车型——哈弗枭龙与哈弗枭龙MAX正式上市，标志着哈弗正式进军新能源", + "boxes": [ + 366, + 530, + 1240, + 560 + ] + }, + { + "text": "汽车市场的决心。", + "boxes": [ + 364, + 554, + 494, + 588 + ] + }, + { + "text": "访问ITBEAR科技资讯", + "boxes": [ + 1426, + 554, + 1596, + 584 + ] + }, + { + "text": "C", + "boxes": [ + 1674, + 554, + 1698, + 576 + ] + }, + { + "text": "哈弗枭龙MAX与哈弗枭龙，作为哈弗品牌的全新力作，不仅继承了哈弗汽车一贯的高性价比优势，更在技术层面实现", + "boxes": [ + 364, + 600, + 1240, + 629 + ] + }, + { + "text": "新疆多地夜空现神秘飞行物，目击", + "boxes": [ + 1444, + 597, + 1666, + 634 + ] + }, + { + "text": "了突破。哈弗枭龙MAX搭载了长城汽车全新的智能四驱电混技术Hi4，推出了三款车型，价格区间为15.98-17.98万", + "boxes": [ + 366, + 625, + 1214, + 655 + ] + }, + { + "text": "者：亮如月亮，几秒加速消失", + "boxes": [ + 1446, + 621, + 1634, + 651 + ] + }, + { + "text": "元。而哈弗枭龙则采用了智能混动DHT技术，同样推出了三款车型，价格区间则为13.98-15.68万元。两款车型均展现", + "boxes": [ + 364, + 651, + 1240, + 681 + ] + }, + { + "text": "小米YU7即将发布，卢伟冰自曝已", + "boxes": [ + 1446, + 663, + 1664, + 693 + ] + }, + { + "text": "出了哈弗在新能源汽车领域的深厚底蕴和技术实力。", + "boxes": [ + 362, + 675, + 750, + 707 + ] + }, + { + "text": "购车并计划添购新车", + "boxes": [ + 1446, + 685, + 1580, + 715 + ] + }, + { + "text": "长安汽车泰国投产，深蓝CEO：增", + "boxes": [ + 1446, + 729, + 1664, + 758 + ] + }, + { + "text": "程车油耗媲美日系混动，驾驶体...", + "boxes": [ + 1446, + 748, + 1660, + 778 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 832, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 878, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 794, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击\"管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置“改变您的选择。欲了解", + "boxes": [ + 28, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 130, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "合弗枭龙MAX", + "boxes": [ + 224, + 1048, + 306, + 1072 + ] + } + ] + }, + { + "frame_id": "70_1_6", + "episode_id": 70, + "env_id": 1, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.284347534179688, + "advantages": -0.24567222595214844, + "generated_text": "{\n \"intent\": \"To start interacting with the environment by exploring various functions in LibreOffice Calc, the first step is to open a file or create a new one.\",\n \"action\": \"DoubleClick(200, 50)\"\n}", + "action_string": "DoubleClick(200, 50)", + "intent_string": "To start interacting with the environment by exploring various functions in LibreOffice Calc, the first step is to open a file or create a new one.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - LibreOffice Calc", + "boxes": [ + 844, + 4, + 1076, + 22 + ] + }, + { + "text": "画··目·目@自、", + "boxes": [ + 12, + 54, + 390, + 83 + ] + }, + { + "text": "&", + "boxes": [ + 371, + 51, + 445, + 84 + ] + }, + { + "text": "←-、", + "boxes": [ + 444, + 56, + 552, + 79 + ] + }, + { + "text": "v田·I·子DI田", + "boxes": [ + 542, + 50, + 1264, + 85 + ] + }, + { + "text": "aaaa旦目日日%00三三图目", + "boxes": [ + 230, + 93, + 1206, + 123 + ] + }, + { + "text": "▼", + "boxes": [ + 134, + 133, + 152, + 147 + ] + }, + { + "text": "游×化", + "boxes": [ + 152, + 129, + 242, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 240 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "O", + "boxes": [ + 1886, + 262, + 1912, + 314 + ] + }, + { + "text": "城", + "boxes": [ + 1890, + 304, + 1910, + 339 + ] + }, + { + "text": "12", + "boxes": [ + 16, + 361, + 30, + 377 + ] + }, + { + "text": "13", + "boxes": [ + 16, + 377, + 30, + 397 + ] + }, + { + "text": "14", + "boxes": [ + 16, + 393, + 30, + 409 + ] + }, + { + "text": "21", + "boxes": [ + 16, + 512, + 30, + 528 + ] + }, + { + "text": "4", + "boxes": [ + 18, + 570, + 30, + 582 + ] + }, + { + "text": "45", + "boxes": [ + 12, + 925, + 30, + 941 + ] + }, + { + "text": "46", + "boxes": [ + 14, + 941, + 32, + 955 + ] + }, + { + "text": "47", + "boxes": [ + 12, + 957, + 32, + 971 + ] + }, + { + "text": "K<→二+", + "boxes": [ + 12, + 995, + 134, + 1018 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "国", + "boxes": [ + 8, + 4, + 22, + 24 + ] + }, + { + "text": "Untitled 1 - Libreoffice Calc", + "boxes": [ + 848, + 2, + 1070, + 26 + ] + }, + { + "text": "---- α", + "boxes": [ + 8, + 48, + 446, + 87 + ] + }, + { + "text": "G-.", + "boxes": [ + 450, + 56, + 548, + 79 + ] + }, + { + "text": "里·听个", + "boxes": [ + 544, + 54, + 848, + 83 + ] + }, + { + "text": "M-EE-B", + "boxes": [ + 850, + 48, + 1264, + 87 + ] + }, + { + "text": "10", + "boxes": [ + 156, + 97, + 186, + 117 + ] + }, + { + "text": "aaaa-==", + "boxes": [ + 238, + 93, + 520, + 123 + ] + }, + { + "text": "", + "boxes": [ + 550, + 101, + 574, + 115 + ] + }, + { + "text": "□B8", + "boxes": [ + 616, + 95, + 720, + 119 + ] + }, + { + "text": "-% 00 ", + "boxes": [ + 724, + 93, + 874, + 123 + ] + }, + { + "text": "十三田二口目", + "boxes": [ + 878, + 93, + 1198, + 123 + ] + }, + { + "text": "紫×", + "boxes": [ + 158, + 129, + 238, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Print", + "boxes": [ + 938, + 262, + 990, + 288 + ] + }, + { + "text": "□□", + "boxes": [ + 1288, + 262, + 1350, + 288 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 266, + 1914, + 314 + ] + }, + { + "text": "8.26 in (A4)", + "boxes": [ + 724, + 290, + 808, + 316 + ] + }, + { + "text": " General ", + "boxes": [ + 950, + 296, + 1014, + 322 + ] + }, + { + "text": "Libreoffice Calc", + "boxes": [ + 1010, + 293, + 1128, + 322 + ] + }, + { + "text": "成", + "boxes": [ + 1890, + 312, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "Printer", + "boxes": [ + 948, + 324, + 1010, + 349 + ] + }, + { + "text": " Generic Printer", + "boxes": [ + 964, + 351, + 1074, + 375 + ] + }, + { + "text": "Status: Default printer", + "boxes": [ + 960, + 385, + 1108, + 409 + ] + }, + { + "text": "O All Pages", + "boxes": [ + 964, + 449, + 1044, + 472 + ] + }, + { + "text": ".8", + "boxes": [ + 18, + 467, + 30, + 478 + ] + }, + { + "text": "O Pages:", + "boxes": [ + 968, + 478, + 1038, + 500 + ] + }, + { + "text": "O Even pages", + "boxes": [ + 965, + 502, + 1061, + 533 + ] + }, + { + "text": "O odd pages", + "boxes": [ + 965, + 528, + 1057, + 558 + ] + }, + { + "text": "more", + "boxes": [ + 968, + 560, + 1016, + 580 + ] + }, + { + "text": "Page Layout", + "boxes": [ + 952, + 588, + 1044, + 611 + ] + }, + { + "text": "Paper size:", + "boxes": [ + 966, + 617, + 1036, + 635 + ] + }, + { + "text": "A4 8.26in x 11.69in", + "boxes": [ + 1048, + 613, + 1182, + 637 + ] + }, + { + "text": "Automatic", + "boxes": [ + 1051, + 647, + 1125, + 670 + ] + }, + { + "text": "Preview", + "boxes": [ + 592, + 754, + 658, + 772 + ] + }, + { + "text": "?", + "boxes": [ + 744, + 756, + 770, + 770 + ] + }, + { + "text": "/10", + "boxes": [ + 818, + 754, + 882, + 774 + ] + }, + { + "text": "?Help", + "boxes": [ + 599, + 789, + 645, + 815 + ] + }, + { + "text": "More Option", + "boxes": [ + 1038, + 796, + 1120, + 814 + ] + }, + { + "text": "OK", + "boxes": [ + 1288, + 796, + 1316, + 810 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 939 + ] + } + ] + }, + { + "frame_id": "70_2_6", + "episode_id": 70, + "env_id": 2, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.22120475769043, + "advantages": 0.29511669278144836, + "generated_text": "{\n \"intent\": \"The current page is related to exploring the Apple website, focusing specifically on the iPhone section. To proceed with our exploration of the Apple website, we should start by accessing the main navigation menu where further exploration options exist.\",\n \"action\": \"DoubleClick(547, 121)\"\n}", + "action_string": "DoubleClick(547, 121)", + "intent_string": "The current page is related to exploring the Apple website, focusing specifically on the iPhone section. To proceed with our exploration of the Apple website, we should start by accessing the main navigation menu where further exploration options exist.", + "before_screen_ocr_results": [ + { + "text": "→", + "boxes": [ + 34, + 68, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1601, + 65, + 1661, + 90 + ] + }, + { + "text": "亦口", + "boxes": [ + 1790, + 69, + 1848, + 87 + ] + }, + { + "text": "a", + "boxes": [ + 1372, + 113, + 1390, + 127 + ] + }, + { + "text": "D", + "boxes": [ + 456, + 159, + 522, + 218 + ] + }, + { + "text": "口", + "boxes": [ + 560, + 155, + 626, + 218 + ] + }, + { + "text": "#", + "boxes": [ + 672, + 153, + 714, + 206 + ] + }, + { + "text": "D", + "boxes": [ + 756, + 157, + 820, + 218 + ] + }, + { + "text": "18", + "boxes": [ + 1106, + 155, + 1168, + 222 + ] + }, + { + "text": "00", + "boxes": [ + 846, + 165, + 920, + 222 + ] + }, + { + "text": "80", + "boxes": [ + 1382, + 157, + 1462, + 220 + ] + }, + { + "text": "9?", + "boxes": [ + 944, + 181, + 992, + 218 + ] + }, + { + "text": "口", + "boxes": [ + 1210, + 173, + 1276, + 220 + ] + }, + { + "text": "·", + "boxes": [ + 1019, + 174, + 1068, + 221 + ] + }, + { + "text": "Los", + "boxes": [ + 1308, + 175, + 1352, + 218 + ] + }, + { + "text": "iphone 16 Pro ", + "boxes": [ + 446, + 216, + 532, + 248 + ] + }, + { + "text": "iphone 16 ", + "boxes": [ + 562, + 216, + 628, + 248 + ] + }, + { + "text": "iphone 16e", + "boxes": [ + 658, + 216, + 726, + 248 + ] + }, + { + "text": "iphone 15 ", + "boxes": [ + 756, + 216, + 822, + 248 + ] + }, + { + "text": "Compare", + "boxes": [ + 850, + 216, + 910, + 248 + ] + }, + { + "text": "AirPods", + "boxes": [ + 941, + 213, + 993, + 247 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1023, + 215, + 1067, + 248 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1102, + 218, + 1172, + 244 + ] + }, + { + "text": "Apple Card", + "boxes": [ + 1208, + 216, + 1276, + 248 + ] + }, + { + "text": "ios 18", + "boxes": [ + 1308, + 218, + 1350, + 244 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 1386, + 218, + 1458, + 244 + ] + }, + { + "text": "iPhone", + "boxes": [ + 100, + 385, + 368, + 490 + ] + }, + { + "text": "Designed to be loved.", + "boxes": [ + 1520, + 433, + 1800, + 474 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 132, + 220, + 768, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1632, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "?", + "boxes": [ + 1592, + 294, + 1614, + 318 + ] + }, + { + "text": "/isit anAppleStore", + "boxes": [ + 1632, + 290, + 1756, + 308 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 378, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 138, + 641, + 720, + 671 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1848, + 752, + 1912, + 786 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1426, + 822, + 1582, + 852 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 826, + 1044 + ] + } + ] + }, + { + "frame_id": "70_3_6", + "episode_id": 70, + "env_id": 3, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.964276313781738, + "advantages": 0.14680802822113037, + "generated_text": "{\n \"intent\": \"I will scroll down to see more details about the product, such as customer reviews or additional specifications.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "I will scroll down to see more details about the product, such as customer reviews or additional specifications.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1872, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1876, + 131 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 304, + 115, + 412, + 139 + ] + }, + { + "text": "EN", + "boxes": [ + 1396, + 121, + 1448, + 147 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1804, + 125, + 1884, + 151 + ] + }, + { + "text": "International Kindle Paperwhite Buy Now,", + "boxes": [ + 752, + 204, + 1072, + 234 + ] + }, + { + "text": " Nintendo SwitchtM with Neon Blue and Neon Red Joy-ConTM ", + "boxes": [ + 640, + 304, + 1298, + 339 + ] + }, + { + "text": "V", + "boxes": [ + 580, + 314, + 614, + 347 + ] + }, + { + "text": "Buy used: $279.99", + "boxes": [ + 1396, + 330, + 1518, + 353 + ] + }, + { + "text": "Platform : Nintendo Switch ", + "boxes": [ + 644, + 351, + 828, + 381 + ] + }, + { + "text": "4.7★★★★☆", + "boxes": [ + 644, + 369, + 776, + 403 + ] + }, + { + "text": "8,882 ratings", + "boxes": [ + 786, + 375, + 876, + 399 + ] + }, + { + "text": "#1 Best Seller", + "boxes": [ + 646, + 401, + 730, + 425 + ] + }, + { + "text": "June 5", + "boxes": [ + 1396, + 415, + 1440, + 435 + ] + }, + { + "text": "3K+ bought in past 1", + "boxes": [ + 646, + 435, + 756, + 453 + ] + }, + { + "text": "Deliver t", + "boxes": [ + 1398, + 445, + 1460, + 463 + ] + }, + { + "text": "$27999", + "boxes": [ + 642, + 474, + 724, + 506 + ] + }, + { + "text": "Used: Very Good | Detail:", + "boxes": [ + 1394, + 474, + 1556, + 498 + ] + }, + { + "text": "SWITCH.", + "boxes": [ + 350, + 498, + 410, + 532 + ] + }, + { + "text": "$192.10 Shipping &", + "boxes": [ + 646, + 512, + 774, + 536 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1810, + 506, + 1864, + 526 + ] + }, + { + "text": "Store", + "boxes": [ + 1398, + 520, + 1432, + 534 + ] + }, + { + "text": "Up to 15% off", + "boxes": [ + 648, + 538, + 750, + 564 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1794, + 548, + 1862, + 574 + ] + }, + { + "text": "Style: Console", + "boxes": [ + 644, + 584, + 740, + 608 + ] + }, + { + "text": "Add to Cart", + "boxes": [ + 1464, + 588, + 1534, + 613 + ] + }, + { + "text": "Color:Black", + "boxes": [ + 646, + 611, + 726, + 635 + ] + }, + { + "text": "About this item", + "boxes": [ + 642, + 657, + 796, + 687 + ] + }, + { + "text": "Add to List", + "boxes": [ + 1399, + 670, + 1479, + 699 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 719, + 1862, + 739 + ] + }, + { + "text": "Other sellers on Amazon", + "boxes": [ + 1388, + 746, + 1576, + 770 + ] + }, + { + "text": "立", + "boxes": [ + 1794, + 768, + 1820, + 788 + ] + }, + { + "text": "1", + "boxes": [ + 1818, + 768, + 1868, + 786 + ] + }, + { + "text": "New & Used (9) from$17200", + "boxes": [ + 1392, + 792, + 1590, + 820 + ] + }, + { + "text": "This product islavailable as Renewed.", + "boxes": [ + 646, + 850, + 960, + 874 + ] + }, + { + "text": "$255.99 + $7.41 shipping", + "boxes": [ + 728, + 893, + 882, + 919 + ] + }, + { + "text": "★★★★☆（449)", + "boxes": [ + 726, + 911, + 848, + 941 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1814, + 925, + 1860, + 945 + ] + }, + { + "text": "Works and looks like new and backed by t", + "boxes": [ + 728, + 939, + 988, + 957 + ] + }, + { + "text": "廿1+", + "boxes": [ + 1790, + 971, + 1886, + 997 + ] + }, + { + "text": "UnwellHydrationfromAlexCooper", + "boxes": [ + 744, + 995, + 974, + 1012 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "30BFJWCYTL/?_encoding=UTF8&pd_rd_w=kzgNd&content-", + "boxes": [ + 866, + 69, + 1276, + 87 + ] + }, + { + "text": "·☆", + "boxes": [ + 1580, + 66, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " Deliver to Canada", + "boxes": [ + 1396, + 97, + 1516, + 121 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "T", + "boxes": [ + 590, + 121, + 604, + 141 + ] + }, + { + "text": "$27999", + "boxes": [ + 642, + 125, + 724, + 163 + ] + }, + { + "text": "Used: Very Good I Detalls", + "boxes": [ + 1392, + 123, + 1562, + 159 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1804, + 125, + 1884, + 151 + ] + }, + { + "text": "Sold by Toys and Video Games", + "boxes": [ + 1392, + 147, + 1592, + 177 + ] + }, + { + "text": "$192.10 Shipping", + "boxes": [ + 644, + 164, + 754, + 193 + ] + }, + { + "text": "ada Details", + "boxes": [ + 920, + 171, + 1004, + 189 + ] + }, + { + "text": "Store", + "boxes": [ + 1398, + 171, + 1434, + 191 + ] + }, + { + "text": "Fulflled by Amazon ", + "boxes": [ + 1396, + 191, + 1526, + 214 + ] + }, + { + "text": "Style: Console", + "boxes": [ + 644, + 238, + 740, + 264 + ] + }, + { + "text": "Add to Cart ", + "boxes": [ + 1460, + 240, + 1542, + 272 + ] + }, + { + "text": "Color: Black", + "boxes": [ + 644, + 266, + 726, + 290 + ] + }, + { + "text": " About this item", + "boxes": [ + 640, + 307, + 798, + 344 + ] + }, + { + "text": "·6.2\" LCD screen", + "boxes": [ + 652, + 341, + 762, + 365 + ] + }, + { + "text": "Add to List", + "boxes": [ + 1410, + 332, + 1472, + 351 + ] + }, + { + "text": "Other sellers on Amazor", + "boxes": [ + 1388, + 401, + 1572, + 425 + ] + }, + { + "text": "w&Used(9)from$17200", + "boxes": [ + 1410, + 448, + 1588, + 471 + ] + }, + { + "text": "This product is available as Renewed.", + "boxes": [ + 642, + 502, + 962, + 532 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1810, + 506, + 1864, + 526 + ] + }, + { + "text": "NintendoSwitchTMwithNeonBlue", + "boxes": [ + 726, + 532, + 942, + 556 + ] + }, + { + "text": "$255.99 + $7.41 shipping", + "boxes": [ + 726, + 546, + 892, + 576 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1794, + 548, + 1862, + 574 + ] + }, + { + "text": "", + "boxes": [ + 676, + 560, + 700, + 574 + ] + }, + { + "text": "★★★★☆(449)", + "boxes": [ + 726, + 568, + 844, + 592 + ] + }, + { + "text": "Works and looks like new and backed by the Ar", + "boxes": [ + 724, + 590, + 1022, + 613 + ] + }, + { + "text": "Unwell Hydration from Alex Cooper", + "boxes": [ + 744, + 647, + 978, + 671 + ] + }, + { + "text": "Hydrate & focus", + "boxes": [ + 744, + 667, + 846, + 691 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 721, + 1862, + 739 + ] + }, + { + "text": " Products related to this item", + "boxes": [ + 150, + 764, + 434, + 804 + ] + }, + { + "text": "立", + "boxes": [ + 1794, + 768, + 1820, + 788 + ] + }, + { + "text": "1", + "boxes": [ + 1816, + 768, + 1868, + 786 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1814, + 925, + 1860, + 945 + ] + }, + { + "text": "1", + "boxes": [ + 1790, + 971, + 1872, + 997 + ] + }, + { + "text": "AceGamer [Pro ", + "boxes": [ + 422, + 987, + 534, + 1016 + ] + }, + { + "text": "COIORVIS Joy Pad", + "boxes": [ + 616, + 985, + 738, + 1014 + ] + }, + { + "text": "Switch Controller", + "boxes": [ + 808, + 984, + 924, + 1015 + ] + }, + { + "text": "Orzly Carry Case", + "boxes": [ + 1004, + 989, + 1112, + 1016 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 1196, + 991, + 1320, + 1014 + ] + }, + { + "text": "FUNLAB Switch", + "boxes": [ + 1390, + 989, + 1490, + 1012 + ] + }, + { + "text": "Y9L2Y/ref=dp_", + "boxes": [ + 240, + 1016, + 342, + 1040 + ] + }, + { + "text": "crwdg titgrad Wireles", + "boxes": [ + 329, + 997, + 551, + 1051 + ] + }, + { + "text": "Controller for ", + "boxes": [ + 618, + 1009, + 710, + 1034 + ] + }, + { + "text": "Charging Dock for ", + "boxes": [ + 808, + 1007, + 930, + 1036 + ] + }, + { + "text": "Compatible with", + "boxes": [ + 1006, + 1009, + 1114, + 1036 + ] + }, + { + "text": "Controller for", + "boxes": [ + 1198, + 1009, + 1288, + 1034 + ] + }, + { + "text": "Handheld Controller for", + "boxes": [ + 1388, + 1007, + 1544, + 1036 + ] + } + ] + }, + { + "frame_id": "70_4_6", + "episode_id": 70, + "env_id": 4, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.590002059936523, + "advantages": -0.06923703104257584, + "generated_text": "{\n \"intent\": \"The current screen shows the Firefox browser's download page. Opening a new browser instance or exploring more of the website by scrolling might provide additional useful information or options.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "The current screen shows the Firefox browser's download page. Opening a new browser instance or exploring more of the website by scrolling might provide additional useful information or options.", + "before_screen_ocr_results": [ + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 238, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "ns and other languages", + "boxes": [ + 440, + 367, + 576, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 492, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 800, + 917, + 1098, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ] + }, + { + "frame_id": "70_5_6", + "episode_id": 70, + "env_id": 5, + "step": 6, + "format_rewards": 1.0, + "rewards": 11.624320030212402, + "advantages": 1.1050471067428589, + "generated_text": "{\n \"intent\": \"I will click on an Apple Watch product link to explore more details about the device, which aligns towards gaining a deeper understanding of the available options.\",\n \"action\": \"DoubleClick(474, 235)\"\n}", + "action_string": "DoubleClick(474, 235)", + "intent_string": "I will click on an Apple Watch product link to explore more details about the device, which aligns towards gaining a deeper understanding of the available options.", + "before_screen_ocr_results": [ + { + "text": "Apple", + "boxes": [ + 120, + 34, + 164, + 52 + ] + }, + { + "text": "0", + "boxes": [ + 374, + 163, + 412, + 206 + ] + }, + { + "text": "0", + "boxes": [ + 470, + 159, + 516, + 212 + ] + }, + { + "text": "0", + "boxes": [ + 576, + 163, + 616, + 204 + ] + }, + { + "text": "口", + "boxes": [ + 672, + 159, + 720, + 208 + ] + }, + { + "text": "6", + "boxes": [ + 770, + 161, + 828, + 212 + ] + }, + { + "text": "0!", + "boxes": [ + 866, + 161, + 938, + 208 + ] + }, + { + "text": "00", + "boxes": [ + 964, + 165, + 1030, + 212 + ] + }, + { + "text": "1", + "boxes": [ + 1050, + 169, + 1078, + 210 + ] + }, + { + "text": "9?", + "boxes": [ + 1114, + 173, + 1168, + 218 + ] + }, + { + "text": "O", + "boxes": [ + 1210, + 173, + 1242, + 208 + ] + }, + { + "text": "?", + "boxes": [ + 1302, + 171, + 1354, + 214 + ] + }, + { + "text": "OS", + "boxes": [ + 1406, + 173, + 1452, + 210 + ] + }, + { + "text": "010", + "boxes": [ + 1480, + 165, + 1562, + 212 + ] + }, + { + "text": "pple Watch", + "boxes": [ + 462, + 216, + 528, + 236 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 556, + 214, + 632, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 662, + 216, + 732, + 234 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 768, + 216, + 838, + 238 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 866, + 212, + 940, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 961, + 211, + 1023, + 244 + ] + }, + { + "text": "Bands", + "boxes": [ + 1052, + 218, + 1080, + 232 + ] + }, + { + "text": "irPods", + "boxes": [ + 1126, + 218, + 1160, + 232 + ] + }, + { + "text": "ShopWatch", + "boxes": [ + 1492, + 216, + 1554, + 234 + ] + }, + { + "text": "Series 10", + "boxes": [ + 364, + 232, + 420, + 250 + ] + }, + { + "text": "Ultra 2", + "boxes": [ + 470, + 232, + 514, + 252 + ] + }, + { + "text": "Nike", + "boxes": [ + 682, + 234, + 708, + 248 + ] + }, + { + "text": "Hermes", + "boxes": [ + 776, + 232, + 822, + 250 + ] + }, + { + "text": "Studio", + "boxes": [ + 878, + 232, + 922, + 252 + ] + }, + { + "text": "Apple Watch :", + "boxes": [ + 1108, + 284, + 1188, + 302 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 111, + 385, + 569, + 487 + ] + }, + { + "text": " The ultimate device", + "boxes": [ + 1538, + 397, + 1796, + 437 + ] + }, + { + "text": "for a healthy life.", + "boxes": [ + 1538, + 431, + 1754, + 472 + ] + }, + { + "text": "浙", + "boxes": [ + 1114, + 653, + 1162, + 681 + ] + }, + { + "text": "10:09:29", + "boxes": [ + 636, + 667, + 1280, + 852 + ] + }, + { + "text": "M2", + "boxes": [ + 918, + 866, + 990, + 905 + ] + }, + { + "text": "65", + "boxes": [ + 1128, + 860, + 1222, + 885 + ] + }, + { + "text": "58", + "boxes": [ + 686, + 870, + 782, + 907 + ] + }, + { + "text": "65", + "boxes": [ + 788, + 939, + 842, + 957 + ] + }, + { + "text": "atch/", + "boxes": [ + 176, + 1024, + 208, + 1038 + ] + }, + { + "text": "Apple Watch .", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 8, + 10, + 20, + 22 + ] + }, + { + "text": " Apple Watch Ultra 2 - Ap × |+", + "boxes": [ + 6, + 28, + 246, + 54 + ] + }, + { + "text": "←→", + "boxes": [ + 4, + 62, + 132, + 93 + ] + }, + { + "text": "目☆", + "boxes": [ + 1538, + 64, + 1664, + 93 + ] + }, + { + "text": "I四@=", + "boxes": [ + 1780, + 59, + 1912, + 96 + ] + }, + { + "text": "Store", + "boxes": [ + 512, + 111, + 538, + 125 + ] + }, + { + "text": " Apple Watch Ultra 2", + "boxes": [ + 458, + 146, + 658, + 191 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 870, + 266, + 1032, + 314 + ] + }, + { + "text": "ULTRA 2", + "boxes": [ + 914, + 308, + 994, + 339 + ] + }, + { + "text": "10:09-29", + "boxes": [ + 687, + 424, + 1223, + 604 + ] + }, + { + "text": "", + "boxes": [ + 1208, + 443, + 1262, + 522 + ] + }, + { + "text": "一", + "boxes": [ + 646, + 546, + 658, + 558 + ] + }, + { + "text": "", + "boxes": [ + 867, + 578, + 1043, + 649 + ] + }, + { + "text": "58", + "boxes": [ + 724, + 600, + 850, + 637 + ] + }, + { + "text": "65″", + "boxes": [ + 1052, + 592, + 1184, + 621 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 216, + 1048, + 302, + 1072 + ] + } + ] + }, + { + "frame_id": "70_6_6", + "episode_id": 70, + "env_id": 6, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.441899299621582, + "advantages": -0.15472747385501862, + "generated_text": "{\n \"intent\": \"Since the current web page is fully loaded and visible, scrolling may not be necessary. The goal could be to explore further content or navigate to another tab. I will click on an element in the navigation bar, which usually contains options to switch between tabs or applications.\",\n \"action\": \"DoubleClick(242, 37)\"\n}", + "action_string": "DoubleClick(242, 37)", + "intent_string": "Since the current web page is fully loaded and visible, scrolling may not be necessary. The goal could be to explore further content or navigate to another tab. I will click on an element in the navigation bar, which usually contains options to switch between tabs or applications.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 2, + 58, + 136, + 101 + ] + }, + { + "text": "①| https://www.apple.com/shop/buy-iphone/iphone-16-pro", + "boxes": [ + 250, + 66, + 692, + 89 + ] + }, + { + "text": "目☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "I四三", + "boxes": [ + 1782, + 64, + 1912, + 93 + ] + }, + { + "text": "2", + "boxes": [ + 102, + 1048, + 152, + 1066 + ] + }, + { + "text": " Buy iPhone 1..", + "boxes": [ + 142, + 1044, + 318, + 1074 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "70_7_6", + "episode_id": 70, + "env_id": 7, + "step": 6, + "format_rewards": 1.0, + "rewards": 10.918482780456543, + "advantages": 0.6976115703582764, + "generated_text": "{\n \"intent\": \"To explore potential applications and their functions, I will open an application by clicking on one of the icons.\",\n \"action\": \"DoubleClick(39, 187)\"\n}", + "action_string": "DoubleClick(39, 187)", + "intent_string": "To explore potential applications and their functions, I will open an application by clicking on one of the icons.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 940, + 42, + 1054, + 60 + ] + }, + { + "text": " Firefox", + "boxes": [ + 22, + 64, + 98, + 95 + ] + }, + { + "text": "Apple", + "boxes": [ + 334, + 71, + 390, + 91 + ] + }, + { + "text": "x+", + "boxes": [ + 522, + 69, + 570, + 89 + ] + }, + { + "text": "C", + "boxes": [ + 390, + 105, + 422, + 127 + ] + }, + { + "text": "仓", + "boxes": [ + 418, + 107, + 448, + 127 + ] + }, + { + "text": "① https:/www.apple.com", + "boxes": [ + 548, + 99, + 768, + 133 + ] + }, + { + "text": "·☆", + "boxes": [ + 1286, + 101, + 1368, + 133 + ] + }, + { + "text": "三四@", + "boxes": [ + 1464, + 99, + 1568, + 131 + ] + }, + { + "text": "→", + "boxes": [ + 356, + 109, + 396, + 125 + ] + }, + { + "text": "iPhone", + "boxes": [ + 696, + 153, + 730, + 167 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1196, + 151, + 1262, + 169 + ] + }, + { + "text": "Support", + "boxes": [ + 1296, + 151, + 1346, + 169 + ] + }, + { + "text": "Q", + "boxes": [ + 1364, + 149, + 1394, + 169 + ] + }, + { + "text": "?", + "boxes": [ + 1420, + 148, + 1444, + 173 + ] + }, + { + "text": "Mousepad", + "boxes": [ + 10, + 173, + 110, + 199 + ] + }, + { + "text": "Terminal", + "boxes": [ + 22, + 282, + 104, + 308 + ] + }, + { + "text": "Calculator", + "boxes": [ + 5, + 387, + 117, + 423 + ] + }, + { + "text": "LibreO.", + "boxes": [ + 20, + 502, + 90, + 528 + ] + }, + { + "text": "Writer", + "boxes": [ + 28, + 609, + 94, + 641 + ] + }, + { + "text": "WATCH", + "boxes": [ + 870, + 659, + 1070, + 715 + ] + }, + { + "text": "國", + "boxes": [ + 38, + 681, + 80, + 719 + ] + }, + { + "text": "Excel", + "boxes": [ + 32, + 721, + 90, + 752 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 904, + 715, + 1032, + 746 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 850, + 752, + 1064, + 788 + ] + }, + { + "text": "", + "boxes": [ + 30, + 771, + 84, + 832 + ] + }, + { + "text": "Learn more", + "boxes": [ + 862, + 814, + 958, + 840 + ] + }, + { + "text": "Draw", + "boxes": [ + 32, + 830, + 88, + 864 + ] + }, + { + "text": "Files", + "boxes": [ + 36, + 941, + 84, + 969 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit Search View", + "boxes": [ + 4, + 22, + 192, + 52 + ] + }, + { + "text": "Untitled 1 - M..", + "boxes": [ + 366, + 1046, + 466, + 1070 + ] + } + ] + }, + { + "frame_id": "70_0_7", + "episode_id": 70, + "env_id": 0, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.442486763000488, + "advantages": -0.15438836812973022, + "generated_text": "{\n \"intent\": \"To continue exploring the webpage by clicking on a link or element that might provide more information about the topic.\",\n \"action\": \"DoubleClick(679, 378)\"\n}", + "action_string": "DoubleClick(679, 378)", + "intent_string": "To continue exploring the webpage by clicking on a link or element that might provide more information about the topic.", + "before_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 56, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 64, + 1874, + 89 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 564, + 117, + 638, + 149 + ] + }, + { + "text": "msn", + "boxes": [ + 58, + 129, + 104, + 149 + ] + }, + { + "text": "Q", + "boxes": [ + 1308, + 123, + 1324, + 139 + ] + }, + { + "text": "中", + "boxes": [ + 1766, + 125, + 1786, + 141 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "资讯 娱乐　 时尚 教育", + "boxes": [ + 344, + 175, + 542, + 204 + ] + }, + { + "text": "科技 萌宠汽车", + "boxes": [ + 532, + 175, + 694, + 204 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1606, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "精确分析", + "boxes": [ + 1440, + 248, + 1590, + 304 + ] + }, + { + "text": "ITBEAR科技资讯+ 关注", + "boxes": [ + 382, + 264, + 632, + 306 + ] + }, + { + "text": "专", + "boxes": [ + 1426, + 260, + 1442, + 274 + ] + }, + { + "text": "60.9K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "绘制图表", + "boxes": [ + 1442, + 290, + 1584, + 334 + ] + }, + { + "text": "助您高效完成数据分析", + "boxes": [ + 1454, + 334, + 1576, + 351 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，能否引领新能源市场新潮流？", + "boxes": [ + 362, + 347, + 1174, + 389 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1456, + 351, + 1506, + 369 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 393, + 218, + 425 + ] + }, + { + "text": "k王者荣耀的报道·15小时· 1分钟读取", + "boxes": [ + 364, + 413, + 706, + 449 + ] + }, + { + "text": "让你成为 Excel 达人，工作学习效", + "boxes": [ + 1422, + 411, + 1702, + 447 + ] + }, + { + "text": "率倍增", + "boxes": [ + 1426, + 437, + 1490, + 469 + ] + }, + { + "text": "B", + "boxes": [ + 192, + 453, + 214, + 476 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 469, + 1530, + 492 + ] + }, + { + "text": "在汽车市场风起云涌的今天，哈弗汽车，这个长久以来以\"性价比\"著称的品牌，正积极布局新能源领域，意图在新的", + "boxes": [ + 364, + 504, + 1232, + 534 + ] + }, + { + "text": "赛道上续写辉煌。近日，哈弗汽车两款全新车型——哈弗枭龙与哈弗枭龙MAX正式上市，标志着哈弗正式进军新能源", + "boxes": [ + 366, + 530, + 1240, + 560 + ] + }, + { + "text": "汽车市场的决心。", + "boxes": [ + 364, + 554, + 494, + 588 + ] + }, + { + "text": "访问ITBEAR科技资讯", + "boxes": [ + 1426, + 554, + 1596, + 584 + ] + }, + { + "text": "G", + "boxes": [ + 1673, + 550, + 1702, + 579 + ] + }, + { + "text": "哈弗枭龙MAX与哈弗枭龙，作为哈弗品牌的全新力作，不仅继承了哈弗汽车一贯的高性价比优势，更在技术层面实现", + "boxes": [ + 364, + 600, + 1240, + 629 + ] + }, + { + "text": "新疆多地夜空现神秘飞行物，目击", + "boxes": [ + 1444, + 597, + 1666, + 634 + ] + }, + { + "text": "了突破。哈弗枭龙MAX搭载了长城汽车全新的智能四驱电混技术Hi4，推出了三款车型，价格区间为15.98-17.98万", + "boxes": [ + 366, + 625, + 1214, + 655 + ] + }, + { + "text": "者：亮如月亮，几秒加速消失", + "boxes": [ + 1446, + 621, + 1634, + 651 + ] + }, + { + "text": "元。而哈弗枭龙则采用了智能混动DHT技术，同样推出了三款车型，价格区间则为13.98-15.68万元。两款车型均展现", + "boxes": [ + 364, + 651, + 1240, + 681 + ] + }, + { + "text": "小米YU7即将发布，卢伟冰自曝已", + "boxes": [ + 1446, + 663, + 1664, + 693 + ] + }, + { + "text": "出了哈弗在新能源汽车领域的深厚底蕴和技术实力。", + "boxes": [ + 362, + 675, + 750, + 707 + ] + }, + { + "text": "购车并计划添购新车", + "boxes": [ + 1446, + 685, + 1580, + 715 + ] + }, + { + "text": "长安汽车泰国投产，深蓝CEO：增", + "boxes": [ + 1446, + 729, + 1664, + 758 + ] + }, + { + "text": "程车油耗媲美日系混动，驾驶体.", + "boxes": [ + 1446, + 748, + 1660, + 778 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 832, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 878, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 794, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击\"管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置“改变您的选择。欲了解", + "boxes": [ + 28, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：", + "boxes": [ + 32, + 971, + 130, + 995 + ] + }, + { + "text": "微软隐私声明", + "boxes": [ + 120, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + }, + { + "text": "合弗枭龙MAX", + "boxes": [ + 224, + 1048, + 306, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "", + "boxes": [ + 54, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 564, + 117, + 638, + 149 + ] + }, + { + "text": "msn", + "boxes": [ + 58, + 129, + 104, + 149 + ] + }, + { + "text": "中", + "boxes": [ + 1762, + 121, + 1788, + 143 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "资讯 娱乐 时尚 教育", + "boxes": [ + 344, + 175, + 542, + 204 + ] + }, + { + "text": "科技萌宠汽车", + "boxes": [ + 532, + 175, + 694, + 204 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1606, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "精确分析", + "boxes": [ + 1440, + 248, + 1590, + 304 + ] + }, + { + "text": "ITBEAR科技资讯+ 关注", + "boxes": [ + 382, + 264, + 632, + 306 + ] + }, + { + "text": "专", + "boxes": [ + 1426, + 260, + 1442, + 274 + ] + }, + { + "text": "60.9K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "绘制图表", + "boxes": [ + 1442, + 290, + 1584, + 334 + ] + }, + { + "text": "助您高效完成数据分析", + "boxes": [ + 1454, + 334, + 1576, + 351 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，能否引领新能源市场新潮流？", + "boxes": [ + 362, + 347, + 1174, + 389 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1456, + 351, + 1506, + 369 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 393, + 218, + 425 + ] + }, + { + "text": "k王者荣耀的报道·15小时·① 1分钟读取", + "boxes": [ + 362, + 411, + 708, + 451 + ] + }, + { + "text": "让你成为Excel达人，工作学习效", + "boxes": [ + 1422, + 411, + 1702, + 447 + ] + }, + { + "text": "率倍增", + "boxes": [ + 1426, + 437, + 1490, + 469 + ] + }, + { + "text": "B", + "boxes": [ + 192, + 453, + 214, + 476 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 469, + 1530, + 492 + ] + }, + { + "text": "在汽车市场风起云涌的今天，哈弗汽车，这个长久以来以\"性价比\"著称的品牌，正积极布局新能源领域，意图在新的", + "boxes": [ + 364, + 504, + 1232, + 534 + ] + }, + { + "text": "赛道上续写辉煌。近日，哈弗汽车两款全新车型——哈弗枭龙与哈弗枭龙MAX正式上市，标志着哈弗正式进军新能源", + "boxes": [ + 366, + 530, + 1240, + 560 + ] + }, + { + "text": "汽车市场的决心。", + "boxes": [ + 364, + 554, + 494, + 588 + ] + }, + { + "text": "访问ITBEAR科技资讯", + "boxes": [ + 1426, + 554, + 1596, + 584 + ] + }, + { + "text": "G", + "boxes": [ + 1673, + 550, + 1702, + 579 + ] + }, + { + "text": "哈弗枭龙MAX与哈弗枭龙，作为哈弗品牌的全新力作，不仅继承了哈弗汽车一贯的高性价比优势，更在技术层面实现", + "boxes": [ + 364, + 600, + 1240, + 629 + ] + }, + { + "text": "新疆多地夜空现神秘飞行物，目击", + "boxes": [ + 1444, + 597, + 1666, + 634 + ] + }, + { + "text": "了突破。哈弗枭龙MAX搭载了长城汽车全新的智能四驱电混技术Hi4，推出了三款车型，价格区间为15.98-17.98万", + "boxes": [ + 366, + 625, + 1214, + 655 + ] + }, + { + "text": "者：亮如月亮，几秒加速消失", + "boxes": [ + 1446, + 621, + 1634, + 651 + ] + }, + { + "text": "元。而哈弗枭龙则采用了智能混动DHT技术，同样推出了三款车型，价格区间则为13.98-15.68万元。两款车型均展现", + "boxes": [ + 364, + 651, + 1240, + 681 + ] + }, + { + "text": "小米YU7即将发布，卢伟冰自曝已", + "boxes": [ + 1446, + 663, + 1664, + 693 + ] + }, + { + "text": "出了哈弗在新能源汽车领域的深厚底蕴和技术实力。", + "boxes": [ + 364, + 677, + 750, + 707 + ] + }, + { + "text": "购车并计划添购新车", + "boxes": [ + 1446, + 685, + 1580, + 715 + ] + }, + { + "text": "长安汽车泰国投产，深蓝CEO：增", + "boxes": [ + 1446, + 729, + 1664, + 758 + ] + }, + { + "text": "程车油耗媲美日系混动，驾驶体..", + "boxes": [ + 1446, + 748, + 1660, + 778 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 832, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 878, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 796, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置\"改变您的选择。欲了解", + "boxes": [ + 30, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：微软隐私声明", + "boxes": [ + 32, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + } + ] + }, + { + "frame_id": "70_1_7", + "episode_id": 70, + "env_id": 1, + "step": 7, + "format_rewards": 1.0, + "rewards": 8.92768383026123, + "advantages": -0.4515518248081207, + "generated_text": "{\n \"intent\": \"To continue exploring various applications within the desktop environment, it's essential to see how different software behaves and interacts with other programs.\",\n \"action\": \"DoubleClick(100, 50)\" }", + "action_string": "DoubleClick(100, 50)", + "intent_string": "To continue exploring various applications within the desktop environment, it's essential to see how different software behaves and interacts with other programs.", + "before_screen_ocr_results": [ + { + "text": "国", + "boxes": [ + 8, + 4, + 22, + 24 + ] + }, + { + "text": "Untitled 1 - Libreoffice Calc", + "boxes": [ + 848, + 2, + 1070, + 26 + ] + }, + { + "text": "---- α", + "boxes": [ + 8, + 48, + 446, + 87 + ] + }, + { + "text": "G-.", + "boxes": [ + 450, + 56, + 548, + 79 + ] + }, + { + "text": "里·听个", + "boxes": [ + 544, + 54, + 848, + 83 + ] + }, + { + "text": "M-EE-B", + "boxes": [ + 850, + 48, + 1264, + 87 + ] + }, + { + "text": "10", + "boxes": [ + 156, + 97, + 186, + 117 + ] + }, + { + "text": "aaaa-==", + "boxes": [ + 238, + 93, + 520, + 123 + ] + }, + { + "text": "", + "boxes": [ + 550, + 101, + 574, + 115 + ] + }, + { + "text": "□B8", + "boxes": [ + 616, + 95, + 720, + 119 + ] + }, + { + "text": "-% 00 ", + "boxes": [ + 724, + 93, + 874, + 123 + ] + }, + { + "text": "十三田二口目", + "boxes": [ + 878, + 93, + 1198, + 123 + ] + }, + { + "text": "紫×", + "boxes": [ + 158, + 129, + 238, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Print", + "boxes": [ + 938, + 262, + 990, + 288 + ] + }, + { + "text": "□□", + "boxes": [ + 1288, + 262, + 1350, + 288 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 266, + 1914, + 314 + ] + }, + { + "text": "8.26 in (A4)", + "boxes": [ + 724, + 290, + 808, + 316 + ] + }, + { + "text": " General ", + "boxes": [ + 950, + 296, + 1014, + 322 + ] + }, + { + "text": "Libreoffice Calc", + "boxes": [ + 1010, + 293, + 1128, + 322 + ] + }, + { + "text": "成", + "boxes": [ + 1890, + 312, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "Printer", + "boxes": [ + 948, + 324, + 1010, + 349 + ] + }, + { + "text": " Generic Printer", + "boxes": [ + 964, + 351, + 1074, + 375 + ] + }, + { + "text": "Status: Default printer", + "boxes": [ + 960, + 385, + 1108, + 409 + ] + }, + { + "text": "O All Pages", + "boxes": [ + 964, + 449, + 1044, + 472 + ] + }, + { + "text": ".8", + "boxes": [ + 18, + 467, + 30, + 478 + ] + }, + { + "text": "O Pages:", + "boxes": [ + 968, + 478, + 1038, + 500 + ] + }, + { + "text": "O Even pages", + "boxes": [ + 965, + 502, + 1061, + 533 + ] + }, + { + "text": "O odd pages", + "boxes": [ + 965, + 528, + 1057, + 558 + ] + }, + { + "text": "more", + "boxes": [ + 968, + 560, + 1016, + 580 + ] + }, + { + "text": "Page Layout", + "boxes": [ + 952, + 588, + 1044, + 611 + ] + }, + { + "text": "Paper size:", + "boxes": [ + 966, + 617, + 1036, + 635 + ] + }, + { + "text": "A4 8.26in x 11.69in", + "boxes": [ + 1048, + 613, + 1182, + 637 + ] + }, + { + "text": "Automatic", + "boxes": [ + 1051, + 647, + 1125, + 670 + ] + }, + { + "text": "Preview", + "boxes": [ + 592, + 754, + 658, + 772 + ] + }, + { + "text": "?", + "boxes": [ + 744, + 756, + 770, + 770 + ] + }, + { + "text": "/10", + "boxes": [ + 818, + 754, + 882, + 774 + ] + }, + { + "text": "?Help", + "boxes": [ + 599, + 789, + 645, + 815 + ] + }, + { + "text": "More Option", + "boxes": [ + 1038, + 796, + 1120, + 814 + ] + }, + { + "text": "OK", + "boxes": [ + 1288, + 796, + 1316, + 810 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 939 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Untitled 1 - Libreoffice Calc", + "boxes": [ + 848, + 2, + 1070, + 26 + ] + }, + { + "text": "--- - α", + "boxes": [ + 8, + 48, + 446, + 87 + ] + }, + { + "text": "G-.", + "boxes": [ + 450, + 56, + 546, + 79 + ] + }, + { + "text": "里·听个", + "boxes": [ + 544, + 54, + 848, + 83 + ] + }, + { + "text": "M-E-EB", + "boxes": [ + 850, + 48, + 1264, + 87 + ] + }, + { + "text": "10", + "boxes": [ + 156, + 97, + 188, + 117 + ] + }, + { + "text": "aaaa--==", + "boxes": [ + 236, + 93, + 520, + 123 + ] + }, + { + "text": "", + "boxes": [ + 550, + 101, + 574, + 115 + ] + }, + { + "text": "□B8", + "boxes": [ + 616, + 95, + 720, + 119 + ] + }, + { + "text": "-% 00 ", + "boxes": [ + 724, + 93, + 874, + 123 + ] + }, + { + "text": "十三至田二口目", + "boxes": [ + 878, + 93, + 1198, + 123 + ] + }, + { + "text": "游×", + "boxes": [ + 156, + 129, + 238, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Print", + "boxes": [ + 938, + 262, + 990, + 288 + ] + }, + { + "text": "□□", + "boxes": [ + 1288, + 262, + 1350, + 288 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 266, + 1914, + 314 + ] + }, + { + "text": "8.26 in (A4)", + "boxes": [ + 724, + 290, + 808, + 316 + ] + }, + { + "text": " General ", + "boxes": [ + 950, + 296, + 1014, + 322 + ] + }, + { + "text": "Libreoffice Calc", + "boxes": [ + 1010, + 293, + 1128, + 322 + ] + }, + { + "text": "成", + "boxes": [ + 1890, + 312, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "Printer", + "boxes": [ + 948, + 324, + 1010, + 349 + ] + }, + { + "text": " Generic Printer", + "boxes": [ + 964, + 351, + 1074, + 375 + ] + }, + { + "text": "Status: Default printer", + "boxes": [ + 960, + 385, + 1108, + 409 + ] + }, + { + "text": "O All Pages", + "boxes": [ + 964, + 449, + 1044, + 472 + ] + }, + { + "text": ".8", + "boxes": [ + 18, + 467, + 30, + 478 + ] + }, + { + "text": "O Pages:", + "boxes": [ + 968, + 478, + 1038, + 500 + ] + }, + { + "text": "O Even pages", + "boxes": [ + 965, + 502, + 1061, + 533 + ] + }, + { + "text": "O odd pages", + "boxes": [ + 965, + 528, + 1057, + 558 + ] + }, + { + "text": "more", + "boxes": [ + 968, + 560, + 1016, + 580 + ] + }, + { + "text": "Page Layout", + "boxes": [ + 952, + 588, + 1044, + 611 + ] + }, + { + "text": "Paper size:", + "boxes": [ + 966, + 617, + 1036, + 635 + ] + }, + { + "text": "A4 8.26in x 11.69in", + "boxes": [ + 1048, + 613, + 1182, + 637 + ] + }, + { + "text": "Automatic", + "boxes": [ + 1051, + 647, + 1125, + 670 + ] + }, + { + "text": "Preview", + "boxes": [ + 592, + 754, + 658, + 772 + ] + }, + { + "text": "?", + "boxes": [ + 744, + 756, + 770, + 770 + ] + }, + { + "text": "/10", + "boxes": [ + 818, + 754, + 882, + 774 + ] + }, + { + "text": "Help", + "boxes": [ + 604, + 796, + 644, + 812 + ] + }, + { + "text": "More Option", + "boxes": [ + 1038, + 796, + 1120, + 814 + ] + }, + { + "text": "OK", + "boxes": [ + 1288, + 796, + 1316, + 810 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 939 + ] + } + ] + }, + { + "frame_id": "70_2_7", + "episode_id": 70, + "env_id": 2, + "step": 7, + "format_rewards": 0.0, + "rewards": 0.0, + "advantages": -5.604943752288818, + "generated_text": "{\n \"intent\": \"The current page shows the main categories offered by Apple, allowing exploration into various products. A logical next step would be to click on one of these product sections to delve deeper into its offerings.\",\n \"action\": \"DoubleClick(iPhone)\"\n}", + "action_string": "None", + "intent_string": null, + "before_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 132, + 220, + 768, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1632, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "?", + "boxes": [ + 1592, + 294, + 1614, + 318 + ] + }, + { + "text": "/isit anAppleStore", + "boxes": [ + 1632, + 290, + 1756, + 308 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 378, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 138, + 641, + 720, + 671 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1848, + 752, + 1912, + 786 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1426, + 822, + 1582, + 852 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 826, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 132, + 220, + 768, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1632, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "?", + "boxes": [ + 1592, + 294, + 1614, + 318 + ] + }, + { + "text": "/isit anAppleStore", + "boxes": [ + 1632, + 290, + 1756, + 308 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 378, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 138, + 641, + 720, + 671 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1843, + 745, + 1915, + 791 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1426, + 822, + 1582, + 852 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 826, + 1044 + ] + } + ] + }, + { + "frame_id": "70_3_7", + "episode_id": 70, + "env_id": 3, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.324941635131836, + "advantages": -0.22223979234695435, + "generated_text": "{\n \"intent\": \"To explore related products or options for purchasing by interacting with icons or toggling product details.\",\n \"action\": \"DoubleClick(1684, 794)\"\n}", + "action_string": "DoubleClick(1684, 794)", + "intent_string": "To explore related products or options for purchasing by interacting with icons or toggling product details.", + "before_screen_ocr_results": [ + { + "text": "30BFJWCYTL/?_encoding=UTF8&pd_rd_w=kzgNd&content-", + "boxes": [ + 866, + 69, + 1276, + 87 + ] + }, + { + "text": "·☆", + "boxes": [ + 1580, + 66, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " Deliver to Canada", + "boxes": [ + 1396, + 97, + 1516, + 121 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "T", + "boxes": [ + 590, + 121, + 604, + 141 + ] + }, + { + "text": "$27999", + "boxes": [ + 642, + 125, + 724, + 163 + ] + }, + { + "text": "Used: Very Good I Detalls", + "boxes": [ + 1392, + 123, + 1562, + 159 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1804, + 125, + 1884, + 151 + ] + }, + { + "text": "Sold by Toys and Video Games", + "boxes": [ + 1392, + 147, + 1592, + 177 + ] + }, + { + "text": "$192.10 Shipping", + "boxes": [ + 644, + 164, + 754, + 193 + ] + }, + { + "text": "ada Details", + "boxes": [ + 920, + 171, + 1004, + 189 + ] + }, + { + "text": "Store", + "boxes": [ + 1398, + 171, + 1434, + 191 + ] + }, + { + "text": "Fulflled by Amazon ", + "boxes": [ + 1396, + 191, + 1526, + 214 + ] + }, + { + "text": "Style: Console", + "boxes": [ + 644, + 238, + 740, + 264 + ] + }, + { + "text": "Add to Cart ", + "boxes": [ + 1460, + 240, + 1542, + 272 + ] + }, + { + "text": "Color: Black", + "boxes": [ + 644, + 266, + 726, + 290 + ] + }, + { + "text": " About this item", + "boxes": [ + 640, + 307, + 798, + 344 + ] + }, + { + "text": "·6.2\" LCD screen", + "boxes": [ + 652, + 341, + 762, + 365 + ] + }, + { + "text": "Add to List", + "boxes": [ + 1410, + 332, + 1472, + 351 + ] + }, + { + "text": "Other sellers on Amazor", + "boxes": [ + 1388, + 401, + 1572, + 425 + ] + }, + { + "text": "w&Used(9)from$17200", + "boxes": [ + 1410, + 448, + 1588, + 471 + ] + }, + { + "text": "This product is available as Renewed.", + "boxes": [ + 642, + 502, + 962, + 532 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1810, + 506, + 1864, + 526 + ] + }, + { + "text": "NintendoSwitchTMwithNeonBlue", + "boxes": [ + 726, + 532, + 942, + 556 + ] + }, + { + "text": "$255.99 + $7.41 shipping", + "boxes": [ + 726, + 546, + 892, + 576 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1794, + 548, + 1862, + 574 + ] + }, + { + "text": "", + "boxes": [ + 676, + 560, + 700, + 574 + ] + }, + { + "text": "★★★★☆(449)", + "boxes": [ + 726, + 568, + 844, + 592 + ] + }, + { + "text": "Works and looks like new and backed by the Ar", + "boxes": [ + 724, + 590, + 1022, + 613 + ] + }, + { + "text": "Unwell Hydration from Alex Cooper", + "boxes": [ + 744, + 647, + 978, + 671 + ] + }, + { + "text": "Hydrate & focus", + "boxes": [ + 744, + 667, + 846, + 691 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 721, + 1862, + 739 + ] + }, + { + "text": " Products related to this item", + "boxes": [ + 150, + 764, + 434, + 804 + ] + }, + { + "text": "立", + "boxes": [ + 1794, + 768, + 1820, + 788 + ] + }, + { + "text": "1", + "boxes": [ + 1816, + 768, + 1868, + 786 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1814, + 925, + 1860, + 945 + ] + }, + { + "text": "1", + "boxes": [ + 1790, + 971, + 1872, + 997 + ] + }, + { + "text": "AceGamer [Pro ", + "boxes": [ + 422, + 987, + 534, + 1016 + ] + }, + { + "text": "COIORVIS Joy Pad", + "boxes": [ + 616, + 985, + 738, + 1014 + ] + }, + { + "text": "Switch Controller", + "boxes": [ + 808, + 984, + 924, + 1015 + ] + }, + { + "text": "Orzly Carry Case", + "boxes": [ + 1004, + 989, + 1112, + 1016 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 1196, + 991, + 1320, + 1014 + ] + }, + { + "text": "FUNLAB Switch", + "boxes": [ + 1390, + 989, + 1490, + 1012 + ] + }, + { + "text": "Y9L2Y/ref=dp_", + "boxes": [ + 240, + 1016, + 342, + 1040 + ] + }, + { + "text": "crwdg titgrad Wireles", + "boxes": [ + 329, + 997, + 551, + 1051 + ] + }, + { + "text": "Controller for ", + "boxes": [ + 618, + 1009, + 710, + 1034 + ] + }, + { + "text": "Charging Dock for ", + "boxes": [ + 808, + 1007, + 930, + 1036 + ] + }, + { + "text": "Compatible with", + "boxes": [ + 1006, + 1009, + 1114, + 1036 + ] + }, + { + "text": "Controller for", + "boxes": [ + 1198, + 1009, + 1288, + 1034 + ] + }, + { + "text": "Handheld Controller for", + "boxes": [ + 1388, + 1007, + 1544, + 1036 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "30BFJWCYTL/?_encoding=UTF8&pd_rd_w=kzgNd&content-", + "boxes": [ + 866, + 69, + 1276, + 87 + ] + }, + { + "text": "·☆", + "boxes": [ + 1580, + 66, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " Deliver to Canada", + "boxes": [ + 1396, + 97, + 1516, + 121 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "T", + "boxes": [ + 590, + 121, + 604, + 141 + ] + }, + { + "text": "$27999", + "boxes": [ + 642, + 125, + 724, + 163 + ] + }, + { + "text": "Used: Very Good | Detalls", + "boxes": [ + 1394, + 127, + 1562, + 157 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1804, + 125, + 1884, + 151 + ] + }, + { + "text": "Sold by Toys and Video Games", + "boxes": [ + 1392, + 147, + 1592, + 177 + ] + }, + { + "text": "$192.10 Shipping", + "boxes": [ + 644, + 164, + 754, + 193 + ] + }, + { + "text": "ada Details", + "boxes": [ + 918, + 171, + 1004, + 189 + ] + }, + { + "text": "Store", + "boxes": [ + 1398, + 171, + 1434, + 191 + ] + }, + { + "text": "Fulflled by Amazon ", + "boxes": [ + 1396, + 191, + 1526, + 214 + ] + }, + { + "text": "Style: Console", + "boxes": [ + 644, + 238, + 740, + 264 + ] + }, + { + "text": "Add to Cart ", + "boxes": [ + 1460, + 240, + 1542, + 272 + ] + }, + { + "text": "Color: Black", + "boxes": [ + 644, + 266, + 726, + 290 + ] + }, + { + "text": " About this item", + "boxes": [ + 640, + 307, + 798, + 344 + ] + }, + { + "text": "·6.2\" LCD screen", + "boxes": [ + 652, + 341, + 762, + 365 + ] + }, + { + "text": "Add to List", + "boxes": [ + 1410, + 332, + 1472, + 351 + ] + }, + { + "text": "Other sellers on Amazor", + "boxes": [ + 1388, + 401, + 1572, + 425 + ] + }, + { + "text": "w&Used(9)from$17200", + "boxes": [ + 1410, + 448, + 1590, + 471 + ] + }, + { + "text": "This product is available as Renewed.", + "boxes": [ + 642, + 502, + 962, + 532 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1810, + 506, + 1864, + 526 + ] + }, + { + "text": "JintendoSwitchTMwithNeonBlue", + "boxes": [ + 732, + 534, + 940, + 552 + ] + }, + { + "text": "$255.99 + $7.41 shipping", + "boxes": [ + 726, + 546, + 892, + 576 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1792, + 548, + 1862, + 574 + ] + }, + { + "text": "", + "boxes": [ + 676, + 560, + 700, + 574 + ] + }, + { + "text": "★★★★☆(449)", + "boxes": [ + 726, + 568, + 844, + 592 + ] + }, + { + "text": "Works and looks like new and backed by the Ar", + "boxes": [ + 724, + 590, + 1022, + 613 + ] + }, + { + "text": "Unwell Hydration from Alex Cooper", + "boxes": [ + 744, + 647, + 978, + 671 + ] + }, + { + "text": "Hydrate & focus", + "boxes": [ + 744, + 667, + 846, + 691 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 721, + 1862, + 739 + ] + }, + { + "text": " Products related to this item", + "boxes": [ + 150, + 764, + 434, + 804 + ] + }, + { + "text": "立", + "boxes": [ + 1793, + 767, + 1819, + 789 + ] + }, + { + "text": "1", + "boxes": [ + 1818, + 768, + 1868, + 786 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1814, + 925, + 1860, + 945 + ] + }, + { + "text": "1", + "boxes": [ + 1790, + 971, + 1872, + 997 + ] + }, + { + "text": "SwitchControllers", + "boxes": [ + 234, + 993, + 348, + 1011 + ] + }, + { + "text": "AceGamer [Pro ", + "boxes": [ + 422, + 987, + 534, + 1016 + ] + }, + { + "text": "COIORVIS Joy Pad", + "boxes": [ + 616, + 985, + 738, + 1014 + ] + }, + { + "text": "Switch Controller", + "boxes": [ + 808, + 984, + 924, + 1015 + ] + }, + { + "text": "Orzly Carry Case", + "boxes": [ + 1004, + 989, + 1112, + 1016 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 1196, + 991, + 1320, + 1014 + ] + }, + { + "text": "FUNLAB Switch ", + "boxes": [ + 1390, + 989, + 1492, + 1012 + ] + }, + { + "text": "apatible with", + "boxes": [ + 258, + 1009, + 340, + 1032 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 422, + 1007, + 548, + 1036 + ] + }, + { + "text": "Controller for ", + "boxes": [ + 618, + 1009, + 710, + 1034 + ] + }, + { + "text": "Charging Dock for ", + "boxes": [ + 808, + 1007, + 930, + 1036 + ] + }, + { + "text": "Compatible with", + "boxes": [ + 1006, + 1009, + 1114, + 1036 + ] + }, + { + "text": "Controller for", + "boxes": [ + 1198, + 1009, + 1288, + 1034 + ] + }, + { + "text": "Handheld Controller for", + "boxes": [ + 1388, + 1007, + 1544, + 1036 + ] + } + ] + }, + { + "frame_id": "70_4_7", + "episode_id": 70, + "env_id": 4, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.31998062133789, + "advantages": -0.2251034677028656, + "generated_text": "{\n \"intent\": \"The current screen shows a browser page about Firefox desktop version. To continue exploring, accessing more information or features might involve interacting further with the page. Double-clicking 'Download Firefox' could lead to more detailed exploration after the download or to additional browser features.\",\n \"action\": \"DoubleClick(315, 735)\"\n}", + "action_string": "DoubleClick(315, 735)", + "intent_string": "The current screen shows a browser page about Firefox desktop version. To continue exploring, accessing more information or features might involve interacting further with the page. Double-clicking 'Download Firefox' could lead to more detailed exploration after the download or to additional browser features.", + "before_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "ns and other languages", + "boxes": [ + 440, + 367, + 576, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 492, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with the new", + "boxes": [ + 800, + 917, + 1098, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 424, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1564, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "is and other languages", + "boxes": [ + 442, + 371, + 576, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 492, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with thenew", + "boxes": [ + 802, + 919, + 1100, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 422, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ] + }, + { + "frame_id": "70_5_7", + "episode_id": 70, + "env_id": 5, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.126825332641602, + "advantages": 0.2406373769044876, + "generated_text": "{\n \"intent\": \"To continue exploring different aspects of the Apple Watch Ultra 2, focusing on the 'Overview' section.\",\n \"action\": \"DoubleClick(1147, 163)\"\n}", + "action_string": "DoubleClick(1147, 163)", + "intent_string": "To continue exploring different aspects of the Apple Watch Ultra 2, focusing on the 'Overview' section.", + "before_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 8, + 10, + 20, + 22 + ] + }, + { + "text": " Apple Watch Ultra 2 - Ap × |+", + "boxes": [ + 6, + 28, + 246, + 54 + ] + }, + { + "text": "←→", + "boxes": [ + 4, + 62, + 132, + 93 + ] + }, + { + "text": "目·", + "boxes": [ + 1538, + 64, + 1664, + 93 + ] + }, + { + "text": "I四@=", + "boxes": [ + 1780, + 59, + 1912, + 96 + ] + }, + { + "text": "Store", + "boxes": [ + 512, + 111, + 538, + 125 + ] + }, + { + "text": "Apple Watch Ultra 2", + "boxes": [ + 460, + 153, + 656, + 187 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 872, + 266, + 1032, + 314 + ] + }, + { + "text": "ULTRA 2", + "boxes": [ + 914, + 308, + 994, + 339 + ] + }, + { + "text": "10:09-30", + "boxes": [ + 688, + 423, + 1232, + 613 + ] + }, + { + "text": "", + "boxes": [ + 867, + 578, + 1039, + 647 + ] + }, + { + "text": "58）", + "boxes": [ + 724, + 600, + 854, + 637 + ] + }, + { + "text": "65”", + "boxes": [ + 1076, + 594, + 1184, + 621 + ] + }, + { + "text": "98FT", + "boxes": [ + 986, + 657, + 1082, + 675 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 218, + 1050, + 302, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 8, + 10, + 20, + 22 + ] + }, + { + "text": " Apple Watch Ultra 2 - Ap × |+", + "boxes": [ + 6, + 28, + 246, + 54 + ] + }, + { + "text": "←→", + "boxes": [ + 4, + 62, + 132, + 93 + ] + }, + { + "text": "目·", + "boxes": [ + 1538, + 64, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1882, + 91 + ] + }, + { + "text": "Store", + "boxes": [ + 514, + 111, + 538, + 125 + ] + }, + { + "text": " Apple Watch Ultra 2", + "boxes": [ + 458, + 153, + 656, + 187 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 872, + 266, + 1032, + 314 + ] + }, + { + "text": "ULTRA 2", + "boxes": [ + 914, + 308, + 994, + 339 + ] + }, + { + "text": "10:09:30", + "boxes": [ + 693, + 432, + 1227, + 604 + ] + }, + { + "text": "58", + "boxes": [ + 734, + 602, + 848, + 633 + ] + }, + { + "text": "65", + "boxes": [ + 818, + 659, + 878, + 677 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 218, + 1050, + 302, + 1068 + ] + } + ] + }, + { + "frame_id": "70_6_7", + "episode_id": 70, + "env_id": 6, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.642447471618652, + "advantages": 0.5382736921310425, + "generated_text": "{\n \"intent\": \"Since this browser window opens new pages easily, clicking any site icon (like Bing or Amazon) would allow us to visit different websites and explore diverse content.\",\n \"action\": \"DoubleClick(673, 505)\"\n}", + "action_string": "DoubleClick(673, 505)", + "intent_string": "Since this browser window opens new pages easily, clicking any site icon (like Bing or Amazon) would allow us to visit different websites and explore diverse content.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 64, + 520, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "ne 16 Pro an", + "boxes": [ + 94, + 32, + 172, + 50 + ] + }, + { + "text": "Search -Microsoft Bing x", + "boxes": [ + 254, + 32, + 444, + 50 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "| https://cn.bing.com", + "boxes": [ + 250, + 64, + 448, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 68, + 1658, + 87 + ] + }, + { + "text": "三四", + "boxes": [ + 1788, + 68, + 1852, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "Translat", + "boxes": [ + 408, + 123, + 452, + 141 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 490, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 800, + 448, + 968, + 477 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 274, + 750 + ] + }, + { + "text": "点击关注", + "boxes": [ + 690, + 750, + 796, + 796 + ] + }, + { + "text": "全球年轻人更理想的家", + "boxes": [ + 1202, + 776, + 1350, + 806 + ] + }, + { + "text": "REALEGIAI", + "boxes": [ + 708, + 788, + 772, + 806 + ] + }, + { + "text": "现已开售", + "boxes": [ + 1278, + 796, + 1324, + 814 + ] + }, + { + "text": "做打事", + "boxes": [ + 914, + 818, + 1018, + 856 + ] + }, + { + "text": "2.2813.2814.28", + "boxes": [ + 1196, + 828, + 1292, + 846 + ] + }, + { + "text": "12.2813.28", + "boxes": [ + 1290, + 828, + 1356, + 846 + ] + }, + { + "text": "被抓包的猫咪", + "boxes": [ + 884, + 846, + 1048, + 883 + ] + }, + { + "text": "选车号·18小时", + "boxes": [ + 1144, + 921, + 1262, + 945 + ] + }, + { + "text": "D", + "boxes": [ + 828, + 933, + 852, + 955 + ] + }, + { + "text": "一点视频/萌宠旅行者·3周", + "boxes": [ + 846, + 931, + 1012, + 961 + ] + }, + { + "text": "激光雷达+800V下放14万级", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "一点视频/萌宠的日", + "boxes": [ + 226, + 959, + 334, + 977 + ] + }, + { + "text": "很显然，猫咪也知道自己在使", + "boxes": [ + 824, + 953, + 1098, + 989 + ] + }, + { + "text": "市场\\\"六边形战士\\\"2026款", + "boxes": [ + 1136, + 967, + 1408, + 1003 + ] + }, + { + "text": "猫咪打针名场面，哪只反应最夸张？", + "boxes": [ + 198, + 981, + 668, + 1022 + ] + }, + { + "text": "坏！", + "boxes": [ + 826, + 983, + 870, + 1011 + ] + }, + { + "text": "零跑C10进阶上市", + "boxes": [ + 1138, + 997, + 1308, + 1026 + ] + }, + { + "text": "3132", + "boxes": [ + 214, + 1028, + 250, + 1042 + ] + }, + { + "text": "AP", + "boxes": [ + 290, + 1022, + 342, + 1040 + ] + }, + { + "text": "10", + "boxes": [ + 832, + 1028, + 864, + 1042 + ] + } + ] + }, + { + "frame_id": "70_7_7", + "episode_id": 70, + "env_id": 7, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.099394798278809, + "advantages": -0.35243383049964905, + "generated_text": "{\n \"intent\": \"In this step, I will check for available files or shortcuts on the desktop by interacting with an icon, which might lead to more exploration opportunities.\",\n \"action\": \"DoubleClick(500, 300)\"\n}", + "action_string": "DoubleClick(500, 300)", + "intent_string": "In this step, I will check for available files or shortcuts on the desktop by interacting with an icon, which might lead to more exploration opportunities.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - M..", + "boxes": [ + 366, + 1046, + 466, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit Search View Document Help", + "boxes": [ + 4, + 22, + 312, + 52 + ] + }, + { + "text": "Untitled 1 - M..", + "boxes": [ + 358, + 1046, + 466, + 1070 + ] + } + ] + }, + { + "frame_id": "70_0_8", + "episode_id": 70, + "env_id": 0, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.079365730285645, + "advantages": -0.36399537324905396, + "generated_text": "{\n \"intent\": \"To continue exploring the web page and gather more information about the topic being discussed.\",\n \"action\": \"ScrollUp(960, 540)\"\n}", + "action_string": "ScrollUp(960, 540)", + "intent_string": "To continue exploring the web page and gather more information about the topic being discussed.", + "before_screen_ocr_results": [ + { + "text": "", + "boxes": [ + 54, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 564, + 117, + 638, + 149 + ] + }, + { + "text": "msn", + "boxes": [ + 58, + 129, + 104, + 149 + ] + }, + { + "text": "中", + "boxes": [ + 1762, + 121, + 1788, + 143 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "资讯 娱乐 时尚 教育", + "boxes": [ + 344, + 175, + 542, + 204 + ] + }, + { + "text": "科技萌宠汽车", + "boxes": [ + 532, + 175, + 694, + 204 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1606, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "精确分析", + "boxes": [ + 1440, + 248, + 1590, + 304 + ] + }, + { + "text": "ITBEAR科技资讯+ 关注", + "boxes": [ + 382, + 264, + 632, + 306 + ] + }, + { + "text": "专", + "boxes": [ + 1426, + 260, + 1442, + 274 + ] + }, + { + "text": "60.9K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "绘制图表", + "boxes": [ + 1442, + 290, + 1584, + 334 + ] + }, + { + "text": "助您高效完成数据分析", + "boxes": [ + 1454, + 334, + 1576, + 351 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，能否引领新能源市场新潮流？", + "boxes": [ + 362, + 347, + 1174, + 389 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1456, + 351, + 1506, + 369 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 393, + 218, + 425 + ] + }, + { + "text": "k王者荣耀的报道·15小时·① 1分钟读取", + "boxes": [ + 362, + 411, + 708, + 451 + ] + }, + { + "text": "让你成为Excel达人，工作学习效", + "boxes": [ + 1422, + 411, + 1702, + 447 + ] + }, + { + "text": "率倍增", + "boxes": [ + 1426, + 437, + 1490, + 469 + ] + }, + { + "text": "B", + "boxes": [ + 192, + 453, + 214, + 476 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 469, + 1530, + 492 + ] + }, + { + "text": "在汽车市场风起云涌的今天，哈弗汽车，这个长久以来以\"性价比\"著称的品牌，正积极布局新能源领域，意图在新的", + "boxes": [ + 364, + 504, + 1232, + 534 + ] + }, + { + "text": "赛道上续写辉煌。近日，哈弗汽车两款全新车型——哈弗枭龙与哈弗枭龙MAX正式上市，标志着哈弗正式进军新能源", + "boxes": [ + 366, + 530, + 1240, + 560 + ] + }, + { + "text": "汽车市场的决心。", + "boxes": [ + 364, + 554, + 494, + 588 + ] + }, + { + "text": "访问ITBEAR科技资讯", + "boxes": [ + 1426, + 554, + 1596, + 584 + ] + }, + { + "text": "G", + "boxes": [ + 1673, + 550, + 1702, + 579 + ] + }, + { + "text": "哈弗枭龙MAX与哈弗枭龙，作为哈弗品牌的全新力作，不仅继承了哈弗汽车一贯的高性价比优势，更在技术层面实现", + "boxes": [ + 364, + 600, + 1240, + 629 + ] + }, + { + "text": "新疆多地夜空现神秘飞行物，目击", + "boxes": [ + 1444, + 597, + 1666, + 634 + ] + }, + { + "text": "了突破。哈弗枭龙MAX搭载了长城汽车全新的智能四驱电混技术Hi4，推出了三款车型，价格区间为15.98-17.98万", + "boxes": [ + 366, + 625, + 1214, + 655 + ] + }, + { + "text": "者：亮如月亮，几秒加速消失", + "boxes": [ + 1446, + 621, + 1634, + 651 + ] + }, + { + "text": "元。而哈弗枭龙则采用了智能混动DHT技术，同样推出了三款车型，价格区间则为13.98-15.68万元。两款车型均展现", + "boxes": [ + 364, + 651, + 1240, + 681 + ] + }, + { + "text": "小米YU7即将发布，卢伟冰自曝已", + "boxes": [ + 1446, + 663, + 1664, + 693 + ] + }, + { + "text": "出了哈弗在新能源汽车领域的深厚底蕴和技术实力。", + "boxes": [ + 364, + 677, + 750, + 707 + ] + }, + { + "text": "购车并计划添购新车", + "boxes": [ + 1446, + 685, + 1580, + 715 + ] + }, + { + "text": "长安汽车泰国投产，深蓝CEO：增", + "boxes": [ + 1446, + 729, + 1664, + 758 + ] + }, + { + "text": "程车油耗媲美日系混动，驾驶体..", + "boxes": [ + 1446, + 748, + 1660, + 778 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 832, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 878, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 796, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置\"改变您的选择。欲了解", + "boxes": [ + 30, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：微软隐私声明", + "boxes": [ + 32, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "", + "boxes": [ + 54, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 564, + 117, + 638, + 149 + ] + }, + { + "text": "msn", + "boxes": [ + 58, + 129, + 104, + 149 + ] + }, + { + "text": "Q", + "boxes": [ + 1308, + 123, + 1326, + 139 + ] + }, + { + "text": "中", + "boxes": [ + 1762, + 121, + 1788, + 143 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "资讯 娱乐 时尚 教育", + "boxes": [ + 344, + 175, + 542, + 204 + ] + }, + { + "text": "科技萌宠汽车", + "boxes": [ + 532, + 175, + 694, + 204 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1606, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "精确分析", + "boxes": [ + 1440, + 248, + 1590, + 304 + ] + }, + { + "text": "ITBEAR科技资讯+ 关注", + "boxes": [ + 382, + 264, + 632, + 306 + ] + }, + { + "text": "专", + "boxes": [ + 1426, + 260, + 1442, + 274 + ] + }, + { + "text": "60.9K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "绘制图表", + "boxes": [ + 1442, + 290, + 1584, + 334 + ] + }, + { + "text": "助您高效完成数据分析", + "boxes": [ + 1454, + 334, + 1576, + 351 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，能否引领新能源市场新潮流？", + "boxes": [ + 362, + 347, + 1174, + 389 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1456, + 351, + 1506, + 369 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 393, + 218, + 425 + ] + }, + { + "text": "k王者荣耀的报道·15小时·①1分钟读取", + "boxes": [ + 362, + 413, + 706, + 449 + ] + }, + { + "text": "让你成为 Excel 达人，工作学习效", + "boxes": [ + 1422, + 411, + 1702, + 447 + ] + }, + { + "text": "率倍增", + "boxes": [ + 1426, + 437, + 1490, + 469 + ] + }, + { + "text": "B", + "boxes": [ + 192, + 453, + 214, + 476 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 469, + 1530, + 492 + ] + }, + { + "text": "在汽车市场风起云涌的今天，哈弗汽车，这个长久以来以\"性价比\"著称的品牌，正积极布局新能源领域，意图在新的", + "boxes": [ + 364, + 504, + 1232, + 534 + ] + }, + { + "text": "赛道上续写辉煌。近日，哈弗汽车两款全新车型——哈弗枭龙与哈弗枭龙MAX正式上市，标志着哈弗正式进军新能源", + "boxes": [ + 366, + 530, + 1238, + 560 + ] + }, + { + "text": "汽车市场的决心。", + "boxes": [ + 364, + 554, + 494, + 588 + ] + }, + { + "text": "访问ITBEAR科技资讯", + "boxes": [ + 1426, + 554, + 1596, + 584 + ] + }, + { + "text": "G", + "boxes": [ + 1673, + 550, + 1702, + 579 + ] + }, + { + "text": "哈弗枭龙MAX与哈弗枭龙，作为哈弗品牌的全新力作，不仅继承了哈弗汽车一贯的高性价比优势，更在技术层面实现", + "boxes": [ + 364, + 600, + 1240, + 629 + ] + }, + { + "text": "新疆多地夜空现神秘飞行物，目击", + "boxes": [ + 1444, + 597, + 1666, + 634 + ] + }, + { + "text": "了突破。哈弗枭龙MAX搭载了长城汽车全新的智能四驱电混技术Hi4，推出了三款车型，价格区间为15.98-17.98万", + "boxes": [ + 366, + 625, + 1214, + 655 + ] + }, + { + "text": "者：亮如月亮，几秒加速消失", + "boxes": [ + 1446, + 621, + 1634, + 651 + ] + }, + { + "text": "元。而哈弗枭龙则采用了智能混动DHT技术，同样推出了三款车型，价格区间则为13.98-15.68万元。两款车型均展现", + "boxes": [ + 364, + 651, + 1240, + 681 + ] + }, + { + "text": "小米YU7即将发布，卢伟冰自曝已", + "boxes": [ + 1446, + 663, + 1664, + 693 + ] + }, + { + "text": "出了哈弗在新能源汽车领域的深厚底蕴和技术实力。", + "boxes": [ + 364, + 677, + 750, + 707 + ] + }, + { + "text": "购车并计划添购新车", + "boxes": [ + 1446, + 685, + 1580, + 715 + ] + }, + { + "text": "长安汽车泰国投产，深蓝CEO：增", + "boxes": [ + 1446, + 729, + 1664, + 758 + ] + }, + { + "text": "程车油耗媲美日系混动，驾驶体..", + "boxes": [ + 1446, + 748, + 1660, + 778 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 832, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 878, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 796, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置\"改变您的选择。欲了解", + "boxes": [ + 30, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：微软隐私声明", + "boxes": [ + 32, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + } + ] + }, + { + "frame_id": "70_1_8", + "episode_id": 70, + "env_id": 1, + "step": 8, + "format_rewards": 1.0, + "rewards": 8.615288734436035, + "advantages": -0.6318778991699219, + "generated_text": "{\n \"intent\": \"To continue exploring other features or applications on the computer by selecting an icon from the desktop.\",\n \"action\": \"DoubleClick(150, 150)\"\n}", + "action_string": "DoubleClick(150, 150)", + "intent_string": "To continue exploring other features or applications on the computer by selecting an icon from the desktop.", + "before_screen_ocr_results": [ + { + "text": "Untitled 1 - Libreoffice Calc", + "boxes": [ + 848, + 2, + 1070, + 26 + ] + }, + { + "text": "--- - α", + "boxes": [ + 8, + 48, + 446, + 87 + ] + }, + { + "text": "G-.", + "boxes": [ + 450, + 56, + 546, + 79 + ] + }, + { + "text": "里·听个", + "boxes": [ + 544, + 54, + 848, + 83 + ] + }, + { + "text": "M-E-EB", + "boxes": [ + 850, + 48, + 1264, + 87 + ] + }, + { + "text": "10", + "boxes": [ + 156, + 97, + 188, + 117 + ] + }, + { + "text": "aaaa--==", + "boxes": [ + 236, + 93, + 520, + 123 + ] + }, + { + "text": "", + "boxes": [ + 550, + 101, + 574, + 115 + ] + }, + { + "text": "□B8", + "boxes": [ + 616, + 95, + 720, + 119 + ] + }, + { + "text": "-% 00 ", + "boxes": [ + 724, + 93, + 874, + 123 + ] + }, + { + "text": "十三至田二口目", + "boxes": [ + 878, + 93, + 1198, + 123 + ] + }, + { + "text": "游×", + "boxes": [ + 156, + 129, + 238, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Print", + "boxes": [ + 938, + 262, + 990, + 288 + ] + }, + { + "text": "□□", + "boxes": [ + 1288, + 262, + 1350, + 288 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 266, + 1914, + 314 + ] + }, + { + "text": "8.26 in (A4)", + "boxes": [ + 724, + 290, + 808, + 316 + ] + }, + { + "text": " General ", + "boxes": [ + 950, + 296, + 1014, + 322 + ] + }, + { + "text": "Libreoffice Calc", + "boxes": [ + 1010, + 293, + 1128, + 322 + ] + }, + { + "text": "成", + "boxes": [ + 1890, + 312, + 1910, + 339 + ] + }, + { + "text": "10", + "boxes": [ + 14, + 330, + 30, + 343 + ] + }, + { + "text": "Printer", + "boxes": [ + 948, + 324, + 1010, + 349 + ] + }, + { + "text": " Generic Printer", + "boxes": [ + 964, + 351, + 1074, + 375 + ] + }, + { + "text": "Status: Default printer", + "boxes": [ + 960, + 385, + 1108, + 409 + ] + }, + { + "text": "O All Pages", + "boxes": [ + 964, + 449, + 1044, + 472 + ] + }, + { + "text": ".8", + "boxes": [ + 18, + 467, + 30, + 478 + ] + }, + { + "text": "O Pages:", + "boxes": [ + 968, + 478, + 1038, + 500 + ] + }, + { + "text": "O Even pages", + "boxes": [ + 965, + 502, + 1061, + 533 + ] + }, + { + "text": "O odd pages", + "boxes": [ + 965, + 528, + 1057, + 558 + ] + }, + { + "text": "more", + "boxes": [ + 968, + 560, + 1016, + 580 + ] + }, + { + "text": "Page Layout", + "boxes": [ + 952, + 588, + 1044, + 611 + ] + }, + { + "text": "Paper size:", + "boxes": [ + 966, + 617, + 1036, + 635 + ] + }, + { + "text": "A4 8.26in x 11.69in", + "boxes": [ + 1048, + 613, + 1182, + 637 + ] + }, + { + "text": "Automatic", + "boxes": [ + 1051, + 647, + 1125, + 670 + ] + }, + { + "text": "Preview", + "boxes": [ + 592, + 754, + 658, + 772 + ] + }, + { + "text": "?", + "boxes": [ + 744, + 756, + 770, + 770 + ] + }, + { + "text": "/10", + "boxes": [ + 818, + 754, + 882, + 774 + ] + }, + { + "text": "Help", + "boxes": [ + 604, + 796, + 644, + 812 + ] + }, + { + "text": "More Option", + "boxes": [ + 1038, + 796, + 1120, + 814 + ] + }, + { + "text": "OK", + "boxes": [ + 1288, + 796, + 1316, + 810 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 939 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "国", + "boxes": [ + 8, + 4, + 24, + 24 + ] + }, + { + "text": "Untitled 1 - Libreoffice Calc", + "boxes": [ + 848, + 2, + 1070, + 26 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 28, + 34, + 42 + ] + }, + { + "text": "--目-@电自-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "Q", + "boxes": [ + 376, + 58, + 442, + 83 + ] + }, + { + "text": "G-.", + "boxes": [ + 450, + 56, + 546, + 79 + ] + }, + { + "text": "里听个", + "boxes": [ + 544, + 54, + 848, + 83 + ] + }, + { + "text": "M-xE-EB", + "boxes": [ + 850, + 48, + 1264, + 87 + ] + }, + { + "text": "10", + "boxes": [ + 154, + 97, + 188, + 117 + ] + }, + { + "text": "aaa.", + "boxes": [ + 236, + 93, + 350, + 123 + ] + }, + { + "text": "三", + "boxes": [ + 512, + 101, + 536, + 115 + ] + }, + { + "text": "", + "boxes": [ + 552, + 101, + 574, + 115 + ] + }, + { + "text": "□B8", + "boxes": [ + 616, + 95, + 720, + 119 + ] + }, + { + "text": ". % 0.0 ", + "boxes": [ + 724, + 93, + 874, + 123 + ] + }, + { + "text": "十三三田二口目", + "boxes": [ + 878, + 93, + 1198, + 123 + ] + }, + { + "text": "浆×", + "boxes": [ + 160, + 129, + 240, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Print", + "boxes": [ + 938, + 262, + 990, + 288 + ] + }, + { + "text": "□□", + "boxes": [ + 1288, + 262, + 1350, + 288 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 266, + 1914, + 314 + ] + }, + { + "text": "8.26 in (A4)", + "boxes": [ + 724, + 290, + 808, + 316 + ] + }, + { + "text": " General ", + "boxes": [ + 950, + 296, + 1014, + 322 + ] + }, + { + "text": "LibreOffice Calc", + "boxes": [ + 1010, + 293, + 1128, + 322 + ] + }, + { + "text": "成", + "boxes": [ + 1890, + 312, + 1910, + 339 + ] + }, + { + "text": "Printer", + "boxes": [ + 948, + 324, + 1010, + 349 + ] + }, + { + "text": " Generic Printer", + "boxes": [ + 964, + 351, + 1074, + 375 + ] + }, + { + "text": "Status: Default printer", + "boxes": [ + 960, + 385, + 1108, + 409 + ] + }, + { + "text": "O All Pages", + "boxes": [ + 964, + 449, + 1044, + 472 + ] + }, + { + "text": ".8", + "boxes": [ + 18, + 467, + 30, + 478 + ] + }, + { + "text": "O Pages:", + "boxes": [ + 968, + 478, + 1038, + 500 + ] + }, + { + "text": "O Even pages", + "boxes": [ + 965, + 502, + 1061, + 533 + ] + }, + { + "text": "O odd pages", + "boxes": [ + 965, + 528, + 1057, + 558 + ] + }, + { + "text": "more", + "boxes": [ + 968, + 560, + 1016, + 580 + ] + }, + { + "text": "Page Layout", + "boxes": [ + 952, + 588, + 1044, + 611 + ] + }, + { + "text": "Paper size:", + "boxes": [ + 966, + 617, + 1036, + 635 + ] + }, + { + "text": "A4 8.26in x 11.69in", + "boxes": [ + 1048, + 613, + 1182, + 637 + ] + }, + { + "text": "Automatic", + "boxes": [ + 1051, + 647, + 1125, + 670 + ] + }, + { + "text": "Preview", + "boxes": [ + 592, + 754, + 658, + 772 + ] + }, + { + "text": "?", + "boxes": [ + 744, + 756, + 770, + 770 + ] + }, + { + "text": "/10", + "boxes": [ + 818, + 754, + 882, + 774 + ] + }, + { + "text": "?Help", + "boxes": [ + 599, + 789, + 645, + 815 + ] + }, + { + "text": "More Option", + "boxes": [ + 1038, + 796, + 1120, + 814 + ] + }, + { + "text": "OK", + "boxes": [ + 1288, + 796, + 1316, + 810 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 939 + ] + } + ] + }, + { + "frame_id": "70_2_8", + "episode_id": 70, + "env_id": 2, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.675932884216309, + "advantages": -0.019634554162621498, + "generated_text": "{\n \"intent\": \"Scrolling down will allow me to explore more products or sections on the Apple Store webpage.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "Scrolling down will allow me to explore more products or sections on the Apple Store webpage.", + "before_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 132, + 220, + 768, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1632, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "?", + "boxes": [ + 1592, + 294, + 1614, + 318 + ] + }, + { + "text": "/isit anAppleStore", + "boxes": [ + 1632, + 290, + 1756, + 308 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 378, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 138, + 641, + 720, + 671 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1846, + 748, + 1914, + 788 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1426, + 822, + 1582, + 852 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 826, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Apple Watch", + "boxes": [ + 604, + 232, + 690, + 256 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 228, + 854, + 260 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 228, + 968, + 260 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 230, + 1110, + 258 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 228, + 1278, + 260 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 230, + 1414, + 256 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 230, + 1566, + 256 + ] + }, + { + "text": " Apple Gift card ", + "boxes": [ + 1612, + 226, + 1724, + 260 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 136, + 353, + 722, + 389 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 164, + 465, + 334, + 500 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 465, + 768, + 502 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 463, + 1112, + 500 + ] + }, + { + "text": "Apple Watch Pride Edition ", + "boxes": [ + 1426, + 463, + 1762, + 504 + ] + }, + { + "text": "Appl", + "boxes": [ + 1846, + 465, + 1914, + 504 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 504, + 326, + 534 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 504, + 740, + 534 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 504, + 1164, + 534 + ] + }, + { + "text": " Sport Band", + "boxes": [ + 1422, + 492, + 1578, + 534 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1848, + 506, + 1910, + 532 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 166, + 528, + 412, + 556 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 584, + 532, + 832, + 562 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1010, + 534, + 1246, + 552 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1424, + 534, + 1582, + 570 + ] + }, + { + "text": "From $35", + "boxes": [ + 1846, + 530, + 1912, + 562 + ] + }, + { + "text": "$49", + "boxes": [ + 1426, + 564, + 1462, + 592 + ] + }, + { + "text": "PIRO", + "boxes": [ + 633, + 670, + 883, + 793 + ] + }, + { + "text": "Help is here. Whenever and however you need it.", + "boxes": [ + 136, + 979, + 742, + 1014 + ] + } + ] + }, + { + "frame_id": "70_3_8", + "episode_id": 70, + "env_id": 3, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.003673553466797, + "advantages": 0.1695495843887329, + "generated_text": "{\n \"intent\": \"Interacting with the page by clicking on elements such as product details or adding items to cart could provide more information about the product. Engaging with these actions might also add items worth considering during the shopping session.\",\n \"action\": \"DoubleClick(588, 530)\"\n}", + "action_string": "DoubleClick(588, 530)", + "intent_string": "Interacting with the page by clicking on elements such as product details or adding items to cart could provide more information about the product. Engaging with these actions might also add items worth considering during the shopping session.", + "before_screen_ocr_results": [ + { + "text": "30BFJWCYTL/?_encoding=UTF8&pd_rd_w=kzgNd&content-", + "boxes": [ + 866, + 69, + 1276, + 87 + ] + }, + { + "text": "·☆", + "boxes": [ + 1580, + 66, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " Deliver to Canada", + "boxes": [ + 1396, + 97, + 1516, + 121 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "T", + "boxes": [ + 590, + 121, + 604, + 141 + ] + }, + { + "text": "$27999", + "boxes": [ + 642, + 125, + 724, + 163 + ] + }, + { + "text": "Used: Very Good | Detalls", + "boxes": [ + 1394, + 127, + 1562, + 157 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1804, + 125, + 1884, + 151 + ] + }, + { + "text": "Sold by Toys and Video Games", + "boxes": [ + 1392, + 147, + 1592, + 177 + ] + }, + { + "text": "$192.10 Shipping", + "boxes": [ + 644, + 164, + 754, + 193 + ] + }, + { + "text": "ada Details", + "boxes": [ + 918, + 171, + 1004, + 189 + ] + }, + { + "text": "Store", + "boxes": [ + 1398, + 171, + 1434, + 191 + ] + }, + { + "text": "Fulflled by Amazon ", + "boxes": [ + 1396, + 191, + 1526, + 214 + ] + }, + { + "text": "Style: Console", + "boxes": [ + 644, + 238, + 740, + 264 + ] + }, + { + "text": "Add to Cart ", + "boxes": [ + 1460, + 240, + 1542, + 272 + ] + }, + { + "text": "Color: Black", + "boxes": [ + 644, + 266, + 726, + 290 + ] + }, + { + "text": " About this item", + "boxes": [ + 640, + 307, + 798, + 344 + ] + }, + { + "text": "·6.2\" LCD screen", + "boxes": [ + 652, + 341, + 762, + 365 + ] + }, + { + "text": "Add to List", + "boxes": [ + 1410, + 332, + 1472, + 351 + ] + }, + { + "text": "Other sellers on Amazor", + "boxes": [ + 1388, + 401, + 1572, + 425 + ] + }, + { + "text": "w&Used(9)from$17200", + "boxes": [ + 1410, + 448, + 1590, + 471 + ] + }, + { + "text": "This product is available as Renewed.", + "boxes": [ + 642, + 502, + 962, + 532 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1810, + 506, + 1864, + 526 + ] + }, + { + "text": "JintendoSwitchTMwithNeonBlue", + "boxes": [ + 732, + 534, + 940, + 552 + ] + }, + { + "text": "$255.99 + $7.41 shipping", + "boxes": [ + 726, + 546, + 892, + 576 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1792, + 548, + 1862, + 574 + ] + }, + { + "text": "", + "boxes": [ + 676, + 560, + 700, + 574 + ] + }, + { + "text": "★★★★☆(449)", + "boxes": [ + 726, + 568, + 844, + 592 + ] + }, + { + "text": "Works and looks like new and backed by the Ar", + "boxes": [ + 724, + 590, + 1022, + 613 + ] + }, + { + "text": "Unwell Hydration from Alex Cooper", + "boxes": [ + 744, + 647, + 978, + 671 + ] + }, + { + "text": "Hydrate & focus", + "boxes": [ + 744, + 667, + 846, + 691 + ] + }, + { + "text": "$14.45", + "boxes": [ + 1810, + 721, + 1862, + 739 + ] + }, + { + "text": " Products related to this item", + "boxes": [ + 150, + 764, + 434, + 804 + ] + }, + { + "text": "立", + "boxes": [ + 1793, + 767, + 1819, + 789 + ] + }, + { + "text": "1", + "boxes": [ + 1818, + 768, + 1868, + 786 + ] + }, + { + "text": "$79.00", + "boxes": [ + 1814, + 925, + 1860, + 945 + ] + }, + { + "text": "1", + "boxes": [ + 1790, + 971, + 1872, + 997 + ] + }, + { + "text": "SwitchControllers", + "boxes": [ + 234, + 993, + 348, + 1011 + ] + }, + { + "text": "AceGamer [Pro ", + "boxes": [ + 422, + 987, + 534, + 1016 + ] + }, + { + "text": "COIORVIS Joy Pad", + "boxes": [ + 616, + 985, + 738, + 1014 + ] + }, + { + "text": "Switch Controller", + "boxes": [ + 808, + 984, + 924, + 1015 + ] + }, + { + "text": "Orzly Carry Case", + "boxes": [ + 1004, + 989, + 1112, + 1016 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 1196, + 991, + 1320, + 1014 + ] + }, + { + "text": "FUNLAB Switch ", + "boxes": [ + 1390, + 989, + 1492, + 1012 + ] + }, + { + "text": "apatible with", + "boxes": [ + 258, + 1009, + 340, + 1032 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 422, + 1007, + 548, + 1036 + ] + }, + { + "text": "Controller for ", + "boxes": [ + 618, + 1009, + 710, + 1034 + ] + }, + { + "text": "Charging Dock for ", + "boxes": [ + 808, + 1007, + 930, + 1036 + ] + }, + { + "text": "Compatible with", + "boxes": [ + 1006, + 1009, + 1114, + 1036 + ] + }, + { + "text": "Controller for", + "boxes": [ + 1198, + 1009, + 1288, + 1034 + ] + }, + { + "text": "Handheld Controller for", + "boxes": [ + 1388, + 1007, + 1544, + 1036 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→e", + "boxes": [ + 22, + 66, + 126, + 89 + ] + }, + { + "text": "n.0f9e35ef-aec2-4925-899e-", + "boxes": [ + 1374, + 69, + 1572, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1576, + 66, + 1662, + 91 + ] + }, + { + "text": "I口三", + "boxes": [ + 1782, + 64, + 1914, + 93 + ] + }, + { + "text": " Deliver to Canada", + "boxes": [ + 1394, + 97, + 1516, + 121 + ] + }, + { + "text": "V", + "boxes": [ + 580, + 113, + 614, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1822, + 105, + 1886, + 131 + ] + }, + { + "text": "$27999", + "boxes": [ + 639, + 120, + 725, + 166 + ] + }, + { + "text": "Used: Very Good | Detalls", + "boxes": [ + 1394, + 127, + 1562, + 157 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1816, + 123, + 1894, + 151 + ] + }, + { + "text": " Sold by Toys and Video Games", + "boxes": [ + 1390, + 145, + 1592, + 177 + ] + }, + { + "text": "Nintendo Switch TM with Neon Blue", + "boxes": [ + 1538, + 236, + 1806, + 266 + ] + }, + { + "text": "and Neon Red Joy-Con TM", + "boxes": [ + 1538, + 256, + 1744, + 286 + ] + }, + { + "text": "Style:Console", + "boxes": [ + 1540, + 288, + 1620, + 314 + ] + }, + { + "text": "Color:Black", + "boxes": [ + 1542, + 322, + 1602, + 341 + ] + }, + { + "text": "NINTENDO", + "boxes": [ + 619, + 460, + 775, + 505 + ] + }, + { + "text": "SWITCH.", + "boxes": [ + 617, + 482, + 779, + 533 + ] + }, + { + "text": "CD", + "boxes": [ + 1078, + 625, + 1132, + 685 + ] + }, + { + "text": "®", + "boxes": [ + 1182, + 643, + 1196, + 661 + ] + }, + { + "text": "SWITCH", + "boxes": [ + 1076, + 677, + 1138, + 711 + ] + }, + { + "text": "SWITCH", + "boxes": [ + 758, + 755, + 890, + 814 + ] + }, + { + "text": " Switch Controllers", + "boxes": [ + 228, + 989, + 348, + 1012 + ] + }, + { + "text": "AceGamer [Pro", + "boxes": [ + 424, + 987, + 536, + 1016 + ] + }, + { + "text": "COIORVIS Joy Pad", + "boxes": [ + 616, + 985, + 740, + 1016 + ] + }, + { + "text": "Switch Controller", + "boxes": [ + 808, + 985, + 926, + 1014 + ] + }, + { + "text": "Orzly carry Case", + "boxes": [ + 1000, + 985, + 1112, + 1016 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 1194, + 987, + 1320, + 1016 + ] + }, + { + "text": "FUNLAB Switch", + "boxes": [ + 1384, + 982, + 1494, + 1017 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 424, + 1007, + 548, + 1038 + ] + }, + { + "text": "Controller for ", + "boxes": [ + 618, + 1009, + 710, + 1032 + ] + }, + { + "text": "Charging Dock for", + "boxes": [ + 808, + 1007, + 930, + 1036 + ] + }, + { + "text": "Compatible with", + "boxes": [ + 1002, + 1007, + 1114, + 1036 + ] + }, + { + "text": "Controller for", + "boxes": [ + 1198, + 1009, + 1288, + 1032 + ] + }, + { + "text": "Handheld Controller for", + "boxes": [ + 1388, + 1007, + 1544, + 1036 + ] + } + ] + }, + { + "frame_id": "70_4_8", + "episode_id": 70, + "env_id": 4, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.452001571655273, + "advantages": 0.42834123969078064, + "generated_text": "{\n \"intent\": \"To continue exploring the Firefox download page by interacting with available content.\",\n \"action\": \"DoubleClick(443, 307)\"\n}", + "action_string": "DoubleClick(443, 307)", + "intent_string": "To continue exploring the Firefox download page by interacting with available content.", + "before_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1564, + 85 + ] + }, + { + "text": "—·☆", + "boxes": [ + 1556, + 64, + 1660, + 89 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 257, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 953, + 107, + 1049, + 146 + ] + }, + { + "text": "About us", + "boxes": [ + 1084, + 107, + 1178, + 145 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 152, + 646, + 215 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress", + "boxes": [ + 344, + 220, + 746, + 250 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 240, + 672, + 270 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 292, + 514, + 324 + ] + }, + { + "text": "Firefox Privacy Notice", + "boxes": [ + 342, + 335, + 478, + 364 + ] + }, + { + "text": "Download options", + "boxes": [ + 346, + 373, + 452, + 391 + ] + }, + { + "text": "is and other languages", + "boxes": [ + 442, + 371, + 576, + 395 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 346, + 403, + 492, + 421 + ] + }, + { + "text": " Dial down the", + "boxes": [ + 418, + 800, + 672, + 854 + ] + }, + { + "text": "Multi-task or", + "boxes": [ + 800, + 806, + 1028, + 850 + ] + }, + { + "text": "Keep tabs on all", + "boxes": [ + 1174, + 800, + 1462, + 856 + ] + }, + { + "text": "noise", + "boxes": [ + 424, + 856, + 526, + 887 + ] + }, + { + "text": "mono-task", + "boxes": [ + 800, + 852, + 996, + 889 + ] + }, + { + "text": "your tabs", + "boxes": [ + 1176, + 845, + 1356, + 894 + ] + }, + { + "text": "Block ads and enhance your privacy", + "boxes": [ + 422, + 899, + 710, + 933 + ] + }, + { + "text": "Go into reading mode, pop out videos", + "boxes": [ + 802, + 901, + 1096, + 931 + ] + }, + { + "text": " Close duplicates, search and pin open", + "boxes": [ + 1178, + 899, + 1480, + 933 + ] + }, + { + "text": "with customizable settings and tons", + "boxes": [ + 424, + 921, + 714, + 951 + ] + }, + { + "text": "and get more organized with thenew", + "boxes": [ + 802, + 919, + 1100, + 955 + ] + }, + { + "text": "of downloadable extensions.", + "boxes": [ + 422, + 939, + 652, + 969 + ] + }, + { + "text": "sidebar layout.", + "boxes": [ + 800, + 937, + 926, + 971 + ] + }, + { + "text": "possibilities for the almost infinite ", + "boxes": [ + 1180, + 939, + 1460, + 969 + ] + }, + { + "text": "number of tabs you have open.", + "boxes": [ + 1182, + 961, + 1424, + 991 + ] + }, + { + "text": "See Release Notes", + "boxes": [ + 880, + 1014, + 1028, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 415, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "70_5_8", + "episode_id": 70, + "env_id": 5, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.595321655273438, + "advantages": -0.06616636365652084, + "generated_text": "{\n \"intent\": \"To ensure navigation support by identifying options such as Store, Mac, iPad, iPhone, Watch, Vision, AirPods, TV & Home, Entertainment, Accessories, Support, which might lead to further exploration or additional insights about the Apple products.\",\n \"action\": \"DoubleClick(750, 93)\"\n}", + "action_string": "DoubleClick(750, 93)", + "intent_string": "To ensure navigation support by identifying options such as Store, Mac, iPad, iPhone, Watch, Vision, AirPods, TV & Home, Entertainment, Accessories, Support, which might lead to further exploration or additional insights about the Apple products.", + "before_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 8, + 10, + 20, + 22 + ] + }, + { + "text": " Apple Watch Ultra 2 - Ap × |+", + "boxes": [ + 6, + 28, + 246, + 54 + ] + }, + { + "text": "←→白", + "boxes": [ + 4, + 62, + 132, + 93 + ] + }, + { + "text": "目☆", + "boxes": [ + 1538, + 64, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1882, + 91 + ] + }, + { + "text": "Apple Watch Ultra 2", + "boxes": [ + 458, + 153, + 656, + 187 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 872, + 266, + 1032, + 314 + ] + }, + { + "text": "ULTRA 2", + "boxes": [ + 914, + 308, + 994, + 339 + ] + }, + { + "text": "10:09:30", + "boxes": [ + 693, + 432, + 1227, + 604 + ] + }, + { + "text": "58", + "boxes": [ + 734, + 602, + 848, + 633 + ] + }, + { + "text": "65", + "boxes": [ + 818, + 659, + 878, + 677 + ] + }, + { + "text": "AppleWatch", + "boxes": [ + 218, + 1050, + 302, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 8, + 10, + 20, + 22 + ] + }, + { + "text": " Apple Watch Ultra 2 - Ap × |+", + "boxes": [ + 6, + 28, + 246, + 54 + ] + }, + { + "text": "←→白", + "boxes": [ + 4, + 62, + 132, + 93 + ] + }, + { + "text": "目☆", + "boxes": [ + 1538, + 64, + 1664, + 93 + ] + }, + { + "text": "I四@三", + "boxes": [ + 1780, + 59, + 1912, + 96 + ] + }, + { + "text": "Apple Watch Ultra 2", + "boxes": [ + 458, + 153, + 656, + 187 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 872, + 266, + 1032, + 314 + ] + }, + { + "text": "ULTRA 2", + "boxes": [ + 914, + 308, + 994, + 339 + ] + }, + { + "text": "10:09:30", + "boxes": [ + 693, + 432, + 1227, + 605 + ] + }, + { + "text": "58", + "boxes": [ + 734, + 602, + 848, + 633 + ] + }, + { + "text": "65", + "boxes": [ + 818, + 659, + 878, + 677 + ] + }, + { + "text": "AppleWatch", + "boxes": [ + 218, + 1050, + 302, + 1068 + ] + } + ] + }, + { + "frame_id": "70_6_8", + "episode_id": 70, + "env_id": 6, + "step": 8, + "format_rewards": 1.0, + "rewards": 11.032890319824219, + "advantages": 0.7636519074440002, + "generated_text": "{\n \"intent\": \"To gain more information about each trending topic displayed on the screen.\",\n \"action\": \"DoubleClick(540, 430)\"\n}", + "action_string": "DoubleClick(540, 430)", + "intent_string": "To gain more information about each trending topic displayed on the screen.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 8, + 64, + 42, + 91 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 444, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 558, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "中电中", + "boxes": [ + 1140, + 383, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 488, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 796, + 448, + 968, + 477 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 192, + 729, + 268, + 746 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1449, + 762, + 1555, + 797 + ] + }, + { + "text": "全球年轻人更理想的家", + "boxes": [ + 1202, + 776, + 1350, + 806 + ] + }, + { + "text": "C10现已开售", + "boxes": [ + 1260, + 796, + 1324, + 814 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1458, + 804, + 1504, + 842 + ] + }, + { + "text": "", + "boxes": [ + 1594, + 810, + 1628, + 832 + ] + }, + { + "text": "+2.09%60", + "boxes": [ + 1632, + 802, + 1698, + 844 + ] + }, + { + "text": "做打事", + "boxes": [ + 914, + 818, + 1014, + 856 + ] + }, + { + "text": "2.2813.2814.28", + "boxes": [ + 1196, + 828, + 1292, + 846 + ] + }, + { + "text": "12.2813.28", + "boxes": [ + 1290, + 828, + 1356, + 846 + ] + }, + { + "text": "被抓包的猫咪", + "boxes": [ + 884, + 846, + 1048, + 883 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1456, + 848, + 1514, + 891 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13, 87%", + "boxes": [ + 1626, + 891, + 1700, + 941 + ] + }, + { + "text": "IT之家·19小时", + "boxes": [ + 230, + 921, + 322, + 945 + ] + }, + { + "text": "选车号·18小时", + "boxes": [ + 1156, + 921, + 1262, + 945 + ] + }, + { + "text": "1", + "boxes": [ + 830, + 937, + 850, + 953 + ] + }, + { + "text": "一点视频/萌宠旅行者·3周", + "boxes": [ + 846, + 931, + 1010, + 961 + ] + }, + { + "text": "千元四盘位，体验新标杆，绿联NAS私有云", + "boxes": [ + 198, + 941, + 792, + 991 + ] + }, + { + "text": "激光雷达+800V下放14万级", + "boxes": [ + 1136, + 941, + 1410, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 935, + 1512, + 989 + ] + }, + { + "text": "r", + "boxes": [ + 1590, + 941, + 1632, + 981 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "很显然，猫咪也知道自己在使", + "boxes": [ + 824, + 953, + 1098, + 989 + ] + }, + { + "text": "市场\\\"六边形战士\\\"2026款", + "boxes": [ + 1136, + 967, + 1408, + 1003 + ] + }, + { + "text": "DH4300Plus开箱图赏", + "boxes": [ + 200, + 979, + 536, + 1025 + ] + }, + { + "text": "坏！", + "boxes": [ + 828, + 983, + 870, + 1011 + ] + }, + { + "text": "中平安", + "boxes": [ + 1452, + 981, + 1522, + 1038 + ] + }, + { + "text": "-11420 ", + "boxes": [ + 1634, + 983, + 1700, + 1030 + ] + }, + { + "text": "零跑C10进阶上市", + "boxes": [ + 1138, + 997, + 1308, + 1026 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "10", + "boxes": [ + 834, + 1028, + 864, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "e 16 Pro and", + "boxes": [ + 100, + 32, + 182, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "国内版国", + "boxes": [ + 184, + 113, + 272, + 145 + ] + }, + { + "text": "国际版", + "boxes": [ + 258, + 115, + 308, + 143 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 48, + 159, + 146, + 185 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 176, + 157, + 312, + 189 + ] + }, + { + "text": "WEB", + "boxes": [ + 178, + 212, + 208, + 232 + ] + }, + { + "text": "IMAGES", + "boxes": [ + 246, + 212, + 288, + 232 + ] + }, + { + "text": "ACADEMI", + "boxes": [ + 410, + 212, + 460, + 230 + ] + }, + { + "text": " About 773,000 results", + "boxes": [ + 174, + 256, + 318, + 288 + ] + }, + { + "text": "腾讯网", + "boxes": [ + 212, + 294, + 262, + 322 + ] + }, + { + "text": "雷军最新演讲内容已回应小米SU7事故，相关视频时长..", + "boxes": [ + 176, + 336, + 676, + 371 + ] + }, + { + "text": "1 day ago·雷军面向小米员工发布的最新演讲被业内外关注，视频内容流出了一部分；流出部分", + "boxes": [ + 178, + 363, + 784, + 393 + ] + }, + { + "text": "主要是雷军对3月底小米SU7交通事故的回应，但就演讲内容来看", + "boxes": [ + 178, + 389, + 580, + 413 + ] + }, + { + "text": "知乎专栏", + "boxes": [ + 212, + 433, + 278, + 465 + ] + }, + { + "text": "人", + "boxes": [ + 536, + 433, + 550, + 449 + ] + }, + { + "text": "知", + "boxes": [ + 186, + 449, + 202, + 465 + ] + }, + { + "text": "https://zhu", + "boxes": [ + 218, + 455, + 276, + 472 + ] + }, + { + "text": " Translate this result ", + "boxes": [ + 384, + 455, + 508, + 478 + ] + }, + { + "text": "雷军回应小米SU7致命事故：自动驾驶时代，谁为生命兜底？", + "boxes": [ + 176, + 476, + 720, + 512 + ] + }, + { + "text": "Apr 2,2025·昨晚，小米雷军对“SU7车祸致3死”事故的回应，瞬间点燃热搜，成为全网焦点。", + "boxes": [ + 176, + 502, + 774, + 532 + ] + }, + { + "text": "这起发生在3月29日晚的悲剧，自曝光以来，便在媒体与大众视野中激起千层浪。起初，乐..", + "boxes": [ + 178, + 530, + 770, + 554 + ] + }, + { + "text": "澎湃新闻", + "boxes": [ + 210, + 574, + 278, + 600 + ] + }, + { + "text": "雷军回应SU7车祸致三人死亡：很多问题此刻还没办法回答..", + "boxes": [ + 178, + 619, + 724, + 649 + ] + }, + { + "text": "4月1日晚间，小米CEO雷军回应广受关注的\"小米SU7高速车祸\"事件。雷军表示：29日晚上的这起", + "boxes": [ + 176, + 643, + 790, + 673 + ] + }, + { + "text": "事故，我的心情非常沉重。三位年轻女孩不幸离世，这样的消息，对她们的家人、朋友，乃至，", + "boxes": [ + 176, + 667, + 766, + 695 + ] + }, + { + "text": "网易", + "boxes": [ + 212, + 717, + 246, + 737 + ] + }, + { + "text": "https://ww.163.com>dy aticle Translate this esult", + "boxes": [ + 214, + 728, + 550, + 763 + ] + }, + { + "text": "雷军首度线下回应小米SU7燃爆事故全程紧张看稿曾引以为.", + "boxes": [ + 176, + 756, + 720, + 792 + ] + }, + { + "text": "1 day ago·近日，网上一段视频显示，雷军首度线下公开回应小米SU7燃爆事故，在台上的发言", + "boxes": [ + 178, + 784, + 780, + 814 + ] + }, + { + "text": "中，雷军全程在紧张看稿，并声称，突如其来的交通事故让小米受到了狂风暴雨般的质疑、批评，", + "boxes": [ + 178, + 810, + 778, + 834 + ] + }, + { + "text": "哗哩哗哩", + "boxes": [ + 210, + 854, + 278, + 879 + ] + }, + { + "text": "https:/www.bibilicom video Translatethisresut", + "boxes": [ + 210, + 872, + 528, + 901 + ] + }, + { + "text": "雷军首度线下回应小米SU7燃爆事故全程紧张看稿曾对质量.", + "boxes": [ + 176, + 897, + 720, + 933 + ] + }, + { + "text": "1 day ago·近日，网上一段视频显示，雷军首度线下公开回应小米SU7燃爆事故，在台上的发言", + "boxes": [ + 176, + 923, + 782, + 953 + ] + }, + { + "text": "中，雷军全程在紧张看稿，并声称，突如其来的交通事故让小米受到了狂风暴雨般的质疑、批评..", + "boxes": [ + 176, + 947, + 780, + 977 + ] + }, + { + "text": "搜狐", + "boxes": [ + 208, + 993, + 250, + 1020 + ] + }, + { + "text": "#", + "boxes": [ + 182, + 1005, + 212, + 1026 + ] + }, + { + "text": "https:/ww.sohu.com Translae thisrsuit", + "boxes": [ + 210, + 1006, + 480, + 1045 + ] + } + ] + }, + { + "frame_id": "70_7_8", + "episode_id": 70, + "env_id": 7, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.407163619995117, + "advantages": -0.17477819323539734, + "generated_text": "{\n \"intent\": \"Since we are already in a text editing application, opening a different file or folder might provide more exploration options. Right-click actions will likely reveal additional menu options relevant to files or folders.\",\n \"action\": \"RightClick(500, 550)\"\n}", + "action_string": "RightClick(500, 550)", + "intent_string": "Since we are already in a text editing application, opening a different file or folder might provide more exploration options. Right-click actions will likely reveal additional menu options relevant to files or folders.", + "before_screen_ocr_results": [ + { + "text": "File Edit Search View Document Help", + "boxes": [ + 4, + 22, + 312, + 52 + ] + }, + { + "text": "Untitled 1 - M..", + "boxes": [ + 358, + 1046, + 466, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Help", + "boxes": [ + 270, + 28, + 308, + 48 + ] + }, + { + "text": "Copy", + "boxes": [ + 520, + 645, + 560, + 661 + ] + }, + { + "text": " Paste", + "boxes": [ + 506, + 669, + 564, + 689 + ] + }, + { + "text": "Paste Special", + "boxes": [ + 526, + 695, + 614, + 719 + ] + }, + { + "text": " Delete", + "boxes": [ + 506, + 723, + 572, + 742 + ] + }, + { + "text": " select All", + "boxes": [ + 504, + 752, + 592, + 778 + ] + }, + { + "text": "Convert ", + "boxes": [ + 524, + 812, + 584, + 838 + ] + }, + { + "text": "Move Selection", + "boxes": [ + 526, + 840, + 630, + 864 + ] + }, + { + "text": "Duplicate Line / Selection", + "boxes": [ + 526, + 865, + 690, + 888 + ] + }, + { + "text": "Untitled 1 - M..", + "boxes": [ + 368, + 1048, + 468, + 1072 + ] + } + ] + }, + { + "frame_id": "70_0_9", + "episode_id": 70, + "env_id": 0, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.431611061096191, + "advantages": -0.16066622734069824, + "generated_text": "{\n \"intent\": \"To explore more articles or resources related to the topic about MAX携Hi4技术, I should scroll down the webpage to see if there are additional information or links.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore more articles or resources related to the topic about MAX携Hi4技术, I should scroll down the webpage to see if there are additional information or links.", + "before_screen_ocr_results": [ + { + "text": "", + "boxes": [ + 54, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 564, + 117, + 638, + 149 + ] + }, + { + "text": "msn", + "boxes": [ + 58, + 129, + 104, + 149 + ] + }, + { + "text": "中", + "boxes": [ + 1762, + 121, + 1788, + 143 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "资讯 娱乐 时尚 教育", + "boxes": [ + 344, + 175, + 542, + 204 + ] + }, + { + "text": "科技萌宠汽车", + "boxes": [ + 532, + 175, + 694, + 204 + ] + }, + { + "text": "个性化设置", + "boxes": [ + 1606, + 171, + 1728, + 208 + ] + }, + { + "text": "发现", + "boxes": [ + 236, + 183, + 266, + 197 + ] + }, + { + "text": "精确分析", + "boxes": [ + 1440, + 248, + 1590, + 304 + ] + }, + { + "text": "ITBEAR科技资讯+ 关注", + "boxes": [ + 382, + 264, + 632, + 306 + ] + }, + { + "text": "专", + "boxes": [ + 1426, + 260, + 1442, + 274 + ] + }, + { + "text": "60.9K关注者", + "boxes": [ + 1098, + 274, + 1174, + 298 + ] + }, + { + "text": "绘制图表", + "boxes": [ + 1442, + 290, + 1584, + 334 + ] + }, + { + "text": "助您高效完成数据分析", + "boxes": [ + 1454, + 334, + 1576, + 351 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，能否引领新能源市场新潮流？", + "boxes": [ + 362, + 347, + 1174, + 389 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1456, + 351, + 1506, + 369 + ] + }, + { + "text": "F", + "boxes": [ + 188, + 393, + 218, + 425 + ] + }, + { + "text": "k王者荣耀 的报道·15小时·① 1 分钟读取", + "boxes": [ + 362, + 413, + 706, + 449 + ] + }, + { + "text": "让你成为Excel达人，工作学习效", + "boxes": [ + 1422, + 411, + 1702, + 447 + ] + }, + { + "text": "率倍增", + "boxes": [ + 1426, + 437, + 1490, + 469 + ] + }, + { + "text": "B", + "boxes": [ + 192, + 453, + 214, + 476 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 469, + 1530, + 492 + ] + }, + { + "text": "在汽车市场风起云涌的今天，哈弗汽车，这个长久以来以\"性价比\"著称的品牌，正积极布局新能源领域，意图在新的", + "boxes": [ + 364, + 504, + 1232, + 534 + ] + }, + { + "text": "赛道上续写辉煌。近日，哈弗汽车两款全新车型——哈弗枭龙与哈弗枭龙MAX正式上市，标志着哈弗正式进军新能源", + "boxes": [ + 366, + 530, + 1238, + 560 + ] + }, + { + "text": "汽车市场的决心。", + "boxes": [ + 364, + 554, + 494, + 588 + ] + }, + { + "text": "访问ITBEAR科技资讯", + "boxes": [ + 1426, + 554, + 1596, + 584 + ] + }, + { + "text": "G", + "boxes": [ + 1673, + 550, + 1702, + 579 + ] + }, + { + "text": "哈弗枭龙MAX与哈弗枭龙，作为哈弗品牌的全新力作，不仅继承了哈弗汽车一贯的高性价比优势，更在技术层面实现", + "boxes": [ + 364, + 600, + 1240, + 629 + ] + }, + { + "text": "新疆多地夜空现神秘飞行物，目击", + "boxes": [ + 1444, + 597, + 1666, + 634 + ] + }, + { + "text": "了突破。哈弗枭龙MAX搭载了长城汽车全新的智能四驱电混技术Hi4，推出了三款车型，价格区间为15.98-17.98万", + "boxes": [ + 366, + 625, + 1214, + 655 + ] + }, + { + "text": "者：亮如月亮，几秒加速消失", + "boxes": [ + 1446, + 621, + 1634, + 651 + ] + }, + { + "text": "元。而哈弗枭龙则采用了智能混动DHT技术，同样推出了三款车型，价格区间则为13.98-15.68万元。两款车型均展现", + "boxes": [ + 364, + 651, + 1240, + 681 + ] + }, + { + "text": "小米YU7即将发布，卢伟冰自曝已", + "boxes": [ + 1446, + 663, + 1664, + 693 + ] + }, + { + "text": "出了哈弗在新能源汽车领域的深厚底蕴和技术实力。", + "boxes": [ + 364, + 677, + 750, + 707 + ] + }, + { + "text": "购车并计划添购新车", + "boxes": [ + 1446, + 685, + 1580, + 715 + ] + }, + { + "text": "长安汽车泰国投产，深蓝CEO：增", + "boxes": [ + 1446, + 729, + 1664, + 758 + ] + }, + { + "text": "程车油耗媲美日系混动，驾驶体..", + "boxes": [ + 1446, + 748, + 1660, + 778 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 832, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1672, + 878, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 796, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别.在设备上存储和/或访问信息。个性化广告和内容、广告和内容衡", + "boxes": [ + 836, + 911, + 1528, + 935 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1668, + 935, + 1728, + 967 + ] + }, + { + "text": "击“管理设置“查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的“管理设置\"改变您的选择。欲了解", + "boxes": [ + 30, + 949, + 796, + 979 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：微软隐私声明", + "boxes": [ + 32, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 72, + 68, + 124, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四@", + "boxes": [ + 1784, + 62, + 1880, + 93 + ] + }, + { + "text": "搜索网页", + "boxes": [ + 564, + 117, + 638, + 149 + ] + }, + { + "text": "长春市", + "boxes": [ + 1636, + 125, + 1686, + 145 + ] + }, + { + "text": "中", + "boxes": [ + 1766, + 125, + 1786, + 141 + ] + }, + { + "text": "登录", + "boxes": [ + 1834, + 127, + 1854, + 141 + ] + }, + { + "text": "在汽车市场风起云涌的今天，哈弗汽车，这个长久以来以\"性价比\"著称的品牌，正积极布局新能源领域，意图在新的", + "boxes": [ + 364, + 173, + 1236, + 202 + ] + }, + { + "text": "赛道上续写辉煌。近日，哈弗汽车两款全新车型一", + "boxes": [ + 364, + 196, + 730, + 230 + ] + }, + { + "text": "精确分析", + "boxes": [ + 1435, + 188, + 1589, + 245 + ] + }, + { + "text": "一哈弗枭龙与哈弗枭龙MAX正式上市，标志着哈弗正式进军新能源", + "boxes": [ + 750, + 201, + 1240, + 230 + ] + }, + { + "text": "?", + "boxes": [ + 1424, + 201, + 1444, + 220 + ] + }, + { + "text": "汽车市场的决心。", + "boxes": [ + 366, + 228, + 490, + 252 + ] + }, + { + "text": "绘制图表", + "boxes": [ + 1442, + 230, + 1588, + 280 + ] + }, + { + "text": "哈弗枭龙MAX与哈弗枭龙，作为哈弗品牌的全新力作，不仅继承了哈弗汽车一贯的高性价比优势，更在技术层面实现", + "boxes": [ + 364, + 268, + 1240, + 298 + ] + }, + { + "text": "功您高效完成数据分析", + "boxes": [ + 1456, + 276, + 1578, + 294 + ] + }, + { + "text": "了突破。哈弗枭龙MAX搭载了长城汽车全新的智能四驱电混技术Hi4，推出了三款车型，价格区间为15.98-17.98万", + "boxes": [ + 366, + 292, + 1216, + 322 + ] + }, + { + "text": "点击下载", + "boxes": [ + 1452, + 288, + 1510, + 314 + ] + }, + { + "text": "元。而哈弗枭龙则采用了智能混动DHT技术，同样推出了三款车型，价格区间则为13.98-15.68万元。两款车型均展现", + "boxes": [ + 364, + 320, + 1238, + 349 + ] + }, + { + "text": "出了哈弗在新能源汽车领域的深厚底蕴和技术实力。", + "boxes": [ + 366, + 343, + 746, + 375 + ] + }, + { + "text": "让你成为 Excel达人，工作学习效", + "boxes": [ + 1422, + 349, + 1704, + 389 + ] + }, + { + "text": "率倍增", + "boxes": [ + 1426, + 379, + 1492, + 411 + ] + }, + { + "text": "广告豆包客户端", + "boxes": [ + 1428, + 411, + 1528, + 435 + ] + }, + { + "text": "?", + "boxes": [ + 194, + 433, + 210, + 449 + ] + }, + { + "text": "®", + "boxes": [ + 1432, + 500, + 1452, + 516 + ] + }, + { + "text": "访问ITBEAR科技资讯", + "boxes": [ + 1444, + 496, + 1594, + 520 + ] + }, + { + "text": "新疆多地夜空现神秘飞行物，目击", + "boxes": [ + 1446, + 544, + 1664, + 574 + ] + }, + { + "text": "者：亮如月亮，几秒加速消失", + "boxes": [ + 1446, + 564, + 1636, + 594 + ] + }, + { + "text": "小米YU7即将发布，卢伟冰自曝已", + "boxes": [ + 1446, + 607, + 1660, + 637 + ] + }, + { + "text": "购车并计划添购新车", + "boxes": [ + 1446, + 627, + 1578, + 657 + ] + }, + { + "text": "长安汽车泰国投产，深蓝CEO：增", + "boxes": [ + 1446, + 671, + 1660, + 695 + ] + }, + { + "text": "程车油耗媲美日系混动，驾驶体.", + "boxes": [ + 1448, + 691, + 1660, + 721 + ] + }, + { + "text": "豆包AI翻译", + "boxes": [ + 1540, + 784, + 1710, + 828 + ] + }, + { + "text": "哈弗枭龙MAX携Hi4技术，能否引领新能源市场新潮流", + "boxes": [ + 390, + 812, + 680, + 830 + ] + }, + { + "text": "微软重视您的隐私", + "boxes": [ + 30, + 878, + 196, + 907 + ] + }, + { + "text": "我们以及合作伙伴处理数据的目的如下：", + "boxes": [ + 834, + 876, + 1090, + 905 + ] + }, + { + "text": "我接受", + "boxes": [ + 1670, + 876, + 1720, + 911 + ] + }, + { + "text": "为了向您提供服务和广告并对其进行维护和改进，微软以及我们的第三方供应商会使用cookies来存储、访问包括唯一识别码（unique", + "boxes": [ + 28, + 909, + 796, + 939 + ] + }, + { + "text": "使用精确的地理位置数据.主动扫描设备特性以进行识别在设备上存储和/或访问信息、个性化广告和内容、广告和内容衡", + "boxes": [ + 834, + 907, + 1530, + 937 + ] + }, + { + "text": "D）等信息。如您同意，微软资讯以及微软必应会向您提供个性化的内容和广告。您可以选择“我同意“从而同意上述使用场景，或者点", + "boxes": [ + 24, + 923, + 800, + 961 + ] + }, + { + "text": "量、受众研究和服务开发.", + "boxes": [ + 836, + 927, + 986, + 957 + ] + }, + { + "text": "全部拒绝", + "boxes": [ + 1670, + 937, + 1726, + 963 + ] + }, + { + "text": "击\"管理设置\"查看您的选项以及在适用情况下行使您反对正当利益的权利。您可以通过本页面底部的\"管理设置\"改变您的选择。欲了解", + "boxes": [ + 30, + 951, + 794, + 975 + ] + }, + { + "text": "合作伙伴（供应商）名单", + "boxes": [ + 836, + 953, + 984, + 983 + ] + }, + { + "text": "详情，请访问：微软隐私声明", + "boxes": [ + 32, + 971, + 210, + 995 + ] + }, + { + "text": "管理设置", + "boxes": [ + 1668, + 993, + 1728, + 1024 + ] + } + ] + }, + { + "frame_id": "70_1_9", + "episode_id": 70, + "env_id": 1, + "step": 9, + "format_rewards": 1.0, + "rewards": 8.962957382202148, + "advantages": -0.4311906099319458, + "generated_text": "{\n \"intent\": \"To close the Print dialog window by clicking the 'OK' button, allowing further exploration or actions in LibreOffice Calc.\",\n \"action\": \"DoubleClick(577, 302)\"\n}", + "action_string": "DoubleClick(577, 302)", + "intent_string": "To close the Print dialog window by clicking the 'OK' button, allowing further exploration or actions in LibreOffice Calc.", + "before_screen_ocr_results": [ + { + "text": "国", + "boxes": [ + 8, + 4, + 24, + 24 + ] + }, + { + "text": "Untitled 1 - Libreoffice Calc", + "boxes": [ + 848, + 2, + 1070, + 26 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 28, + 34, + 42 + ] + }, + { + "text": "--目-@电自-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "Q", + "boxes": [ + 376, + 58, + 442, + 83 + ] + }, + { + "text": "G-.", + "boxes": [ + 450, + 56, + 546, + 79 + ] + }, + { + "text": "里听个", + "boxes": [ + 544, + 54, + 848, + 83 + ] + }, + { + "text": "M-xE-EB", + "boxes": [ + 850, + 48, + 1264, + 87 + ] + }, + { + "text": "10", + "boxes": [ + 154, + 97, + 188, + 117 + ] + }, + { + "text": "aaa.", + "boxes": [ + 236, + 93, + 350, + 123 + ] + }, + { + "text": "三", + "boxes": [ + 512, + 101, + 536, + 115 + ] + }, + { + "text": "", + "boxes": [ + 552, + 101, + 574, + 115 + ] + }, + { + "text": "□B8", + "boxes": [ + 616, + 95, + 720, + 119 + ] + }, + { + "text": ". % 0.0 ", + "boxes": [ + 724, + 93, + 874, + 123 + ] + }, + { + "text": "十三三田二口目", + "boxes": [ + 878, + 93, + 1198, + 123 + ] + }, + { + "text": "浆×", + "boxes": [ + 160, + 129, + 240, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Print", + "boxes": [ + 938, + 262, + 990, + 288 + ] + }, + { + "text": "□□", + "boxes": [ + 1288, + 262, + 1350, + 288 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 266, + 1914, + 314 + ] + }, + { + "text": "8.26 in (A4)", + "boxes": [ + 724, + 290, + 808, + 316 + ] + }, + { + "text": " General ", + "boxes": [ + 950, + 296, + 1014, + 322 + ] + }, + { + "text": "LibreOffice Calc", + "boxes": [ + 1010, + 293, + 1128, + 322 + ] + }, + { + "text": "成", + "boxes": [ + 1890, + 312, + 1910, + 339 + ] + }, + { + "text": "Printer", + "boxes": [ + 948, + 324, + 1010, + 349 + ] + }, + { + "text": " Generic Printer", + "boxes": [ + 964, + 351, + 1074, + 375 + ] + }, + { + "text": "Status: Default printer", + "boxes": [ + 960, + 385, + 1108, + 409 + ] + }, + { + "text": "O All Pages", + "boxes": [ + 964, + 449, + 1044, + 472 + ] + }, + { + "text": ".8", + "boxes": [ + 18, + 467, + 30, + 478 + ] + }, + { + "text": "O Pages:", + "boxes": [ + 968, + 478, + 1038, + 500 + ] + }, + { + "text": "O Even pages", + "boxes": [ + 965, + 502, + 1061, + 533 + ] + }, + { + "text": "O odd pages", + "boxes": [ + 965, + 528, + 1057, + 558 + ] + }, + { + "text": "more", + "boxes": [ + 968, + 560, + 1016, + 580 + ] + }, + { + "text": "Page Layout", + "boxes": [ + 952, + 588, + 1044, + 611 + ] + }, + { + "text": "Paper size:", + "boxes": [ + 966, + 617, + 1036, + 635 + ] + }, + { + "text": "A4 8.26in x 11.69in", + "boxes": [ + 1048, + 613, + 1182, + 637 + ] + }, + { + "text": "Automatic", + "boxes": [ + 1051, + 647, + 1125, + 670 + ] + }, + { + "text": "Preview", + "boxes": [ + 592, + 754, + 658, + 772 + ] + }, + { + "text": "?", + "boxes": [ + 744, + 756, + 770, + 770 + ] + }, + { + "text": "/10", + "boxes": [ + 818, + 754, + 882, + 774 + ] + }, + { + "text": "Help", + "boxes": [ + 604, + 796, + 644, + 812 + ] + }, + { + "text": "More Option", + "boxes": [ + 1038, + 796, + 1120, + 814 + ] + }, + { + "text": "OK", + "boxes": [ + 1288, + 796, + 1316, + 810 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 939 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "国", + "boxes": [ + 8, + 4, + 24, + 24 + ] + }, + { + "text": "Untitled 1 - Libreoffice Calc", + "boxes": [ + 848, + 2, + 1070, + 26 + ] + }, + { + "text": "File", + "boxes": [ + 10, + 28, + 34, + 42 + ] + }, + { + "text": "--目-@电自-", + "boxes": [ + 12, + 54, + 376, + 83 + ] + }, + { + "text": "Q", + "boxes": [ + 376, + 58, + 442, + 83 + ] + }, + { + "text": "G-.", + "boxes": [ + 452, + 56, + 542, + 79 + ] + }, + { + "text": "用目·听抄个子", + "boxes": [ + 598, + 54, + 848, + 83 + ] + }, + { + "text": "M-xE-EB", + "boxes": [ + 850, + 48, + 1264, + 87 + ] + }, + { + "text": "10", + "boxes": [ + 154, + 97, + 188, + 117 + ] + }, + { + "text": "aaa.", + "boxes": [ + 236, + 93, + 350, + 123 + ] + }, + { + "text": "三", + "boxes": [ + 512, + 101, + 536, + 115 + ] + }, + { + "text": "", + "boxes": [ + 552, + 101, + 574, + 115 + ] + }, + { + "text": "□8 8", + "boxes": [ + 618, + 95, + 718, + 119 + ] + }, + { + "text": "-% 00 ", + "boxes": [ + 724, + 93, + 874, + 123 + ] + }, + { + "text": "+", + "boxes": [ + 880, + 95, + 948, + 121 + ] + }, + { + "text": "三三", + "boxes": [ + 936, + 95, + 1026, + 119 + ] + }, + { + "text": "田·黑·口·昌", + "boxes": [ + 1014, + 95, + 1196, + 119 + ] + }, + { + "text": "紫×化", + "boxes": [ + 156, + 129, + 240, + 155 + ] + }, + { + "text": "T", + "boxes": [ + 1888, + 208, + 1910, + 238 + ] + }, + { + "text": "自", + "boxes": [ + 1888, + 240, + 1910, + 272 + ] + }, + { + "text": "Print", + "boxes": [ + 938, + 262, + 990, + 288 + ] + }, + { + "text": "□□", + "boxes": [ + 1288, + 262, + 1350, + 288 + ] + }, + { + "text": "?", + "boxes": [ + 1886, + 266, + 1914, + 314 + ] + }, + { + "text": "8.26 in (A4)", + "boxes": [ + 724, + 290, + 808, + 316 + ] + }, + { + "text": " General ", + "boxes": [ + 950, + 296, + 1014, + 322 + ] + }, + { + "text": "Libreoffice Calc", + "boxes": [ + 1010, + 293, + 1128, + 322 + ] + }, + { + "text": "成", + "boxes": [ + 1890, + 312, + 1910, + 339 + ] + }, + { + "text": "Printer", + "boxes": [ + 948, + 324, + 1010, + 349 + ] + }, + { + "text": " Generic Printer", + "boxes": [ + 964, + 351, + 1074, + 375 + ] + }, + { + "text": "Status: Default printer", + "boxes": [ + 960, + 385, + 1108, + 409 + ] + }, + { + "text": "O All Pages", + "boxes": [ + 964, + 449, + 1044, + 472 + ] + }, + { + "text": ".8", + "boxes": [ + 18, + 467, + 30, + 478 + ] + }, + { + "text": "O Pages:", + "boxes": [ + 968, + 478, + 1038, + 500 + ] + }, + { + "text": "O Even pages", + "boxes": [ + 965, + 502, + 1061, + 533 + ] + }, + { + "text": "O odd pages", + "boxes": [ + 965, + 528, + 1057, + 558 + ] + }, + { + "text": "more", + "boxes": [ + 968, + 560, + 1016, + 580 + ] + }, + { + "text": "Page Layout", + "boxes": [ + 952, + 588, + 1044, + 611 + ] + }, + { + "text": "Paper size:", + "boxes": [ + 966, + 617, + 1036, + 635 + ] + }, + { + "text": "A4 8.26in x 11.69in", + "boxes": [ + 1048, + 613, + 1182, + 637 + ] + }, + { + "text": "Automatic", + "boxes": [ + 1051, + 647, + 1125, + 670 + ] + }, + { + "text": "Preview", + "boxes": [ + 592, + 754, + 658, + 772 + ] + }, + { + "text": "?", + "boxes": [ + 744, + 756, + 770, + 770 + ] + }, + { + "text": "/10", + "boxes": [ + 818, + 754, + 882, + 774 + ] + }, + { + "text": "Help", + "boxes": [ + 604, + 796, + 644, + 812 + ] + }, + { + "text": "More Option:", + "boxes": [ + 1038, + 796, + 1122, + 814 + ] + }, + { + "text": "OK", + "boxes": [ + 1288, + 796, + 1316, + 810 + ] + }, + { + "text": "45", + "boxes": [ + 14, + 925, + 30, + 939 + ] + } + ] + }, + { + "frame_id": "70_2_9", + "episode_id": 70, + "env_id": 2, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.246135711669922, + "advantages": 0.3095077872276306, + "generated_text": "{\n \"intent\": \"To start exploring different sections of the Apple Store website, the most valuable action would be loading a new section like 'Mac,' 'iPhone,' etc., by clicking on one of the icons.\",\n \"action\": \"DoubleClick(201, 481)\"\n}", + "action_string": "DoubleClick(201, 481)", + "intent_string": "To start exploring different sections of the Apple Store website, the most valuable action would be loading a new section like 'Mac,' 'iPhone,' etc., by clicking on one of the icons.", + "before_screen_ocr_results": [ + { + "text": "Apple Watch", + "boxes": [ + 604, + 232, + 690, + 256 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 228, + 854, + 260 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 228, + 968, + 260 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 230, + 1110, + 258 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 228, + 1278, + 260 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 230, + 1414, + 256 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 230, + 1566, + 256 + ] + }, + { + "text": " Apple Gift card ", + "boxes": [ + 1612, + 226, + 1724, + 260 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 136, + 353, + 722, + 389 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 164, + 465, + 334, + 500 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 465, + 768, + 502 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 463, + 1112, + 500 + ] + }, + { + "text": "Apple Watch Pride Edition ", + "boxes": [ + 1426, + 463, + 1762, + 504 + ] + }, + { + "text": "Appl", + "boxes": [ + 1846, + 465, + 1914, + 504 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 504, + 326, + 534 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 504, + 740, + 534 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 504, + 1164, + 534 + ] + }, + { + "text": " Sport Band", + "boxes": [ + 1422, + 492, + 1578, + 534 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1848, + 506, + 1910, + 532 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 166, + 528, + 412, + 556 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 584, + 532, + 832, + 562 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1010, + 534, + 1246, + 552 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1424, + 534, + 1582, + 570 + ] + }, + { + "text": "From $35", + "boxes": [ + 1846, + 530, + 1912, + 562 + ] + }, + { + "text": "$49", + "boxes": [ + 1426, + 564, + 1462, + 592 + ] + }, + { + "text": "PIRO", + "boxes": [ + 633, + 670, + 883, + 793 + ] + }, + { + "text": "Help is here. Whenever and however you need it.", + "boxes": [ + 136, + 979, + 742, + 1014 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "BuyMacBookAir-", + "boxes": [ + 782, + 4, + 936, + 22 + ] + }, + { + "text": "目☆", + "boxes": [ + 1538, + 62, + 1662, + 91 + ] + }, + { + "text": "Phone", + "boxes": [ + 696, + 113, + 728, + 127 + ] + }, + { + "text": "Watch", + "boxes": [ + 766, + 113, + 800, + 127 + ] + }, + { + "text": "Vision", + "boxes": [ + 838, + 113, + 870, + 127 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 948, + 129 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 984, + 109, + 1048, + 129 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1078, + 109, + 1164, + 133 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 111, + 1262, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1296, + 111, + 1342, + 131 + ] + }, + { + "text": "Q", + "boxes": [ + 1368, + 109, + 1396, + 131 + ] + }, + { + "text": "6", + "boxes": [ + 1418, + 111, + 1442, + 135 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 460, + 155, + 588, + 185 + ] + }, + { + "text": "Overview", + "boxes": [ + 1254, + 161, + 1316, + 181 + ] + }, + { + "text": " macos", + "boxes": [ + 1322, + 161, + 1372, + 179 + ] + }, + { + "text": "Compare", + "boxes": [ + 1386, + 161, + 1442, + 179 + ] + }, + { + "text": "Choose your new MacBook Air.", + "boxes": [ + 622, + 322, + 1286, + 381 + ] + }, + { + "text": "B", + "boxes": [ + 934, + 407, + 974, + 437 + ] + }, + { + "text": "Have questions about buying a Mac?", + "boxes": [ + 804, + 439, + 1106, + 469 + ] + }, + { + "text": "Chat with a Mac Specialist 7", + "boxes": [ + 844, + 467, + 1066, + 490 + ] + }, + { + "text": "13-inch", + "boxes": [ + 848, + 550, + 922, + 582 + ] + }, + { + "text": "15-inch", + "boxes": [ + 986, + 550, + 1062, + 582 + ] + }, + { + "text": "From $999", + "boxes": [ + 847, + 571, + 921, + 602 + ] + }, + { + "text": "From $1199", + "boxes": [ + 988, + 576, + 1060, + 602 + ] + } + ] + }, + { + "frame_id": "70_3_9", + "episode_id": 70, + "env_id": 3, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.812726020812988, + "advantages": 0.05932754650712013, + "generated_text": "{\n \"intent\": \"To explore more about the Nintendo Switch product details or related items, I will scroll down slightly.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To explore more about the Nintendo Switch product details or related items, I will scroll down slightly.", + "before_screen_ocr_results": [ + { + "text": "→e", + "boxes": [ + 22, + 66, + 126, + 89 + ] + }, + { + "text": "n.0f9e35ef-aec2-4925-899e-", + "boxes": [ + 1374, + 69, + 1572, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1576, + 66, + 1662, + 91 + ] + }, + { + "text": "I口三", + "boxes": [ + 1782, + 64, + 1914, + 93 + ] + }, + { + "text": " Deliver to Canada", + "boxes": [ + 1394, + 97, + 1516, + 121 + ] + }, + { + "text": "V", + "boxes": [ + 580, + 113, + 614, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1822, + 105, + 1886, + 131 + ] + }, + { + "text": "$27999", + "boxes": [ + 639, + 120, + 725, + 166 + ] + }, + { + "text": "Used: Very Good | Detalls", + "boxes": [ + 1394, + 127, + 1562, + 157 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1816, + 123, + 1894, + 151 + ] + }, + { + "text": " Sold by Toys and Video Games", + "boxes": [ + 1390, + 145, + 1592, + 177 + ] + }, + { + "text": "Nintendo Switch TM with Neon Blue", + "boxes": [ + 1538, + 236, + 1806, + 266 + ] + }, + { + "text": "and Neon Red Joy-Con TM", + "boxes": [ + 1538, + 256, + 1744, + 286 + ] + }, + { + "text": "Style:Console", + "boxes": [ + 1540, + 288, + 1620, + 314 + ] + }, + { + "text": "Color:Black", + "boxes": [ + 1542, + 322, + 1602, + 341 + ] + }, + { + "text": "NINTENDO", + "boxes": [ + 619, + 460, + 775, + 505 + ] + }, + { + "text": "SWITCH.", + "boxes": [ + 617, + 482, + 779, + 533 + ] + }, + { + "text": "CD", + "boxes": [ + 1078, + 625, + 1132, + 685 + ] + }, + { + "text": "®", + "boxes": [ + 1182, + 643, + 1196, + 661 + ] + }, + { + "text": "SWITCH", + "boxes": [ + 1076, + 677, + 1138, + 711 + ] + }, + { + "text": "SWITCH", + "boxes": [ + 758, + 755, + 890, + 814 + ] + }, + { + "text": " Switch Controllers", + "boxes": [ + 228, + 989, + 348, + 1012 + ] + }, + { + "text": "AceGamer [Pro", + "boxes": [ + 424, + 987, + 536, + 1016 + ] + }, + { + "text": "COIORVIS Joy Pad", + "boxes": [ + 616, + 985, + 740, + 1016 + ] + }, + { + "text": "Switch Controller", + "boxes": [ + 808, + 985, + 926, + 1014 + ] + }, + { + "text": "Orzly carry Case", + "boxes": [ + 1000, + 985, + 1112, + 1016 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 1194, + 987, + 1320, + 1016 + ] + }, + { + "text": "FUNLAB Switch", + "boxes": [ + 1384, + 982, + 1494, + 1017 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 424, + 1007, + 548, + 1038 + ] + }, + { + "text": "Controller for ", + "boxes": [ + 618, + 1009, + 710, + 1032 + ] + }, + { + "text": "Charging Dock for", + "boxes": [ + 808, + 1007, + 930, + 1036 + ] + }, + { + "text": "Compatible with", + "boxes": [ + 1002, + 1007, + 1114, + 1036 + ] + }, + { + "text": "Controller for", + "boxes": [ + 1198, + 1009, + 1288, + 1032 + ] + }, + { + "text": "Handheld Controller for", + "boxes": [ + 1388, + 1007, + 1544, + 1036 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→c", + "boxes": [ + 12, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 66, + 1662, + 91 + ] + }, + { + "text": "I口三", + "boxes": [ + 1782, + 64, + 1914, + 93 + ] + }, + { + "text": " Subtotal", + "boxes": [ + 1820, + 103, + 1888, + 135 + ] + }, + { + "text": "2799", + "boxes": [ + 654, + 133, + 714, + 153 + ] + }, + { + "text": "$14,177.72", + "boxes": [ + 1822, + 129, + 1888, + 147 + ] + }, + { + "text": "α", + "boxes": [ + 1838, + 157, + 1878, + 187 + ] + }, + { + "text": "Nintendo SwitchM with Neon Blue", + "boxes": [ + 1538, + 197, + 1806, + 226 + ] + }, + { + "text": "and Neon Red Joy-Con TM", + "boxes": [ + 1538, + 216, + 1738, + 246 + ] + }, + { + "text": "Style:Console", + "boxes": [ + 1540, + 250, + 1616, + 274 + ] + }, + { + "text": "NINTENDO", + "boxes": [ + 619, + 418, + 775, + 467 + ] + }, + { + "text": "SWITCH.", + "boxes": [ + 617, + 442, + 779, + 491 + ] + }, + { + "text": "D", + "boxes": [ + 1078, + 588, + 1134, + 643 + ] + }, + { + "text": "®", + "boxes": [ + 1182, + 602, + 1196, + 625 + ] + }, + { + "text": "SWITCH", + "boxes": [ + 1068, + 629, + 1144, + 677 + ] + }, + { + "text": "SWITCH", + "boxes": [ + 765, + 723, + 883, + 770 + ] + }, + { + "text": "lo", + "boxes": [ + 1390, + 943, + 1544, + 985 + ] + }, + { + "text": "ML", + "boxes": [ + 252, + 959, + 280, + 973 + ] + }, + { + "text": "?", + "boxes": [ + 300, + 959, + 324, + 973 + ] + }, + { + "text": "MR", + "boxes": [ + 356, + 959, + 378, + 973 + ] + }, + { + "text": "廿1+", + "boxes": [ + 1800, + 969, + 1900, + 1001 + ] + }, + { + "text": "Switch Controllers", + "boxes": [ + 232, + 991, + 350, + 1014 + ] + }, + { + "text": "AceGamer [Pro", + "boxes": [ + 421, + 984, + 535, + 1019 + ] + }, + { + "text": "COIORVIS Joy Pad", + "boxes": [ + 616, + 985, + 738, + 1016 + ] + }, + { + "text": "Switch Controller", + "boxes": [ + 808, + 987, + 924, + 1016 + ] + }, + { + "text": "Orzly Carry Case", + "boxes": [ + 1004, + 991, + 1108, + 1014 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 1196, + 991, + 1318, + 1014 + ] + }, + { + "text": "FUNLAB Switch", + "boxes": [ + 1388, + 989, + 1496, + 1016 + ] + }, + { + "text": "Upgraded Wireless", + "boxes": [ + 424, + 1007, + 546, + 1038 + ] + }, + { + "text": "Controller for", + "boxes": [ + 618, + 1009, + 710, + 1034 + ] + }, + { + "text": "Charging Dock for", + "boxes": [ + 808, + 1007, + 930, + 1038 + ] + }, + { + "text": "Compatible with", + "boxes": [ + 1004, + 1007, + 1112, + 1038 + ] + }, + { + "text": "Controller for", + "boxes": [ + 1198, + 1009, + 1288, + 1034 + ] + }, + { + "text": "Handheld Controller for", + "boxes": [ + 1388, + 1007, + 1544, + 1036 + ] + } + ] + }, + { + "frame_id": "70_4_9", + "episode_id": 70, + "env_id": 4, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.1264009475708, + "advantages": -0.33684489130973816, + "generated_text": "{\n \"intent\": \"Since my goal is to explore the desktop environment, accessing a download page within Firefox gives me an opportunity to browse different web pages, which could lead to discovering more information or applications. Selecting a download option will open the folder where the file is saved.\",\n \"action\": \"DoubleClick(1152, 671)\"\n}", + "action_string": "DoubleClick(1152, 671)", + "intent_string": "Since my goal is to explore the desktop environment, accessing a download page within Firefox gives me an opportunity to browse different web pages, which could lead to discovering more information or applications. Selecting a download option will open the folder where the file is saved.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 415, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1086, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 471 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": " Download for Linux 64-bit", + "boxes": [ + 846, + 635, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 728, + 675, + 1176, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1118, + 731 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "70_5_9", + "episode_id": 70, + "env_id": 5, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.704813003540039, + "advantages": -0.002963872393593192, + "generated_text": "{\n \"intent\": \"To continue exploring different sections of the Apple website by scrolling down, which will reveal more content or information.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring different sections of the Apple website by scrolling down, which will reveal more content or information.", + "before_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 8, + 10, + 20, + 22 + ] + }, + { + "text": " Apple Watch Ultra 2 - Ap × |+", + "boxes": [ + 6, + 28, + 246, + 54 + ] + }, + { + "text": "←→白", + "boxes": [ + 4, + 62, + 132, + 93 + ] + }, + { + "text": "目☆", + "boxes": [ + 1538, + 64, + 1664, + 93 + ] + }, + { + "text": "I四@三", + "boxes": [ + 1780, + 59, + 1912, + 96 + ] + }, + { + "text": "Apple Watch Ultra 2", + "boxes": [ + 458, + 153, + 656, + 187 + ] + }, + { + "text": "‘WATCH", + "boxes": [ + 872, + 266, + 1032, + 314 + ] + }, + { + "text": "ULTRA 2", + "boxes": [ + 914, + 308, + 994, + 339 + ] + }, + { + "text": "10:09:30", + "boxes": [ + 693, + 432, + 1227, + 605 + ] + }, + { + "text": "58", + "boxes": [ + 734, + 602, + 848, + 633 + ] + }, + { + "text": "65", + "boxes": [ + 818, + 659, + 878, + 677 + ] + }, + { + "text": "AppleWatch", + "boxes": [ + 218, + 1050, + 302, + 1068 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 8, + 10, + 20, + 22 + ] + }, + { + "text": "回", + "boxes": [ + 1860, + 4, + 1912, + 22 + ] + }, + { + "text": "● Apple Watch Uitra 2 - Ap ×+", + "boxes": [ + 6, + 24, + 248, + 56 + ] + }, + { + "text": "←→", + "boxes": [ + 4, + 62, + 132, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1536, + 62, + 1664, + 93 + ] + }, + { + "text": "I四=", + "boxes": [ + 1780, + 59, + 1914, + 94 + ] + }, + { + "text": "Apple Watch Ultra 2", + "boxes": [ + 460, + 111, + 658, + 141 + ] + }, + { + "text": "10:09:30", + "boxes": [ + 694, + 151, + 1234, + 314 + ] + }, + { + "text": "", + "boxes": [ + 1195, + 150, + 1289, + 309 + ] + }, + { + "text": "lilili", + "boxes": [ + 625, + 225, + 679, + 299 + ] + }, + { + "text": "需", + "boxes": [ + 918, + 305, + 991, + 355 + ] + }, + { + "text": "58", + "boxes": [ + 712, + 318, + 840, + 349 + ] + }, + { + "text": "65", + "boxes": [ + 1074, + 310, + 1182, + 334 + ] + }, + { + "text": "FT", + "boxes": [ + 1106, + 336, + 1160, + 353 + ] + }, + { + "text": "65FT", + "boxes": [ + 818, + 373, + 924, + 391 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 216, + 1048, + 304, + 1072 + ] + } + ] + }, + { + "frame_id": "70_6_9", + "episode_id": 70, + "env_id": 6, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.135990142822266, + "advantages": 0.24592764675617218, + "generated_text": "{\n \"intent\": \"To explore the search results further, I will click on one of the links to access more detailed information about the event.\",\n \"action\": \"DoubleClick(303, 401)\"\n}", + "action_string": "DoubleClick(303, 401)", + "intent_string": "To explore the search results further, I will click on one of the links to access more detailed information about the event.", + "before_screen_ocr_results": [ + { + "text": "e 16 Pro and", + "boxes": [ + 100, + 32, + 182, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "国内版国", + "boxes": [ + 184, + 113, + 272, + 145 + ] + }, + { + "text": "国际版", + "boxes": [ + 258, + 115, + 308, + 143 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 48, + 159, + 146, + 185 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 176, + 157, + 312, + 189 + ] + }, + { + "text": "WEB", + "boxes": [ + 178, + 212, + 208, + 232 + ] + }, + { + "text": "IMAGES", + "boxes": [ + 246, + 212, + 288, + 232 + ] + }, + { + "text": "ACADEMI", + "boxes": [ + 410, + 212, + 460, + 230 + ] + }, + { + "text": " About 773,000 results", + "boxes": [ + 174, + 256, + 318, + 288 + ] + }, + { + "text": "腾讯网", + "boxes": [ + 212, + 294, + 262, + 322 + ] + }, + { + "text": "雷军最新演讲内容已回应小米SU7事故，相关视频时长..", + "boxes": [ + 176, + 336, + 676, + 371 + ] + }, + { + "text": "1 day ago·雷军面向小米员工发布的最新演讲被业内外关注，视频内容流出了一部分；流出部分", + "boxes": [ + 178, + 363, + 784, + 393 + ] + }, + { + "text": "主要是雷军对3月底小米SU7交通事故的回应，但就演讲内容来看", + "boxes": [ + 178, + 389, + 580, + 413 + ] + }, + { + "text": "知乎专栏", + "boxes": [ + 212, + 433, + 278, + 465 + ] + }, + { + "text": "人", + "boxes": [ + 536, + 433, + 550, + 449 + ] + }, + { + "text": "知", + "boxes": [ + 186, + 449, + 202, + 465 + ] + }, + { + "text": "https://zhu", + "boxes": [ + 218, + 455, + 276, + 472 + ] + }, + { + "text": " Translate this result ", + "boxes": [ + 384, + 455, + 508, + 478 + ] + }, + { + "text": "雷军回应小米SU7致命事故：自动驾驶时代，谁为生命兜底？", + "boxes": [ + 176, + 476, + 720, + 512 + ] + }, + { + "text": "Apr 2,2025·昨晚，小米雷军对“SU7车祸致3死”事故的回应，瞬间点燃热搜，成为全网焦点。", + "boxes": [ + 176, + 502, + 774, + 532 + ] + }, + { + "text": "这起发生在3月29日晚的悲剧，自曝光以来，便在媒体与大众视野中激起千层浪。起初，乐..", + "boxes": [ + 178, + 530, + 770, + 554 + ] + }, + { + "text": "澎湃新闻", + "boxes": [ + 210, + 574, + 278, + 600 + ] + }, + { + "text": "雷军回应SU7车祸致三人死亡：很多问题此刻还没办法回答..", + "boxes": [ + 178, + 619, + 724, + 649 + ] + }, + { + "text": "4月1日晚间，小米CEO雷军回应广受关注的\"小米SU7高速车祸\"事件。雷军表示：29日晚上的这起", + "boxes": [ + 176, + 643, + 790, + 673 + ] + }, + { + "text": "事故，我的心情非常沉重。三位年轻女孩不幸离世，这样的消息，对她们的家人、朋友，乃至，", + "boxes": [ + 176, + 667, + 766, + 695 + ] + }, + { + "text": "网易", + "boxes": [ + 212, + 717, + 246, + 737 + ] + }, + { + "text": "https://ww.163.com>dy aticle Translate this esult", + "boxes": [ + 214, + 728, + 550, + 763 + ] + }, + { + "text": "雷军首度线下回应小米SU7燃爆事故全程紧张看稿曾引以为.", + "boxes": [ + 176, + 756, + 720, + 792 + ] + }, + { + "text": "1 day ago·近日，网上一段视频显示，雷军首度线下公开回应小米SU7燃爆事故，在台上的发言", + "boxes": [ + 178, + 784, + 780, + 814 + ] + }, + { + "text": "中，雷军全程在紧张看稿，并声称，突如其来的交通事故让小米受到了狂风暴雨般的质疑、批评，", + "boxes": [ + 178, + 810, + 778, + 834 + ] + }, + { + "text": "哗哩哗哩", + "boxes": [ + 210, + 854, + 278, + 879 + ] + }, + { + "text": "https:/www.bibilicom video Translatethisresut", + "boxes": [ + 210, + 872, + 528, + 901 + ] + }, + { + "text": "雷军首度线下回应小米SU7燃爆事故全程紧张看稿曾对质量.", + "boxes": [ + 176, + 897, + 720, + 933 + ] + }, + { + "text": "1 day ago·近日，网上一段视频显示，雷军首度线下公开回应小米SU7燃爆事故，在台上的发言", + "boxes": [ + 176, + 923, + 782, + 953 + ] + }, + { + "text": "中，雷军全程在紧张看稿，并声称，突如其来的交通事故让小米受到了狂风暴雨般的质疑、批评..", + "boxes": [ + 176, + 947, + 780, + 977 + ] + }, + { + "text": "搜狐", + "boxes": [ + 208, + 993, + 250, + 1020 + ] + }, + { + "text": "#", + "boxes": [ + 182, + 1005, + 212, + 1026 + ] + }, + { + "text": "https:/ww.sohu.com Translae thisrsuit", + "boxes": [ + 210, + 1006, + 480, + 1045 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 40, + 66, + 130, + 91 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1870, + 91 + ] + }, + { + "text": "腾讯网", + "boxes": [ + 28, + 117, + 136, + 149 + ] + }, + { + "text": "首页", + "boxes": [ + 150, + 119, + 246, + 145 + ] + }, + { + "text": "要闻", + "boxes": [ + 234, + 119, + 284, + 145 + ] + }, + { + "text": "", + "boxes": [ + 276, + 125, + 306, + 139 + ] + }, + { + "text": "长春", + "boxes": [ + 298, + 119, + 370, + 145 + ] + }, + { + "text": "财经", + "boxes": [ + 378, + 119, + 426, + 145 + ] + }, + { + "text": "科技", + "boxes": [ + 444, + 119, + 492, + 145 + ] + }, + { + "text": "日", + "boxes": [ + 482, + 121, + 516, + 141 + ] + }, + { + "text": "电视剧更多", + "boxes": [ + 504, + 117, + 632, + 149 + ] + }, + { + "text": "天和Auto", + "boxes": [ + 1044, + 119, + 1114, + 145 + ] + }, + { + "text": "Q搜索", + "boxes": [ + 1372, + 119, + 1432, + 145 + ] + }, + { + "text": "客户端游戏无障碍快捷访问", + "boxes": [ + 1472, + 117, + 1790, + 147 + ] + }, + { + "text": "登录", + "boxes": [ + 1816, + 119, + 1862, + 147 + ] + }, + { + "text": "腾讯新闻电脑版全新上线", + "boxes": [ + 916, + 208, + 1272, + 250 + ] + }, + { + "text": "广告×", + "boxes": [ + 1438, + 201, + 1468, + 222 + ] + }, + { + "text": "安装体验", + "boxes": [ + 1332, + 226, + 1428, + 264 + ] + }, + { + "text": "1", + "boxes": [ + 510, + 246, + 532, + 260 + ] + }, + { + "text": "", + "boxes": [ + 826, + 246, + 858, + 266 + ] + }, + { + "text": "全网热点，即时推送丨电脑看新闻，大屏更过瘾", + "boxes": [ + 914, + 246, + 1270, + 276 + ] + }, + { + "text": "?", + "boxes": [ + 360, + 326, + 380, + 345 + ] + }, + { + "text": "雷军最新演讲内容已回应小米SU7事故，", + "boxes": [ + 428, + 330, + 1084, + 377 + ] + }, + { + "text": "作者其他文章", + "boxes": [ + 1154, + 334, + 1282, + 365 + ] + }, + { + "text": "轮胎知识讲解：磨损到极限", + "boxes": [ + 1274, + 373, + 1478, + 403 + ] + }, + { + "text": "相关视频时长超过四分钟", + "boxes": [ + 426, + 385, + 842, + 433 + ] + }, + { + "text": "的轮胎主要影响排水性能", + "boxes": [ + 1276, + 395, + 1464, + 429 + ] + }, + { + "text": "8小时前", + "boxes": [ + 1280, + 429, + 1326, + 455 + ] + }, + { + "text": "天和Auto", + "boxes": [ + 472, + 461, + 540, + 486 + ] + }, + { + "text": "+关注", + "boxes": [ + 1040, + 467, + 1096, + 500 + ] + }, + { + "text": "大排量摩托车有没有降价空", + "boxes": [ + 1276, + 462, + 1478, + 496 + ] + }, + { + "text": "2025-05-1614:22 发布于安徽汽车领域创作者", + "boxes": [ + 472, + 472, + 726, + 506 + ] + }, + { + "text": "间，未来能低到几千元-", + "boxes": [ + 1278, + 490, + 1448, + 520 + ] + }, + { + "text": "☆", + "boxes": [ + 356, + 500, + 388, + 530 + ] + }, + { + "text": "Al划重点·全文约1631字，阅读需5分钟", + "boxes": [ + 446, + 534, + 700, + 564 + ] + }, + { + "text": "小时前", + "boxes": [ + 1286, + 528, + 1322, + 542 + ] + }, + { + "text": "「六周年庆典」，「育婴圣", + "boxes": [ + 1282, + 558, + 1478, + 588 + ] + }, + { + "text": "", + "boxes": [ + 354, + 588, + 388, + 617 + ] + }, + { + "text": "堂许愿墙」活动开启！", + "boxes": [ + 1278, + 582, + 1440, + 611 + ] + }, + { + "text": "雷军~面向小米员工发布的最新演讲被业内外关注，视频内容流出了一部分；流出部", + "boxes": [ + 428, + 602, + 1110, + 631 + ] + }, + { + "text": "广告明日方舟", + "boxes": [ + 1277, + 611, + 1367, + 640 + ] + }, + { + "text": "分主要是雷军对3月底小米SU7交通事故的回应，但就演讲内容来看，并没有对造成", + "boxes": [ + 428, + 629, + 1108, + 659 + ] + }, + { + "text": "交通事故和后果的原因进行明确说明。", + "boxes": [ + 430, + 665, + 732, + 689 + ] + }, + { + "text": "精选视频", + "boxes": [ + 1154, + 667, + 1244, + 699 + ] + }, + { + "text": "日", + "boxes": [ + 358, + 679, + 382, + 707 + ] + }, + { + "text": "手机看", + "boxes": [ + 350, + 715, + 390, + 741 + ] + }, + { + "text": "来看视频：", + "boxes": [ + 426, + 713, + 520, + 744 + ] + }, + { + "text": "雷军演讲回应SU7事故", + "boxes": [ + 654, + 772, + 870, + 808 + ] + }, + { + "text": "公众期待和要求远超想象", + "boxes": [ + 654, + 792, + 886, + 834 + ] + }, + { + "text": "00:00", + "boxes": [ + 1220, + 854, + 1262, + 879 + ] + }, + { + "text": "", + "boxes": [ + 1394, + 860, + 1416, + 874 + ] + }, + { + "text": "C", + "boxes": [ + 1435, + 856, + 1458, + 877 + ] + }, + { + "text": "国台办发言人陈斌华：“台独\"翻不了天..", + "boxes": [ + 1164, + 897, + 1462, + 933 + ] + }, + { + "text": "歼10CE首次取得实战战果专家解析歼..", + "boxes": [ + 1170, + 945, + 1458, + 975 + ] + }, + { + "text": "暖心一幕！潘展乐获得全国冠军孙杨..", + "boxes": [ + 1168, + 989, + 1448, + 1024 + ] + }, + { + "text": "雷军最新演讲", + "boxes": [ + 220, + 1048, + 302, + 1072 + ] + } + ] + }, + { + "frame_id": "70_7_9", + "episode_id": 70, + "env_id": 7, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.029364585876465, + "advantages": -0.3928578794002533, + "generated_text": "{\n \"intent\": \"To continue the exploration, clicking on an item from the context menu would provide further interaction and information.\",\n \"action\": \"DoubleClick(170, 688)\"\n}", + "action_string": "DoubleClick(170, 688)", + "intent_string": "To continue the exploration, clicking on an item from the context menu would provide further interaction and information.", + "before_screen_ocr_results": [ + { + "text": "Help", + "boxes": [ + 270, + 28, + 308, + 48 + ] + }, + { + "text": "Copy", + "boxes": [ + 520, + 645, + 560, + 661 + ] + }, + { + "text": " Paste", + "boxes": [ + 506, + 669, + 564, + 689 + ] + }, + { + "text": "Paste Special", + "boxes": [ + 526, + 695, + 614, + 719 + ] + }, + { + "text": " Delete", + "boxes": [ + 506, + 723, + 572, + 742 + ] + }, + { + "text": " select All", + "boxes": [ + 504, + 752, + 592, + 778 + ] + }, + { + "text": "Convert ", + "boxes": [ + 524, + 812, + 584, + 838 + ] + }, + { + "text": "Move Selection", + "boxes": [ + 526, + 840, + 630, + 864 + ] + }, + { + "text": "Duplicate Line / Selection", + "boxes": [ + 526, + 865, + 690, + 888 + ] + }, + { + "text": "Untitled 1 - M..", + "boxes": [ + 368, + 1048, + 468, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "File Edit Search View Document Help", + "boxes": [ + 4, + 22, + 312, + 52 + ] + }, + { + "text": "Untitled 1 - M..", + "boxes": [ + 358, + 1046, + 466, + 1070 + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_70/scores_v3.json b/ScreenExplorer-7B-E1/episode_70/scores_v3.json new file mode 100644 index 0000000000000000000000000000000000000000..67a4f9384c49c12eb5994f4d67069172adc920be --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_70/scores_v3.json @@ -0,0 +1,19 @@ +{ + "directory_path": "logs/ScreenExplorer_2025-05-17_22-11-27ab2072/episode_70", + "single_trajectory_image_diversity": 0.40894097788259387, + "single_trajectory_text_diversity": 0.31867133639752865, + "single_trajectory_image_diversity_max": 0.701171875, + "single_trajectory_image_diversity_min": 0.0, + "single_trajectory_image_diversity_std": 0.23583984375, + "single_trajectory_text_diversity_max": 0.4951171875, + "single_trajectory_text_diversity_min": 0.017333984375, + "single_trajectory_text_diversity_std": 0.1415863037109375, + "group_trajectories_image_diversity": 0.5359733700752258, + "group_trajectories_text_diversity": 0.4472617506980896, + "group_trajectories_image_diversity_max": 1.546875, + "group_trajectories_image_diversity_min": -0.0078125, + "group_trajectories_image_diversity_std": 0.361328125, + "group_trajectories_text_diversity_max": 0.734375, + "group_trajectories_text_diversity_min": -0.0009765625, + "group_trajectories_text_diversity_std": 0.1341552734375 +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_80/game_record_20250518_073212.json b/ScreenExplorer-7B-E1/episode_80/game_record_20250518_073212.json new file mode 100644 index 0000000000000000000000000000000000000000..577789ecf54eeb37575acff26eaa6c87024cf885 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_80/game_record_20250518_073212.json @@ -0,0 +1,43308 @@ +{ + "start_time": "2025-05-18T07:32:12.998140", + "end_time": "2025-05-18T07:39:41.413427", + "buffer_size": 80, + "all_records": [ + { + "frame_id": "80_0_0", + "episode_id": 80, + "env_id": 0, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.406159400939941, + "advantages": 1.9170118570327759, + "generated_text": "{\n \"intent\": \"The first step in exploring this desktop environment involves interacting with an application. Firefox is prominently located near the top left, suggesting it could be a useful starting point for web browsing or testing the environment.\",\n \"action\": \"DoubleClick(59, 42)\"\n}", + "action_string": "DoubleClick(59, 42)", + "intent_string": "The first step in exploring this desktop environment involves interacting with an application. Firefox is prominently located near the top left, suggesting it could be a useful starting point for web browsing or testing the environment.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_1_0", + "episode_id": 80, + "env_id": 1, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.789002418518066, + "advantages": 2.423792839050293, + "generated_text": "{\n \"intent\": \"I will start by clicking one of the applications on the desktop to see what kind of programs are available for exploration.\",\n \"action\": \"DoubleClick(48, 79)\"\n}", + "action_string": "DoubleClick(48, 79)", + "intent_string": "I will start by clicking one of the applications on the desktop to see what kind of programs are available for exploration.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "√C", + "boxes": [ + 40, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_2_0", + "episode_id": 80, + "env_id": 2, + "step": 0, + "format_rewards": 1.0, + "rewards": 9.457181930541992, + "advantages": -0.6629093289375305, + "generated_text": "{\n \"intent\": \"Starting the exploration by opening a productivity application from the desktop environment to gather information or perform tasks.\",\n \"action\": \"DoubleClick(25, 607)\"\n}", + "action_string": "DoubleClick(25, 607)", + "intent_string": "Starting the exploration by opening a productivity application from the desktop environment to gather information or perform tasks.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 6, + 385, + 118, + 423 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 18, + 606, + 98, + 643 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ] + }, + { + "frame_id": "80_3_0", + "episode_id": 80, + "env_id": 3, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.392218589782715, + "advantages": 1.8985579013824463, + "generated_text": "{\n \"intent\": \"Starting the browser will provide access to various applications or content available online.\",\n \"action\": \"DoubleClick(53, 45)\"\n}", + "action_string": "DoubleClick(53, 45)", + "intent_string": "Starting the browser will provide access to various applications or content available online.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_4_0", + "episode_id": 80, + "env_id": 4, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.391105651855469, + "advantages": 1.8970847129821777, + "generated_text": "{\n \"intent\": \"Double-clicking on Firefox will allow us to open a web browser, providing access to the internet which can be a useful resource for gathering information or setting up the desktop even further.\",\n \"action\": \"DoubleClick(55, 37)\"\n}", + "action_string": "DoubleClick(55, 37)", + "intent_string": "Double-clicking on Firefox will allow us to open a web browser, providing access to the internet which can be a useful resource for gathering information or setting up the desktop even further.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_5_0", + "episode_id": 80, + "env_id": 5, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.142377853393555, + "advantages": 1.5678361654281616, + "generated_text": "{\n \"intent\": \"To start exploring various applications by opening them from the desktop.\",\n \"action\": \"DoubleClick(55, 37)\"\n}", + "action_string": "DoubleClick(55, 37)", + "intent_string": "To start exploring various applications by opening them from the desktop.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "Q仓", + "boxes": [ + 70, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_6_0", + "episode_id": 80, + "env_id": 6, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.458223342895508, + "advantages": 1.9859304428100586, + "generated_text": "{\n \"intent\": \"To start exploring the available applications, selecting an icon like Firefox would be beneficial. This will allow us to open further analysis.\",\n \"action\": \"DoubleClick(70, 55)\"\n}", + "action_string": "DoubleClick(70, 55)", + "intent_string": "To start exploring the available applications, selecting an icon like Firefox would be beneficial. This will allow us to open further analysis.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 200, + 97, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_7_0", + "episode_id": 80, + "env_id": 7, + "step": 0, + "format_rewards": 1.0, + "rewards": 11.234184265136719, + "advantages": 1.6893631219863892, + "generated_text": "{\n \"intent\": \"To start exploring the desktop environment by interacting with available applications, the best initial step is to open one of the available programs.\",\n \"action\": \"DoubleClick(45, 43)\"\n}", + "action_string": "DoubleClick(45, 43)", + "intent_string": "To start exploring the desktop environment by interacting with available applications, the best initial step is to open one of the available programs.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 3, + 383, + 121, + 425 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 26, + 609, + 96, + 641 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 89, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "D", + "boxes": [ + 30, + 885, + 80, + 943 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 96 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_0_1", + "episode_id": 80, + "env_id": 0, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.50985336303711, + "advantages": -0.5931865572929382, + "generated_text": "{\n \"intent\": \"Opening a new tab in Firefox allows me to browse the web and gather more information about the environment or explore different websites.\",\n \"action\": \"DoubleClick(76, 34)\"\n}", + "action_string": "DoubleClick(76, 34)", + "intent_string": "Opening a new tab in Firefox allows me to browse the web and gather more information about the environment or explore different websites.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q earch with Bing or enter address", + "boxes": [ + 244, + 57, + 522, + 97 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "New Ta", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_1_1", + "episode_id": 80, + "env_id": 1, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.114648818969727, + "advantages": 0.20739972591400146, + "generated_text": "{\n \"intent\": \"To start exploring by visiting a top site, I will initially select an icon from the 'Top Sites' section.\",\n \"action\": \"DoubleClick(575, 518)\"\n}", + "action_string": "DoubleClick(575, 518)", + "intent_string": "To start exploring by visiting a top site, I will initially select an icon from the 'Top Sites' section.", + "before_screen_ocr_results": [ + { + "text": "New Tab", + "boxes": [ + 26, + 32, + 86, + 50 + ] + }, + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "√C", + "boxes": [ + 40, + 66, + 130, + 91 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 184, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1298, + 111, + 1342, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 189, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 824, + 320, + 930, + 357 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "80_2_1", + "episode_id": 80, + "env_id": 2, + "step": 1, + "format_rewards": 1.0, + "rewards": 11.32597827911377, + "advantages": 1.8108736276626587, + "generated_text": "{\n \"intent\": \"To begin exploring the desktop environment, I should focus on interacting with an application icon. The intention here is to open program icons like Firefox, Mousepad, or Calculator, which could provide immediate feedback about the environment.\",\n \"action\": \"DoubleClick(40, 38)\"\n}", + "action_string": "DoubleClick(40, 38)", + "intent_string": "To begin exploring the desktop environment, I should focus on interacting with an application icon. The intention here is to open program icons like Firefox, Mousepad, or Calculator, which could provide immediate feedback about the environment.", + "before_screen_ocr_results": [ + { + "text": " Firefox ", + "boxes": [ + 20, + 62, + 100, + 99 + ] + }, + { + "text": "D", + "boxes": [ + 36, + 123, + 80, + 163 + ] + }, + { + "text": " Mousepad", + "boxes": [ + 1, + 164, + 117, + 207 + ] + }, + { + "text": "二", + "boxes": [ + 40, + 234, + 76, + 262 + ] + }, + { + "text": "Terminal", + "boxes": [ + 20, + 282, + 104, + 308 + ] + }, + { + "text": "", + "boxes": [ + 28, + 336, + 82, + 379 + ] + }, + { + "text": "Calculator", + "boxes": [ + 6, + 385, + 118, + 423 + ] + }, + { + "text": "Libreo...", + "boxes": [ + 16, + 498, + 98, + 530 + ] + }, + { + "text": "酒", + "boxes": [ + 30, + 554, + 84, + 611 + ] + }, + { + "text": "Writer", + "boxes": [ + 18, + 606, + 98, + 643 + ] + }, + { + "text": "", + "boxes": [ + 30, + 665, + 84, + 723 + ] + }, + { + "text": "Excel", + "boxes": [ + 30, + 719, + 92, + 752 + ] + }, + { + "text": "", + "boxes": [ + 26, + 766, + 88, + 836 + ] + }, + { + "text": "Draw", + "boxes": [ + 28, + 826, + 90, + 866 + ] + }, + { + "text": "Files", + "boxes": [ + 34, + 939, + 88, + 973 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 48, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_3_1", + "episode_id": 80, + "env_id": 3, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.037863731384277, + "advantages": 0.1057569608092308, + "generated_text": "{\n \"intent\": \"By clicking on 'New Tab', we can open a new browsing tab which allows us to visit different websites and access various content on the desktop more efficiently. This action will enable further exploration into the web and possibly find useful or interesting resources.\",\n \"action\": \"DoubleClick(456, 72)\"\n}", + "action_string": "DoubleClick(456, 72)", + "intent_string": "By clicking on 'New Tab', we can open a new browsing tab which allows us to visit different websites and access various content on the desktop more efficiently. This action will enable further exploration into the web and possibly find useful or interesting resources.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C仓", + "boxes": [ + 74, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "Q Search with Bing or entel address", + "boxes": [ + 246, + 60, + 524, + 95 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing 一 bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "iqiyi.com", + "boxes": [ + 602, + 220, + 662, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1225, + 673, + 1417, + 735 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ] + }, + { + "frame_id": "80_4_1", + "episode_id": 80, + "env_id": 4, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.710211753845215, + "advantages": 0.9957646131515503, + "generated_text": "{\n \"intent\": \"To continue exploring the desktop environment, a logical next step is to open a new tab or explore linked websites. Opening the 'Getting Started' link might provide useful information.\",\n \"action\": \"DoubleClick(254, 115)\"\n}", + "action_string": "DoubleClick(254, 115)", + "intent_string": "To continue exploring the desktop environment, a logical next step is to open a new tab or explore linked websites. Opening the 'Getting Started' link might provide useful information.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 260, + 69, + 276, + 85 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "*Mozillo I", + "boxes": [ + 245, + 102, + 377, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 236, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ] + }, + { + "frame_id": "80_5_1", + "episode_id": 80, + "env_id": 5, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.985191345214844, + "advantages": 1.359763503074646, + "generated_text": "{\n \"intent\": \"Selecting an app from Top Sites or Highlights will allow us to explore its content or functionalities. Since there are multiple options provided, choosing one will help in diversifying the exploration.\",\n \"action\": \"DoubleClick(764, 714)\"\n}", + "action_string": "DoubleClick(764, 714)", + "intent_string": "Selecting an app from Top Sites or Highlights will allow us to explore its content or functionalities. Since there are multiple options provided, choosing one will help in diversifying the exploration.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "①| https://cn.bing.com", + "boxes": [ + 248, + 62, + 446, + 91 + ] + }, + { + "text": " Microsoft Bing", + "boxes": [ + 20, + 109, + 184, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 558, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 472, + 300, + 612, + 330 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "中忠中", + "boxes": [ + 1320, + 377, + 1348, + 401 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1652, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1106, + 411, + 1278, + 441 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 488, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 449, + 972, + 478 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 188, + 725, + 274, + 750 + ] + }, + { + "text": "C", + "boxes": [ + 468, + 766, + 482, + 782 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1453, + 760, + 1555, + 797 + ] + }, + { + "text": "孤品", + "boxes": [ + 236, + 784, + 296, + 816 + ] + }, + { + "text": "7", + "boxes": [ + 206, + 794, + 226, + 808 + ] + }, + { + "text": "主属性", + "boxes": [ + 236, + 804, + 294, + 836 + ] + }, + { + "text": "伤害", + "boxes": [ + 424, + 806, + 464, + 836 + ] + }, + { + "text": "防御", + "boxes": [ + 520, + 806, + 558, + 836 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 840 + ] + }, + { + "text": "+2 0936 ", + "boxes": [ + 1632, + 798, + 1700, + 846 + ] + }, + { + "text": "2", + "boxes": [ + 204, + 814, + 226, + 828 + ] + }, + { + "text": "法伤", + "boxes": [ + 336, + 810, + 368, + 832 + ] + }, + { + "text": "副属性", + "boxes": [ + 236, + 830, + 290, + 856 + ] + }, + { + "text": "法伤", + "boxes": [ + 332, + 828, + 368, + 856 + ] + }, + { + "text": "伤害", + "boxes": [ + 422, + 824, + 466, + 860 + ] + }, + { + "text": "防御", + "boxes": [ + 518, + 824, + 560, + 860 + ] + }, + { + "text": "3", + "boxes": [ + 206, + 834, + 226, + 850 + ] + }, + { + "text": "副属性", + "boxes": [ + 238, + 850, + 290, + 876 + ] + }, + { + "text": "法伤", + "boxes": [ + 332, + 850, + 370, + 878 + ] + }, + { + "text": "伤害", + "boxes": [ + 426, + 850, + 462, + 878 + ] + }, + { + "text": "防御", + "boxes": [ + 520, + 850, + 558, + 879 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 852, + 1512, + 887 + ] + }, + { + "text": "261029 ", + "boxes": [ + 1632, + 846, + 1700, + 893 + ] + }, + { + "text": "极品", + "boxes": [ + 238, + 868, + 278, + 895 + ] + }, + { + "text": "法防", + "boxes": [ + 328, + 885, + 372, + 943 + ] + }, + { + "text": "信锯", + "boxes": [ + 416, + 881, + 472, + 965 + ] + }, + { + "text": "湖", + "boxes": [ + 512, + 881, + 566, + 967 + ] + }, + { + "text": "主属性", + "boxes": [ + 238, + 891, + 294, + 917 + ] + }, + { + "text": "13,67", + "boxes": [ + 1628, + 891, + 1698, + 941 + ] + }, + { + "text": "昊性", + "boxes": [ + 234, + 913, + 294, + 939 + ] + }, + { + "text": "1周", + "boxes": [ + 292, + 925, + 314, + 939 + ] + }, + { + "text": "一点视频/萌宠的日常·2天", + "boxes": [ + 846, + 929, + 1010, + 959 + ] + }, + { + "text": "游戏天地·1周", + "boxes": [ + 1146, + 921, + 1254, + 945 + ] + }, + { + "text": "法爆", + "boxes": [ + 334, + 931, + 368, + 953 + ] + }, + { + "text": "1", + "boxes": [ + 826, + 930, + 853, + 958 + ] + }, + { + "text": "探秘浮世万千之前世今生新手", + "boxes": [ + 1134, + 941, + 1414, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "*322", + "boxes": [ + 1628, + 935, + 1702, + 991 + ] + }, + { + "text": "那些和主人长得很像的萌宠，", + "boxes": [ + 826, + 955, + 1088, + 985 + ] + }, + { + "text": "值装备", + "boxes": [ + 193, + 970, + 301, + 1027 + ] + }, + { + "text": "金触趣", + "boxes": [ + 512, + 967, + 564, + 1028 + ] + }, + { + "text": "防御", + "boxes": [ + 708, + 975, + 748, + 1001 + ] + }, + { + "text": "修真全流程通关秘籍", + "boxes": [ + 1138, + 967, + 1330, + 1003 + ] + }, + { + "text": "法伤", + "boxes": [ + 336, + 979, + 368, + 999 + ] + }, + { + "text": "气血", + "boxes": [ + 620, + 979, + 652, + 999 + ] + }, + { + "text": "看看哪只更搞笑？", + "boxes": [ + 824, + 979, + 994, + 1014 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "法爆", + "boxes": [ + 330, + 993, + 370, + 1020 + ] + }, + { + "text": "法伤", + "boxes": [ + 426, + 993, + 466, + 1022 + ] + }, + { + "text": "防御", + "boxes": [ + 614, + 993, + 654, + 1022 + ] + }, + { + "text": "气血", + "boxes": [ + 708, + 993, + 746, + 1022 + ] + }, + { + "text": "法爆", + "boxes": [ + 328, + 1005, + 374, + 1046 + ] + }, + { + "text": "5", + "boxes": [ + 208, + 1018, + 290, + 1042 + ] + }, + { + "text": "法伤", + "boxes": [ + 424, + 1011, + 472, + 1048 + ] + }, + { + "text": "伤", + "boxes": [ + 516, + 1012, + 564, + 1050 + ] + }, + { + "text": "防御", + "boxes": [ + 612, + 1011, + 656, + 1048 + ] + }, + { + "text": "气血", + "boxes": [ + 708, + 1012, + 746, + 1042 + ] + } + ] + }, + { + "frame_id": "80_6_1", + "episode_id": 80, + "env_id": 6, + "step": 1, + "format_rewards": 1.0, + "rewards": 9.306852340698242, + "advantages": -0.8619052171707153, + "generated_text": "{\n \"intent\": \"I am currently exploring the Firefox browser homepage. To broaden my exploration, I will click the 'New Tab' button to open a new tab, thus expanding the available tabs and potentially accessing different resources.\",\n \"action\": \"Click(43, 42)\"\n}", + "action_string": "Click(43, 42)", + "intent_string": "I am currently exploring the Firefox browser homepage. To broaden my exploration, I will click the 'New Tab' button to open a new tab, thus expanding the available tabs and potentially accessing different resources.", + "before_screen_ocr_results": [ + { + "text": "Qbearch with Bing orenteraddress", + "boxes": [ + 244, + 57, + 522, + 100 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_7_1", + "episode_id": 80, + "env_id": 7, + "step": 1, + "format_rewards": 1.0, + "rewards": 10.035078048706055, + "advantages": 0.10206946730613708, + "generated_text": "{\"intent\": \"To explore more websites or applications beyond the initial view, I'll open a new tab by clicking \\\"New Tab.\\\" This will allow me to access additional content or search sites without losing track of my current activity.\", \"action\": \"DoubleClick(280, 68)\"}", + "action_string": "DoubleClick(280, 68)", + "intent_string": "To explore more websites or applications beyond the initial view, I'll open a new tab by clicking \"New Tab.\" This will allow me to access additional content or search sites without losing track of my current activity.", + "before_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 198, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 68, + 128, + 91 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing 一 bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 524, + 488, + 560, + 530 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1223, + 668, + 1419, + 737 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 748, + 772, + 792, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ] + }, + { + "frame_id": "80_0_2", + "episode_id": 80, + "env_id": 0, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.66312026977539, + "advantages": 0.933428168296814, + "generated_text": "{\n \"intent\": \"Since the task is to explore this environment as much as possible, clicking on one of the shortcuts would be beneficial to open relevant sites or applications quickly.\",\n \"action\": \"DoubleClick(786, 538)\"\n}", + "action_string": "DoubleClick(786, 538)", + "intent_string": "Since the task is to explore this environment as much as possible, clicking on one of the shortcuts would be beneficial to open relevant sites or applications quickly.", + "before_screen_ocr_results": [ + { + "text": "New Ta", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 68, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": " Shop for your houhe essentials", + "boxes": [ + 608, + 524, + 918, + 562 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1028 + ] + } + ] + }, + { + "frame_id": "80_1_2", + "episode_id": 80, + "env_id": 1, + "step": 2, + "format_rewards": 1.0, + "rewards": 11.055768013000488, + "advantages": 1.4531880617141724, + "generated_text": "{\n \"intent\": \"The goal is to continue exploring the desktop environment and potentially interact with this website to gather more information. Clicking 'Learn more' could provide additional details about the iPhone 16 family or initiate further exploration.\",\n \"action\": \"DoubleClick(870, 336)\"\n}", + "action_string": "DoubleClick(870, 336)", + "intent_string": "The goal is to continue exploring the desktop environment and potentially interact with this website to gather more information. Clicking 'Learn more' could provide additional details about the iPhone 16 family or initiate further exploration.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1298, + 111, + 1342, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 189, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 824, + 320, + 930, + 357 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Apple", + "boxes": [ + 82, + 32, + 126, + 50 + ] + }, + { + "text": "?", + "boxes": [ + 262, + 73, + 274, + 85 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1082, + 111, + 1158, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "D", + "boxes": [ + 458, + 161, + 518, + 216 + ] + }, + { + "text": "C0", + "boxes": [ + 560, + 155, + 630, + 226 + ] + }, + { + "text": "0", + "boxes": [ + 670, + 163, + 714, + 218 + ] + }, + { + "text": "C", + "boxes": [ + 754, + 157, + 818, + 224 + ] + }, + { + "text": "00", + "boxes": [ + 843, + 161, + 921, + 226 + ] + }, + { + "text": "18", + "boxes": [ + 1106, + 155, + 1168, + 222 + ] + }, + { + "text": "?", + "boxes": [ + 942, + 179, + 994, + 222 + ] + }, + { + "text": "·", + "boxes": [ + 1026, + 179, + 1066, + 218 + ] + }, + { + "text": "口", + "boxes": [ + 1210, + 173, + 1276, + 220 + ] + }, + { + "text": "[os", + "boxes": [ + 1310, + 173, + 1350, + 216 + ] + }, + { + "text": "iphone 16 Pro", + "boxes": [ + 448, + 218, + 530, + 244 + ] + }, + { + "text": "ipPhone 16 ", + "boxes": [ + 564, + 218, + 626, + 244 + ] + }, + { + "text": "iphone 16e", + "boxes": [ + 660, + 218, + 726, + 244 + ] + }, + { + "text": "iPhone 15", + "boxes": [ + 758, + 218, + 820, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 852, + 218, + 910, + 244 + ] + }, + { + "text": "AirPods", + "boxes": [ + 944, + 218, + 994, + 246 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1026, + 218, + 1066, + 246 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1102, + 218, + 1172, + 244 + ] + }, + { + "text": "Apple Card", + "boxes": [ + 1210, + 218, + 1274, + 244 + ] + }, + { + "text": "ios 18", + "boxes": [ + 1308, + 218, + 1350, + 244 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 1386, + 218, + 1458, + 244 + ] + }, + { + "text": "Get $170-$630 in c", + "boxes": [ + 592, + 282, + 718, + 306 + ] + }, + { + "text": "人", + "boxes": [ + 868, + 338, + 882, + 355 + ] + }, + { + "text": "iPhone", + "boxes": [ + 101, + 389, + 365, + 481 + ] + }, + { + "text": "Designed to be loved.", + "boxes": [ + 1520, + 433, + 1800, + 474 + ] + }, + { + "text": "Monday, September 9", + "boxes": [ + 836, + 562, + 1066, + 598 + ] + }, + { + "text": "9:41", + "boxes": [ + 844, + 594, + 1056, + 699 + ] + } + ] + }, + { + "frame_id": "80_2_2", + "episode_id": 80, + "env_id": 2, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.518259048461914, + "advantages": 0.7416709661483765, + "generated_text": "{\n \"intent\": \"To initiate a web search or access a specific site, I will click on one of the website icons.\",\n \"action\": \"DoubleClick(754, 498)\"\n}", + "action_string": "DoubleClick(754, 498)", + "intent_string": "To initiate a web search or access a specific site, I will click on one of the website icons.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "C仓", + "boxes": [ + 56, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "You are on ama", + "boxes": [ + 510, + 461, + 606, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 916, + 560 + ] + }, + { + "text": " Shop deals in Fashion ", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 240, + 969, + 538, + 999 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 967, + 870, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 610, + 997, + 726, + 1028 + ] + } + ] + }, + { + "frame_id": "80_3_2", + "episode_id": 80, + "env_id": 3, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.901972770690918, + "advantages": -0.07412606477737427, + "generated_text": "{\n \"intent\": \"The current action aims to navigate away from the browser's search bar by clicking on the 'New Tab' icon, potentially preparing to open a new website or browsing through additional tabs.\",\n \"action\": \"Click(72, 36)\"\n}", + "action_string": "Click(72, 36)", + "intent_string": "The current action aims to navigate away from the browser's search bar by clicking on the 'New Tab' icon, potentially preparing to open a new website or browsing through additional tabs.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "Q Search with Bing or entel address", + "boxes": [ + 246, + 60, + 524, + 95 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing 一 bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "iqiyi.com", + "boxes": [ + 602, + 220, + 662, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1225, + 673, + 1417, + 735 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 68, + 69, + 124, + 87 + ] + }, + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 62, + 522, + 91 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1225, + 673, + 1417, + 735 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ] + }, + { + "frame_id": "80_4_2", + "episode_id": 80, + "env_id": 4, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.720772743225098, + "advantages": 1.0097445249557495, + "generated_text": "{\n \"intent\": \"The webpage displayed appears to be about downloading Mozilla Firefox for desktop. Double clicking this button would likely take me to the download page, which would further progress my exploration by potentially downloading and installing the browser.\",\n \"action\": \"DoubleClick(440, 593)\"\n}", + "action_string": "DoubleClick(440, 593)", + "intent_string": "The webpage displayed appears to be about downloading Mozilla Firefox for desktop. Double clicking this button would likely take me to the download page, which would further progress my exploration by potentially downloading and installing the browser.", + "before_screen_ocr_results": [ + { + "text": "?", + "boxes": [ + 260, + 69, + 276, + 85 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "*Mozillo I", + "boxes": [ + 245, + 102, + 377, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": "About us", + "boxes": [ + 1088, + 113, + 1174, + 139 + ] + }, + { + "text": "Firefox", + "boxes": [ + 236, + 171, + 328, + 197 + ] + }, + { + "text": "Desktop Mobile Features Compare Browsers", + "boxes": [ + 1264, + 169, + 1630, + 193 + ] + }, + { + "text": "Firefox Browser", + "boxes": [ + 398, + 311, + 608, + 354 + ] + }, + { + "text": "Take control of", + "boxes": [ + 340, + 389, + 680, + 443 + ] + }, + { + "text": "your internet", + "boxes": [ + 338, + 436, + 646, + 499 + ] + }, + { + "text": "Go online with fewer distractions, noise and stress.", + "boxes": [ + 342, + 502, + 748, + 536 + ] + }, + { + "text": "Think of us as bubble wrap for your brain.", + "boxes": [ + 344, + 526, + 674, + 556 + ] + }, + { + "text": "Download Firefox", + "boxes": [ + 368, + 578, + 514, + 609 + ] + }, + { + "text": "Firefox Browser support", + "boxes": [ + 342, + 685, + 494, + 711 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "80_5_2", + "episode_id": 80, + "env_id": 5, + "step": 2, + "format_rewards": 1.0, + "rewards": 11.026019096374512, + "advantages": 1.4138084650039673, + "generated_text": "{\n \"intent\": \"The goal is to explore more about the browser and its features by accessing different sections such as Images, Videos, Translate, Maps, etc. Clicking on one of these options will allow us to see what each tab offers.\",\n \"action\": \"DoubleClick(277, 132)\"\n}", + "action_string": "DoubleClick(277, 132)", + "intent_string": "The goal is to explore more about the browser and its features by accessing different sections such as Images, Videos, Translate, Maps, etc. Clicking on one of these options will allow us to see what each tab offers.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 62, + 132, + 93 + ] + }, + { + "text": "① https://cn.bing.com", + "boxes": [ + 254, + 66, + 446, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生姐", + "boxes": [ + 488, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 448, + 970, + 477 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "DISCOVER", + "boxes": [ + 190, + 725, + 270, + 750 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1451, + 762, + 1555, + 797 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.096 ", + "boxes": [ + 1632, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "点资讯·19小时", + "boxes": [ + 240, + 921, + 334, + 945 + ] + }, + { + "text": "一点视频/萌宠的日常·2天", + "boxes": [ + 846, + 929, + 1010, + 959 + ] + }, + { + "text": "游戏天地·1周", + "boxes": [ + 1142, + 921, + 1254, + 945 + ] + }, + { + "text": "物理学的新边界：科学家直接设计“时空结", + "boxes": [ + 200, + 945, + 752, + 987 + ] + }, + { + "text": "D", + "boxes": [ + 825, + 929, + 854, + 958 + ] + }, + { + "text": "探秘浮世万千之前世今生新手", + "boxes": [ + 1136, + 941, + 1416, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "那些和主人长得很像的萌宠，", + "boxes": [ + 826, + 955, + 1088, + 985 + ] + }, + { + "text": "修真全流程通关秘籍", + "boxes": [ + 1136, + 967, + 1330, + 1003 + ] + }, + { + "text": "", + "boxes": [ + 1581, + 931, + 1641, + 990 + ] + }, + { + "text": "构”，来控制“光\"", + "boxes": [ + 196, + 978, + 438, + 1025 + ] + }, + { + "text": "看看哪只更搞笑？", + "boxes": [ + 826, + 979, + 994, + 1014 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "73", + "boxes": [ + 832, + 1028, + 866, + 1042 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国际版", + "boxes": [ + 260, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ] + }, + { + "frame_id": "80_6_2", + "episode_id": 80, + "env_id": 6, + "step": 2, + "format_rewards": 1.0, + "rewards": 9.315329551696777, + "advantages": -0.8506836295127869, + "generated_text": "{\n \"intent\": \"Exploring the browser features or entering a new search term may be beneficial, but first visually exploring other applications on the desktop could provide important information or resources needed.\",\n \"action\": \"ScrollDown(123, 456)\"\n}", + "action_string": "ScrollDown(123, 456)", + "intent_string": "Exploring the browser features or entering a new search term may be beneficial, but first visually exploring other applications on the desktop could provide important information or resources needed.", + "before_screen_ocr_results": [ + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 188, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlight", + "boxes": [ + 492, + 619, + 572, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 95 + ] + }, + { + "text": "→e", + "boxes": [ + 46, + 69, + 126, + 87 + ] + }, + { + "text": " Getting Started ", + "boxes": [ + 186, + 99, + 314, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_7_2", + "episode_id": 80, + "env_id": 7, + "step": 2, + "format_rewards": 1.0, + "rewards": 10.230111122131348, + "advantages": 0.36024072766304016, + "generated_text": "{\n \"intent\": \"The current goal is to explore the desktop environment by navigating through various applications or web pages. A meaningful first step would be to open a new browser tab to further explore content and possibly download additional applications.\",\n \"action\": \"DoubleClick(1018, 722)\"\n}", + "action_string": "DoubleClick(1018, 722)", + "intent_string": "The current goal is to explore the desktop environment by navigating through various applications or web pages. A meaningful first step would be to open a new browser tab to further explore content and possibly download additional applications.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 26, + 32, + 84, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 36, + 68, + 128, + 91 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing 一 bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 524, + 488, + 560, + 530 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1223, + 668, + 1419, + 737 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 506, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 748, + 772, + 792, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caivarta", + "boxes": [ + 134, + 109, + 218, + 149 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "EN+", + "boxes": [ + 1516, + 114, + 1582, + 154 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 918, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1412, + 486, + 1550, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 916, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1204, + 562 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 1350, + 530, + 1608, + 560 + ] + }, + { + "text": "appliances", + "boxes": [ + 1350, + 558, + 1464, + 588 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 240, + 967, + 426, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 967, + 1544, + 1001 + ] + } + ] + }, + { + "frame_id": "80_0_3", + "episode_id": 80, + "env_id": 0, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.283681869506836, + "advantages": 0.43115395307540894, + "generated_text": "{\n \"intent\": \"The goal is to explore different sections of the Amazon homepage and find interesting categories or deals. The 'Shop Books' section is currently visible, but there might be more areas to explore further on the page.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "The goal is to explore different sections of the Amazon homepage and find interesting categories or deals. The 'Shop Books' section is currently visible, but there might be more areas to explore further on the page.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 249, + 63, + 313, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your hone essentials", + "boxes": [ + 608, + 524, + 918, + 562 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Gaming accessories", + "boxes": [ + 240, + 183, + 440, + 215 + ] + }, + { + "text": "Shopfor your home essentials", + "boxes": [ + 614, + 187, + 916, + 210 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 982, + 185, + 1204, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 185, + 1536, + 214 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 544, + 1058, + 568 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1349, + 541, + 1437, + 572 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 625, + 1544, + 655 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 653, + 354, + 683 + ] + }, + { + "text": "Explore all products in Kitcher", + "boxes": [ + 246, + 987, + 418, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Seemore", + "boxes": [ + 1354, + 987, + 1406, + 1007 + ] + } + ] + }, + { + "frame_id": "80_1_3", + "episode_id": 80, + "env_id": 1, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.334896087646484, + "advantages": 0.49894776940345764, + "generated_text": "{\n \"intent\": \"To continue exploring the website, I will click on an icon to see more details or options related to iPhones.\",\n \"action\": \"DoubleClick(593, 229)\"\n}", + "action_string": "DoubleClick(593, 229)", + "intent_string": "To continue exploring the website, I will click on an icon to see more details or options related to iPhones.", + "before_screen_ocr_results": [ + { + "text": "iPhone - Apple", + "boxes": [ + 30, + 32, + 126, + 50 + ] + }, + { + "text": "×+", + "boxes": [ + 200, + 32, + 246, + 50 + ] + }, + { + "text": "☆", + "boxes": [ + 1608, + 68, + 1658, + 87 + ] + }, + { + "text": "Pad", + "boxes": [ + 638, + 113, + 658, + 127 + ] + }, + { + "text": "Watch", + "boxes": [ + 768, + 113, + 800, + 127 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 948, + 129 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1082, + 111, + 1160, + 129 + ] + }, + { + "text": "Support", + "boxes": [ + 1296, + 111, + 1342, + 129 + ] + }, + { + "text": "Q", + "boxes": [ + 1372, + 111, + 1392, + 127 + ] + }, + { + "text": "B", + "boxes": [ + 456, + 157, + 522, + 218 + ] + }, + { + "text": "B", + "boxes": [ + 562, + 157, + 630, + 224 + ] + }, + { + "text": "0", + "boxes": [ + 670, + 163, + 716, + 214 + ] + }, + { + "text": "口", + "boxes": [ + 748, + 153, + 824, + 224 + ] + }, + { + "text": "18", + "boxes": [ + 1106, + 155, + 1170, + 222 + ] + }, + { + "text": "00", + "boxes": [ + 846, + 165, + 920, + 222 + ] + }, + { + "text": "·", + "boxes": [ + 1026, + 179, + 1066, + 218 + ] + }, + { + "text": "口", + "boxes": [ + 1210, + 173, + 1276, + 220 + ] + }, + { + "text": "[os", + "boxes": [ + 1308, + 171, + 1354, + 218 + ] + }, + { + "text": "9?", + "boxes": [ + 944, + 181, + 992, + 218 + ] + }, + { + "text": "iphone 16 Pro", + "boxes": [ + 448, + 218, + 530, + 244 + ] + }, + { + "text": "ipPhone 16 ", + "boxes": [ + 564, + 218, + 626, + 244 + ] + }, + { + "text": "iphone 16e", + "boxes": [ + 660, + 218, + 726, + 244 + ] + }, + { + "text": "iPhone 15", + "boxes": [ + 758, + 218, + 820, + 244 + ] + }, + { + "text": "Compare", + "boxes": [ + 852, + 218, + 910, + 244 + ] + }, + { + "text": "AirPods", + "boxes": [ + 944, + 218, + 994, + 244 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1026, + 218, + 1066, + 246 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1102, + 218, + 1172, + 244 + ] + }, + { + "text": "Apple Card", + "boxes": [ + 1212, + 222, + 1272, + 242 + ] + }, + { + "text": "ios 18", + "boxes": [ + 1308, + 218, + 1350, + 244 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 1386, + 218, + 1458, + 244 + ] + }, + { + "text": "", + "boxes": [ + 866, + 338, + 882, + 355 + ] + }, + { + "text": "iPhone", + "boxes": [ + 103, + 390, + 365, + 479 + ] + }, + { + "text": "Designed to be loved.", + "boxes": [ + 1520, + 433, + 1800, + 474 + ] + }, + { + "text": "\"", + "boxes": [ + 1842, + 975, + 1858, + 993 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 × |+", + "boxes": [ + 8, + 26, + 246, + 50 + ] + }, + { + "text": "→日", + "boxes": [ + 0, + 51, + 139, + 106 + ] + }, + { + "text": "| https://www.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 550, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "iphone 16 an..", + "boxes": [ + 216, + 1048, + 314, + 1072 + ] + } + ] + }, + { + "frame_id": "80_2_3", + "episode_id": 80, + "env_id": 2, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.984258651733398, + "advantages": 0.0347982719540596, + "generated_text": "{\n \"intent\": \"To continue exploring the Amazon website, I should click on a category to see more product options or navigate further into the site.\",\n \"action\": \"Click(340, 547)\"\n}", + "action_string": "Click(340, 547)", + "intent_string": "To continue exploring the Amazon website, I should click on a category to see more product options or navigate further into the site.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 249, + 63, + 313, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 240, + 969, + 538, + 999 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 967, + 870, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 610, + 997, + 724, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 136, + 143 + ] + }, + { + "text": " Caivarta", + "boxes": [ + 134, + 109, + 218, + 149 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Card", + "boxes": [ + 358, + 169, + 416, + 187 + ] + }, + { + "text": "Kitchen essentials", + "boxes": [ + 522, + 274, + 900, + 328 + ] + }, + { + "text": "Under $50 ", + "boxes": [ + 522, + 318, + 638, + 355 + ] + }, + { + "text": "on Canada for millions of products with fast local delivery. Click here to go to a", + "boxes": [ + 838, + 461, + 1340, + 478 + ] + }, + { + "text": "Gaming afcessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shopfor your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "New home arrivals under $5o", + "boxes": [ + 240, + 969, + 538, + 999 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 967, + 870, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 610, + 997, + 724, + 1028 + ] + } + ] + }, + { + "frame_id": "80_3_3", + "episode_id": 80, + "env_id": 3, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.282093048095703, + "advantages": 0.4290507733821869, + "generated_text": "{\n \"intent\": \"To visit Amazon, which might be the site referenced by one of the shortcuts or as a general step towards exploring online resources.\",\n \"action\": \"DoubleClick(795, 507)\"\n}", + "action_string": "DoubleClick(795, 507)", + "intent_string": "To visit Amazon, which might be the site referenced by one of the shortcuts or as a general step towards exploring online resources.", + "before_screen_ocr_results": [ + { + "text": "C", + "boxes": [ + 68, + 69, + 124, + 87 + ] + }, + { + "text": "Q Search with Bing or enter adress", + "boxes": [ + 248, + 62, + 522, + 91 + ] + }, + { + "text": " Apple -- tt://www.apple.com", + "boxes": [ + 254, + 107, + 480, + 137 + ] + }, + { + "text": " bing - bing.com", + "boxes": [ + 252, + 141, + 396, + 173 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "爱奇艺-在线视频网站-海量正版高清视频在线观看", + "boxes": [ + 272, + 214, + 590, + 238 + ] + }, + { + "text": "知zhihu", + "boxes": [ + 258, + 252, + 324, + 272 + ] + }, + { + "text": "QIYI", + "boxes": [ + 860, + 459, + 934, + 506 + ] + }, + { + "text": "知", + "boxes": [ + 978, + 451, + 1060, + 518 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "。", + "boxes": [ + 1236, + 484, + 1268, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 634, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "amazon", + "boxes": [ + 986, + 669, + 1174, + 723 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1225, + 673, + 1417, + 735 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 542, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1232, + 772, + 1268, + 792 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 116, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "Retuns", + "boxes": [ + 1738, + 117, + 1780, + 137 + ] + }, + { + "text": "", + "boxes": [ + 1826, + 115, + 1858, + 135 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " Sign in ", + "boxes": [ + 1624, + 179, + 1678, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "You are on ama", + "boxes": [ + 510, + 461, + 606, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 916, + 554 + ] + }, + { + "text": " Shop deals in Fashion ", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": "See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1028 + ] + } + ] + }, + { + "frame_id": "80_4_3", + "episode_id": 80, + "env_id": 4, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.130852699279785, + "advantages": 0.22884930670261383, + "generated_text": "{\n \"intent\": \"The goal is to explore the browser window and its contents. The highlighted part suggests selecting a download option, which could potentially lead to downloading Firefox. To begin exploring the browser effectively, I will double-click on one of the download options.\",\n \"action\": \"DoubleClick(956, 610)\"\n}", + "action_string": "DoubleClick(956, 610)", + "intent_string": "The goal is to explore the browser window and its contents. The highlighted part suggests selecting a download option, which could potentially lead to downloading Firefox. To begin exploring the browser effectively, I will double-click on one of the download options.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1604, + 68, + 1658, + 87 + ] + }, + { + "text": "Mozillα /", + "boxes": [ + 255, + 104, + 375, + 152 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 740, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Almost there!", + "boxes": [ + 804, + 365, + 1108, + 413 + ] + }, + { + "text": "Select a download option", + "boxes": [ + 670, + 413, + 1236, + 469 + ] + }, + { + "text": "below", + "boxes": [ + 874, + 467, + 1032, + 522 + ] + }, + { + "text": " Download for Linux 32-bit", + "boxes": [ + 846, + 588, + 1058, + 617 + ] + }, + { + "text": "Download for Linux 64-bit", + "boxes": [ + 848, + 639, + 1064, + 669 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 730, + 677, + 1178, + 715 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 701, + 1118, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now", + "boxes": [ + 764, + 594, + 1102, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository insteadl.", + "boxes": [ + 786, + 703, + 1122, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "80_5_3", + "episode_id": 80, + "env_id": 5, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.94706916809082, + "advantages": 1.3093000650405884, + "generated_text": "{\n \"intent\": \"I will click on one of the image results to explore more details or potentially download an interesting picture related to world tourist attractions.\",\n \"action\": \"DoubleClick(1180, 666)\"\n}", + "action_string": "DoubleClick(1180, 666)", + "intent_string": "I will click on one of the image results to explore more details or potentially download an interesting picture related to world tourist attractions.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1658, + 87 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 244, + 139 + ] + }, + { + "text": "国际版", + "boxes": [ + 260, + 119, + 302, + 147 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 338, + 183 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "ngland Th - vrogue.co", + "boxes": [ + 1122, + 945, + 1266, + 963 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→×", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 69, + 1658, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "crosoftBing", + "boxes": [ + 160, + 175, + 228, + 193 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": " Number 1 Tourist Spot In The World - vrogue.co", + "boxes": [ + 1238, + 238, + 1608, + 270 + ] + }, + { + "text": " vrogue.co | 1200 x 936 jpeg I 1 yr ago", + "boxes": [ + 1244, + 264, + 1518, + 290 + ] + }, + { + "text": "Visit site ", + "boxes": [ + 1246, + 296, + 1304, + 322 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 84, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 84, + 963 + ] + } + ] + }, + { + "frame_id": "80_6_3", + "episode_id": 80, + "env_id": 6, + "step": 3, + "format_rewards": 1.0, + "rewards": 9.861613273620605, + "advantages": -0.12755116820335388, + "generated_text": "{\n \"intent\": \"To start exploring websites from the browser's homepage, I will double-click one of the top site icons.\",\n \"action\": \"DoubleClick(567, 486)\"\n}", + "action_string": "DoubleClick(567, 486)", + "intent_string": "To start exploring websites from the browser's homepage, I will double-click one of the top site icons.", + "before_screen_ocr_results": [ + { + "text": "×十", + "boxes": [ + 200, + 32, + 248, + 50 + ] + }, + { + "text": "→", + "boxes": [ + 40, + 66, + 128, + 89 + ] + }, + { + "text": "QSearch with Bing or enteraddress", + "boxes": [ + 246, + 60, + 522, + 95 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highligh", + "boxes": [ + 492, + 619, + 566, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1298, + 111, + 1342, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 189, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 824, + 320, + 930, + 357 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "80_7_3", + "episode_id": 80, + "env_id": 7, + "step": 3, + "format_rewards": 1.0, + "rewards": 10.072942733764648, + "advantages": 0.1521921008825302, + "generated_text": "{\n \"intent\": \"Clicking 'Sign in' will allow me to go through any new customer registration process which might introduce me to potential functionalities or services different from those explicitly advertised to signed-out users.\",\n \"action\": \"DoubleClick(1629, 117)\"\n}", + "action_string": "DoubleClick(1629, 117)", + "intent_string": "Clicking 'Sign in' will allow me to go through any new customer registration process which might introduce me to potential functionalities or services different from those explicitly advertised to signed-out users.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1474, + 508 + ] + }, + { + "text": "", + "boxes": [ + 1522, + 488, + 1564, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 1350, + 530, + 1608, + 560 + ] + }, + { + "text": "appliances", + "boxes": [ + 1350, + 556, + 1462, + 586 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 240, + 967, + 426, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1656, + 87 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1872, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 806, + 434, + 930, + 471 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "80_0_4", + "episode_id": 80, + "env_id": 0, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.413484573364258, + "advantages": 0.6029777526855469, + "generated_text": "{\n \"intent\": \"I will start by clicking one of these suggested product categories to see if there's anything interesting or useful I can find.\",\n \"action\": \"DoubleClick(688, 370)\"\n}", + "action_string": "DoubleClick(688, 370)", + "intent_string": "I will start by clicking one of these suggested product categories to see if there's anything interesting or useful I can find.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1606, + 69, + 1658, + 87 + ] + }, + { + "text": "Gamingaccessories", + "boxes": [ + 244, + 189, + 438, + 212 + ] + }, + { + "text": "Shopfor your home essentials", + "boxes": [ + 614, + 187, + 916, + 210 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 982, + 185, + 1202, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 185, + 1534, + 214 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 544, + 1058, + 568 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1349, + 541, + 1437, + 572 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 625, + 1544, + 655 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 653, + 354, + 683 + ] + }, + { + "text": "Explore all products in Kitcher", + "boxes": [ + 246, + 987, + 418, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "Seemore", + "boxes": [ + 1354, + 987, + 1406, + 1007 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→×", + "boxes": [ + 36, + 66, + 128, + 89 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 302, + 115, + 404, + 141 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 174, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 70, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Household", + "boxes": [ + 10, + 322, + 86, + 347 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 134, + 435 + ] + }, + { + "text": "★★★★☆&p", + "boxes": [ + 14, + 435, + 142, + 459 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 132, + 604 + ] + }, + { + "text": "nsored", + "boxes": [ + 628, + 594, + 680, + 611 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning ", + "boxes": [ + 314, + 611, + 574, + 643 + ] + }, + { + "text": "TreeLen Long Handle Broom and ", + "boxes": [ + 608, + 617, + 854, + 641 + ] + }, + { + "text": "6Pcs Steel Wool Scrubbers - Flexible ", + "boxes": [ + 902, + 617, + 1172, + 641 + ] + }, + { + "text": " Shark StainStriker HairPro Pet,", + "boxes": [ + 1192, + 613, + 1422, + 643 + ] + }, + { + "text": " BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1486, + 613, + 1748, + 637 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 623, + 102, + 649 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 635, + 534, + 665 + ] + }, + { + "text": "Dustpan Set,Upright Dust Pan", + "boxes": [ + 606, + 639, + 832, + 669 + ] + }, + { + "text": " Scrubber Pads for Cleaning Dishes,", + "boxes": [ + 898, + 637, + 1160, + 669 + ] + }, + { + "text": "Portable Spot, Stain, & Odor", + "boxes": [ + 1194, + 639, + 1412, + 669 + ] + }, + { + "text": "handheld portable steam cleaner for", + "boxes": [ + 1486, + 637, + 1760, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 5, + 648, + 103, + 688 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 663, + 524, + 693 + ] + }, + { + "text": "Combo for Home, Kitchen, Room,", + "boxes": [ + 606, + 661, + 856, + 691 + ] + }, + { + "text": "Pots, Pans, Grills and Sinks", + "boxes": [ + 900, + 663, + 1100, + 693 + ] + }, + { + "text": " Eliminator for Carpets, Area Rugs.", + "boxes": [ + 1192, + 659, + 1446, + 693 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1486, + 661, + 1728, + 691 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": "Reusable and Washable, Pack of...", + "boxes": [ + 314, + 689, + 566, + 713 + ] + }, + { + "text": "Office, Lobby Floor Use Without..", + "boxes": [ + 602, + 681, + 858, + 721 + ] + }, + { + "text": "★★★★1.708", + "boxes": [ + 904, + 691, + 1044, + 715 + ] + }, + { + "text": "Upholstery, Cars, with Bonus...", + "boxes": [ + 1194, + 687, + 1420, + 717 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 683, + 1726, + 715 + ] + }, + { + "text": "Household Cleaning Sponges", + "boxes": [ + 22, + 697, + 214, + 725 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Options: 4 sizes ", + "boxes": [ + 316, + 717, + 406, + 741 + ] + }, + { + "text": "★★★★☆42.011", + "boxes": [ + 608, + 713, + 760, + 742 + ] + }, + { + "text": "★★★★★2.759", + "boxes": [ + 1196, + 713, + 1338, + 742 + ] + }, + { + "text": "★★★★☆2.828", + "boxes": [ + 1488, + 711, + 1630, + 739 + ] + }, + { + "text": "4K+ bought in past month", + "boxes": [ + 1194, + 732, + 1366, + 760 + ] + }, + { + "text": "30K+ bought in past month", + "boxes": [ + 1486, + 728, + 1668, + 763 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "★★★★79.824", + "boxes": [ + 316, + 738, + 462, + 767 + ] + }, + { + "text": "$728 ($1.21/count)", + "boxes": [ + 897, + 738, + 1035, + 783 + ] + }, + { + "text": "$16999", + "boxes": [ + 1190, + 764, + 1278, + 802 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 14, + 782, + 108, + 800 + ] + }, + { + "text": "Delivery Fri, May 23 ", + "boxes": [ + 902, + 776, + 1038, + 806 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 800, + 750, + 830 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 902, + 800, + 996, + 826 + ] + }, + { + "text": " Delivery Wed, May 21", + "boxes": [ + 1192, + 800, + 1340, + 830 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 834, + 60, + 852 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 312, + 826, + 454, + 860 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 610, + 826, + 700, + 850 + ] + }, + { + "text": "Ad to cart", + "boxes": [ + 903, + 825, + 993, + 865 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1570, + 846 + ] + }, + { + "text": " Bissel", + "boxes": [ + 12, + 856, + 72, + 876 + ] + }, + { + "text": "Add to car", + "boxes": [ + 616, + 856, + 686, + 881 + ] + }, + { + "text": "Addto", + "boxes": [ + 1212, + 862, + 1252, + 878 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "二", + "boxes": [ + 14, + 878, + 36, + 891 + ] + }, + { + "text": " 1 sustai", + "boxes": [ + 320, + 874, + 390, + 899 + ] + }, + { + "text": "o-cedar", + "boxes": [ + 9, + 891, + 87, + 922 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1486, + 883, + 1630, + 913 + ] + }, + { + "text": " scrub Daddy", + "boxes": [ + 7, + 909, + 121, + 944 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 328, + 907, + 400, + 933 + ] + }, + { + "text": "swiffer", + "boxes": [ + 9, + 930, + 83, + 960 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + } + ] + }, + { + "frame_id": "80_1_4", + "episode_id": 80, + "env_id": 1, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.103520393371582, + "advantages": -1.1310619115829468, + "generated_text": "{\n \"intent\": \"To continue exploring the webpage, I will scroll down to see more content.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring the webpage, I will scroll down to see more content.", + "before_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 × |+", + "boxes": [ + 8, + 26, + 246, + 50 + ] + }, + { + "text": "→日", + "boxes": [ + 0, + 51, + 139, + 106 + ] + }, + { + "text": "| https://www.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 550, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "iphone 16 an..", + "boxes": [ + 216, + 1048, + 314, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 ×+", + "boxes": [ + 6, + 24, + 248, + 54 + ] + }, + { + "text": "①→白", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "| https://ww.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 552, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四◎", + "boxes": [ + 1786, + 66, + 1880, + 91 + ] + }, + { + "text": "?", + "boxes": [ + 96, + 1044, + 216, + 1074 + ] + }, + { + "text": " iphone 16 an..", + "boxes": [ + 208, + 1048, + 316, + 1072 + ] + } + ] + }, + { + "frame_id": "80_2_4", + "episode_id": 80, + "env_id": 2, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.445547103881836, + "advantages": -0.6783106923103333, + "generated_text": "{\n \"intent\": \"To continue exploring the Amazon website and find more products or information by双clicking on one of the categories.\",\n \"action\": \"DoubleClick(500, 150)\"\n}", + "action_string": "DoubleClick(500, 150)", + "intent_string": "To continue exploring the Amazon website and find more products or information by双clicking on one of the categories.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1586, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " canarta", + "boxes": [ + 132, + 107, + 220, + 153 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 416, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 117, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": "New arrivals in Toys", + "boxes": [ + 770, + 252, + 1192, + 314 + ] + }, + { + "text": "Gaming afcessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 528, + 1202, + 558 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 240, + 969, + 538, + 999 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 608, + 967, + 868, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": " Refresh your space", + "boxes": [ + 1346, + 965, + 1544, + 1003 + ] + }, + { + "text": " appliances", + "boxes": [ + 608, + 994, + 724, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 660, + 294, + 944, + 349 + ] + }, + { + "text": "You are on amazon.com. Yo", + "boxes": [ + 510, + 461, + 684, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1544, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 240, + 969, + 538, + 999 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 967, + 870, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": " Refresh your space", + "boxes": [ + 1348, + 965, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 610, + 997, + 724, + 1028 + ] + } + ] + }, + { + "frame_id": "80_3_4", + "episode_id": 80, + "env_id": 3, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.542123794555664, + "advantages": -0.550469160079956, + "generated_text": "{\n \"intent\": \"Selecting a category to explore different product options and gain more insight into the site's offerings.\",\n \"action\": \"DoubleClick(474, 472)\"\n}", + "action_string": "DoubleClick(474, 472)", + "intent_string": "Selecting a category to explore different product options and gain more insight into the site's offerings.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "①", + "boxes": [ + 249, + 63, + 313, + 94 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 109, + 414, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "mEN+", + "boxes": [ + 1520, + 119, + 1582, + 151 + ] + }, + { + "text": "Returns", + "boxes": [ + 1736, + 113, + 1782, + 139 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 169, + 246, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Sign in", + "boxes": [ + 1622, + 177, + 1680, + 208 + ] + }, + { + "text": "New customer? Start here", + "boxes": [ + 1586, + 216, + 1714, + 234 + ] + }, + { + "text": "Shop Books", + "boxes": [ + 652, + 284, + 838, + 322 + ] + }, + { + "text": "explore titles", + "boxes": [ + 670, + 317, + 818, + 360 + ] + }, + { + "text": "roducts withfast local delivery. Click here to go to amazon.ca", + "boxes": [ + 1010, + 455, + 1400, + 484 + ] + }, + { + "text": "", + "boxes": [ + 1434, + 488, + 1476, + 508 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 136, + 143 + ] + }, + { + "text": " Caivarta", + "boxes": [ + 134, + 109, + 218, + 149 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Card", + "boxes": [ + 358, + 169, + 416, + 187 + ] + }, + { + "text": "Kitchen essentials", + "boxes": [ + 522, + 274, + 900, + 328 + ] + }, + { + "text": "Under $50", + "boxes": [ + 522, + 318, + 636, + 355 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ] + }, + { + "frame_id": "80_4_4", + "episode_id": 80, + "env_id": 4, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.772886276245117, + "advantages": -0.24500180780887604, + "generated_text": "{\n \"intent\": \"The current action will open Firefox by clicking on the Firefox browser link. This will provide access to the Firefox application page, where I can explore more options or download the Firefox browser.\",\n \"action\": \"DoubleClick(789, 908)\"\n}", + "action_string": "DoubleClick(789, 908)", + "intent_string": "The current action will open Firefox by clicking on the Firefox browser link. This will provide access to the Firefox application page, where I can explore more options or download the Firefox browser.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 956, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with/usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now", + "boxes": [ + 764, + 594, + 1102, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 734, + 685, + 1172, + 709 + ] + }, + { + "text": "You can set up our APT repository insteadl.", + "boxes": [ + 786, + 703, + 1122, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1788, + 66, + 1880, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 351, + 1088, + 381 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 924, + 413 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which s: TAR Archive (72.4 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwnload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for files like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system", + "boxes": [ + 704, + 887, + 1204, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "80_5_4", + "episode_id": 80, + "env_id": 5, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.595006942749023, + "advantages": -0.4804661273956299, + "generated_text": "{\n \"intent\": \"To continue exploring various world tourist attractions and gather more information, it makes sense to click on one of the thumbnails or links provided next on the page. This will likely lead to a new webpage with additional details about tourist attractions.\",\n \"action\": \"DoubleClick(1050, 197)\"\n}", + "action_string": "DoubleClick(1050, 197)", + "intent_string": "To continue exploring various world tourist attractions and gather more information, it makes sense to click on one of the thumbnails or links provided next on the page. This will likely lead to a new webpage with additional details about tourist attractions.", + "before_screen_ocr_results": [ + { + "text": "→×", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 69, + 1656, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "rosoftBing", + "boxes": [ + 166, + 175, + 228, + 193 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": " Number 1 Tourist Spot In The World - vrogue.co", + "boxes": [ + 1238, + 238, + 1608, + 270 + ] + }, + { + "text": " vrogue.co | 1200 × 936 jpeg I 1 yr ago", + "boxes": [ + 1242, + 264, + 1518, + 290 + ] + }, + { + "text": "Visit site ", + "boxes": [ + 1246, + 296, + 1304, + 322 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 84, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "个", + "boxes": [ + 14, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 69, + 1656, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "crosoftBing", + "boxes": [ + 160, + 175, + 228, + 193 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": " Number 1 Tourist Spot In The World - vrogue.co", + "boxes": [ + 1238, + 238, + 1608, + 270 + ] + }, + { + "text": " vrogue.co | 1200 x 936 jpeg I 1 yr ago", + "boxes": [ + 1244, + 264, + 1518, + 290 + ] + }, + { + "text": "Vist sit", + "boxes": [ + 1244, + 292, + 1306, + 324 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 84, + 465 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 943, + 76, + 963 + ] + } + ] + }, + { + "frame_id": "80_6_4", + "episode_id": 80, + "env_id": 6, + "step": 4, + "format_rewards": 1.0, + "rewards": 10.039026260375977, + "advantages": 0.10729583352804184, + "generated_text": "{\n \"intent\": \"Double clicking the 'Shop iPhone' button will likely lead to more detailed information about the iPhone family.\",\n \"action\": \"DoubleClick(1034, 338)\"\n}", + "action_string": "DoubleClick(1034, 338)", + "intent_string": "Double clicking the 'Shop iPhone' button will likely lead to more detailed information about the iPhone family.", + "before_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1298, + 111, + 1342, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1366, + 109, + 1394, + 129 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1440, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 189, + 1053, + 265 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 824, + 320, + 930, + 357 + ] + }, + { + "text": " Shop iPhone", + "boxes": [ + 972, + 320, + 1082, + 357 + ] + }, + { + "text": "Built for Apple Intelligence.", + "boxes": [ + 830, + 383, + 1078, + 419 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 992, + 1060, + 1035 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1658, + 87 + ] + }, + { + "text": "Support", + "boxes": [ + 1300, + 113, + 1340, + 129 + ] + }, + { + "text": "α", + "boxes": [ + 1368, + 113, + 1392, + 127 + ] + }, + { + "text": "2", + "boxes": [ + 1422, + 115, + 1438, + 129 + ] + }, + { + "text": "iPhone", + "boxes": [ + 857, + 191, + 1053, + 263 + ] + }, + { + "text": " Meet the iPhone 16 family.", + "boxes": [ + 792, + 260, + 1112, + 300 + ] + }, + { + "text": " Learn more", + "boxes": [ + 824, + 320, + 930, + 357 + ] + }, + { + "text": " Shop iPgone", + "boxes": [ + 972, + 322, + 1082, + 359 + ] + }, + { + "text": " Built for Apple Intelligence.", + "boxes": [ + 828, + 379, + 1078, + 421 + ] + }, + { + "text": "WATCH", + "boxes": [ + 888, + 909, + 1070, + 959 + ] + }, + { + "text": "SERIES 10", + "boxes": [ + 902, + 959, + 1032, + 997 + ] + }, + { + "text": "Thinstant classic.", + "boxes": [ + 848, + 994, + 1060, + 1035 + ] + } + ] + }, + { + "frame_id": "80_7_4", + "episode_id": 80, + "env_id": 7, + "step": 4, + "format_rewards": 1.0, + "rewards": 9.067477226257324, + "advantages": -1.1787734031677246, + "generated_text": "{\n \"intent\": \"I need to continue exploring the desktop environment by interacting with elements commonly found there. The displayed browser suggests we may be on a web page, but since the goal is to explore the desktop environment, clicking on the Firefox icon or navigating elements such as the address bar would be more informative.\",\n \"action\": \"Click(184, 74)\"\n}", + "action_string": "Click(184, 74)", + "intent_string": "I need to continue exploring the desktop environment by interacting with elements commonly found there. The displayed browser suggests we may be on a web page, but since the goal is to explore the desktop environment, clicking on the Firefox icon or navigating elements such as the address bar would be more informative.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "1", + "boxes": [ + 16, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 69, + 1656, + 87 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1872, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1392, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "80_0_5", + "episode_id": 80, + "env_id": 0, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.370940208435059, + "advantages": -0.7770701050758362, + "generated_text": "{\n \"intent\": \"The user has opened an Amazon search page for 'cleaning tools'. The next logical step is to explore the items in detail by clicking on one to see more information about it.\",\n \"action\": \"DoubleClick(417, 752)\"\n}", + "action_string": "DoubleClick(417, 752)", + "intent_string": "The user has opened an Amazon search page for 'cleaning tools'. The next logical step is to explore the items in detail by clicking on one to see more information about it.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1872, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1884, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 254, + 172, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 70, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Checke", + "boxes": [ + 312, + 294, + 352, + 310 + ] + }, + { + "text": "Household", + "boxes": [ + 10, + 322, + 86, + 347 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 134, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 132, + 604 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1486, + 613, + 1748, + 637 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 623, + 100, + 649 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning ", + "boxes": [ + 316, + 641, + 574, + 665 + ] + }, + { + "text": "TreeLen Long Handle Broom and", + "boxes": [ + 608, + 641, + 852, + 665 + ] + }, + { + "text": "6Pcs Steel Wool Scrubbers - Flexible ", + "boxes": [ + 900, + 635, + 1174, + 665 + ] + }, + { + "text": " Shark StainStriker HairPro Pet,", + "boxes": [ + 1194, + 641, + 1422, + 665 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1484, + 635, + 1758, + 665 + ] + }, + { + "text": " Department", + "boxes": [ + 7, + 651, + 101, + 686 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 659, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Dust Pan", + "boxes": [ + 606, + 663, + 832, + 693 + ] + }, + { + "text": "Scrubber Pads for Cleaning Dishes,", + "boxes": [ + 900, + 661, + 1158, + 693 + ] + }, + { + "text": "Portable Spot, Stain,& Odor", + "boxes": [ + 1194, + 661, + 1412, + 691 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1486, + 661, + 1728, + 691 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": "Combo for Home, Kitchen, Room,", + "boxes": [ + 604, + 681, + 856, + 719 + ] + }, + { + "text": "Pots, Pans, Grills and Sinks", + "boxes": [ + 900, + 687, + 1100, + 717 + ] + }, + { + "text": "EliminatorforCarpets, AreaRugs", + "boxes": [ + 1190, + 681, + 1448, + 721 + ] + }, + { + "text": "kitchens, fabric steaming tool..", + "boxes": [ + 1486, + 685, + 1726, + 715 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Reusable and Washable, Pack of..", + "boxes": [ + 314, + 713, + 562, + 737 + ] + }, + { + "text": "Office, Lobby Floor Use Without...", + "boxes": [ + 608, + 711, + 858, + 741 + ] + }, + { + "text": "★★★★★1.708", + "boxes": [ + 902, + 713, + 1046, + 742 + ] + }, + { + "text": "Upholstery, Cars, with Bonus.", + "boxes": [ + 1194, + 711, + 1414, + 741 + ] + }, + { + "text": "★★★★☆2.828", + "boxes": [ + 1488, + 711, + 1630, + 739 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": "★★★★★2.759", + "boxes": [ + 1196, + 737, + 1336, + 766 + ] + }, + { + "text": "★★★★★79.824", + "boxes": [ + 314, + 760, + 464, + 794 + ] + }, + { + "text": "$728 ($1.21/count)", + "boxes": [ + 895, + 761, + 1037, + 807 + ] + }, + { + "text": "4K+", + "boxes": [ + 1202, + 764, + 1230, + 778 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 1226, + 762, + 1364, + 780 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 12, + 780, + 110, + 804 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "ught in past month", + "boxes": [ + 370, + 790, + 490, + 808 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "$16999", + "boxes": [ + 1190, + 788, + 1276, + 826 + ] + }, + { + "text": "DeliveryFri, May 23", + "boxes": [ + 902, + 798, + 1034, + 832 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1628, + 830 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 608, + 823, + 748, + 856 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 826, + 992, + 850 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1568, + 846 + ] + }, + { + "text": "Bissell", + "boxes": [ + 7, + 848, + 75, + 882 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 610, + 852, + 694, + 870 + ] + }, + { + "text": "Add to car", + "boxes": [ + 912, + 860, + 976, + 878 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1198, + 852, + 1282, + 870 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "L", + "boxes": [ + 16, + 878, + 36, + 891 + ] + }, + { + "text": "Adto cart", + "boxes": [ + 605, + 871, + 703, + 916 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1198, + 876, + 1284, + 907 + ] + }, + { + "text": "o-cedar", + "boxes": [ + 9, + 891, + 87, + 922 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1486, + 883, + 1630, + 913 + ] + }, + { + "text": " 1 sustaina", + "boxes": [ + 320, + 895, + 406, + 921 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 7, + 907, + 121, + 944 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1684, + 931 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1809, + 913, + 1863, + 938 + ] + }, + { + "text": " swiffer", + "boxes": [ + 9, + 928, + 85, + 962 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 322, + 931, + 402, + 957 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 254, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1884, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 174, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 70, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Household", + "boxes": [ + 10, + 322, + 86, + 347 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 136, + 435 + ] + }, + { + "text": "★★★★☆&p", + "boxes": [ + 14, + 435, + 142, + 459 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "$59.99", + "boxes": [ + 1820, + 508, + 1856, + 522 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 134, + 604 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1486, + 613, + 1748, + 637 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 623, + 100, + 649 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning", + "boxes": [ + 312, + 637, + 576, + 669 + ] + }, + { + "text": "TreeLen Long Handle Broom and", + "boxes": [ + 606, + 639, + 850, + 663 + ] + }, + { + "text": "6Pcs Steel Wool Scrubbers - Flexible", + "boxes": [ + 900, + 635, + 1174, + 665 + ] + }, + { + "text": " Shark StainStriker HairPro Pet,", + "boxes": [ + 1194, + 641, + 1422, + 665 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1486, + 637, + 1760, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 7, + 651, + 101, + 686 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 661, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Dust Pan", + "boxes": [ + 606, + 663, + 832, + 693 + ] + }, + { + "text": "Scrubber Pads for Cleaning Dishes,", + "boxes": [ + 900, + 661, + 1158, + 693 + ] + }, + { + "text": "Portable Spot, Stain, & Odor ", + "boxes": [ + 1196, + 665, + 1410, + 689 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1484, + 659, + 1726, + 689 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": "Combo for Home, Kitchen, Room,", + "boxes": [ + 604, + 683, + 856, + 717 + ] + }, + { + "text": "Pots, Pans, Grills and Sinks", + "boxes": [ + 900, + 687, + 1100, + 717 + ] + }, + { + "text": "Eliminator forCarpets,AreaRug", + "boxes": [ + 1190, + 681, + 1446, + 719 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1726, + 715 + ] + }, + { + "text": "Household Cleaning Sponges", + "boxes": [ + 22, + 697, + 214, + 725 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Reusable and Washable, Pack of..", + "boxes": [ + 314, + 713, + 562, + 737 + ] + }, + { + "text": "Office, Lobby Floor Use Without..", + "boxes": [ + 608, + 711, + 858, + 741 + ] + }, + { + "text": "★★★★1.708", + "boxes": [ + 904, + 715, + 1044, + 739 + ] + }, + { + "text": "Upholstery, Cars, with Bonus..", + "boxes": [ + 1194, + 711, + 1414, + 741 + ] + }, + { + "text": "★★★★☆2,828", + "boxes": [ + 1490, + 715, + 1630, + 739 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "Options: 4 sizes", + "boxes": [ + 318, + 741, + 410, + 764 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": "★★★★★2.759", + "boxes": [ + 1196, + 737, + 1336, + 766 + ] + }, + { + "text": "★★★★★79.824", + "boxes": [ + 311, + 758, + 465, + 795 + ] + }, + { + "text": "$728 ($1.21/count)", + "boxes": [ + 895, + 761, + 1037, + 807 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 1226, + 762, + 1364, + 780 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 12, + 780, + 110, + 804 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "pought in past month", + "boxes": [ + 358, + 790, + 490, + 808 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "$16999", + "boxes": [ + 1190, + 788, + 1276, + 826 + ] + }, + { + "text": "Delivery Fri, May 23 ", + "boxes": [ + 902, + 800, + 1036, + 830 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": "$499 ($0.35/count) Typical: $65.64", + "boxes": [ + 309, + 813, + 559, + 860 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 822, + 748, + 856 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 834, + 60, + 852 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 826, + 992, + 850 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1568, + 846 + ] + }, + { + "text": " Bissel", + "boxes": [ + 12, + 856, + 70, + 876 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 610, + 852, + 694, + 870 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 848, + 1284, + 872 + ] + }, + { + "text": "Add to car", + "boxes": [ + 912, + 860, + 976, + 878 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "L", + "boxes": [ + 16, + 878, + 36, + 891 + ] + }, + { + "text": "Adto cart", + "boxes": [ + 605, + 871, + 703, + 916 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1198, + 876, + 1286, + 907 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1486, + 883, + 1630, + 913 + ] + }, + { + "text": "o-Ceda", + "boxes": [ + 14, + 897, + 84, + 919 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 12, + 911, + 118, + 941 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1686, + 931 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1812, + 917, + 1862, + 937 + ] + }, + { + "text": "swiffer", + "boxes": [ + 11, + 933, + 81, + 957 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 326, + 931, + 402, + 957 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ] + }, + { + "frame_id": "80_1_5", + "episode_id": 80, + "env_id": 1, + "step": 5, + "format_rewards": 1.0, + "rewards": 8.87983226776123, + "advantages": -1.4271647930145264, + "generated_text": "{\n \"intent\": \"To scroll through the webpage to explore more content related to the product being displayed.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To scroll through the webpage to explore more content related to the product being displayed.", + "before_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 ×+", + "boxes": [ + 6, + 24, + 248, + 54 + ] + }, + { + "text": "①→白", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "| https://ww.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 552, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四◎", + "boxes": [ + 1786, + 66, + 1880, + 91 + ] + }, + { + "text": "?", + "boxes": [ + 96, + 1044, + 216, + 1074 + ] + }, + { + "text": " iphone 16 an..", + "boxes": [ + 208, + 1048, + 316, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 ×+", + "boxes": [ + 6, + 24, + 248, + 54 + ] + }, + { + "text": "①→白", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "| https://ww.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 552, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1882, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 94, + 1044, + 216, + 1074 + ] + }, + { + "text": "iphone 16 an..", + "boxes": [ + 214, + 1048, + 316, + 1072 + ] + } + ] + }, + { + "frame_id": "80_2_5", + "episode_id": 80, + "env_id": 2, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.34641170501709, + "advantages": 0.5141913294792175, + "generated_text": "{\n \"intent\": \"To navigate through Amazon's homepage and explore potential purchasing options or new products.\",\n \"action\": \"DoubleClick(375, 125)\"\n}", + "action_string": "DoubleClick(375, 125)", + "intent_string": "To navigate through Amazon's homepage and explore potential purchasing options or new products.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": "@ Caharta", + "boxes": [ + 129, + 100, + 221, + 154 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 414, + 141 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "You are on amazon.com. Yo", + "boxes": [ + 510, + 461, + 684, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 240, + 969, + 538, + 999 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 967, + 870, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": " Refresh your space", + "boxes": [ + 1348, + 965, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 610, + 997, + 724, + 1028 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Anlazon", + "boxes": [ + 294, + 115, + 412, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1445, + 107, + 1483, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "19", + "boxes": [ + 1822, + 113, + 1862, + 139 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 299, + 148, + 421, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "+", + "boxes": [ + 1446, + 149, + 1480, + 183 + ] + }, + { + "text": "gaming headsets", + "boxes": [ + 304, + 189, + 440, + 218 + ] + }, + { + "text": "×× ", + "boxes": [ + 1450, + 183, + 1476, + 262 + ] + }, + { + "text": "beauty", + "boxes": [ + 299, + 217, + 367, + 255 + ] + }, + { + "text": "home decor", + "boxes": [ + 302, + 256, + 400, + 288 + ] + }, + { + "text": "××", + "boxes": [ + 1450, + 252, + 1476, + 326 + ] + }, + { + "text": "pots and pans ", + "boxes": [ + 302, + 294, + 418, + 326 + ] + }, + { + "text": "gaming", + "boxes": [ + 304, + 332, + 368, + 357 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 326, + 1478, + 361 + ] + }, + { + "text": "bedding", + "boxes": [ + 299, + 356, + 375, + 398 + ] + }, + { + "text": "×", + "boxes": [ + 1452, + 359, + 1474, + 393 + ] + }, + { + "text": "kitchen and dining", + "boxes": [ + 302, + 393, + 454, + 429 + ] + }, + { + "text": "x", + "boxes": [ + 1452, + 391, + 1474, + 423 + ] + }, + { + "text": " nintendo switch games ", + "boxes": [ + 302, + 433, + 486, + 463 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1202, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 236, + 963, + 542, + 1003 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 969, + 866, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 608, + 994, + 724, + 1029 + ] + } + ] + }, + { + "frame_id": "80_3_5", + "episode_id": 80, + "env_id": 3, + "step": 5, + "format_rewards": 1.0, + "rewards": 10.452469825744629, + "advantages": 0.654583752155304, + "generated_text": "{\n \"intent\": \"Opening the Amazon search bar will allow me to type in a keyword or product name to explore further options within this website.\",\n \"action\": \"DoubleClick(360, 125)\"\n}", + "action_string": "DoubleClick(360, 125)", + "intent_string": "Opening the Amazon search bar will allow me to type in a keyword or product name to explore further options within this website.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1586, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 130, + 143 + ] + }, + { + "text": " canarta", + "boxes": [ + 132, + 107, + 220, + 153 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 416, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1450, + 117, + 1476, + 137 + ] + }, + { + "text": "三EN+", + "boxes": [ + 1520, + 119, + 1580, + 151 + ] + }, + { + "text": "三All", + "boxes": [ + 24, + 167, + 68, + 195 + ] + }, + { + "text": "Registry", + "boxes": [ + 190, + 165, + 248, + 197 + ] + }, + { + "text": "Gift Caro", + "boxes": [ + 356, + 169, + 412, + 187 + ] + }, + { + "text": " New arrivals in Toys", + "boxes": [ + 768, + 252, + 1192, + 314 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 528, + 1202, + 558 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": " Top categories in Kitchen", + "boxes": [ + 238, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": " Refresh your space", + "boxes": [ + 1346, + 965, + 1544, + 1003 + ] + }, + { + "text": " appliances", + "boxes": [ + 238, + 994, + 356, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四@", + "boxes": [ + 1779, + 59, + 1881, + 96 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Kmazon", + "boxes": [ + 296, + 115, + 414, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1445, + 107, + 1483, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "19", + "boxes": [ + 1822, + 113, + 1862, + 139 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 299, + 148, + 421, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "+", + "boxes": [ + 1446, + 149, + 1480, + 183 + ] + }, + { + "text": "gaming headsets", + "boxes": [ + 304, + 189, + 440, + 218 + ] + }, + { + "text": "×× ", + "boxes": [ + 1450, + 183, + 1476, + 262 + ] + }, + { + "text": "beauty", + "boxes": [ + 299, + 217, + 367, + 255 + ] + }, + { + "text": "home decor", + "boxes": [ + 302, + 256, + 400, + 288 + ] + }, + { + "text": "××", + "boxes": [ + 1450, + 250, + 1476, + 326 + ] + }, + { + "text": "pots and pans", + "boxes": [ + 302, + 294, + 416, + 326 + ] + }, + { + "text": "gaming", + "boxes": [ + 302, + 330, + 370, + 361 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 326, + 1478, + 363 + ] + }, + { + "text": "bedding", + "boxes": [ + 299, + 356, + 375, + 398 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 357, + 1476, + 397 + ] + }, + { + "text": "kitchen and dining", + "boxes": [ + 302, + 393, + 454, + 429 + ] + }, + { + "text": "x", + "boxes": [ + 1452, + 391, + 1474, + 423 + ] + }, + { + "text": " nintendo switch games ", + "boxes": [ + 302, + 433, + 486, + 463 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1202, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 242, + 969, + 498, + 999 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 354, + 1026 + ] + } + ] + }, + { + "frame_id": "80_4_5", + "episode_id": 80, + "env_id": 4, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.269573211669922, + "advantages": -0.9112527370452881, + "generated_text": "{\n \"intent\": \"The goal is to explore the desktop and possibly install Firefox. The current action involves interacting with a dialog box related to opening an archive file for Firefox. To proceed, I should probably right-click to get more options, such as opening or canceling the process.\",\n \"action\": \"RightClick(753, 573)\"\n}", + "action_string": "RightClick(753, 573)", + "intent_string": "The goal is to explore the desktop and possibly install Firefox. The current action involves interacting with a dialog box related to opening an archive file for Firefox. To proceed, I should probably right-click to get more options, such as opening or canceling the process.", + "before_screen_ocr_results": [ + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1788, + 66, + 1882, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 351, + 1088, + 381 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 924, + 413 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which s: TAR Archive (72.4 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwnload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for files like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distribution?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system", + "boxes": [ + 704, + 887, + 1204, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with]/usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "Save File", + "boxes": [ + 768, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for files like this from now on", + "boxes": [ + 762, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distrbuton?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 706, + 889, + 1204, + 913 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ] + }, + { + "frame_id": "80_5_5", + "episode_id": 80, + "env_id": 5, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.70692253112793, + "advantages": -0.33232003450393677, + "generated_text": "{\n \"intent\": \"To continue exploring the exploration goal by interacting visually with the pop-up image gallery. doubleClick will likely lead to more images or functionalities related to this image.\",\n \"action\": \"DoubleClick(679, 542)\"\n}", + "action_string": "DoubleClick(679, 542)", + "intent_string": "To continue exploring the exploration goal by interacting visually with the pop-up image gallery. doubleClick will likely lead to more images or functionalities related to this image.", + "before_screen_ocr_results": [ + { + "text": "个", + "boxes": [ + 14, + 69, + 34, + 85 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "☆", + "boxes": [ + 1598, + 69, + 1656, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 190, + 117, + 238, + 143 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "crosoftBing", + "boxes": [ + 160, + 175, + 228, + 193 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": " Number 1 Tourist Spot In The World - vrogue.co", + "boxes": [ + 1238, + 238, + 1608, + 270 + ] + }, + { + "text": " vrogue.co | 1200 x 936 jpeg I 1 yr ago", + "boxes": [ + 1244, + 264, + 1518, + 290 + ] + }, + { + "text": "Vist sit", + "boxes": [ + 1244, + 292, + 1306, + 324 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 84, + 465 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 943, + 76, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "·☆", + "boxes": [ + 1590, + 69, + 1656, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 187, + 112, + 239, + 144 + ] + }, + { + "text": "国际版", + "boxes": [ + 266, + 119, + 304, + 139 + ] + }, + { + "text": "rosoftBing", + "boxes": [ + 166, + 175, + 228, + 193 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 262, + 169, + 432, + 199 + ] + }, + { + "text": "Most Beautiful Tourist Spot In The World - Tanya Tanya", + "boxes": [ + 1240, + 236, + 1664, + 270 + ] + }, + { + "text": "Visit site ", + "boxes": [ + 1248, + 296, + 1304, + 322 + ] + }, + { + "text": "These Are ", + "boxes": [ + 18, + 447, + 90, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 302, + 1072 + ] + } + ] + }, + { + "frame_id": "80_6_5", + "episode_id": 80, + "env_id": 6, + "step": 5, + "format_rewards": 1.0, + "rewards": 9.774094581604004, + "advantages": -0.24340234696865082, + "generated_text": "{\n \"intent\": \"Since my goal is to explore the computer desktop environment, clicking on an icon or link would be beneficial. The 'Buy' button under different iPhone models seems like a logical choice for further exploration.\",\n \"action\": \"DoubleClick(271, 984)\"\n}", + "action_string": "DoubleClick(271, 984)", + "intent_string": "Since my goal is to explore the computer desktop environment, clicking on an icon or link would be beneficial. The 'Buy' button under different iPhone models seems like a logical choice for further exploration.", + "before_screen_ocr_results": [ + { + "text": "目", + "boxes": [ + 1546, + 69, + 1562, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1660, + 89 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 944, + 127 + ] + }, + { + "text": "TV & Hon", + "boxes": [ + 986, + 111, + 1036, + 129 + ] + }, + { + "text": "Shop iPhone", + "boxes": [ + 134, + 218, + 412, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1600, + 230, + 1742, + 254 + ] + }, + { + "text": "ecialist7", + "boxes": [ + 1714, + 254, + 1766, + 272 + ] + }, + { + "text": "Visit an Apple Store", + "boxes": [ + 1596, + 284, + 1732, + 312 + ] + }, + { + "text": "?", + "boxes": [ + 1562, + 294, + 1588, + 318 + ] + }, + { + "text": "Find one near you 7", + "boxes": [ + 1598, + 303, + 1726, + 332 + ] + }, + { + "text": "All Models ", + "boxes": [ + 144, + 389, + 230, + 413 + ] + }, + { + "text": " Shopping guides ", + "boxes": [ + 260, + 387, + 400, + 417 + ] + }, + { + "text": "Ways to Save", + "boxes": [ + 430, + 389, + 540, + 413 + ] + }, + { + "text": " Sustainability", + "boxes": [ + 570, + 387, + 682, + 417 + ] + }, + { + "text": "Accessories", + "boxes": [ + 716, + 389, + 814, + 413 + ] + }, + { + "text": " Setup and Support ", + "boxes": [ + 846, + 387, + 1000, + 417 + ] + }, + { + "text": "The iPhone Experience", + "boxes": [ + 1032, + 389, + 1214, + 413 + ] + }, + { + "text": "Special Stores ", + "boxes": [ + 1248, + 387, + 1366, + 417 + ] + }, + { + "text": "All models. Take your pick.", + "boxes": [ + 138, + 472, + 464, + 508 + ] + }, + { + "text": "PPLE INTELLIGENCE", + "boxes": [ + 180, + 560, + 296, + 578 + ] + }, + { + "text": "APPLE INTELLIGENCE", + "boxes": [ + 1012, + 556, + 1140, + 580 + ] + }, + { + "text": "iPhone 16 Pro &", + "boxes": [ + 164, + 580, + 376, + 621 + ] + }, + { + "text": "iPhone 16 &", + "boxes": [ + 586, + 582, + 746, + 619 + ] + }, + { + "text": "iPhone 16e", + "boxes": [ + 1004, + 579, + 1156, + 622 + ] + }, + { + "text": "iPhone 15 &", + "boxes": [ + 1424, + 580, + 1582, + 617 + ] + }, + { + "text": "Expl", + "boxes": [ + 1841, + 579, + 1917, + 624 + ] + }, + { + "text": "iPhone 16 Pro Max", + "boxes": [ + 164, + 611, + 408, + 653 + ] + }, + { + "text": "iPhone 16 Plus", + "boxes": [ + 586, + 613, + 778, + 651 + ] + }, + { + "text": "iPhone 15 Plus", + "boxes": [ + 1424, + 611, + 1618, + 653 + ] + }, + { + "text": "acce", + "boxes": [ + 1846, + 619, + 1914, + 651 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "BuyiPhone16", + "boxes": [ + 674, + 4, + 804, + 22 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1128, + 2, + 1246, + 26 + ] + }, + { + "text": "→", + "boxes": [ + 8, + 60, + 130, + 91 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1880, + 91 + ] + }, + { + "text": "Store", + "boxes": [ + 512, + 113, + 546, + 127 + ] + }, + { + "text": "Mac", + "boxes": [ + 572, + 113, + 602, + 127 + ] + }, + { + "text": "iPad", + "boxes": [ + 636, + 113, + 660, + 127 + ] + }, + { + "text": "iPhone", + "boxes": [ + 692, + 113, + 732, + 129 + ] + }, + { + "text": "Watch", + "boxes": [ + 764, + 113, + 804, + 129 + ] + }, + { + "text": "Vision", + "boxes": [ + 834, + 111, + 874, + 131 + ] + }, + { + "text": "AirPods", + "boxes": [ + 904, + 111, + 948, + 131 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 982, + 111, + 1048, + 131 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1078, + 109, + 1164, + 133 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 111, + 1264, + 133 + ] + }, + { + "text": "Support", + "boxes": [ + 1294, + 111, + 1344, + 131 + ] + }, + { + "text": "Q", + "boxes": [ + 1368, + 109, + 1396, + 131 + ] + }, + { + "text": "6", + "boxes": [ + 1418, + 111, + 1442, + 135 + ] + }, + { + "text": "Carrer Deals at Apple", + "boxes": [ + 496, + 170, + 654, + 207 + ] + }, + { + "text": "", + "boxes": [ + 672, + 187, + 706, + 214 + ] + }, + { + "text": "S", + "boxes": [ + 864, + 191, + 892, + 210 + ] + }, + { + "text": "T", + "boxes": [ + 1058, + 189, + 1084, + 212 + ] + }, + { + "text": "p to $1000 afte", + "boxes": [ + 1324, + 183, + 1410, + 201 + ] + }, + { + "text": "See all deals ①", + "boxes": [ + 500, + 197, + 606, + 220 + ] + }, + { + "text": "V", + "boxes": [ + 1256, + 193, + 1274, + 206 + ] + } + ] + }, + { + "frame_id": "80_7_5", + "episode_id": 80, + "env_id": 7, + "step": 5, + "format_rewards": 1.0, + "rewards": 8.860170364379883, + "advantages": -1.4531917572021484, + "generated_text": "{\n \"intent\": \"To continue exploring, selecting an icon on the desktop is logical since it could lead to new applications or additional resources.\",\n \"action\": \"DoubleClick(58, 83)\"\n}", + "action_string": "DoubleClick(58, 83)", + "intent_string": "To continue exploring, selecting an icon on the desktop is logical since it could lead to new applications or additional resources.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1392, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "?", + "boxes": [ + 262, + 73, + 274, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "80_0_6", + "episode_id": 80, + "env_id": 0, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.359620094299316, + "advantages": -0.7920548915863037, + "generated_text": "{\n \"intent\": \"To begin exploring different cleaning tools available on Amazon, I will start by right-clicking near one of the product listings to open the context menu and gather more information about the item.\",\n \"action\": \"RightClick(780, 454)\"\n}", + "action_string": "RightClick(780, 454)", + "intent_string": "To begin exploring different cleaning tools available on Amazon, I will start by right-clicking near one of the product listings to open the context menu and gather more information about the item.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 254, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1884, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 174, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 70, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Household", + "boxes": [ + 10, + 322, + 86, + 347 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 136, + 435 + ] + }, + { + "text": "★★★★☆&p", + "boxes": [ + 14, + 435, + 142, + 459 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "$59.99", + "boxes": [ + 1820, + 508, + 1856, + 522 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 134, + 604 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1486, + 613, + 1748, + 637 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 623, + 100, + 649 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning", + "boxes": [ + 312, + 637, + 576, + 669 + ] + }, + { + "text": "TreeLen Long Handle Broom and", + "boxes": [ + 606, + 639, + 850, + 663 + ] + }, + { + "text": "6Pcs Steel Wool Scrubbers - Flexible", + "boxes": [ + 900, + 635, + 1174, + 665 + ] + }, + { + "text": " Shark StainStriker HairPro Pet,", + "boxes": [ + 1194, + 641, + 1422, + 665 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1486, + 637, + 1760, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 7, + 651, + 101, + 686 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 661, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Dust Pan", + "boxes": [ + 606, + 663, + 832, + 693 + ] + }, + { + "text": "Scrubber Pads for Cleaning Dishes,", + "boxes": [ + 900, + 661, + 1158, + 693 + ] + }, + { + "text": "Portable Spot, Stain, & Odor ", + "boxes": [ + 1196, + 665, + 1410, + 689 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1484, + 659, + 1726, + 689 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": "Combo for Home, Kitchen, Room,", + "boxes": [ + 604, + 683, + 856, + 717 + ] + }, + { + "text": "Pots, Pans, Grills and Sinks", + "boxes": [ + 900, + 687, + 1102, + 717 + ] + }, + { + "text": "Eliminator forCarpets,AreaRug", + "boxes": [ + 1190, + 681, + 1446, + 719 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1726, + 715 + ] + }, + { + "text": "Household Cleaning Sponges", + "boxes": [ + 22, + 697, + 214, + 725 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Reusable and Washable, Pack of..", + "boxes": [ + 314, + 713, + 562, + 737 + ] + }, + { + "text": "Office, Lobby Floor Use Without..", + "boxes": [ + 608, + 711, + 858, + 741 + ] + }, + { + "text": "★★★★1,708", + "boxes": [ + 904, + 715, + 1044, + 739 + ] + }, + { + "text": "Upholstery, Cars, with Bonus..", + "boxes": [ + 1194, + 711, + 1414, + 741 + ] + }, + { + "text": "★★★★☆2.828", + "boxes": [ + 1488, + 711, + 1630, + 739 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "Options: 4 sizes", + "boxes": [ + 318, + 741, + 410, + 764 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": "★★★★★2.759", + "boxes": [ + 1196, + 737, + 1336, + 766 + ] + }, + { + "text": "★★★★★79.824", + "boxes": [ + 311, + 758, + 465, + 795 + ] + }, + { + "text": "$728 ($1.21/count)", + "boxes": [ + 895, + 761, + 1037, + 807 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 1226, + 762, + 1364, + 780 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 12, + 780, + 110, + 804 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "pought in past month", + "boxes": [ + 358, + 790, + 490, + 808 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "$16999", + "boxes": [ + 1190, + 788, + 1276, + 826 + ] + }, + { + "text": "Delivery Fri, May 23 ", + "boxes": [ + 902, + 800, + 1036, + 830 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": "$499 ($0.35/count) Typical: $65.64", + "boxes": [ + 309, + 813, + 559, + 860 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 822, + 748, + 856 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 834, + 60, + 852 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 826, + 992, + 850 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1568, + 846 + ] + }, + { + "text": " Bissel", + "boxes": [ + 12, + 856, + 70, + 876 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 610, + 852, + 694, + 870 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 848, + 1284, + 872 + ] + }, + { + "text": "Add to car", + "boxes": [ + 912, + 860, + 976, + 878 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "L", + "boxes": [ + 16, + 878, + 36, + 891 + ] + }, + { + "text": "Adto cart", + "boxes": [ + 605, + 871, + 703, + 916 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1198, + 876, + 1286, + 907 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1486, + 883, + 1630, + 913 + ] + }, + { + "text": "o-Ceda", + "boxes": [ + 14, + 897, + 84, + 919 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 12, + 911, + 118, + 941 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1686, + 931 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1812, + 917, + 1862, + 937 + ] + }, + { + "text": "swiffer", + "boxes": [ + 11, + 933, + 81, + 957 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 326, + 931, + 402, + 957 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 254, + 172, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 378, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 68, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Checke", + "boxes": [ + 312, + 294, + 352, + 310 + ] + }, + { + "text": "Household", + "boxes": [ + 8, + 318, + 88, + 349 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 38, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 132, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 52, + 496 + ] + }, + { + "text": "Open Link in New Window", + "boxes": [ + 812, + 486, + 982, + 510 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "Open Link in New Private Windo", + "boxes": [ + 808, + 514, + 1022, + 538 + ] + }, + { + "text": "Bookmark This Link ", + "boxes": [ + 806, + 540, + 944, + 570 + ] + }, + { + "text": "Save Link As..", + "boxes": [ + 808, + 570, + 902, + 594 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 132, + 604 + ] + }, + { + "text": "Save Link to Pocket", + "boxes": [ + 806, + 594, + 942, + 623 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 14, + 623, + 100, + 649 + ] + }, + { + "text": " Copy Link Location", + "boxes": [ + 804, + 621, + 940, + 651 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1488, + 615, + 1750, + 639 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning", + "boxes": [ + 312, + 635, + 576, + 669 + ] + }, + { + "text": "Treelen Long Handle Br", + "boxes": [ + 606, + 639, + 790, + 669 + ] + }, + { + "text": "ibbers - Flexible ", + "boxes": [ + 1050, + 637, + 1174, + 667 + ] + }, + { + "text": " Shark StainStriker HairPro Pet,", + "boxes": [ + 1192, + 637, + 1424, + 667 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1484, + 635, + 1758, + 665 + ] + }, + { + "text": "Department", + "boxes": [ + 12, + 655, + 98, + 681 + ] + }, + { + "text": "View Image", + "boxes": [ + 804, + 649, + 892, + 681 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 661, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Du", + "boxes": [ + 606, + 663, + 790, + 693 + ] + }, + { + "text": "eaning Dishes,", + "boxes": [ + 1052, + 665, + 1160, + 693 + ] + }, + { + "text": "Portable Spot, Stain, & Odor", + "boxes": [ + 1196, + 665, + 1410, + 689 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1486, + 661, + 1728, + 691 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": " Saye Image As..", + "boxes": [ + 801, + 673, + 917, + 709 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": " Combo for Home, Kitch", + "boxes": [ + 604, + 683, + 788, + 721 + ] + }, + { + "text": "I Sinks", + "boxes": [ + 1050, + 687, + 1100, + 713 + ] + }, + { + "text": "Eliminator for Carpets,Area Rugs.", + "boxes": [ + 1190, + 681, + 1448, + 721 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1724, + 715 + ] + }, + { + "text": "Household Cleaning Sponges", + "boxes": [ + 24, + 699, + 214, + 723 + ] + }, + { + "text": " Reusable and Washable, Pack of..", + "boxes": [ + 312, + 711, + 558, + 735 + ] + }, + { + "text": "Office, Lobby Floor Use", + "boxes": [ + 606, + 707, + 788, + 742 + ] + }, + { + "text": "Copy Image", + "boxes": [ + 806, + 705, + 894, + 731 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Upholstery, Cars, with Bonus.", + "boxes": [ + 1194, + 711, + 1412, + 741 + ] + }, + { + "text": "★★★★☆2,828", + "boxes": [ + 1490, + 715, + 1630, + 739 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": "Copy Image Location", + "boxes": [ + 806, + 729, + 950, + 760 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 22, + 739, + 168, + 762 + ] + }, + { + "text": "Options: 4 sizes", + "boxes": [ + 318, + 739, + 406, + 764 + ] + }, + { + "text": "★★★★★2.759", + "boxes": [ + 1196, + 737, + 1336, + 766 + ] + }, + { + "text": "★★★★★79.824", + "boxes": [ + 314, + 760, + 464, + 794 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 634, + 762, + 776, + 780 + ] + }, + { + "text": "Email i", + "boxes": [ + 806, + 758, + 856, + 786 + ] + }, + { + "text": "Image", + "boxes": [ + 846, + 762, + 892, + 782 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 1226, + 762, + 1364, + 780 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 14, + 782, + 108, + 800 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 354, + 790, + 490, + 808 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Copy", + "boxes": [ + 804, + 786, + 846, + 814 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "$16999", + "boxes": [ + 1190, + 788, + 1276, + 826 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1628, + 830 + ] + }, + { + "text": "$499 ($0.35/count) Typical: $63.64", + "boxes": [ + 309, + 813, + 557, + 860 + ] + }, + { + "text": "Send Link to Device", + "boxes": [ + 804, + 812, + 946, + 842 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 822, + 750, + 856 + ] + }, + { + "text": "Brands", + "boxes": [ + 12, + 832, + 62, + 858 + ] + }, + { + "text": "Delivery Wed, May 2", + "boxes": [ + 1196, + 826, + 1330, + 850 + ] + }, + { + "text": "Ships to Cana", + "boxes": [ + 1494, + 828, + 1568, + 846 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 610, + 850, + 696, + 874 + ] + }, + { + "text": "Print Selection ", + "boxes": [ + 806, + 842, + 910, + 866 + ] + }, + { + "text": " Bissell", + "boxes": [ + 10, + 852, + 74, + 878 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 850, + 1284, + 874 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "Adto cart", + "boxes": [ + 605, + 871, + 703, + 916 + ] + }, + { + "text": "View Selection Source", + "boxes": [ + 806, + 866, + 960, + 895 + ] + }, + { + "text": "", + "boxes": [ + 16, + 878, + 38, + 891 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1198, + 876, + 1286, + 907 + ] + }, + { + "text": "o-ceda", + "boxes": [ + 9, + 891, + 85, + 922 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1488, + 883, + 1630, + 913 + ] + }, + { + "text": " 1 sustainal", + "boxes": [ + 320, + 895, + 408, + 921 + ] + }, + { + "text": "Inspect Accessibility Properties", + "boxes": [ + 808, + 897, + 1014, + 921 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 7, + 907, + 121, + 944 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1684, + 931 + ] + }, + { + "text": "Inspect Element (Q)", + "boxes": [ + 806, + 921, + 948, + 951 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1812, + 917, + 1862, + 935 + ] + }, + { + "text": " swiffer", + "boxes": [ + 9, + 928, + 83, + 962 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 322, + 931, + 402, + 957 + ] + }, + { + "text": "Holikme", + "boxes": [ + 14, + 957, + 88, + 975 + ] + }, + { + "text": " Take a Screenshot ", + "boxes": [ + 782, + 947, + 934, + 979 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ] + }, + { + "frame_id": "80_1_6", + "episode_id": 80, + "env_id": 1, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.39342212677002, + "advantages": -0.7473101019859314, + "generated_text": "{\n \"intent\": \"To view more details about the iPhone 16 and 16 Plus products by scrolling down the page.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To view more details about the iPhone 16 and 16 Plus products by scrolling down the page.", + "before_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 ×+", + "boxes": [ + 6, + 24, + 248, + 54 + ] + }, + { + "text": "①→白", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "| https://ww.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 552, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1882, + 91 + ] + }, + { + "text": "0", + "boxes": [ + 94, + 1044, + 216, + 1074 + ] + }, + { + "text": "iphone 16 an..", + "boxes": [ + 214, + 1048, + 316, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 ×+", + "boxes": [ + 6, + 24, + 248, + 54 + ] + }, + { + "text": "①→白", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "| https://ww.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 552, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1882, + 91 + ] + }, + { + "text": "?", + "boxes": [ + 96, + 1044, + 216, + 1074 + ] + }, + { + "text": "iphone 16 an..", + "boxes": [ + 214, + 1048, + 316, + 1072 + ] + } + ] + }, + { + "frame_id": "80_2_6", + "episode_id": 80, + "env_id": 2, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.330093383789062, + "advantages": -0.8311403393745422, + "generated_text": "{\n \"intent\": \"The current goal is to explore the computer desktop environment. Since we are already on an Amazon webpage, double clicking an icon could lead us to detailed information about that item. This action combines movement and interaction.\"\n ,\n \"action\": \"DoubleClick(367, 125)\"\n}", + "action_string": "DoubleClick(367, 125)", + "intent_string": "The current goal is to explore the computer desktop environment. Since we are already on an Amazon webpage, double clicking an icon could lead us to detailed information about that item. This action combines movement and interaction.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Anlazon", + "boxes": [ + 294, + 115, + 412, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1445, + 107, + 1483, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "19", + "boxes": [ + 1822, + 113, + 1862, + 139 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 299, + 148, + 421, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "+", + "boxes": [ + 1446, + 149, + 1480, + 183 + ] + }, + { + "text": "gaming headsets", + "boxes": [ + 304, + 189, + 440, + 218 + ] + }, + { + "text": "×× ", + "boxes": [ + 1450, + 183, + 1476, + 262 + ] + }, + { + "text": "beauty", + "boxes": [ + 299, + 217, + 367, + 255 + ] + }, + { + "text": "home decor", + "boxes": [ + 302, + 256, + 400, + 288 + ] + }, + { + "text": "××", + "boxes": [ + 1450, + 252, + 1476, + 326 + ] + }, + { + "text": "pots and pans ", + "boxes": [ + 302, + 294, + 418, + 326 + ] + }, + { + "text": "gaming", + "boxes": [ + 304, + 332, + 368, + 357 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 326, + 1478, + 361 + ] + }, + { + "text": "bedding", + "boxes": [ + 299, + 356, + 375, + 398 + ] + }, + { + "text": "×", + "boxes": [ + 1452, + 359, + 1474, + 393 + ] + }, + { + "text": "kitchen and dining", + "boxes": [ + 302, + 393, + 454, + 429 + ] + }, + { + "text": "x", + "boxes": [ + 1452, + 391, + 1474, + 423 + ] + }, + { + "text": " nintendo switch games ", + "boxes": [ + 302, + 433, + 486, + 463 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1202, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 236, + 963, + 542, + 1003 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 969, + 866, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 608, + 994, + 724, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search AInazon", + "boxes": [ + 296, + 115, + 414, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1445, + 107, + 1483, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "19", + "boxes": [ + 1822, + 113, + 1862, + 139 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 299, + 148, + 421, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "+", + "boxes": [ + 1446, + 149, + 1480, + 183 + ] + }, + { + "text": "gaming headsets", + "boxes": [ + 304, + 189, + 440, + 218 + ] + }, + { + "text": "×× ", + "boxes": [ + 1450, + 183, + 1476, + 262 + ] + }, + { + "text": "beauty", + "boxes": [ + 299, + 217, + 367, + 255 + ] + }, + { + "text": "home decor", + "boxes": [ + 302, + 256, + 400, + 288 + ] + }, + { + "text": "××", + "boxes": [ + 1450, + 252, + 1476, + 326 + ] + }, + { + "text": "pots and pans", + "boxes": [ + 302, + 294, + 416, + 326 + ] + }, + { + "text": "gaming", + "boxes": [ + 304, + 332, + 368, + 357 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 326, + 1478, + 361 + ] + }, + { + "text": "bedding", + "boxes": [ + 299, + 356, + 375, + 398 + ] + }, + { + "text": "×", + "boxes": [ + 1452, + 359, + 1474, + 393 + ] + }, + { + "text": "kitchen and dining", + "boxes": [ + 302, + 393, + 454, + 429 + ] + }, + { + "text": "x", + "boxes": [ + 1452, + 391, + 1474, + 423 + ] + }, + { + "text": " nintendo switch games ", + "boxes": [ + 302, + 433, + 486, + 463 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1202, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 236, + 963, + 542, + 1003 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 969, + 866, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 608, + 994, + 724, + 1029 + ] + } + ] + }, + { + "frame_id": "80_3_6", + "episode_id": 80, + "env_id": 3, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.932437896728516, + "advantages": -0.033798445016145706, + "generated_text": "{\n \"intent\": \"Analyze the product list by clicking through different categories to explore more selections.\",\n \"action\": \"DoubleClick(364, 676)\"\n}", + "action_string": "DoubleClick(364, 676)", + "intent_string": "Analyze the product list by clicking through different categories to explore more selections.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四@", + "boxes": [ + 1779, + 59, + 1881, + 96 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Kmazon", + "boxes": [ + 296, + 115, + 414, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1445, + 107, + 1483, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "19", + "boxes": [ + 1822, + 113, + 1862, + 139 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 299, + 148, + 421, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "+", + "boxes": [ + 1446, + 149, + 1480, + 183 + ] + }, + { + "text": "gaming headsets", + "boxes": [ + 304, + 189, + 440, + 218 + ] + }, + { + "text": "×× ", + "boxes": [ + 1450, + 183, + 1476, + 262 + ] + }, + { + "text": "beauty", + "boxes": [ + 299, + 217, + 367, + 255 + ] + }, + { + "text": "home decor", + "boxes": [ + 302, + 256, + 400, + 288 + ] + }, + { + "text": "××", + "boxes": [ + 1450, + 250, + 1476, + 326 + ] + }, + { + "text": "pots and pans", + "boxes": [ + 302, + 294, + 416, + 326 + ] + }, + { + "text": "gaming", + "boxes": [ + 302, + 330, + 370, + 361 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 326, + 1478, + 363 + ] + }, + { + "text": "bedding", + "boxes": [ + 299, + 356, + 375, + 398 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 357, + 1476, + 397 + ] + }, + { + "text": "kitchen and dining", + "boxes": [ + 302, + 393, + 454, + 429 + ] + }, + { + "text": "x", + "boxes": [ + 1452, + 391, + 1474, + 423 + ] + }, + { + "text": " nintendo switch games ", + "boxes": [ + 302, + 433, + 486, + 463 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1202, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 242, + 969, + 498, + 999 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 612, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 354, + 1026 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 134, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 416, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三Al", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 660, + 294, + 942, + 349 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1462, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1518, + 1005, + 1546, + 1018 + ] + }, + { + "text": "", + "boxes": [ + 1555, + 996, + 1647, + 1025 + ] + }, + { + "text": "a-4a72-4a5f-b71a-0", + "boxes": [ + 1514, + 1020, + 1642, + 1038 + ] + } + ] + }, + { + "frame_id": "80_4_6", + "episode_id": 80, + "env_id": 4, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.174381256103516, + "advantages": -1.0372612476348877, + "generated_text": "{\n \"intent\": \"The current window is prompting me to choose how to open or proceed with the Firefox download file. To continue exploring the browser, I will need to select an option, as selecting \\\"Save File\\\" might be necessary if further actions are required. This action will help in progressing through the setup process.\",\n \"action\": \"DoubleClick(708, 576)\"\n}", + "action_string": "DoubleClick(708, 576)", + "intent_string": "The current window is prompting me to choose how to open or proceed with the Firefox download file. To continue exploring the browser, I will need to select an option, as selecting \"Save File\" might be necessary if further actions are required. This action will help in progressing through the setup process.", + "before_screen_ocr_results": [ + { + "text": "Get Firefox for desktop", + "boxes": [ + 738, + 2, + 928, + 26 + ] + }, + { + "text": "- Mozilla Firefox", + "boxes": [ + 1040, + 2, + 1182, + 26 + ] + }, + { + "text": "0", + "boxes": [ + 1854, + 0, + 1920, + 30 + ] + }, + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1591, + 61, + 1663, + 92 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1882, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": " Products", + "boxes": [ + 954, + 109, + 1048, + 141 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1176, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 824, + 353, + 1102, + 381 + ] + }, + { + "text": "口口区", + "boxes": [ + 1120, + 359, + 1184, + 379 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 381, + 926, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 465 + ] + }, + { + "text": "from:https:/dwoad-istallecmoillet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Open with]/usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "Save File", + "boxes": [ + 768, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for files like this from now on", + "boxes": [ + 762, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "OK", + "boxes": [ + 1126, + 659, + 1138, + 671 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distrbuton?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 706, + 889, + 1204, + 913 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1018, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1880, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 353, + 1088, + 383 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 383, + 922, + 411 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this fle?", + "boxes": [ + 746, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "80_5_6", + "episode_id": 80, + "env_id": 5, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.387584686279297, + "advantages": -0.7550373077392578, + "generated_text": "{\n \"intent\": \"To continue exploring different **World tourist attractions** by scrolling through more images, I will use **ScrollUp(960, 540)**.\",\n \"action\": \"ScrollUp(960, 540)\"\n}", + "action_string": "ScrollUp(960, 540)", + "intent_string": "To continue exploring different **World tourist attractions** by scrolling through more images, I will use **ScrollUp(960, 540)**.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "·☆", + "boxes": [ + 1590, + 69, + 1656, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 187, + 112, + 239, + 144 + ] + }, + { + "text": "国际版", + "boxes": [ + 266, + 119, + 304, + 139 + ] + }, + { + "text": "rosoftBing", + "boxes": [ + 166, + 175, + 228, + 193 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 262, + 169, + 432, + 199 + ] + }, + { + "text": "Most Beautiful Tourist Spot In The World - Tanya Tanya", + "boxes": [ + 1240, + 236, + 1664, + 270 + ] + }, + { + "text": "Visit site ", + "boxes": [ + 1248, + 296, + 1304, + 322 + ] + }, + { + "text": "These Are ", + "boxes": [ + 18, + 447, + 90, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 302, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "·☆", + "boxes": [ + 1590, + 69, + 1656, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 187, + 112, + 239, + 144 + ] + }, + { + "text": "国际版", + "boxes": [ + 266, + 119, + 304, + 139 + ] + }, + { + "text": "rosoftBing", + "boxes": [ + 166, + 175, + 228, + 193 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 262, + 169, + 432, + 199 + ] + }, + { + "text": "Most Beautiful Tourist Spot In The World - Tanya Tanya", + "boxes": [ + 1240, + 236, + 1664, + 270 + ] + }, + { + "text": "Visit site ", + "boxes": [ + 1248, + 296, + 1304, + 322 + ] + }, + { + "text": "These Are ", + "boxes": [ + 18, + 447, + 90, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 302, + 1072 + ] + } + ] + }, + { + "frame_id": "80_6_6", + "episode_id": 80, + "env_id": 6, + "step": 6, + "format_rewards": 1.0, + "rewards": 9.702980995178223, + "advantages": -0.33753758668899536, + "generated_text": "{\n \"intent\": \"To continue exploring the environment, clicking on an area seems necessary. The top bar contains navigation links such as 'Store', 'Mac', etc., which could lead to more information about different products.\",\n \"action\": \"Click(539, 106)\"\n}", + "action_string": "Click(539, 106)", + "intent_string": "To continue exploring the environment, clicking on an area seems necessary. The top bar contains navigation links such as 'Store', 'Mac', etc., which could lead to more information about different products.", + "before_screen_ocr_results": [ + { + "text": "BuyiPhone16", + "boxes": [ + 674, + 4, + 804, + 22 + ] + }, + { + "text": "MozillaFirefox", + "boxes": [ + 1128, + 2, + 1246, + 26 + ] + }, + { + "text": "→", + "boxes": [ + 8, + 60, + 130, + 91 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1880, + 91 + ] + }, + { + "text": "Store", + "boxes": [ + 512, + 113, + 546, + 127 + ] + }, + { + "text": "Mac", + "boxes": [ + 572, + 113, + 602, + 127 + ] + }, + { + "text": "iPad", + "boxes": [ + 636, + 113, + 660, + 127 + ] + }, + { + "text": "iPhone", + "boxes": [ + 692, + 113, + 732, + 129 + ] + }, + { + "text": "Watch", + "boxes": [ + 764, + 113, + 804, + 129 + ] + }, + { + "text": "Vision", + "boxes": [ + 834, + 111, + 874, + 131 + ] + }, + { + "text": "AirPods", + "boxes": [ + 904, + 111, + 948, + 131 + ] + }, + { + "text": "TV & Home", + "boxes": [ + 982, + 111, + 1048, + 131 + ] + }, + { + "text": "Entertainment", + "boxes": [ + 1078, + 109, + 1164, + 133 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1194, + 111, + 1264, + 133 + ] + }, + { + "text": "Support", + "boxes": [ + 1294, + 111, + 1344, + 131 + ] + }, + { + "text": "Q", + "boxes": [ + 1368, + 109, + 1396, + 131 + ] + }, + { + "text": "6", + "boxes": [ + 1418, + 111, + 1442, + 135 + ] + }, + { + "text": "Carrer Deals at Apple", + "boxes": [ + 496, + 170, + 654, + 207 + ] + }, + { + "text": "", + "boxes": [ + 672, + 187, + 706, + 214 + ] + }, + { + "text": "S", + "boxes": [ + 864, + 191, + 892, + 210 + ] + }, + { + "text": "T", + "boxes": [ + 1058, + 189, + 1084, + 212 + ] + }, + { + "text": "p to $1000 afte", + "boxes": [ + 1324, + 183, + 1410, + 201 + ] + }, + { + "text": "See all deals ①", + "boxes": [ + 500, + 197, + 606, + 220 + ] + }, + { + "text": "V", + "boxes": [ + 1256, + 193, + 1274, + 206 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 134, + 222, + 768, + 276 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1632, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "?", + "boxes": [ + 1592, + 294, + 1614, + 318 + ] + }, + { + "text": "/isit anAppleStore", + "boxes": [ + 1632, + 290, + 1756, + 308 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 378, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 138, + 641, + 720, + 671 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1843, + 745, + 1915, + 791 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1426, + 822, + 1582, + 852 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 826, + 1044 + ] + } + ] + }, + { + "frame_id": "80_7_6", + "episode_id": 80, + "env_id": 7, + "step": 6, + "format_rewards": 1.0, + "rewards": 8.90633773803711, + "advantages": -1.3920786380767822, + "generated_text": "{\n \"intent\": \"To continue exploring the environment by investigating any clickable elements or icons on the screen.\",\n \"action\": \"DoubleClick(593, 270)\"\n}", + "action_string": "DoubleClick(593, 270)", + "intent_string": "To continue exploring the environment by investigating any clickable elements or icons on the screen.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "?", + "boxes": [ + 262, + 73, + 274, + 85 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account", + "boxes": [ + 804, + 179, + 1044, + 214 + ] + }, + { + "text": " Enter mobile number or email", + "boxes": [ + 808, + 220, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 925, + 291, + 991, + 322 + ] + }, + { + "text": "By continuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1100, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use ", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1392, + 66, + 1582, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 806, + 181, + 1042, + 210 + ] + }, + { + "text": "Enter mobile number or email", + "boxes": [ + 806, + 216, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 928, + 294, + 990, + 320 + ] + }, + { + "text": "Bycontinuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1102, + 361 + ] + }, + { + "text": "of Use and Privacy Notice.", + "boxes": [ + 810, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "80_0_7", + "episode_id": 80, + "env_id": 0, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.419734001159668, + "advantages": -0.7124803066253662, + "generated_text": "{\n \"intent\": \"The current screen shows various cleaning tools listed on Amazon. It's beneficial to explore more of these products by scrolling through the list. Such exploration could help identify other useful cleaning tools or different product options.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "The current screen shows various cleaning tools listed on Amazon. It's beneficial to explore more of these products by scrolling through the list. Such exploration could help identify other useful cleaning tools or different product options.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1872, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1812, + 105, + 1874, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 254, + 172, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 378, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 68, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Checke", + "boxes": [ + 312, + 294, + 352, + 310 + ] + }, + { + "text": "Household", + "boxes": [ + 8, + 318, + 88, + 349 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 38, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 132, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 52, + 496 + ] + }, + { + "text": "Open Link in New Window", + "boxes": [ + 812, + 486, + 982, + 510 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "Open Link in New Private Windo", + "boxes": [ + 808, + 514, + 1022, + 538 + ] + }, + { + "text": "Bookmark This Link ", + "boxes": [ + 806, + 540, + 944, + 570 + ] + }, + { + "text": "Save Link As..", + "boxes": [ + 808, + 570, + 902, + 594 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 132, + 604 + ] + }, + { + "text": "Save Link to Pocket", + "boxes": [ + 806, + 594, + 942, + 623 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 14, + 623, + 100, + 649 + ] + }, + { + "text": " Copy Link Location", + "boxes": [ + 804, + 621, + 940, + 651 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1488, + 615, + 1750, + 639 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning", + "boxes": [ + 312, + 635, + 576, + 669 + ] + }, + { + "text": "Treelen Long Handle Br", + "boxes": [ + 606, + 639, + 790, + 669 + ] + }, + { + "text": "ibbers - Flexible ", + "boxes": [ + 1050, + 637, + 1174, + 667 + ] + }, + { + "text": " Shark StainStriker HairPro Pet,", + "boxes": [ + 1192, + 637, + 1424, + 667 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1484, + 635, + 1758, + 665 + ] + }, + { + "text": "Department", + "boxes": [ + 12, + 655, + 98, + 681 + ] + }, + { + "text": "View Image", + "boxes": [ + 804, + 649, + 892, + 681 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 661, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Du", + "boxes": [ + 606, + 663, + 790, + 693 + ] + }, + { + "text": "eaning Dishes,", + "boxes": [ + 1052, + 665, + 1160, + 693 + ] + }, + { + "text": "Portable Spot, Stain, & Odor", + "boxes": [ + 1196, + 665, + 1410, + 689 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1486, + 661, + 1728, + 691 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": " Saye Image As..", + "boxes": [ + 801, + 673, + 917, + 709 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": " Combo for Home, Kitch", + "boxes": [ + 604, + 683, + 788, + 721 + ] + }, + { + "text": "I Sinks", + "boxes": [ + 1050, + 687, + 1100, + 713 + ] + }, + { + "text": "Eliminator for Carpets,Area Rugs.", + "boxes": [ + 1190, + 681, + 1448, + 721 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1724, + 715 + ] + }, + { + "text": "Household Cleaning Sponges", + "boxes": [ + 24, + 699, + 214, + 723 + ] + }, + { + "text": " Reusable and Washable, Pack of..", + "boxes": [ + 312, + 711, + 558, + 735 + ] + }, + { + "text": "Office, Lobby Floor Use", + "boxes": [ + 606, + 707, + 788, + 742 + ] + }, + { + "text": "Copy Image", + "boxes": [ + 806, + 705, + 894, + 731 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Upholstery, Cars, with Bonus.", + "boxes": [ + 1194, + 711, + 1412, + 741 + ] + }, + { + "text": "★★★★☆2,828", + "boxes": [ + 1490, + 715, + 1630, + 739 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": "Copy Image Location", + "boxes": [ + 806, + 729, + 950, + 760 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 22, + 739, + 168, + 762 + ] + }, + { + "text": "Options: 4 sizes", + "boxes": [ + 318, + 739, + 406, + 764 + ] + }, + { + "text": "★★★★★2.759", + "boxes": [ + 1196, + 737, + 1336, + 766 + ] + }, + { + "text": "★★★★★79.824", + "boxes": [ + 314, + 760, + 464, + 794 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 634, + 762, + 776, + 780 + ] + }, + { + "text": "Email i", + "boxes": [ + 806, + 758, + 856, + 786 + ] + }, + { + "text": "Image", + "boxes": [ + 846, + 762, + 892, + 782 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 1226, + 762, + 1364, + 780 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 14, + 782, + 108, + 800 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 354, + 790, + 490, + 808 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Copy", + "boxes": [ + 804, + 786, + 846, + 814 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "$16999", + "boxes": [ + 1190, + 788, + 1276, + 826 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1628, + 830 + ] + }, + { + "text": "$499 ($0.35/count) Typical: $63.64", + "boxes": [ + 309, + 813, + 557, + 860 + ] + }, + { + "text": "Send Link to Device", + "boxes": [ + 804, + 812, + 946, + 842 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 822, + 750, + 856 + ] + }, + { + "text": "Brands", + "boxes": [ + 12, + 832, + 62, + 858 + ] + }, + { + "text": "Delivery Wed, May 2", + "boxes": [ + 1196, + 826, + 1330, + 850 + ] + }, + { + "text": "Ships to Cana", + "boxes": [ + 1494, + 828, + 1568, + 846 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 610, + 850, + 696, + 874 + ] + }, + { + "text": "Print Selection ", + "boxes": [ + 806, + 842, + 910, + 866 + ] + }, + { + "text": " Bissell", + "boxes": [ + 10, + 852, + 74, + 878 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1196, + 850, + 1284, + 874 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "Adto cart", + "boxes": [ + 605, + 871, + 703, + 916 + ] + }, + { + "text": "View Selection Source", + "boxes": [ + 806, + 866, + 960, + 895 + ] + }, + { + "text": "", + "boxes": [ + 16, + 878, + 38, + 891 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1198, + 876, + 1286, + 907 + ] + }, + { + "text": "o-cedar", + "boxes": [ + 9, + 891, + 87, + 922 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1488, + 883, + 1630, + 913 + ] + }, + { + "text": " 1 sustainal", + "boxes": [ + 320, + 895, + 408, + 921 + ] + }, + { + "text": "Inspect Accessibility Properties", + "boxes": [ + 808, + 897, + 1014, + 921 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 7, + 907, + 121, + 944 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1684, + 931 + ] + }, + { + "text": "Inspect Element (Q)", + "boxes": [ + 806, + 921, + 948, + 951 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1812, + 917, + 1862, + 935 + ] + }, + { + "text": " swiffer", + "boxes": [ + 9, + 928, + 83, + 962 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 322, + 931, + 402, + 957 + ] + }, + { + "text": "Holikme", + "boxes": [ + 14, + 957, + 88, + 975 + ] + }, + { + "text": " Take a Screenshot ", + "boxes": [ + 782, + 947, + 934, + 979 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 254, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1872, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 254, + 172, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 68, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Household", + "boxes": [ + 10, + 322, + 86, + 347 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 136, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": " Open Link in New Iab", + "boxes": [ + 806, + 461, + 950, + 484 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "Open Link in New Window", + "boxes": [ + 806, + 484, + 986, + 512 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": " Open Link in New Private Window", + "boxes": [ + 804, + 512, + 1034, + 542 + ] + }, + { + "text": " Bookmark This Link ", + "boxes": [ + 800, + 540, + 948, + 570 + ] + }, + { + "text": "Save Link As..", + "boxes": [ + 808, + 570, + 902, + 596 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 132, + 604 + ] + }, + { + "text": "Save Link to Pocket", + "boxes": [ + 806, + 594, + 942, + 623 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 623, + 100, + 649 + ] + }, + { + "text": " Copy Link Location", + "boxes": [ + 804, + 621, + 940, + 651 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1488, + 615, + 1750, + 639 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning", + "boxes": [ + 312, + 637, + 576, + 669 + ] + }, + { + "text": "Treelen Long Handle Br", + "boxes": [ + 606, + 639, + 790, + 669 + ] + }, + { + "text": "ibbers - Flexible ", + "boxes": [ + 1050, + 637, + 1174, + 667 + ] + }, + { + "text": " Shark StainStriker HairPro Pet,", + "boxes": [ + 1192, + 637, + 1424, + 667 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1486, + 637, + 1760, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 7, + 649, + 101, + 686 + ] + }, + { + "text": "View Image", + "boxes": [ + 804, + 649, + 892, + 681 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 661, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Du", + "boxes": [ + 606, + 663, + 790, + 693 + ] + }, + { + "text": "eaning Dishes,", + "boxes": [ + 1052, + 665, + 1160, + 693 + ] + }, + { + "text": "Portable Spot, Stain, & Odor ", + "boxes": [ + 1196, + 665, + 1410, + 689 + ] + }, + { + "text": " tile, grout, windows, bathrooms,", + "boxes": [ + 1484, + 659, + 1726, + 689 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": " Combo for Home, Kich", + "boxes": [ + 604, + 681, + 788, + 721 + ] + }, + { + "text": " Saye Image As..", + "boxes": [ + 801, + 673, + 917, + 709 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": "I Sinks", + "boxes": [ + 1050, + 687, + 1100, + 713 + ] + }, + { + "text": "Eliminator for Carpets, Area Rugs.", + "boxes": [ + 1188, + 681, + 1446, + 719 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1724, + 715 + ] + }, + { + "text": " Reusable and Washable, Pack of..", + "boxes": [ + 312, + 711, + 558, + 735 + ] + }, + { + "text": "Office, Lobby Floor Use", + "boxes": [ + 606, + 707, + 788, + 742 + ] + }, + { + "text": "Copy Image", + "boxes": [ + 806, + 705, + 894, + 731 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Upholstery, Cars, with Bonus..", + "boxes": [ + 1194, + 711, + 1414, + 741 + ] + }, + { + "text": "★★★★☆2,828", + "boxes": [ + 1490, + 715, + 1630, + 739 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": "Copy Image Location", + "boxes": [ + 806, + 729, + 950, + 760 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "★★★★★2.759", + "boxes": [ + 1196, + 737, + 1336, + 766 + ] + }, + { + "text": "★★★★☆79.824", + "boxes": [ + 312, + 760, + 464, + 794 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 634, + 762, + 776, + 780 + ] + }, + { + "text": "Email i", + "boxes": [ + 806, + 758, + 856, + 786 + ] + }, + { + "text": "Image", + "boxes": [ + 846, + 762, + 892, + 782 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 1226, + 762, + 1364, + 780 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "ought in past month", + "boxes": [ + 366, + 790, + 490, + 808 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Copy", + "boxes": [ + 802, + 782, + 848, + 816 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "$16999", + "boxes": [ + 1190, + 788, + 1276, + 826 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": "$499 ($0.35/count) Typical: $65.6", + "boxes": [ + 309, + 813, + 555, + 860 + ] + }, + { + "text": "Send Link to Device", + "boxes": [ + 804, + 812, + 946, + 842 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 822, + 750, + 856 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1568, + 846 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 608, + 850, + 698, + 874 + ] + }, + { + "text": "Print Selection ", + "boxes": [ + 806, + 842, + 910, + 866 + ] + }, + { + "text": " Bissel", + "boxes": [ + 12, + 856, + 70, + 876 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "Addt cart", + "boxes": [ + 608, + 874, + 702, + 911 + ] + }, + { + "text": "View Selection Source", + "boxes": [ + 806, + 866, + 960, + 895 + ] + }, + { + "text": "L", + "boxes": [ + 16, + 878, + 36, + 891 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1198, + 876, + 1286, + 907 + ] + }, + { + "text": "o-ceda", + "boxes": [ + 9, + 891, + 85, + 922 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1488, + 883, + 1630, + 913 + ] + }, + { + "text": "Inspect Accessibility Properties", + "boxes": [ + 808, + 897, + 1014, + 921 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 9, + 909, + 121, + 944 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1684, + 931 + ] + }, + { + "text": "Inspect Element (Q)", + "boxes": [ + 806, + 921, + 948, + 951 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1812, + 917, + 1862, + 937 + ] + }, + { + "text": "swiffer", + "boxes": [ + 11, + 933, + 81, + 957 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 322, + 931, + 402, + 957 + ] + }, + { + "text": " Take a Screenshot ", + "boxes": [ + 782, + 947, + 934, + 979 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ] + }, + { + "frame_id": "80_1_7", + "episode_id": 80, + "env_id": 1, + "step": 7, + "format_rewards": 1.0, + "rewards": 8.953507423400879, + "advantages": -1.3296387195587158, + "generated_text": "{\n \"intent\": \"The current task is to explore efficiently by viewing different content on the website. If there are more interactive elements or information below the visible area, scrolling might reveal additional content.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "The current task is to explore efficiently by viewing different content on the website. If there are more interactive elements or information below the visible area, scrolling might reveal additional content.", + "before_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 ×+", + "boxes": [ + 6, + 24, + 248, + 54 + ] + }, + { + "text": "①→白", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "| https://ww.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 552, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1882, + 91 + ] + }, + { + "text": "?", + "boxes": [ + 96, + 1044, + 216, + 1074 + ] + }, + { + "text": "iphone 16 an..", + "boxes": [ + 214, + 1048, + 316, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 ×+", + "boxes": [ + 6, + 24, + 248, + 54 + ] + }, + { + "text": "①→白", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "| https://ww.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 552, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1884, + 91 + ] + }, + { + "text": "?", + "boxes": [ + 96, + 1044, + 216, + 1074 + ] + }, + { + "text": "iphone 16 an..", + "boxes": [ + 214, + 1048, + 316, + 1072 + ] + } + ] + }, + { + "frame_id": "80_2_7", + "episode_id": 80, + "env_id": 2, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.352883338928223, + "advantages": -0.800972580909729, + "generated_text": "{\n \"intent\": \"The current action will open the search bar on Amazon where I can start exploring various products by entering different categories or keywords.\",\n \"action\": \"DoubleClick(378, 131)\"\n}", + "action_string": "DoubleClick(378, 131)", + "intent_string": "The current action will open the search bar on Amazon where I can start exploring various products by entering different categories or keywords.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search AInazon", + "boxes": [ + 296, + 115, + 414, + 139 + ] + }, + { + "text": "Q", + "boxes": [ + 1445, + 107, + 1483, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "+", + "boxes": [ + 1826, + 115, + 1860, + 137 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 299, + 148, + 421, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "+", + "boxes": [ + 1446, + 149, + 1480, + 183 + ] + }, + { + "text": "gaming headsets", + "boxes": [ + 304, + 189, + 440, + 218 + ] + }, + { + "text": "×× ", + "boxes": [ + 1450, + 183, + 1476, + 262 + ] + }, + { + "text": "beauty", + "boxes": [ + 299, + 217, + 367, + 255 + ] + }, + { + "text": "home decor", + "boxes": [ + 302, + 256, + 400, + 288 + ] + }, + { + "text": "××", + "boxes": [ + 1450, + 252, + 1476, + 326 + ] + }, + { + "text": "pots and pans", + "boxes": [ + 302, + 294, + 416, + 326 + ] + }, + { + "text": "gaming", + "boxes": [ + 304, + 332, + 368, + 357 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 326, + 1478, + 361 + ] + }, + { + "text": "bedding", + "boxes": [ + 299, + 356, + 375, + 398 + ] + }, + { + "text": "×", + "boxes": [ + 1452, + 359, + 1474, + 393 + ] + }, + { + "text": "kitchen and dining", + "boxes": [ + 302, + 393, + 454, + 429 + ] + }, + { + "text": "x", + "boxes": [ + 1452, + 391, + 1474, + 423 + ] + }, + { + "text": " nintendo switch games ", + "boxes": [ + 302, + 433, + 486, + 463 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1202, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 236, + 963, + 542, + 1003 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 969, + 866, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 608, + 994, + 724, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": " earch Anyazon", + "boxes": [ + 291, + 108, + 415, + 150 + ] + }, + { + "text": "Q", + "boxes": [ + 1445, + 107, + 1483, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "+", + "boxes": [ + 1826, + 115, + 1860, + 137 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 299, + 148, + 421, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "+", + "boxes": [ + 1446, + 149, + 1480, + 183 + ] + }, + { + "text": "gaming headsets", + "boxes": [ + 304, + 189, + 440, + 218 + ] + }, + { + "text": "×× ", + "boxes": [ + 1450, + 183, + 1476, + 262 + ] + }, + { + "text": "beauty", + "boxes": [ + 299, + 217, + 367, + 255 + ] + }, + { + "text": "home decor", + "boxes": [ + 302, + 256, + 400, + 288 + ] + }, + { + "text": "××", + "boxes": [ + 1450, + 252, + 1476, + 326 + ] + }, + { + "text": "pots and pans ", + "boxes": [ + 302, + 294, + 418, + 326 + ] + }, + { + "text": "gaming", + "boxes": [ + 304, + 332, + 368, + 357 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 326, + 1478, + 361 + ] + }, + { + "text": "bedding", + "boxes": [ + 299, + 356, + 375, + 398 + ] + }, + { + "text": "×", + "boxes": [ + 1452, + 359, + 1474, + 393 + ] + }, + { + "text": "kitchen and dining", + "boxes": [ + 302, + 393, + 454, + 429 + ] + }, + { + "text": "x", + "boxes": [ + 1452, + 391, + 1474, + 423 + ] + }, + { + "text": " nintendo switch games ", + "boxes": [ + 302, + 433, + 486, + 463 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1202, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 236, + 963, + 542, + 1003 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 969, + 866, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 608, + 994, + 724, + 1029 + ] + } + ] + }, + { + "frame_id": "80_3_7", + "episode_id": 80, + "env_id": 3, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.077404022216797, + "advantages": 0.15809765458106995, + "generated_text": "{\n \"intent\": \"To continue exploring the environment, I will scroll down the webpage to view more content available on Amazon.\",\n \"action\": \"ScrollDown(960, 540)\"\n}", + "action_string": "ScrollDown(960, 540)", + "intent_string": "To continue exploring the environment, I will scroll down the webpage to view more content available on Amazon.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 134, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 416, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三Al", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 660, + 294, + 942, + 349 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Home", + "boxes": [ + 612, + 889, + 750, + 913 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 240, + 967, + 500, + 1003 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 610, + 969, + 908, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 984, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 997, + 356, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1462, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1518, + 1005, + 1546, + 1018 + ] + }, + { + "text": "", + "boxes": [ + 1555, + 996, + 1647, + 1025 + ] + }, + { + "text": "a-4a72-4a5f-b71a-0", + "boxes": [ + 1514, + 1020, + 1644, + 1038 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Gamingaccessorles", + "boxes": [ + 246, + 191, + 434, + 208 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 614, + 187, + 916, + 210 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 982, + 185, + 1202, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 185, + 1534, + 214 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 544, + 1058, + 568 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1349, + 541, + 1435, + 572 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 625, + 1544, + 655 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 653, + 352, + 683 + ] + }, + { + "text": "Explore all products in Kitche", + "boxes": [ + 246, + 987, + 414, + 1005 + ] + }, + { + "text": "Shop the latest from Hon", + "boxes": [ + 616, + 987, + 762, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "See more", + "boxes": [ + 1354, + 987, + 1406, + 1007 + ] + } + ] + }, + { + "frame_id": "80_4_7", + "episode_id": 80, + "env_id": 4, + "step": 7, + "format_rewards": 1.0, + "rewards": 8.948359489440918, + "advantages": -1.3364531993865967, + "generated_text": "{\n \"intent\": \"The goal is to continue exploring the environment by clicking through the various elements and actions available. Opening Firefox from the download page will provide a new context and reveal more about the program's functionalities.\",\n \"action\": \"DoubleClick(617, 485)\"\n}", + "action_string": "DoubleClick(617, 485)", + "intent_string": "The goal is to continue exploring the environment by clicking through the various elements and actions available. Opening Firefox from the download page will provide a new context and reveal more about the program's functionalities.", + "before_screen_ocr_results": [ + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1880, + 89 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 353, + 1088, + 383 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 383, + 922, + 411 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which is: TAR Archive (72.4 MB)", + "boxes": [ + 772, + 431, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this fle?", + "boxes": [ + 746, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 760, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1880, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 353, + 1088, + 383 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 381, + 924, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 676, + 425, + 732, + 459 + ] + }, + { + "text": "Which s: TAR Archive (72.4 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwnload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "80_5_7", + "episode_id": 80, + "env_id": 5, + "step": 7, + "format_rewards": 1.0, + "rewards": 10.80754280090332, + "advantages": 1.1246047019958496, + "generated_text": "{\n \"intent\": \"Opening a new image gallery to continue exploring world tourist attractions.\",\n \"action\": \"DoubleClick(92, 267)\"\n}", + "action_string": "DoubleClick(92, 267)", + "intent_string": "Opening a new image gallery to continue exploring world tourist attractions.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "·☆", + "boxes": [ + 1590, + 69, + 1656, + 87 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 187, + 112, + 239, + 144 + ] + }, + { + "text": "国际版", + "boxes": [ + 266, + 119, + 304, + 139 + ] + }, + { + "text": "rosoftBing", + "boxes": [ + 166, + 175, + 228, + 193 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 262, + 169, + 432, + 199 + ] + }, + { + "text": "Most Beautiful Tourist Spot In The World - Tanya Tanya", + "boxes": [ + 1240, + 236, + 1664, + 270 + ] + }, + { + "text": "Visit site ", + "boxes": [ + 1248, + 296, + 1304, + 322 + ] + }, + { + "text": "These Are ", + "boxes": [ + 18, + 447, + 90, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 302, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 139 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 336, + 183 + ] + }, + { + "text": "Worl", + "boxes": [ + 20, + 703, + 50, + 723 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ] + }, + { + "frame_id": "80_6_7", + "episode_id": 80, + "env_id": 6, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.10208511352539, + "advantages": -1.1329618692398071, + "generated_text": "{\n \"intent\": \"To move across different products on the Apple Store page and explore the available categories and items. This helps in gathering information about different products and their features.\",\n \"action\": \"Move(600, 500)\" \n}", + "action_string": "Move(600, 500)", + "intent_string": "To move across different products on the Apple Store page and explore the available categories and items. This helps in gathering information about different products and their features.", + "before_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 134, + 222, + 768, + 276 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1632, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "?", + "boxes": [ + 1592, + 294, + 1614, + 318 + ] + }, + { + "text": "/isit anAppleStore", + "boxes": [ + 1632, + 290, + 1756, + 308 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 378, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 138, + 641, + 720, + 671 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1843, + 745, + 1915, + 791 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1426, + 822, + 1582, + 852 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 826, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 132, + 220, + 768, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1630, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "?", + "boxes": [ + 1592, + 294, + 1614, + 318 + ] + }, + { + "text": "/isit anAppleStore", + "boxes": [ + 1632, + 290, + 1756, + 308 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 380, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 134, + 635, + 722, + 675 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1848, + 752, + 1912, + 786 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1426, + 822, + 1582, + 852 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 828, + 1044 + ] + } + ] + }, + { + "frame_id": "80_7_7", + "episode_id": 80, + "env_id": 7, + "step": 7, + "format_rewards": 1.0, + "rewards": 9.08461856842041, + "advantages": -1.1560828685760498, + "generated_text": "{\n \"intent\": \"Since the signing into Amazon's account lifecycle is not a direct exploration task, I will proceed by exiting the current webpage to navigate or explore other areas of the desktop. This ensures progression through the environment, including accessing different programs or directories.\",\n \"action\": \"DoubleClick(100, 40)\"\n}", + "action_string": "DoubleClick(100, 40)", + "intent_string": "Since the signing into Amazon's account lifecycle is not a direct exploration task, I will proceed by exiting the current webpage to navigate or explore other areas of the desktop. This ensures progression through the environment, including accessing different programs or directories.", + "before_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 900, + 4, + 1020, + 22 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 126, + 89 + ] + }, + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1392, + 66, + 1582, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 806, + 181, + 1042, + 210 + ] + }, + { + "text": "Enter mobile number or email", + "boxes": [ + 806, + 216, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 928, + 294, + 990, + 320 + ] + }, + { + "text": "Bycontinuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1102, + 361 + ] + }, + { + "text": "of Use and Privacy Notice.", + "boxes": [ + 810, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": "Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1384, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 806, + 181, + 1042, + 210 + ] + }, + { + "text": "Enter mobile number or email", + "boxes": [ + 806, + 216, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 928, + 294, + 990, + 320 + ] + }, + { + "text": "Bycontinuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1102, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ] + }, + { + "frame_id": "80_0_8", + "episode_id": 80, + "env_id": 0, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.38735294342041, + "advantages": 0.5683865547180176, + "generated_text": "{\n \"intent\": \"To continue exploring cleaning tools available on Amazon, I will click on one of the product listings to view more details.\",\n \"action\": \"DoubleClick(753, 657)\"\n}", + "action_string": "DoubleClick(753, 657)", + "intent_string": "To continue exploring cleaning tools available on Amazon, I will click on one of the product listings to view more details.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 254, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1872, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1874, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 254, + 172, + 284 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 68, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Household", + "boxes": [ + 10, + 322, + 86, + 347 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 136, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": " Open Link in New Iab", + "boxes": [ + 806, + 461, + 952, + 484 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "Open Link in New Window", + "boxes": [ + 806, + 484, + 986, + 512 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": " Open Link in New Private Window", + "boxes": [ + 804, + 512, + 1034, + 542 + ] + }, + { + "text": " Bookmark This Link ", + "boxes": [ + 800, + 540, + 948, + 570 + ] + }, + { + "text": "Save Link As..", + "boxes": [ + 808, + 570, + 902, + 596 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 132, + 604 + ] + }, + { + "text": "Save Link to Pocket", + "boxes": [ + 806, + 594, + 942, + 623 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 623, + 100, + 649 + ] + }, + { + "text": " Copy Link Location", + "boxes": [ + 804, + 621, + 940, + 651 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1488, + 615, + 1750, + 639 + ] + }, + { + "text": " Amazon Basics Microfiber Cleaning ", + "boxes": [ + 312, + 637, + 576, + 669 + ] + }, + { + "text": "Treelen Long Handle Br", + "boxes": [ + 606, + 639, + 790, + 669 + ] + }, + { + "text": "ibbers - Flexible ", + "boxes": [ + 1050, + 637, + 1174, + 667 + ] + }, + { + "text": " Shark StainStriker HairPro Pet,", + "boxes": [ + 1192, + 637, + 1424, + 667 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1486, + 637, + 1760, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 7, + 649, + 101, + 686 + ] + }, + { + "text": "View Image", + "boxes": [ + 804, + 649, + 892, + 681 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 661, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Du", + "boxes": [ + 606, + 663, + 790, + 693 + ] + }, + { + "text": "eaning Dishes,", + "boxes": [ + 1052, + 665, + 1160, + 693 + ] + }, + { + "text": "Portable Spot, Stain, & Odor", + "boxes": [ + 1196, + 665, + 1410, + 689 + ] + }, + { + "text": " tile, grout, windows, bathrooms,", + "boxes": [ + 1484, + 659, + 1726, + 689 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": " Combo for Home, Kich", + "boxes": [ + 604, + 681, + 788, + 721 + ] + }, + { + "text": " Saye Image As..", + "boxes": [ + 801, + 673, + 917, + 709 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": "I Sinks", + "boxes": [ + 1050, + 687, + 1100, + 713 + ] + }, + { + "text": "Eliminatorfor Carpets, Area Rugs.", + "boxes": [ + 1188, + 681, + 1446, + 719 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1724, + 715 + ] + }, + { + "text": " Reusable and Washable, Pack of..", + "boxes": [ + 312, + 711, + 558, + 735 + ] + }, + { + "text": "Office, Lobby Floor Use", + "boxes": [ + 606, + 707, + 788, + 742 + ] + }, + { + "text": "Copy Image", + "boxes": [ + 806, + 705, + 894, + 731 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Upholstery, Cars, with Bonus..", + "boxes": [ + 1194, + 711, + 1414, + 741 + ] + }, + { + "text": "★★★★☆2,828", + "boxes": [ + 1490, + 715, + 1630, + 739 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": "Copy Image Location", + "boxes": [ + 806, + 729, + 950, + 760 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "★★★★★2.759", + "boxes": [ + 1196, + 737, + 1336, + 766 + ] + }, + { + "text": "★★★★☆79.824", + "boxes": [ + 312, + 760, + 464, + 794 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 634, + 762, + 776, + 780 + ] + }, + { + "text": "Email i", + "boxes": [ + 806, + 758, + 856, + 786 + ] + }, + { + "text": "Image", + "boxes": [ + 846, + 762, + 892, + 782 + ] + }, + { + "text": "bought in past month", + "boxes": [ + 1226, + 762, + 1364, + 780 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "ought in past month", + "boxes": [ + 366, + 790, + 490, + 808 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Copy", + "boxes": [ + 802, + 782, + 848, + 816 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "$16999", + "boxes": [ + 1190, + 788, + 1276, + 826 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": "$499 ($0.35/count) Typical: $65.6", + "boxes": [ + 309, + 813, + 555, + 860 + ] + }, + { + "text": "Send Link to Device", + "boxes": [ + 804, + 812, + 946, + 842 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 822, + 750, + 856 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1568, + 846 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 608, + 850, + 698, + 874 + ] + }, + { + "text": "Print Selection ", + "boxes": [ + 806, + 842, + 910, + 866 + ] + }, + { + "text": " Bissel", + "boxes": [ + 12, + 856, + 70, + 876 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "Addt cart", + "boxes": [ + 608, + 874, + 702, + 911 + ] + }, + { + "text": "View Selection Source", + "boxes": [ + 806, + 866, + 960, + 895 + ] + }, + { + "text": "L", + "boxes": [ + 16, + 878, + 36, + 891 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 1198, + 876, + 1286, + 907 + ] + }, + { + "text": "o-ceda", + "boxes": [ + 9, + 891, + 85, + 922 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1488, + 883, + 1630, + 913 + ] + }, + { + "text": " 1 sustainab", + "boxes": [ + 320, + 895, + 414, + 921 + ] + }, + { + "text": "Inspect Accessibility Properties", + "boxes": [ + 808, + 897, + 1014, + 921 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 9, + 909, + 121, + 944 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1684, + 931 + ] + }, + { + "text": "Inspect Element (Q)", + "boxes": [ + 806, + 921, + 948, + 951 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1812, + 917, + 1862, + 937 + ] + }, + { + "text": "swiffer", + "boxes": [ + 11, + 933, + 81, + 957 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 322, + 931, + 402, + 957 + ] + }, + { + "text": " Take a Screenshot ", + "boxes": [ + 782, + 947, + 934, + 979 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "曰』", + "boxes": [ + 252, + 64, + 310, + 89 + ] + }, + { + "text": "7-814b-0240l", + "boxes": [ + 1474, + 69, + 1564, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 302, + 115, + 406, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1400, + 125, + 1444, + 145 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 125, + 1884, + 151 + ] + }, + { + "text": "三ALl", + "boxes": [ + 24, + 167, + 66, + 193 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 420, + 187 + ] + }, + { + "text": "International Kindle Paperwhite Buy Now", + "boxes": [ + 754, + 204, + 1072, + 234 + ] + }, + { + "text": "TreeLen Long Handle Broom and Dustpan ", + "boxes": [ + 834, + 306, + 1300, + 341 + ] + }, + { + "text": "", + "boxes": [ + 782, + 324, + 802, + 343 + ] + }, + { + "text": "$2099", + "boxes": [ + 1392, + 318, + 1458, + 357 + ] + }, + { + "text": "Set,Upright Dust Pan Combo for Home,", + "boxes": [ + 834, + 338, + 1270, + 373 + ] + }, + { + "text": "Kitchen, Room, Office, Lobby Floor Use ", + "boxes": [ + 834, + 369, + 1268, + 405 + ] + }, + { + "text": "No Import Charges & $8.21", + "boxes": [ + 1392, + 363, + 1576, + 393 + ] + }, + { + "text": "Shipping to Canada Details v", + "boxes": [ + 1394, + 383, + 1590, + 413 + ] + }, + { + "text": "Without Bending", + "boxes": [ + 830, + 395, + 1030, + 439 + ] + }, + { + "text": "Delivery Tuesday, May 20.", + "boxes": [ + 1392, + 403, + 1566, + 433 + ] + }, + { + "text": "Order within 7 hrs 10 mins", + "boxes": [ + 1392, + 423, + 1568, + 453 + ] + }, + { + "text": "4.6★★★★☆", + "boxes": [ + 836, + 447, + 968, + 476 + ] + }, + { + "text": "42,011 ratings", + "boxes": [ + 976, + 446, + 1074, + 477 + ] + }, + { + "text": " Deliver to Canada", + "boxes": [ + 1394, + 457, + 1512, + 480 + ] + }, + { + "text": "Amazon's Choice", + "boxes": [ + 838, + 476, + 934, + 502 + ] + }, + { + "text": "In Stock", + "boxes": [ + 1394, + 494, + 1466, + 520 + ] + }, + { + "text": "2K+bought in pa", + "boxes": [ + 840, + 512, + 932, + 530 + ] + }, + { + "text": "$59.99", + "boxes": [ + 1814, + 506, + 1860, + 524 + ] + }, + { + "text": "Quantity: 1", + "boxes": [ + 1402, + 536, + 1482, + 560 + ] + }, + { + "text": "-13% $2099", + "boxes": [ + 838, + 552, + 966, + 584 + ] + }, + { + "text": "Add to Cart ", + "boxes": [ + 1458, + 580, + 1542, + 604 + ] + }, + { + "text": "& $8.21 Ship", + "boxes": [ + 960, + 613, + 1044, + 631 + ] + }, + { + "text": "Buy Now", + "boxes": [ + 1464, + 619, + 1532, + 645 + ] + }, + { + "text": "Ships from Amazon ", + "boxes": [ + 1398, + 657, + 1522, + 687 + ] + }, + { + "text": "Sold by", + "boxes": [ + 1400, + 677, + 1450, + 705 + ] + }, + { + "text": "HomeBest Direct", + "boxes": [ + 1466, + 679, + 1566, + 705 + ] + }, + { + "text": "Returns", + "boxes": [ + 1400, + 701, + 1450, + 727 + ] + }, + { + "text": "seuees", + "boxes": [ + 1460, + 695, + 1550, + 748 + ] + }, + { + "text": "Packaging", + "boxes": [ + 1400, + 741, + 1460, + 758 + ] + }, + { + "text": "Brand", + "boxes": [ + 836, + 780, + 880, + 806 + ] + }, + { + "text": "TreeLen", + "boxes": [ + 960, + 778, + 1020, + 810 + ] + }, + { + "text": "Color", + "boxes": [ + 836, + 808, + 876, + 834 + ] + }, + { + "text": "Blue Broom and Dustpan", + "boxes": [ + 960, + 806, + 1124, + 834 + ] + }, + { + "text": "口", + "boxes": [ + 1396, + 804, + 1420, + 828 + ] + }, + { + "text": "Floor Cleaning", + "boxes": [ + 960, + 834, + 1058, + 858 + ] + }, + { + "text": "Add a gift receipt for", + "boxes": [ + 1416, + 826, + 1550, + 850 + ] + }, + { + "text": "lended", + "boxes": [ + 892, + 836, + 938, + 856 + ] + }, + { + "text": "returns", + "boxes": [ + 1420, + 848, + 1466, + 868 + ] + }, + { + "text": "UsesForProduct", + "boxes": [ + 840, + 858, + 946, + 876 + ] + }, + { + "text": "BristleMaterial", + "boxes": [ + 840, + 883, + 940, + 901 + ] + }, + { + "text": "Plastic", + "boxes": [ + 962, + 883, + 1006, + 901 + ] + }, + { + "text": "Handle Material", + "boxes": [ + 840, + 903, + 952, + 933 + ] + }, + { + "text": "Metal", + "boxes": [ + 960, + 905, + 1002, + 931 + ] + }, + { + "text": " Ad to List", + "boxes": [ + 1402, + 897, + 1476, + 923 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1810, + 917, + 1864, + 937 + ] + }, + { + "text": "Bristle Type", + "boxes": [ + 833, + 929, + 919, + 959 + ] + }, + { + "text": "Unflagged Bristle", + "boxes": [ + 966, + 935, + 1070, + 953 + ] + }, + { + "text": "VSeemo", + "boxes": [ + 838, + 967, + 896, + 985 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1794, + 963, + 1864, + 985 + ] + }, + { + "text": "About this item", + "boxes": [ + 834, + 1002, + 988, + 1039 + ] + } + ] + }, + { + "frame_id": "80_1_8", + "episode_id": 80, + "env_id": 1, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.61362075805664, + "advantages": -0.45582646131515503, + "generated_text": "{\n \"intent\": \"Since the goal is to explore the desktop environment, можно apologise right now by clicking the close button on the highlighted application window.\",\n \"action\": \"DoubleClick(370, 38)\"\n}", + "action_string": "DoubleClick(370, 38)", + "intent_string": "Since the goal is to explore the desktop environment, можно apologise right now by clicking the close button on the highlighted application window.", + "before_screen_ocr_results": [ + { + "text": "● iPhone 16 and iPhone 16 ×+", + "boxes": [ + 6, + 24, + 248, + 54 + ] + }, + { + "text": "①→白", + "boxes": [ + 0, + 58, + 136, + 101 + ] + }, + { + "text": "| https://ww.apple.com/iphone-16/", + "boxes": [ + 248, + 64, + 552, + 93 + ] + }, + { + "text": "目·☆", + "boxes": [ + 1538, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1882, + 91 + ] + }, + { + "text": "?", + "boxes": [ + 96, + 1044, + 216, + 1074 + ] + }, + { + "text": "iphone 16 an..", + "boxes": [ + 214, + 1048, + 316, + 1072 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 250, + 32, + 314, + 52 + ] + }, + { + "text": "→白", + "boxes": [ + 40, + 66, + 130, + 91 + ] + }, + { + "text": "Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 520, + 93 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_2_8", + "episode_id": 80, + "env_id": 2, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.769988059997559, + "advantages": -0.24883827567100525, + "generated_text": "{\n \"intent\": \"The goal is to explore different products available on the Amazon website by searching for specific items. The dropdown menu offers suggestions like 'cleaning tools.' To proceed with exploring more specific categories or products, clicking one of these suggestions will direct me towards relevant pages.\",\n \"action\": \"DoubleClick(528, 857)\"\n}", + "action_string": "DoubleClick(528, 857)", + "intent_string": "The goal is to explore different products available on the Amazon website by searching for specific items. The dropdown menu offers suggestions like 'cleaning tools.' To proceed with exploring more specific categories or products, clicking one of these suggestions will direct me towards relevant pages.", + "before_screen_ocr_results": [ + { + "text": "→白", + "boxes": [ + 8, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1580, + 62, + 1666, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": "三", + "boxes": [ + 1894, + 71, + 1906, + 83 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 132, + 105, + 220, + 151 + ] + }, + { + "text": "Search Angazon", + "boxes": [ + 293, + 108, + 415, + 150 + ] + }, + { + "text": "Q", + "boxes": [ + 1445, + 107, + 1483, + 145 + ] + }, + { + "text": "EN", + "boxes": [ + 1520, + 121, + 1578, + 147 + ] + }, + { + "text": "tums", + "boxes": [ + 1746, + 117, + 1780, + 137 + ] + }, + { + "text": "+", + "boxes": [ + 1826, + 115, + 1860, + 137 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 299, + 148, + 421, + 185 + ] + }, + { + "text": "三AIl", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "+", + "boxes": [ + 1446, + 149, + 1480, + 183 + ] + }, + { + "text": "gaming headsets", + "boxes": [ + 304, + 189, + 440, + 218 + ] + }, + { + "text": "×× ", + "boxes": [ + 1450, + 183, + 1476, + 262 + ] + }, + { + "text": "beauty", + "boxes": [ + 299, + 217, + 367, + 255 + ] + }, + { + "text": "home decor", + "boxes": [ + 302, + 256, + 400, + 288 + ] + }, + { + "text": "××", + "boxes": [ + 1450, + 252, + 1476, + 326 + ] + }, + { + "text": "pots and pans ", + "boxes": [ + 302, + 294, + 418, + 326 + ] + }, + { + "text": "gaming", + "boxes": [ + 304, + 332, + 368, + 357 + ] + }, + { + "text": "×", + "boxes": [ + 1450, + 326, + 1478, + 361 + ] + }, + { + "text": "bedding", + "boxes": [ + 299, + 356, + 375, + 398 + ] + }, + { + "text": "×", + "boxes": [ + 1452, + 359, + 1474, + 393 + ] + }, + { + "text": "kitchen and dining", + "boxes": [ + 302, + 393, + 454, + 429 + ] + }, + { + "text": "x", + "boxes": [ + 1452, + 391, + 1474, + 423 + ] + }, + { + "text": " nintendo switch games ", + "boxes": [ + 302, + 433, + 486, + 463 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 612, + 530, + 918, + 560 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 980, + 526, + 1202, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 298, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": " New home arrivals under $50", + "boxes": [ + 236, + 963, + 542, + 1003 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 969, + 866, + 999 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 965, + 1544, + 1005 + ] + }, + { + "text": " appliances", + "boxes": [ + 608, + 994, + 724, + 1029 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 134, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 416, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三Al", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chairs", + "boxes": [ + 408, + 840, + 438, + 860 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 240, + 965, + 540, + 1001 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 967, + 870, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 610, + 997, + 724, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1458, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1557, + 996, + 1647, + 1025 + ] + }, + { + "text": "=UTF8&k", + "boxes": [ + 262, + 1020, + 324, + 1040 + ] + }, + { + "text": "chairs&ref=nb_sb_", + "boxes": [ + 388, + 1020, + 518, + 1038 + ] + }, + { + "text": "rC", + "boxes": [ + 582, + 1024, + 594, + 1036 + ] + } + ] + }, + { + "frame_id": "80_3_8", + "episode_id": 80, + "env_id": 3, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.200217247009277, + "advantages": 0.3206692636013031, + "generated_text": "{\n \"intent\": \"To explore more sections of the Amazon website, I will start by clicking on one of the suggested categories to delve deeper into its offerings.\",\n \"action\": \"DoubleClick(678, 339)\"\n}", + "action_string": "DoubleClick(678, 339)", + "intent_string": "To explore more sections of the Amazon website, I will start by clicking on one of the suggested categories to delve deeper into its offerings.", + "before_screen_ocr_results": [ + { + "text": "Gamingaccessorles", + "boxes": [ + 246, + 191, + 434, + 208 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 614, + 187, + 916, + 210 + ] + }, + { + "text": "Shop deals in Fashion", + "boxes": [ + 982, + 185, + 1202, + 214 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 185, + 1534, + 214 + ] + }, + { + "text": "See all deals", + "boxes": [ + 984, + 544, + 1058, + 568 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1349, + 541, + 1435, + 572 + ] + }, + { + "text": " Top categories in Kitchen ", + "boxes": [ + 238, + 621, + 500, + 657 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 608, + 621, + 910, + 657 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 980, + 621, + 1222, + 657 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1350, + 625, + 1544, + 655 + ] + }, + { + "text": "appliances", + "boxes": [ + 240, + 653, + 352, + 683 + ] + }, + { + "text": "Explore all products in Kitche", + "boxes": [ + 246, + 987, + 414, + 1005 + ] + }, + { + "text": "Shop the latest from Hon", + "boxes": [ + 616, + 987, + 762, + 1005 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 985, + 1060, + 1009 + ] + }, + { + "text": "See more", + "boxes": [ + 1354, + 987, + 1406, + 1007 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 38, + 87 + ] + }, + { + "text": "→×", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1586, + 66, + 1662, + 91 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 68, + 1848, + 87 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 132, + 105, + 222, + 155 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "EN", + "boxes": [ + 1406, + 127, + 1442, + 141 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 176, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 10, + 278, + 72, + 310 + ] + }, + { + "text": "Check e", + "boxes": [ + 310, + 292, + 354, + 310 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Household", + "boxes": [ + 5, + 315, + 89, + 352 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 134, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 132, + 604 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": "Today's Deals", + "boxes": [ + 12, + 623, + 102, + 649 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1488, + 615, + 1750, + 639 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning ", + "boxes": [ + 312, + 635, + 576, + 669 + ] + }, + { + "text": "TreeLen Long Handle Broom and", + "boxes": [ + 608, + 641, + 852, + 665 + ] + }, + { + "text": " Multifunctional Cleaning Cart and", + "boxes": [ + 898, + 637, + 1154, + 667 + ] + }, + { + "text": " 6Pcs Steel Wool Scrubbers- Flexible", + "boxes": [ + 1192, + 635, + 1462, + 665 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1484, + 637, + 1760, + 667 + ] + }, + { + "text": "Department", + "boxes": [ + 10, + 655, + 98, + 681 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 659, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Dust Pan", + "boxes": [ + 606, + 663, + 832, + 693 + ] + }, + { + "text": " Tool Organizer - Broom and Mop", + "boxes": [ + 898, + 659, + 1148, + 693 + ] + }, + { + "text": " Scrubber Pads for Cleaning Dishes,", + "boxes": [ + 1192, + 659, + 1454, + 693 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1484, + 659, + 1726, + 689 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 308, + 681, + 526, + 719 + ] + }, + { + "text": "Combo for Home,Kitchen, Room,", + "boxes": [ + 602, + 681, + 858, + 721 + ] + }, + { + "text": "Holder", + "boxes": [ + 898, + 685, + 960, + 717 + ] + }, + { + "text": "Pots, Pans, Grills and Sinks ", + "boxes": [ + 1194, + 687, + 1396, + 717 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1728, + 715 + ] + }, + { + "text": "Household Cleaning Sponges", + "boxes": [ + 22, + 697, + 214, + 725 + ] + }, + { + "text": "Reusable and Washable, Pack of..", + "boxes": [ + 314, + 713, + 562, + 737 + ] + }, + { + "text": "Office, Lobby Floor Use Without..", + "boxes": [ + 608, + 709, + 860, + 739 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "★★★★☆6", + "boxes": [ + 904, + 715, + 1012, + 742 + ] + }, + { + "text": "★★★★★1.708", + "boxes": [ + 1196, + 713, + 1336, + 742 + ] + }, + { + "text": "★★★★☆2,828", + "boxes": [ + 1486, + 708, + 1632, + 743 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": "3K+ bought in past month ", + "boxes": [ + 1196, + 735, + 1368, + 758 + ] + }, + { + "text": "30K+ bought in past month", + "boxes": [ + 1486, + 728, + 1670, + 765 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "$14999", + "boxes": [ + 895, + 740, + 985, + 783 + ] + }, + { + "text": "Vacuum Cleaners & Floor Care", + "boxes": [ + 12, + 758, + 204, + 782 + ] + }, + { + "text": "★★★★★79.824", + "boxes": [ + 316, + 760, + 462, + 792 + ] + }, + { + "text": "$728 ($1.21/count)", + "boxes": [ + 1191, + 763, + 1329, + 807 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 12, + 780, + 110, + 804 + ] + }, + { + "text": "Save 5% with coupon ", + "boxes": [ + 902, + 776, + 1046, + 806 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "ught in past month", + "boxes": [ + 368, + 786, + 494, + 810 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 900, + 800, + 1046, + 830 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 1194, + 800, + 1332, + 830 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1484, + 800, + 1628, + 830 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 822, + 750, + 856 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 834, + 60, + 852 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 826, + 992, + 850 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1198, + 826, + 1286, + 850 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1570, + 846 + ] + }, + { + "text": " Bissel", + "boxes": [ + 12, + 856, + 72, + 876 + ] + }, + { + "text": " Delivery Thu, May 22", + "boxes": [ + 312, + 852, + 456, + 881 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 608, + 850, + 696, + 874 + ] + }, + { + "text": "Add to ca", + "boxes": [ + 912, + 860, + 970, + 879 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "二", + "boxes": [ + 14, + 878, + 36, + 891 + ] + }, + { + "text": "Addt cart", + "boxes": [ + 608, + 874, + 702, + 911 + ] + }, + { + "text": "o-cedar", + "boxes": [ + 9, + 891, + 87, + 922 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1486, + 883, + 1630, + 913 + ] + }, + { + "text": " 1 sustaina", + "boxes": [ + 320, + 895, + 406, + 921 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 7, + 907, + 121, + 944 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1488, + 903, + 1686, + 933 + ] + }, + { + "text": "swiffer", + "boxes": [ + 9, + 930, + 85, + 960 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 320, + 927, + 404, + 959 + ] + }, + { + "text": "Holikme", + "boxes": [ + 14, + 957, + 88, + 975 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + } + ] + }, + { + "frame_id": "80_4_8", + "episode_id": 80, + "env_id": 4, + "step": 8, + "format_rewards": 1.0, + "rewards": 8.94424057006836, + "advantages": -1.3419054746627808, + "generated_text": "{\n \"intent\": \"To open Firefox from its download link, I need to double-click on the Firefox icon which likely means retrieving and installing the web browser.\",\n \"action\": \"DoubleClick(83, 24)\"\n}", + "action_string": "DoubleClick(83, 24)", + "intent_string": "To open Firefox from its download link, I need to double-click on the Firefox icon which likely means retrieving and installing the web browser.", + "before_screen_ocr_results": [ + { + "text": "↑→", + "boxes": [ + 10, + 62, + 130, + 91 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 314, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1880, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 353, + 1088, + 383 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 381, + 924, + 413 + ] + }, + { + "text": " firefox-138.0.4.tar.xz", + "boxes": [ + 754, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 676, + 425, + 732, + 459 + ] + }, + { + "text": "Which s: TAR Archive (72.4 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwnload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 558, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompan", + "boxes": [ + 738, + 1030, + 790, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1882, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 351, + 1088, + 381 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 924, + 413 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which s: TAR Archive (72.4 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwnload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompany", + "boxes": [ + 738, + 1030, + 792, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "80_5_8", + "episode_id": 80, + "env_id": 5, + "step": 8, + "format_rewards": 1.0, + "rewards": 10.55665111541748, + "advantages": 0.7924916744232178, + "generated_text": "{\n \"intent\": \"My intention is to explore more images related to world tourist attractions by clicking one of the image thumbnails. This will likely lead to a detailed view of the selected attraction.\",\n \"action\": \"DoubleClick(52, 336)\"\n}", + "action_string": "DoubleClick(52, 336)", + "intent_string": "My intention is to explore more images related to world tourist attractions by clicking one of the image thumbnails. This will likely lead to a detailed view of the selected attraction.", + "before_screen_ocr_results": [ + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 139 + ] + }, + { + "text": "World tourist attractior", + "boxes": [ + 182, + 159, + 336, + 183 + ] + }, + { + "text": "Worl", + "boxes": [ + 20, + 703, + 50, + 723 + ] + }, + { + "text": "World Tour", + "boxes": [ + 1454, + 927, + 1520, + 945 + ] + }, + { + "text": "Places Hd", + "boxes": [ + 1454, + 943, + 1516, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 310, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 141 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 173, + 236, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "These Are The 10 Most Popular Tourist Attractions in the", + "boxes": [ + 1322, + 242, + 1750, + 266 + ] + }, + { + "text": "World -Journeyz", + "boxes": [ + 1322, + 262, + 1454, + 286 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 84, + 465 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 943, + 76, + 963 + ] + } + ] + }, + { + "frame_id": "80_6_8", + "episode_id": 80, + "env_id": 6, + "step": 8, + "format_rewards": 1.0, + "rewards": 8.988359451293945, + "advantages": -1.2835040092468262, + "generated_text": "{\n \"intent\": \"To explore the available products and features by clicking on an interesting category or item.\",\n \"action\": \"DoubleClick(1587, 306)\"\n}", + "action_string": "DoubleClick(1587, 306)", + "intent_string": "To explore the available products and features by clicking on an interesting category or item.", + "before_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 132, + 220, + 768, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1630, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "?", + "boxes": [ + 1592, + 294, + 1614, + 318 + ] + }, + { + "text": "/isit anAppleStore", + "boxes": [ + 1632, + 290, + 1756, + 308 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 380, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 134, + 635, + 722, + 675 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1846, + 748, + 1914, + 788 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": "Show your stripes.", + "boxes": [ + 1426, + 822, + 1582, + 852 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 828, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 132, + 220, + 768, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1630, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "O", + "boxes": [ + 1588, + 296, + 1612, + 320 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 378, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 138, + 641, + 720, + 671 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1843, + 745, + 1915, + 791 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": " Show your stripes.", + "boxes": [ + 1424, + 818, + 1582, + 854 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 828, + 1044 + ] + } + ] + }, + { + "frame_id": "80_7_8", + "episode_id": 80, + "env_id": 7, + "step": 8, + "format_rewards": 1.0, + "rewards": 9.44474983215332, + "advantages": -0.6793660521507263, + "generated_text": "{\n \"intent\": \"To continue exploring within the browser by interacting with available elements. Since I appear to be in Amazon’s sign-in page, which does not necessarily progress towards the goal of desktop exploration or online activities, the next logical step could be to explore further within Amazon or browse another website.\",\n \"action\": \"DoubleClick(487, 46)\"\n}", + "action_string": "DoubleClick(487, 46)", + "intent_string": "To continue exploring within the browser by interacting with available elements. Since I appear to be in Amazon’s sign-in page, which does not necessarily progress towards the goal of desktop exploration or online activities, the next logical step could be to explore further within Amazon or browse another website.", + "before_screen_ocr_results": [ + { + "text": "&openid.identity=http%3A", + "boxes": [ + 1384, + 66, + 1584, + 89 + ] + }, + { + "text": "☆", + "boxes": [ + 1572, + 66, + 1660, + 89 + ] + }, + { + "text": "三目?", + "boxes": [ + 1786, + 66, + 1874, + 89 + ] + }, + { + "text": "amazon", + "boxes": [ + 904, + 107, + 1018, + 139 + ] + }, + { + "text": " Sign in or create account ", + "boxes": [ + 806, + 181, + 1042, + 210 + ] + }, + { + "text": "Enter mobile number or email", + "boxes": [ + 806, + 216, + 1012, + 244 + ] + }, + { + "text": "Continue", + "boxes": [ + 928, + 294, + 990, + 320 + ] + }, + { + "text": "Bycontinuing youagreeto mazon's Conditons", + "boxes": [ + 806, + 332, + 1102, + 361 + ] + }, + { + "text": " of Use and Privacy Notice.", + "boxes": [ + 808, + 349, + 966, + 373 + ] + }, + { + "text": "Need help?", + "boxes": [ + 808, + 381, + 886, + 407 + ] + }, + { + "text": "Buying for work?", + "boxes": [ + 808, + 437, + 930, + 467 + ] + }, + { + "text": "Create a free business account", + "boxes": [ + 810, + 465, + 1006, + 488 + ] + }, + { + "text": "Conditions of Use", + "boxes": [ + 834, + 582, + 942, + 606 + ] + }, + { + "text": " Privacy Notice Help", + "boxes": [ + 936, + 582, + 1090, + 606 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 250, + 32, + 312, + 52 + ] + }, + { + "text": "Q Search with Bing or enter addres", + "boxes": [ + 248, + 63, + 516, + 91 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ] + }, + { + "frame_id": "80_0_9", + "episode_id": 80, + "env_id": 0, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.121102333068848, + "advantages": 0.21594244241714478, + "generated_text": "{\n \"intent\": \"The user may want to continue browsing or explore more products on Amazon. Given the high rating and low price, clicking 'Add to Cart' could be beneficial if researching future purchases.\",\n \"action\": \"DoubleClick(1498, 595)\"\n}", + "action_string": "DoubleClick(1498, 595)", + "intent_string": "The user may want to continue browsing or explore more products on Amazon. Given the high rating and low price, clicking 'Add to Cart' could be beneficial if researching future purchases.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 130, + 91 + ] + }, + { + "text": "曰』", + "boxes": [ + 252, + 64, + 310, + 89 + ] + }, + { + "text": "7-814b-0240l", + "boxes": [ + 1474, + 69, + 1564, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1584, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1874, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": "Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": " cleaning tools", + "boxes": [ + 302, + 115, + 406, + 141 + ] + }, + { + "text": "EN", + "boxes": [ + 1400, + 125, + 1444, + 145 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 125, + 1884, + 151 + ] + }, + { + "text": "三ALl", + "boxes": [ + 24, + 167, + 66, + 193 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 358, + 169, + 420, + 187 + ] + }, + { + "text": "International Kindle Paperwhite Buy Now", + "boxes": [ + 754, + 204, + 1072, + 234 + ] + }, + { + "text": "TreeLen Long Handle Broom and Dustpan ", + "boxes": [ + 834, + 306, + 1300, + 341 + ] + }, + { + "text": "", + "boxes": [ + 782, + 324, + 802, + 343 + ] + }, + { + "text": "$2099", + "boxes": [ + 1392, + 318, + 1458, + 357 + ] + }, + { + "text": "Set,Upright Dust Pan Combo for Home,", + "boxes": [ + 834, + 338, + 1270, + 373 + ] + }, + { + "text": "Kitchen, Room, Office, Lobby Floor Use ", + "boxes": [ + 834, + 369, + 1268, + 405 + ] + }, + { + "text": "No Import Charges & $8.21", + "boxes": [ + 1392, + 363, + 1576, + 393 + ] + }, + { + "text": "Shipping to Canada Details v", + "boxes": [ + 1394, + 383, + 1590, + 413 + ] + }, + { + "text": "Without Bending", + "boxes": [ + 830, + 395, + 1030, + 439 + ] + }, + { + "text": "Delivery Tuesday, May 20.", + "boxes": [ + 1392, + 403, + 1566, + 433 + ] + }, + { + "text": "Order within 7 hrs 10 mins", + "boxes": [ + 1392, + 423, + 1568, + 453 + ] + }, + { + "text": "4.6★★★★☆", + "boxes": [ + 836, + 447, + 968, + 476 + ] + }, + { + "text": "42,011 ratings", + "boxes": [ + 976, + 446, + 1074, + 477 + ] + }, + { + "text": " Deliver to Canada", + "boxes": [ + 1394, + 457, + 1512, + 480 + ] + }, + { + "text": "Amazon's Choice", + "boxes": [ + 838, + 476, + 934, + 502 + ] + }, + { + "text": "In Stock", + "boxes": [ + 1394, + 494, + 1466, + 520 + ] + }, + { + "text": "2K+bought in pa", + "boxes": [ + 840, + 512, + 932, + 530 + ] + }, + { + "text": "$59.99", + "boxes": [ + 1814, + 506, + 1860, + 524 + ] + }, + { + "text": "Quantity: 1", + "boxes": [ + 1402, + 536, + 1482, + 560 + ] + }, + { + "text": "-13% $2099", + "boxes": [ + 838, + 552, + 966, + 584 + ] + }, + { + "text": "Add to Cart ", + "boxes": [ + 1458, + 580, + 1542, + 604 + ] + }, + { + "text": "& $8.21 Ship", + "boxes": [ + 960, + 613, + 1044, + 631 + ] + }, + { + "text": "Buy Now", + "boxes": [ + 1464, + 619, + 1532, + 645 + ] + }, + { + "text": "Ships from Amazon ", + "boxes": [ + 1398, + 657, + 1522, + 687 + ] + }, + { + "text": "Sold by", + "boxes": [ + 1400, + 677, + 1450, + 705 + ] + }, + { + "text": "HomeBest Direct", + "boxes": [ + 1466, + 679, + 1566, + 705 + ] + }, + { + "text": "Returns", + "boxes": [ + 1400, + 701, + 1450, + 727 + ] + }, + { + "text": "seuees", + "boxes": [ + 1460, + 695, + 1550, + 748 + ] + }, + { + "text": "Packaging", + "boxes": [ + 1400, + 741, + 1460, + 758 + ] + }, + { + "text": "Brand", + "boxes": [ + 836, + 780, + 880, + 806 + ] + }, + { + "text": "TreeLen", + "boxes": [ + 960, + 778, + 1020, + 810 + ] + }, + { + "text": "Color", + "boxes": [ + 836, + 808, + 876, + 834 + ] + }, + { + "text": "Blue Broom and Dustpan", + "boxes": [ + 960, + 806, + 1124, + 834 + ] + }, + { + "text": "口", + "boxes": [ + 1396, + 804, + 1420, + 828 + ] + }, + { + "text": "Floor Cleaning", + "boxes": [ + 960, + 834, + 1058, + 858 + ] + }, + { + "text": "Add a gift receipt for", + "boxes": [ + 1416, + 826, + 1550, + 850 + ] + }, + { + "text": "lended", + "boxes": [ + 892, + 836, + 938, + 856 + ] + }, + { + "text": "returns", + "boxes": [ + 1420, + 848, + 1466, + 868 + ] + }, + { + "text": "UsesForProduct", + "boxes": [ + 840, + 858, + 946, + 876 + ] + }, + { + "text": "BristleMaterial", + "boxes": [ + 840, + 883, + 940, + 901 + ] + }, + { + "text": "Plastic", + "boxes": [ + 962, + 883, + 1006, + 901 + ] + }, + { + "text": "Handle Material", + "boxes": [ + 840, + 903, + 952, + 933 + ] + }, + { + "text": "Metal", + "boxes": [ + 960, + 905, + 1002, + 931 + ] + }, + { + "text": " Ad to List", + "boxes": [ + 1402, + 897, + 1476, + 923 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1810, + 917, + 1864, + 937 + ] + }, + { + "text": "Bristle Type", + "boxes": [ + 833, + 929, + 919, + 959 + ] + }, + { + "text": "Unflagged Bristle", + "boxes": [ + 966, + 935, + 1070, + 953 + ] + }, + { + "text": "VSeemo", + "boxes": [ + 838, + 967, + 896, + 985 + ] + }, + { + "text": "Qty:13", + "boxes": [ + 1794, + 963, + 1864, + 985 + ] + }, + { + "text": "About this item", + "boxes": [ + 834, + 1002, + 988, + 1039 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→", + "boxes": [ + 8, + 62, + 130, + 93 + ] + }, + { + "text": "sw_refres", + "boxes": [ + 1044, + 69, + 1116, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1586, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1872, + 91 + ] + }, + { + "text": "amazon", + "boxes": [ + 18, + 111, + 128, + 143 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1808, + 105, + 1876, + 131 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 304, + 115, + 412, + 139 + ] + }, + { + "text": "EN", + "boxes": [ + 1392, + 121, + 1450, + 147 + ] + }, + { + "text": "$14,452.18", + "boxes": [ + 1804, + 125, + 1882, + 151 + ] + }, + { + "text": "三AlL", + "boxes": [ + 24, + 167, + 66, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 360, + 169, + 420, + 187 + ] + }, + { + "text": "Cart Subtotal: $14,45218", + "boxes": [ + 1344, + 234, + 1568, + 264 + ] + }, + { + "text": "Added to cart", + "boxes": [ + 682, + 278, + 848, + 308 + ] + }, + { + "text": "Proceed to checkout (135 items)", + "boxes": [ + 1376, + 278, + 1582, + 302 + ] + }, + { + "text": "$20.99", + "boxes": [ + 1818, + 302, + 1858, + 318 + ] + }, + { + "text": "Go to Cart", + "boxes": [ + 1448, + 324, + 1512, + 343 + ] + }, + { + "text": " Important messages about items in your Cart:", + "boxes": [ + 164, + 431, + 588, + 469 + ] + }, + { + "text": "V See more", + "boxes": [ + 169, + 468, + 247, + 501 + ] + }, + { + "text": "$347.99", + "boxes": [ + 1812, + 506, + 1858, + 524 + ] + }, + { + "text": ". also bought these items from other categories", + "boxes": [ + 682, + 558, + 1070, + 576 + ] + }, + { + "text": "Page 1 of 3", + "boxes": [ + 1668, + 556, + 1742, + 580 + ] + }, + { + "text": "esKitchen", + "boxes": [ + 392, + 782, + 452, + 800 + ] + }, + { + "text": "Duster,7 PCS Reusable Bend...", + "boxes": [ + 84, + 798, + 274, + 822 + ] + }, + { + "text": "Knifeset, stallesstee..", + "boxes": [ + 280, + 792, + 480, + 828 + ] + }, + { + "text": "BowlBrush Combofor Bath..", + "boxes": [ + 484, + 790, + 674, + 826 + ] + }, + { + "text": "Small Trash Can with Lid, 10..", + "boxes": [ + 686, + 794, + 880, + 824 + ] + }, + { + "text": "Countertop, Stainless stee.", + "boxes": [ + 886, + 790, + 1072, + 828 + ] + }, + { + "text": "Grippers, 2 Hangers and 4 C.", + "boxes": [ + 1092, + 796, + 1276, + 826 + ] + }, + { + "text": "Kitchen Sinkink Caddyr..", + "boxes": [ + 1290, + 790, + 1484, + 826 + ] + }, + { + "text": "with 2 Banana Hangers, Cou", + "boxes": [ + 1498, + 798, + 1678, + 822 + ] + }, + { + "text": "★★★★★3,255", + "boxes": [ + 82, + 818, + 200, + 848 + ] + }, + { + "text": "★★★★★ 2,136", + "boxes": [ + 284, + 820, + 406, + 850 + ] + }, + { + "text": "★★★★★710.981", + "boxes": [ + 486, + 820, + 614, + 850 + ] + }, + { + "text": "★★★★★494", + "boxes": [ + 688, + 818, + 800, + 850 + ] + }, + { + "text": "★★★★★4.387", + "boxes": [ + 892, + 816, + 1010, + 850 + ] + }, + { + "text": "★★★★★38.018", + "boxes": [ + 1092, + 820, + 1220, + 850 + ] + }, + { + "text": "★★★★★5.939", + "boxes": [ + 1294, + 820, + 1416, + 850 + ] + }, + { + "text": "★★★★★5.499", + "boxes": [ + 1498, + 818, + 1616, + 850 + ] + }, + { + "text": "$2199", + "boxes": [ + 82, + 844, + 136, + 874 + ] + }, + { + "text": "$1899", + "boxes": [ + 284, + 846, + 336, + 872 + ] + }, + { + "text": "$3899", + "boxes": [ + 484, + 842, + 544, + 874 + ] + }, + { + "text": "$2399", + "boxes": [ + 685, + 839, + 743, + 876 + ] + }, + { + "text": "11% off", + "boxes": [ + 894, + 848, + 950, + 874 + ] + }, + { + "text": "$1309", + "boxes": [ + 1090, + 842, + 1148, + 874 + ] + }, + { + "text": "$1499", + "boxes": [ + 1294, + 844, + 1346, + 870 + ] + }, + { + "text": "$2089", + "boxes": [ + 1494, + 842, + 1552, + 874 + ] + }, + { + "text": "Limited time deal", + "boxes": [ + 954, + 852, + 1054, + 870 + ] + }, + { + "text": "$1423", + "boxes": [ + 890, + 876, + 944, + 905 + ] + }, + { + "text": "List Price: $45.99", + "boxes": [ + 888, + 901, + 988, + 925 + ] + }, + { + "text": "See buying optior", + "boxes": [ + 99, + 968, + 209, + 1001 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 300, + 967, + 382, + 999 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 502, + 967, + 582, + 999 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 704, + 967, + 786, + 999 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 906, + 967, + 988, + 999 + ] + }, + { + "text": "Add to cat ", + "boxes": [ + 1106, + 967, + 1190, + 999 + ] + }, + { + "text": "Ad to cart", + "boxes": [ + 1312, + 971, + 1386, + 997 + ] + } + ] + }, + { + "frame_id": "80_1_9", + "episode_id": 80, + "env_id": 1, + "step": 9, + "format_rewards": 1.0, + "rewards": 10.858362197875977, + "advantages": 1.19187593460083, + "generated_text": "{\n \"intent\": \"To start exploring other websites or information by visiting one of the highlighted or top sites.\",\n \"action\": \"DoubleClick(770, 735)\"\n}", + "action_string": "DoubleClick(770, 735)", + "intent_string": "To start exploring other websites or information by visiting one of the highlighted or top sites.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 250, + 32, + 312, + 52 + ] + }, + { + "text": "→白", + "boxes": [ + 40, + 66, + 130, + 91 + ] + }, + { + "text": "Q earch with Bing or enter address", + "boxes": [ + 244, + 57, + 522, + 97 + ] + }, + { + "text": " Import bookmarks..", + "boxes": [ + 12, + 99, + 162, + 121 + ] + }, + { + "text": " Getting Started", + "boxes": [ + 186, + 99, + 312, + 123 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 996, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": " iPhone 16 and iPhone 16 × ", + "boxes": [ + 26, + 32, + 220, + 50 + ] + }, + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1610, + 69, + 1658, + 87 + ] + }, + { + "text": " Microsoft Bing ", + "boxes": [ + 20, + 109, + 186, + 151 + ] + }, + { + "text": "国内版", + "boxes": [ + 490, + 250, + 560, + 282 + ] + }, + { + "text": "国际版", + "boxes": [ + 550, + 250, + 638, + 282 + ] + }, + { + "text": "雷军回应SU7事故", + "boxes": [ + 478, + 300, + 612, + 330 + ] + }, + { + "text": "Q", + "boxes": [ + 1340, + 300, + 1360, + 326 + ] + }, + { + "text": "Trending on Bing", + "boxes": [ + 482, + 371, + 644, + 401 + ] + }, + { + "text": "中亭", + "boxes": [ + 1318, + 375, + 1352, + 403 + ] + }, + { + "text": "@Next", + "boxes": [ + 1358, + 373, + 1418, + 405 + ] + }, + { + "text": "礼", + "boxes": [ + 1546, + 367, + 1564, + 383 + ] + }, + { + "text": "电电中", + "boxes": [ + 1140, + 385, + 1174, + 413 + ] + }, + { + "text": "兄", + "boxes": [ + 1636, + 385, + 1654, + 399 + ] + }, + { + "text": "1 雷军回应SU7事故热", + "boxes": [ + 484, + 411, + 664, + 441 + ] + }, + { + "text": "2汽车新规来了", + "boxes": [ + 792, + 411, + 916, + 441 + ] + }, + { + "text": "福", + "boxes": [ + 980, + 407, + 996, + 421 + ] + }, + { + "text": "3朱婷妹妹保送被质疑", + "boxes": [ + 1104, + 407, + 1278, + 443 + ] + }, + { + "text": "4爆料正新鸡排生", + "boxes": [ + 486, + 451, + 634, + 474 + ] + }, + { + "text": "5取款身亡银行赔10万", + "boxes": [ + 794, + 444, + 970, + 479 + ] + }, + { + "text": "①图卢兹的暮色", + "boxes": [ + 1518, + 675, + 1632, + 705 + ] + }, + { + "text": "自选股建议", + "boxes": [ + 1451, + 762, + 1555, + 797 + ] + }, + { + "text": "特斯拉", + "boxes": [ + 1460, + 804, + 1504, + 842 + ] + }, + { + "text": "+2.00%6 ", + "boxes": [ + 1634, + 802, + 1698, + 842 + ] + }, + { + "text": "贵州茅台", + "boxes": [ + 1458, + 852, + 1512, + 887 + ] + }, + { + "text": "1,614023", + "boxes": [ + 1628, + 850, + 1698, + 891 + ] + }, + { + "text": "13,87", + "boxes": [ + 1628, + 893, + 1700, + 941 + ] + }, + { + "text": "一点视频/萌宠的日常·2天", + "boxes": [ + 846, + 929, + 1010, + 959 + ] + }, + { + "text": "游戏天地·1周", + "boxes": [ + 1142, + 921, + 1254, + 945 + ] + }, + { + "text": "你还不知道怎么选梦幻西游命魂之玉吗？一篇", + "boxes": [ + 200, + 945, + 794, + 987 + ] + }, + { + "text": "1.", + "boxes": [ + 832, + 937, + 850, + 951 + ] + }, + { + "text": "探秘浮世万千之前世今生新手", + "boxes": [ + 1136, + 941, + 1416, + 977 + ] + }, + { + "text": "比", + "boxes": [ + 1454, + 937, + 1512, + 989 + ] + }, + { + "text": "+3.282", + "boxes": [ + 1626, + 939, + 1700, + 989 + ] + }, + { + "text": "那些和主人长得很像的萌宠，", + "boxes": [ + 828, + 955, + 1088, + 985 + ] + }, + { + "text": "修真全流程通关秘籍", + "boxes": [ + 1138, + 967, + 1330, + 1003 + ] + }, + { + "text": "", + "boxes": [ + 1584, + 935, + 1636, + 987 + ] + }, + { + "text": "带你看懂", + "boxes": [ + 195, + 976, + 333, + 1025 + ] + }, + { + "text": "看看哪只更搞笑？", + "boxes": [ + 828, + 983, + 992, + 1012 + ] + }, + { + "text": "中国平安", + "boxes": [ + 1454, + 983, + 1520, + 1034 + ] + }, + { + "text": "1.102 ", + "boxes": [ + 1634, + 983, + 1700, + 1032 + ] + }, + { + "text": "LLM", + "boxes": [ + 1594, + 1003, + 1628, + 1022 + ] + }, + { + "text": "373", + "boxes": [ + 836, + 1028, + 858, + 1042 + ] + }, + { + "text": "3", + "boxes": [ + 188, + 1052, + 204, + 1066 + ] + } + ] + }, + { + "frame_id": "80_2_9", + "episode_id": 80, + "env_id": 2, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.473836898803711, + "advantages": -0.6408626437187195, + "generated_text": "{\n \"intent\": \"The displayed Amazon homepage offers various categories, such as gaming accessories, home essentials, fashion deals, and more. Clicking on one of these categories will allow us to explore specific sections and potentially discover interesting products.\",\n \"action\": \"Click(1031, 907)\"\n}", + "action_string": "Click(1031, 907)", + "intent_string": "The displayed Amazon homepage offers various categories, such as gaming accessories, home essentials, fashion deals, and more. Clicking on one of these categories will allow us to explore specific sections and potentially discover interesting products.", + "before_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 134, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 416, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三Al", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "Gift Cards", + "boxes": [ + 356, + 169, + 420, + 187 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 658, + 290, + 942, + 350 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chairs", + "boxes": [ + 408, + 840, + 438, + 860 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": " See all deals", + "boxes": [ + 980, + 889, + 1060, + 913 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1438, + 915 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 240, + 965, + 540, + 1001 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 967, + 870, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1220, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 963, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 610, + 997, + 724, + 1026 + ] + }, + { + "text": "", + "boxes": [ + 1356, + 1001, + 1458, + 1024 + ] + }, + { + "text": "", + "boxes": [ + 1557, + 996, + 1647, + 1025 + ] + }, + { + "text": "=UTF8&k", + "boxes": [ + 262, + 1020, + 324, + 1040 + ] + }, + { + "text": "chairs&ref=nb_sb_", + "boxes": [ + 388, + 1020, + 518, + 1038 + ] + }, + { + "text": "rC", + "boxes": [ + 582, + 1024, + 594, + 1036 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←→白", + "boxes": [ + 6, + 62, + 132, + 93 + ] + }, + { + "text": "☆", + "boxes": [ + 1582, + 62, + 1664, + 93 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1878, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 133, + 145 + ] + }, + { + "text": " Caharta", + "boxes": [ + 134, + 105, + 220, + 151 + ] + }, + { + "text": "Search Amazon", + "boxes": [ + 294, + 113, + 416, + 143 + ] + }, + { + "text": "Q", + "boxes": [ + 1454, + 119, + 1474, + 135 + ] + }, + { + "text": "EN", + "boxes": [ + 1522, + 121, + 1578, + 147 + ] + }, + { + "text": "三Al", + "boxes": [ + 24, + 167, + 70, + 195 + ] + }, + { + "text": "egistry", + "boxes": [ + 200, + 167, + 246, + 193 + ] + }, + { + "text": "GiftCards", + "boxes": [ + 354, + 169, + 420, + 187 + ] + }, + { + "text": "Father's Day", + "boxes": [ + 654, + 240, + 920, + 306 + ] + }, + { + "text": "gifts he'll love", + "boxes": [ + 656, + 290, + 942, + 350 + ] + }, + { + "text": "You are on amazon.com. Yo", + "boxes": [ + 510, + 461, + 684, + 478 + ] + }, + { + "text": "", + "boxes": [ + 1448, + 486, + 1542, + 510 + ] + }, + { + "text": "Gaming accessories", + "boxes": [ + 242, + 530, + 440, + 560 + ] + }, + { + "text": "Shop for your home essentials", + "boxes": [ + 610, + 530, + 918, + 560 + ] + }, + { + "text": " Shop deals in Fashion", + "boxes": [ + 978, + 526, + 1204, + 562 + ] + }, + { + "text": "Get your game on", + "boxes": [ + 1352, + 530, + 1536, + 560 + ] + }, + { + "text": "Chair", + "boxes": [ + 410, + 844, + 436, + 858 + ] + }, + { + "text": "See more", + "boxes": [ + 242, + 891, + 300, + 911 + ] + }, + { + "text": "Discover more in Hom", + "boxes": [ + 614, + 891, + 746, + 909 + ] + }, + { + "text": "See all deals", + "boxes": [ + 982, + 891, + 1058, + 909 + ] + }, + { + "text": "Shop gaming", + "boxes": [ + 1350, + 889, + 1436, + 915 + ] + }, + { + "text": "New home arrivals under $50", + "boxes": [ + 240, + 969, + 538, + 999 + ] + }, + { + "text": "Top categories in Kitchen", + "boxes": [ + 610, + 967, + 870, + 1003 + ] + }, + { + "text": "Gifts for Dad under $25", + "boxes": [ + 982, + 969, + 1222, + 999 + ] + }, + { + "text": "Refresh your space", + "boxes": [ + 1348, + 965, + 1546, + 1005 + ] + }, + { + "text": "appliances", + "boxes": [ + 610, + 997, + 724, + 1028 + ] + } + ] + }, + { + "frame_id": "80_3_9", + "episode_id": 80, + "env_id": 3, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.461929321289062, + "advantages": -0.6566250324249268, + "generated_text": "{\n \"intent\": \"Clicking on 'Go' will adjust the filter conditions based on customer reviews, price, deals, discounts, department, and brand preferences. This will help refine the search results to better match the user's interests and potentially find more relevant cleaning tools.\",\n \"action\": \"DoubleClick(272, 553)\"\n}", + "action_string": "DoubleClick(272, 553)", + "intent_string": "Clicking on 'Go' will adjust the filter conditions based on customer reviews, price, deals, discounts, department, and brand preferences. This will help refine the search results to better match the user's interests and potentially find more relevant cleaning tools.", + "before_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1596, + 66, + 1662, + 91 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1872, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1884, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 174, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 10, + 278, + 72, + 310 + ] + }, + { + "text": "Check e", + "boxes": [ + 310, + 292, + 354, + 310 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Household", + "boxes": [ + 10, + 322, + 86, + 347 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 134, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 134, + 604 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 94, + 623 + ] + }, + { + "text": " Today's Deals", + "boxes": [ + 7, + 617, + 105, + 654 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1488, + 615, + 1750, + 639 + ] + }, + { + "text": " Amazon Basics Microfiber Cleaning ", + "boxes": [ + 312, + 637, + 576, + 669 + ] + }, + { + "text": "TreeLen Long Handle Broom and", + "boxes": [ + 606, + 639, + 850, + 663 + ] + }, + { + "text": "Multifunctional Cleaning Cart and ", + "boxes": [ + 900, + 635, + 1154, + 665 + ] + }, + { + "text": "6Pcs SteelWool Scrubbers -Flexibe", + "boxes": [ + 1192, + 633, + 1464, + 671 + ] + }, + { + "text": " handheld portable steam cleaner for", + "boxes": [ + 1484, + 637, + 1760, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 7, + 649, + 101, + 686 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 659, + 536, + 693 + ] + }, + { + "text": "Dustpan Set,Upright Dust Pan", + "boxes": [ + 606, + 663, + 832, + 693 + ] + }, + { + "text": "Tool Organizer - Broom and Mop", + "boxes": [ + 900, + 663, + 1146, + 691 + ] + }, + { + "text": " Scrubber Pads for Cleaning Dishes,", + "boxes": [ + 1192, + 661, + 1454, + 693 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1486, + 661, + 1728, + 691 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": "Combo for Home, Kitchen, Room,", + "boxes": [ + 604, + 681, + 856, + 721 + ] + }, + { + "text": "Holder", + "boxes": [ + 898, + 685, + 960, + 717 + ] + }, + { + "text": "Pots, Pans, Grills and Sinks", + "boxes": [ + 1194, + 687, + 1396, + 717 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1728, + 715 + ] + }, + { + "text": "Reusable and Washable, Pack of..", + "boxes": [ + 314, + 713, + 562, + 737 + ] + }, + { + "text": "Office, Lobby Floor Use Without..", + "boxes": [ + 608, + 709, + 860, + 739 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "★★★★☆6", + "boxes": [ + 904, + 715, + 1012, + 742 + ] + }, + { + "text": "★★★★★1.708", + "boxes": [ + 1196, + 713, + 1336, + 742 + ] + }, + { + "text": "★★★★☆2.828", + "boxes": [ + 1488, + 711, + 1630, + 739 + ] + }, + { + "text": "★★★★★42.011", + "boxes": [ + 610, + 737, + 760, + 766 + ] + }, + { + "text": " 3K+ bought in past month", + "boxes": [ + 1194, + 732, + 1366, + 760 + ] + }, + { + "text": "30K+ bought in past month", + "boxes": [ + 1486, + 728, + 1668, + 765 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "$14999", + "boxes": [ + 895, + 740, + 983, + 783 + ] + }, + { + "text": "★★★★★79.824", + "boxes": [ + 314, + 760, + 464, + 794 + ] + }, + { + "text": " bought in past month", + "boxes": [ + 632, + 760, + 778, + 784 + ] + }, + { + "text": "$728 ($1.21/count) ", + "boxes": [ + 1194, + 768, + 1328, + 800 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 12, + 780, + 110, + 804 + ] + }, + { + "text": "Save 5% with coupon ", + "boxes": [ + 902, + 776, + 1046, + 806 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "ught in past month", + "boxes": [ + 368, + 786, + 494, + 810 + ] + }, + { + "text": "$209 Typlcal: 523.99", + "boxes": [ + 605, + 786, + 771, + 830 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": " Delivery Thu, May 22", + "boxes": [ + 898, + 798, + 1046, + 832 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 1194, + 800, + 1332, + 830 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": "Delivery Tue, May 20", + "boxes": [ + 606, + 822, + 750, + 856 + ] + }, + { + "text": "Brands", + "boxes": [ + 12, + 832, + 62, + 858 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 826, + 992, + 850 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1198, + 826, + 1286, + 850 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1568, + 846 + ] + }, + { + "text": " Bissell", + "boxes": [ + 10, + 852, + 74, + 878 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 608, + 850, + 698, + 874 + ] + }, + { + "text": "Add to car", + "boxes": [ + 912, + 860, + 978, + 878 + ] + }, + { + "text": "Add to car", + "boxes": [ + 1206, + 858, + 1272, + 878 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1494, + 856, + 1580, + 881 + ] + }, + { + "text": "L", + "boxes": [ + 16, + 878, + 36, + 891 + ] + }, + { + "text": "Addt cart", + "boxes": [ + 608, + 874, + 702, + 911 + ] + }, + { + "text": "o-cedar", + "boxes": [ + 9, + 891, + 87, + 922 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1488, + 883, + 1630, + 913 + ] + }, + { + "text": " 1 sustainab", + "boxes": [ + 320, + 895, + 414, + 921 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 7, + 907, + 121, + 944 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1686, + 931 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1809, + 912, + 1859, + 938 + ] + }, + { + "text": " swiffer", + "boxes": [ + 9, + 928, + 85, + 962 + ] + }, + { + "text": "Add to cart ", + "boxes": [ + 322, + 931, + 402, + 957 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 87 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①』", + "boxes": [ + 256, + 68, + 308, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1602, + 68, + 1660, + 87 + ] + }, + { + "text": "三四?", + "boxes": [ + 1784, + 66, + 1872, + 91 + ] + }, + { + "text": " amazon", + "boxes": [ + 13, + 103, + 131, + 145 + ] + }, + { + "text": " Carta", + "boxes": [ + 134, + 105, + 220, + 153 + ] + }, + { + "text": "cleaning tools", + "boxes": [ + 297, + 111, + 407, + 147 + ] + }, + { + "text": "", + "boxes": [ + 1694, + 113, + 1734, + 139 + ] + }, + { + "text": "Subtotal", + "boxes": [ + 1810, + 105, + 1876, + 131 + ] + }, + { + "text": "$14,471.17", + "boxes": [ + 1804, + 127, + 1882, + 151 + ] + }, + { + "text": "三 AIl ", + "boxes": [ + 22, + 167, + 70, + 195 + ] + }, + { + "text": "Popular Shopping Ideas", + "boxes": [ + 10, + 256, + 174, + 286 + ] + }, + { + "text": "Results", + "boxes": [ + 306, + 264, + 380, + 290 + ] + }, + { + "text": "Window", + "boxes": [ + 12, + 282, + 70, + 308 + ] + }, + { + "text": "Bathing", + "boxes": [ + 12, + 302, + 70, + 328 + ] + }, + { + "text": "Household", + "boxes": [ + 10, + 322, + 86, + 347 + ] + }, + { + "text": "Tile", + "boxes": [ + 14, + 343, + 40, + 365 + ] + }, + { + "text": "Customer Reviews", + "boxes": [ + 12, + 409, + 136, + 435 + ] + }, + { + "text": "★★★★☆&", + "boxes": [ + 12, + 433, + 144, + 463 + ] + }, + { + "text": "Price", + "boxes": [ + 10, + 471, + 54, + 496 + ] + }, + { + "text": "$0 - $10,600+", + "boxes": [ + 10, + 496, + 108, + 520 + ] + }, + { + "text": "Deals & Discounts", + "boxes": [ + 12, + 580, + 134, + 604 + ] + }, + { + "text": "All Discounts", + "boxes": [ + 14, + 606, + 96, + 623 + ] + }, + { + "text": " Today's Deals", + "boxes": [ + 7, + 617, + 105, + 654 + ] + }, + { + "text": "BISSELL@ Steam ShotTM OmniReach", + "boxes": [ + 1488, + 615, + 1750, + 639 + ] + }, + { + "text": "Amazon Basics Microfiber Cleaning ", + "boxes": [ + 312, + 635, + 576, + 669 + ] + }, + { + "text": "Multifunctional Cleaning Cart and", + "boxes": [ + 606, + 637, + 856, + 667 + ] + }, + { + "text": "6Pcs Steel Wool Scrubbers - Flexible", + "boxes": [ + 902, + 635, + 1174, + 665 + ] + }, + { + "text": " Electric Ear Wax Removal System,", + "boxes": [ + 1194, + 641, + 1440, + 665 + ] + }, + { + "text": "handheld portable steam cleaner for", + "boxes": [ + 1486, + 637, + 1760, + 667 + ] + }, + { + "text": " Department", + "boxes": [ + 7, + 649, + 101, + 686 + ] + }, + { + "text": "Cloths for Cars, Non-Abrasive,", + "boxes": [ + 312, + 661, + 536, + 693 + ] + }, + { + "text": " Tool Organizer - Broom and Mop", + "boxes": [ + 604, + 659, + 854, + 693 + ] + }, + { + "text": " Scrubber Pads for Cleaning Dishes,", + "boxes": [ + 900, + 663, + 1158, + 691 + ] + }, + { + "text": "Water Powered Ear Cleaning Tool", + "boxes": [ + 1192, + 659, + 1446, + 693 + ] + }, + { + "text": "tile, grout, windows, bathrooms,", + "boxes": [ + 1486, + 661, + 1728, + 691 + ] + }, + { + "text": "Household Cleaning Tools", + "boxes": [ + 10, + 677, + 184, + 707 + ] + }, + { + "text": "Highly Absorbent, Lint Free,", + "boxes": [ + 312, + 687, + 524, + 717 + ] + }, + { + "text": "Holder", + "boxes": [ + 604, + 685, + 664, + 717 + ] + }, + { + "text": "Pots, Pans, Grills and Sinks", + "boxes": [ + 900, + 687, + 1100, + 717 + ] + }, + { + "text": "with rrigation Flusher, 6 Tips, 4..", + "boxes": [ + 1194, + 687, + 1444, + 717 + ] + }, + { + "text": "kitchens, fabric steaming tool...", + "boxes": [ + 1486, + 685, + 1726, + 715 + ] + }, + { + "text": "Dusting Tools", + "boxes": [ + 24, + 719, + 118, + 744 + ] + }, + { + "text": "Reusable and Washable, Pack of..", + "boxes": [ + 314, + 713, + 562, + 737 + ] + }, + { + "text": "★★★★☆6", + "boxes": [ + 608, + 713, + 718, + 744 + ] + }, + { + "text": "★★★★☆1,708", + "boxes": [ + 902, + 713, + 1044, + 737 + ] + }, + { + "text": "★★★★☆427", + "boxes": [ + 1196, + 713, + 1328, + 742 + ] + }, + { + "text": "★★★★☆2.828", + "boxes": [ + 1488, + 711, + 1630, + 739 + ] + }, + { + "text": "Household Squeegees", + "boxes": [ + 24, + 739, + 168, + 762 + ] + }, + { + "text": "400+ bought in past month", + "boxes": [ + 1198, + 735, + 1374, + 758 + ] + }, + { + "text": " 30K+ bought in past month", + "boxes": [ + 1486, + 730, + 1670, + 762 + ] + }, + { + "text": "$14999", + "boxes": [ + 604, + 742, + 690, + 780 + ] + }, + { + "text": "Vacuum Cleaners & Floor Care", + "boxes": [ + 10, + 756, + 208, + 786 + ] + }, + { + "text": "★★★★★79.824", + "boxes": [ + 314, + 760, + 464, + 794 + ] + }, + { + "text": "$728 (1.2/count)", + "boxes": [ + 899, + 762, + 1033, + 807 + ] + }, + { + "text": "$549 ($54.99/cout List:$8.99", + "boxes": [ + 1189, + 762, + 1431, + 809 + ] + }, + { + "text": "$4999", + "boxes": [ + 1486, + 764, + 1552, + 798 + ] + }, + { + "text": "Paint Scrapers", + "boxes": [ + 12, + 780, + 110, + 804 + ] + }, + { + "text": "Save 5% with coupon", + "boxes": [ + 608, + 776, + 750, + 806 + ] + }, + { + "text": "300+", + "boxes": [ + 322, + 790, + 346, + 804 + ] + }, + { + "text": "ought in past month", + "boxes": [ + 364, + 786, + 494, + 810 + ] + }, + { + "text": "Household Cleaning", + "boxes": [ + 10, + 796, + 144, + 824 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 608, + 804, + 748, + 828 + ] + }, + { + "text": "Delivery Fri, May 23", + "boxes": [ + 900, + 800, + 1036, + 830 + ] + }, + { + "text": " Delivery Fri, May 23", + "boxes": [ + 1192, + 800, + 1330, + 830 + ] + }, + { + "text": " Delivery Tue, May 20", + "boxes": [ + 1486, + 800, + 1630, + 830 + ] + }, + { + "text": "$499 ($0.35/count Typical: $65.64", + "boxes": [ + 309, + 813, + 561, + 860 + ] + }, + { + "text": "Brands", + "boxes": [ + 14, + 834, + 60, + 852 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 610, + 826, + 698, + 850 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 904, + 826, + 992, + 850 + ] + }, + { + "text": "Ships to Canada", + "boxes": [ + 1198, + 826, + 1286, + 850 + ] + }, + { + "text": "Ships to Canac", + "boxes": [ + 1492, + 828, + 1568, + 846 + ] + }, + { + "text": "Delivery Thu, May 22", + "boxes": [ + 314, + 852, + 456, + 881 + ] + }, + { + "text": " Add to car", + "boxes": [ + 620, + 860, + 684, + 879 + ] + }, + { + "text": "Add to ca", + "boxes": [ + 914, + 860, + 970, + 878 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 1204, + 856, + 1278, + 879 + ] + }, + { + "text": " Add to cart", + "boxes": [ + 1492, + 856, + 1580, + 881 + ] + }, + { + "text": "L", + "boxes": [ + 16, + 878, + 36, + 891 + ] + }, + { + "text": " 1 sustalnab", + "boxes": [ + 317, + 891, + 415, + 926 + ] + }, + { + "text": "More Buying Choices ", + "boxes": [ + 1488, + 883, + 1630, + 913 + ] + }, + { + "text": "o-Ceda", + "boxes": [ + 14, + 897, + 84, + 919 + ] + }, + { + "text": "scrub Daddy", + "boxes": [ + 12, + 911, + 118, + 941 + ] + }, + { + "text": "$39.74 (2+ used & new offers)", + "boxes": [ + 1490, + 907, + 1684, + 931 + ] + }, + { + "text": "$129.99", + "boxes": [ + 1810, + 917, + 1862, + 937 + ] + }, + { + "text": "Add to cart", + "boxes": [ + 326, + 931, + 402, + 957 + ] + }, + { + "text": " scotch-Brite", + "boxes": [ + 9, + 970, + 117, + 1003 + ] + }, + { + "text": "Qty: 13", + "boxes": [ + 1790, + 961, + 1864, + 985 + ] + } + ] + }, + { + "frame_id": "80_4_9", + "episode_id": 80, + "env_id": 4, + "step": 9, + "format_rewards": 1.0, + "rewards": 8.870145797729492, + "advantages": -1.439987063407898, + "generated_text": "{\n \"intent\": \"The current goal is to explore this environment, which may include interacting with files or applications. Opening external applications like Firefox requires a double-click, so clicking on the Firefox icon would be a logical first step in continuing exploration.\",\n \"action\": \"DoubleClick(70, 36)\"\n}", + "action_string": "DoubleClick(70, 36)", + "intent_string": "The current goal is to explore this environment, which may include interacting with files or applications. Opening external applications like Firefox requires a double-click, so clicking on the Firefox icon would be a logical first step in continuing exploration.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 34, + 66, + 128, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1882, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 351, + 1088, + 381 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": "You have chosen to open:", + "boxes": [ + 744, + 383, + 922, + 411 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which s: TAR Archive (72.4 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwnload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default)", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompany", + "boxes": [ + 738, + 1030, + 792, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "et Fikefox for desktop", + "boxes": [ + 44, + 32, + 186, + 50 + ] + }, + { + "text": "→C", + "boxes": [ + 32, + 66, + 126, + 89 + ] + }, + { + "text": "①", + "boxes": [ + 250, + 62, + 312, + 93 + ] + }, + { + "text": "·☆", + "boxes": [ + 1594, + 64, + 1662, + 89 + ] + }, + { + "text": "三四?", + "boxes": [ + 1786, + 66, + 1882, + 91 + ] + }, + { + "text": "Mozilla I3", + "boxes": [ + 253, + 102, + 379, + 154 + ] + }, + { + "text": " Firefox browsers", + "boxes": [ + 738, + 111, + 920, + 141 + ] + }, + { + "text": "Products", + "boxes": [ + 958, + 113, + 1046, + 139 + ] + }, + { + "text": " About us", + "boxes": [ + 1084, + 109, + 1174, + 141 + ] + }, + { + "text": "Opening firefox-138.0.4.tar.xz", + "boxes": [ + 838, + 351, + 1088, + 381 + ] + }, + { + "text": "口", + "boxes": [ + 1120, + 355, + 1188, + 381 + ] + }, + { + "text": " You have chosen to open: ", + "boxes": [ + 742, + 379, + 924, + 413 + ] + }, + { + "text": "frefox-138.0.4.tar.xz", + "boxes": [ + 756, + 411, + 942, + 441 + ] + }, + { + "text": "Se", + "boxes": [ + 674, + 421, + 734, + 461 + ] + }, + { + "text": "Which s: TAR Archive (72.4 MB) ", + "boxes": [ + 770, + 429, + 1000, + 469 + ] + }, + { + "text": "on", + "boxes": [ + 1182, + 425, + 1242, + 467 + ] + }, + { + "text": "from: https:/dwload-istallercd.moillanet", + "boxes": [ + 770, + 450, + 1098, + 491 + ] + }, + { + "text": "What should Firefox do with this file?", + "boxes": [ + 744, + 502, + 1034, + 532 + ] + }, + { + "text": "Qpen with /usr/bin/xarchiver (default) ", + "boxes": [ + 758, + 528, + 1050, + 558 + ] + }, + { + "text": "O Save File", + "boxes": [ + 766, + 560, + 846, + 584 + ] + }, + { + "text": " Do this automatically for fles like this from now on", + "boxes": [ + 764, + 594, + 1122, + 617 + ] + }, + { + "text": " Cancel", + "boxes": [ + 1010, + 653, + 1064, + 673 + ] + }, + { + "text": "Using Debian, Ubuntu or any Debian-based distributon?", + "boxes": [ + 732, + 681, + 1174, + 711 + ] + }, + { + "text": "You can set up our APT repository instead.", + "boxes": [ + 786, + 703, + 1120, + 733 + ] + }, + { + "text": "Download in another language or for another operating system.", + "boxes": [ + 704, + 887, + 1202, + 917 + ] + }, + { + "text": "ompany", + "boxes": [ + 738, + 1030, + 792, + 1048 + ] + }, + { + "text": "Sunnort", + "boxes": [ + 958, + 1028, + 1020, + 1048 + ] + }, + { + "text": "Follow @Mozilla", + "boxes": [ + 1512, + 1022, + 1644, + 1052 + ] + }, + { + "text": "ning firef.", + "boxes": [ + 398, + 1052, + 460, + 1070 + ] + } + ] + }, + { + "frame_id": "80_5_9", + "episode_id": 80, + "env_id": 5, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.893980026245117, + "advantages": -0.08470630645751953, + "generated_text": "{\n \"intent\": \"To continue exploring images of world tourist attractions and gather more information about them.\",\n \"action\": \"DoubleClick(1047, 309)\"\n}", + "action_string": "DoubleClick(1047, 309)", + "intent_string": "To continue exploring images of world tourist attractions and gather more information about them.", + "before_screen_ocr_results": [ + { + "text": "→C", + "boxes": [ + 32, + 66, + 128, + 91 + ] + }, + { + "text": "①P", + "boxes": [ + 256, + 69, + 310, + 87 + ] + }, + { + "text": "三口", + "boxes": [ + 1788, + 66, + 1862, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 141 + ] + }, + { + "text": "国际版", + "boxes": [ + 264, + 119, + 304, + 139 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 173, + 236, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "These Are The 10 Most Popular Tourist Attractions in the", + "boxes": [ + 1322, + 242, + 1750, + 266 + ] + }, + { + "text": "World -Journeyz", + "boxes": [ + 1322, + 262, + 1454, + 286 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 84, + 465 + ] + }, + { + "text": "2023 rev", + "boxes": [ + 18, + 943, + 76, + 963 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "←", + "boxes": [ + 12, + 66, + 36, + 89 + ] + }, + { + "text": "→", + "boxes": [ + 30, + 66, + 128, + 91 + ] + }, + { + "text": "三口?", + "boxes": [ + 1788, + 66, + 1870, + 89 + ] + }, + { + "text": "国内版", + "boxes": [ + 192, + 121, + 236, + 141 + ] + }, + { + "text": "国际版", + "boxes": [ + 262, + 117, + 306, + 143 + ] + }, + { + "text": "MicrosoftBing", + "boxes": [ + 118, + 171, + 234, + 197 + ] + }, + { + "text": "World tourist attraction", + "boxes": [ + 260, + 169, + 432, + 199 + ] + }, + { + "text": "Tourist Places Wallpapers - Wallpaper Cave", + "boxes": [ + 1320, + 240, + 1650, + 270 + ] + }, + { + "text": "", + "boxes": [ + 1452, + 294, + 1518, + 320 + ] + }, + { + "text": "", + "boxes": [ + 1532, + 290, + 1564, + 304 + ] + }, + { + "text": "ave.com I| 2500 x 1666 ipeg I 4 yrs ago", + "boxes": [ + 1408, + 337, + 1668, + 369 + ] + }, + { + "text": "wal", + "boxes": [ + 1334, + 347, + 1362, + 361 + ] + }, + { + "text": "Visit site", + "boxes": [ + 1330, + 375, + 1374, + 395 + ] + }, + { + "text": "These Are", + "boxes": [ + 18, + 447, + 88, + 465 + ] + }, + { + "text": "2023 reve", + "boxes": [ + 18, + 943, + 82, + 963 + ] + }, + { + "text": "World tourist", + "boxes": [ + 216, + 1048, + 310, + 1072 + ] + } + ] + }, + { + "frame_id": "80_6_9", + "episode_id": 80, + "env_id": 6, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.593218803405762, + "advantages": -0.4828331470489502, + "generated_text": "{\n \"intent\": \"Clicking on one of the product categories like Mac will allow me to explore more details about those specific products.\",\n \"action\": \"Click(210, 479)\"\n}", + "action_string": "Click(210, 479)", + "intent_string": "Clicking on one of the product categories like Mac will allow me to explore more details about those specific products.", + "before_screen_ocr_results": [ + { + "text": "Store. The best way to buy the", + "boxes": [ + 132, + 220, + 768, + 278 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1630, + 230, + 1770, + 254 + ] + }, + { + "text": "products you love.", + "boxes": [ + 138, + 280, + 526, + 328 + ] + }, + { + "text": "O", + "boxes": [ + 1588, + 296, + 1612, + 320 + ] + }, + { + "text": "iPhone", + "boxes": [ + 328, + 516, + 378, + 542 + ] + }, + { + "text": "iPad ", + "boxes": [ + 480, + 516, + 520, + 542 + ] + }, + { + "text": "Apple Watch", + "boxes": [ + 604, + 516, + 692, + 542 + ] + }, + { + "text": "Apple Vision Pro", + "boxes": [ + 734, + 514, + 854, + 546 + ] + }, + { + "text": "AirPods", + "boxes": [ + 910, + 516, + 966, + 542 + ] + }, + { + "text": "AirTag", + "boxes": [ + 1060, + 516, + 1110, + 544 + ] + }, + { + "text": "Apple TV 4K ", + "boxes": [ + 1186, + 514, + 1280, + 546 + ] + }, + { + "text": "HomePod", + "boxes": [ + 1340, + 516, + 1414, + 542 + ] + }, + { + "text": "Accessories", + "boxes": [ + 1482, + 516, + 1566, + 542 + ] + }, + { + "text": "Apple Gift Card ", + "boxes": [ + 1616, + 516, + 1724, + 544 + ] + }, + { + "text": "The latest. Take a look at what's new, right now.", + "boxes": [ + 138, + 641, + 720, + 671 + ] + }, + { + "text": " MacBook Air", + "boxes": [ + 162, + 744, + 334, + 787 + ] + }, + { + "text": "iPhone 16 Pro", + "boxes": [ + 584, + 748, + 768, + 786 + ] + }, + { + "text": "iPad Air ", + "boxes": [ + 1004, + 748, + 1112, + 786 + ] + }, + { + "text": " Apple Watch Pride Edition", + "boxes": [ + 1422, + 744, + 1760, + 788 + ] + }, + { + "text": "Appl", + "boxes": [ + 1843, + 745, + 1915, + 791 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 168, + 788, + 324, + 818 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 588, + 790, + 738, + 820 + ] + }, + { + "text": "Apple Intelligence", + "boxes": [ + 1008, + 788, + 1164, + 818 + ] + }, + { + "text": "Sport Band", + "boxes": [ + 1428, + 784, + 1576, + 816 + ] + }, + { + "text": "Thinsta", + "boxes": [ + 1846, + 786, + 1912, + 818 + ] + }, + { + "text": "From $999 or $83.25/mo. for 12 mo.t", + "boxes": [ + 168, + 816, + 412, + 840 + ] + }, + { + "text": "From $999 or $41.62/mo. for 24 mo.*", + "boxes": [ + 588, + 820, + 828, + 844 + ] + }, + { + "text": "From $599 or $49.91/mo. for 12 mo.t", + "boxes": [ + 1008, + 816, + 1248, + 840 + ] + }, + { + "text": " Show your stripes.", + "boxes": [ + 1424, + 818, + 1582, + 854 + ] + }, + { + "text": "From $39", + "boxes": [ + 1848, + 818, + 1910, + 844 + ] + }, + { + "text": "$49", + "boxes": [ + 1428, + 848, + 1462, + 876 + ] + }, + { + "text": "PIRC", + "boxes": [ + 650, + 967, + 828, + 1044 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "Apple", + "boxes": [ + 106, + 34, + 134, + 48 + ] + }, + { + "text": "目", + "boxes": [ + 1542, + 68, + 1568, + 87 + ] + }, + { + "text": "☆", + "boxes": [ + 1576, + 66, + 1660, + 89 + ] + }, + { + "text": "AirPods", + "boxes": [ + 908, + 113, + 944, + 127 + ] + }, + { + "text": "TV&H", + "boxes": [ + 988, + 113, + 1018, + 127 + ] + }, + { + "text": "Shop Mac", + "boxes": [ + 133, + 218, + 355, + 281 + ] + }, + { + "text": "Need shopping help?", + "boxes": [ + 1626, + 230, + 1766, + 254 + ] + }, + { + "text": "·", + "boxes": [ + 1586, + 292, + 1614, + 320 + ] + }, + { + "text": "Visit an Apple Store", + "boxes": [ + 1620, + 284, + 1756, + 312 + ] + }, + { + "text": "near you 7", + "boxes": [ + 1682, + 312, + 1750, + 330 + ] + }, + { + "text": "All Models ", + "boxes": [ + 144, + 389, + 230, + 413 + ] + }, + { + "text": " Shopping guides", + "boxes": [ + 256, + 387, + 398, + 417 + ] + }, + { + "text": "Ways to Save", + "boxes": [ + 432, + 389, + 540, + 413 + ] + }, + { + "text": "The Apple Store Difference", + "boxes": [ + 574, + 389, + 788, + 413 + ] + }, + { + "text": "Accessories", + "boxes": [ + 822, + 389, + 920, + 413 + ] + }, + { + "text": " Setup and Support ", + "boxes": [ + 952, + 389, + 1108, + 413 + ] + }, + { + "text": " The Mac Experience ", + "boxes": [ + 1136, + 389, + 1302, + 413 + ] + }, + { + "text": "Special Stores", + "boxes": [ + 1334, + 389, + 1448, + 413 + ] + }, + { + "text": "All mddels. Take your pick.", + "boxes": [ + 138, + 468, + 468, + 508 + ] + }, + { + "text": "NEW", + "boxes": [ + 170, + 558, + 200, + 580 + ] + }, + { + "text": "NEW", + "boxes": [ + 1846, + 556, + 1884, + 584 + ] + }, + { + "text": "MacBook Air", + "boxes": [ + 164, + 577, + 334, + 620 + ] + }, + { + "text": "MacBook Pro", + "boxes": [ + 582, + 579, + 762, + 622 + ] + }, + { + "text": "iMac", + "boxes": [ + 1001, + 577, + 1079, + 622 + ] + }, + { + "text": "Mac mini", + "boxes": [ + 1424, + 582, + 1550, + 619 + ] + }, + { + "text": "Mac", + "boxes": [ + 1846, + 584, + 1910, + 617 + ] + }, + { + "text": "From $1599 or $133.25/mo. for 12 mo.", + "boxes": [ + 588, + 979, + 838, + 1003 + ] + }, + { + "text": "From $19", + "boxes": [ + 1850, + 981, + 1906, + 1001 + ] + } + ] + }, + { + "frame_id": "80_7_9", + "episode_id": 80, + "env_id": 7, + "step": 9, + "format_rewards": 1.0, + "rewards": 9.262516975402832, + "advantages": -0.92059326171875, + "generated_text": "{\n \"intent\": \"Since I am exploring a desktop environment, clicking on 'Import bookmarks' will allow me to see saved web pages or bookmarks. This operation will help me organize my activities more effectively by accessing saved content.\",\n \"action\": \"DoubleClick(90, 109)\"\n}", + "action_string": "DoubleClick(90, 109)", + "intent_string": "Since I am exploring a desktop environment, clicking on 'Import bookmarks' will allow me to see saved web pages or bookmarks. This operation will help me organize my activities more effectively by accessing saved content.", + "before_screen_ocr_results": [ + { + "text": " New Tab", + "boxes": [ + 250, + 32, + 312, + 50 + ] + }, + { + "text": "Q pearch with Bing or enter address", + "boxes": [ + 248, + 64, + 518, + 93 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Search the Web", + "boxes": [ + 656, + 286, + 768, + 316 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 592, + 425 + ] + }, + { + "text": "知", + "boxes": [ + 976, + 445, + 1064, + 518 + ] + }, + { + "text": "Y", + "boxes": [ + 858, + 455, + 938, + 508 + ] + }, + { + "text": "8", + "boxes": [ + 526, + 490, + 558, + 526 + ] + }, + { + "text": "8", + "boxes": [ + 1236, + 484, + 1274, + 504 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "Amazon", + "boxes": [ + 738, + 532, + 810, + 558 + ] + }, + { + "text": "iqiyi", + "boxes": [ + 882, + 534, + 916, + 556 + ] + }, + { + "text": " zhihu", + "boxes": [ + 994, + 534, + 1038, + 554 + ] + }, + { + "text": " ifeng", + "boxes": [ + 1112, + 532, + 1160, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1232, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "Highlig", + "boxes": [ + 492, + 619, + 558, + 637 + ] + }, + { + "text": "amazon", + "boxes": [ + 990, + 673, + 1172, + 717 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1228, + 677, + 1416, + 733 + ] + }, + { + "text": "", + "boxes": [ + 1148, + 744, + 1172, + 770 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 766 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 750, + 774, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 904, + 814 + ] + }, + { + "text": "Firefox Priva", + "boxes": [ + 1226, + 792, + 1310, + 816 + ] + } + ], + "after_screen_ocr_results": [ + { + "text": "MozillaFirefox", + "boxes": [ + 904, + 4, + 1020, + 22 + ] + }, + { + "text": " New Tab", + "boxes": [ + 250, + 32, + 312, + 50 + ] + }, + { + "text": "Q仓", + "boxes": [ + 68, + 68, + 126, + 87 + ] + }, + { + "text": " Q Search with Bing or enter address", + "boxes": [ + 246, + 59, + 518, + 93 + ] + }, + { + "text": " Impor", + "boxes": [ + 14, + 103, + 70, + 121 + ] + }, + { + "text": "X", + "boxes": [ + 1884, + 149, + 1898, + 165 + ] + }, + { + "text": "Import Wizard", + "boxes": [ + 894, + 308, + 1032, + 338 + ] + }, + { + "text": "Import Settings and Data", + "boxes": [ + 716, + 336, + 910, + 359 + ] + }, + { + "text": " Top Sites ", + "boxes": [ + 490, + 401, + 584, + 425 + ] + }, + { + "text": "No programs that contain bookmarks, history or password data", + "boxes": [ + 744, + 401, + 1166, + 425 + ] + }, + { + "text": "®", + "boxes": [ + 518, + 483, + 564, + 531 + ] + }, + { + "text": "?", + "boxes": [ + 1242, + 486, + 1266, + 500 + ] + }, + { + "text": "Apple", + "boxes": [ + 520, + 532, + 562, + 558 + ] + }, + { + "text": "bing", + "boxes": [ + 632, + 532, + 678, + 558 + ] + }, + { + "text": "weibo", + "boxes": [ + 1236, + 532, + 1286, + 558 + ] + }, + { + "text": "ctrip", + "boxes": [ + 1362, + 534, + 1396, + 556 + ] + }, + { + "text": "moz://a", + "boxes": [ + 1225, + 672, + 1415, + 735 + ] + }, + { + "text": "", + "boxes": [ + 906, + 744, + 932, + 764 + ] + }, + { + "text": "", + "boxes": [ + 1142, + 740, + 1177, + 774 + ] + }, + { + "text": "?", + "boxes": [ + 676, + 752, + 688, + 764 + ] + }, + { + "text": "?", + "boxes": [ + 1392, + 750, + 1408, + 764 + ] + }, + { + "text": "apple", + "boxes": [ + 508, + 772, + 540, + 792 + ] + }, + { + "text": "cn.bing", + "boxes": [ + 746, + 770, + 790, + 790 + ] + }, + { + "text": "nozilla", + "boxes": [ + 1234, + 774, + 1266, + 788 + ] + }, + { + "text": "Apple", + "boxes": [ + 504, + 790, + 550, + 818 + ] + }, + { + "text": "Search - Microsoft Bing", + "boxes": [ + 746, + 790, + 902, + 814 + ] + }, + { + "text": "Firefox Priv", + "boxes": [ + 1228, + 790, + 1302, + 814 + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_80/scores_v3.json b/ScreenExplorer-7B-E1/episode_80/scores_v3.json new file mode 100644 index 0000000000000000000000000000000000000000..386a10049523b49e8be50face9f8fee037e502b3 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_80/scores_v3.json @@ -0,0 +1,19 @@ +{ + "directory_path": "logs/ScreenExplorer_2025-05-17_22-11-27ab2072/episode_80", + "single_trajectory_image_diversity": 0.3646050402894616, + "single_trajectory_text_diversity": 0.29539795964956284, + "single_trajectory_image_diversity_max": 0.7197265625, + "single_trajectory_image_diversity_min": 0.0, + "single_trajectory_image_diversity_std": 0.22711181640625, + "single_trajectory_text_diversity_max": 0.52935791015625, + "single_trajectory_text_diversity_min": 0.00909423828125, + "single_trajectory_text_diversity_std": 0.1655426025390625, + "group_trajectories_image_diversity": 0.5267466902732849, + "group_trajectories_text_diversity": 0.4342447519302368, + "group_trajectories_image_diversity_max": 1.234375, + "group_trajectories_image_diversity_min": -0.0078125, + "group_trajectories_image_diversity_std": 0.333984375, + "group_trajectories_text_diversity_max": 0.6669921875, + "group_trajectories_text_diversity_min": 0.0, + "group_trajectories_text_diversity_std": 0.1444091796875 +} \ No newline at end of file diff --git a/ScreenExplorer-7B-E1/episode_90/recording_env_rank_0_7b881.h5 b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_0_7b881.h5 new file mode 100644 index 0000000000000000000000000000000000000000..e67a13d5105403d75a6f1c66c39b7afc8ea01c33 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_0_7b881.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a715458cfda8e985fb04f62411fe54134f8fe93b9d8f594d5f8f70afb495b8ae +size 10284065 diff --git a/ScreenExplorer-7B-E1/episode_90/recording_env_rank_1_d5eb1.h5 b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_1_d5eb1.h5 new file mode 100644 index 0000000000000000000000000000000000000000..4d50440663928d07a4afc5775ce7ef48f003ad15 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_1_d5eb1.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7045033344981e198a9bcf51a150444cc3a3d907999eaa231d2a1d9971f0baf3 +size 8243105 diff --git a/ScreenExplorer-7B-E1/episode_90/recording_env_rank_2_5f828.h5 b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_2_5f828.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8ec6e00cc00c95c0150834ef2d71d361fb31089f --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_2_5f828.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:930aa70215ec295858f93f55c14fce981057150edba278a0cc7a3d5fd84b0f5e +size 7512728 diff --git a/ScreenExplorer-7B-E1/episode_90/recording_env_rank_3_09f57.h5 b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_3_09f57.h5 new file mode 100644 index 0000000000000000000000000000000000000000..00cb0b22abbdcec207f702c9b73f7aada8cee557 --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_3_09f57.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f795af9f8cf0b129ebdc125a46d977a8e292af33fc5664ba961e7c492231de19 +size 7619950 diff --git a/ScreenExplorer-7B-E1/episode_90/recording_env_rank_5_8482b.h5 b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_5_8482b.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8a9868f401048f8333be3872f426042fc56b206a --- /dev/null +++ b/ScreenExplorer-7B-E1/episode_90/recording_env_rank_5_8482b.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:574b2345a3351620df9941f079206511ddb2ab604b687bc18d247c735bee7c8f +size 14487567