| { |
| "schema": "betterwright-frozen-reserve-eval-v1", |
| "policy_tuned_on_reserve": false, |
| "config": { |
| "schema": "betterwright-relevance-config-v1", |
| "relevance_threshold": 0.25, |
| "confidence_threshold": 0.45, |
| "max_retained_fraction": 1.0, |
| "max_ranked_lines": 0, |
| "strategy": "coarse-ref-context", |
| "max_ranked_windows": 8, |
| "ref_context_lines": 3, |
| "window_chars": 1800, |
| "minimum_validation_recall": 0.999, |
| "minimum_validation_perfect_task_recall": 0.995, |
| "validated_max_chars": 10000, |
| "fallback_on_error": true, |
| "model": "ProCreations/betterwright-encoder-350m" |
| }, |
| "metrics": { |
| "tasks": 161, |
| "target_evidence_node_recall": 1.0, |
| "mean_task_node_recall": 1.0, |
| "perfect_task_recall": 1.0, |
| "mean_char_savings": 0.0650058016826541, |
| "p50_char_savings": 0.0, |
| "fallback_rate": 0.6645962732919255, |
| "pruning_coverage": 0.3354037267080745, |
| "eligible_tasks": 161, |
| "non_fallback_tasks": 54, |
| "non_fallback_target_evidence_node_recall": 1.0, |
| "non_fallback_perfect_task_recall": 1.0, |
| "mean_token_savings": 0.05626420853717087, |
| "p50_token_savings": 0.0, |
| "original_tokens": 1079517, |
| "retained_tokens": 1053351, |
| "token_weighted_savings": 0.024238617826305653, |
| "eligible_mean_token_savings": 0.05626420853717087, |
| "non_fallback_mean_token_savings": 0.16775069582378718, |
| "domains": 17, |
| "by_kind": { |
| "action": { |
| "tasks": 81, |
| "target_evidence_node_recall": 1.0, |
| "mean_task_node_recall": 1.0, |
| "perfect_task_recall": 1.0, |
| "mean_char_savings": 0.047894415954788536, |
| "p50_char_savings": 0.0, |
| "fallback_rate": 0.7283950617283951, |
| "pruning_coverage": 0.2716049382716049, |
| "eligible_tasks": 81, |
| "non_fallback_tasks": 22, |
| "non_fallback_target_evidence_node_recall": 1.0, |
| "non_fallback_perfect_task_recall": 1.0, |
| "mean_token_savings": 0.040223486962387485, |
| "p50_token_savings": 0.0, |
| "original_tokens": 556762, |
| "retained_tokens": 547396, |
| "token_weighted_savings": 0.01682226876115822, |
| "eligible_mean_token_savings": 0.040223486962387485, |
| "non_fallback_mean_token_savings": 0.14809556563424484 |
| }, |
| "evidence": { |
| "tasks": 67, |
| "target_evidence_node_recall": 1.0, |
| "mean_task_node_recall": 1.0, |
| "perfect_task_recall": 1.0, |
| "mean_char_savings": 0.07259474593610533, |
| "p50_char_savings": 0.0, |
| "fallback_rate": 0.6417910447761194, |
| "pruning_coverage": 0.35820895522388063, |
| "eligible_tasks": 67, |
| "non_fallback_tasks": 24, |
| "non_fallback_target_evidence_node_recall": 1.0, |
| "non_fallback_perfect_task_recall": 1.0, |
| "mean_token_savings": 0.06368484640893213, |
| "p50_token_savings": 0.0, |
| "original_tokens": 436257, |
| "retained_tokens": 424249, |
| "token_weighted_savings": 0.027525059769814564, |
| "eligible_mean_token_savings": 0.06368484640893213, |
| "non_fallback_mean_token_savings": 0.1777868628916022 |
| }, |
| "mixed": { |
| "tasks": 13, |
| "target_evidence_node_recall": 1.0, |
| "mean_task_node_recall": 1.0, |
| "perfect_task_recall": 1.0, |
| "mean_char_savings": 0.13251064621926026, |
| "p50_char_savings": 0.16570836082571405, |
| "fallback_rate": 0.38461538461538464, |
| "pruning_coverage": 0.6153846153846154, |
| "eligible_tasks": 13, |
| "non_fallback_tasks": 8, |
| "non_fallback_target_evidence_node_recall": 1.0, |
| "non_fallback_perfect_task_recall": 1.0, |
| "mean_token_savings": 0.11796541701020548, |
| "p50_token_savings": 0.1470926885434658, |
| "original_tokens": 86498, |
| "retained_tokens": 81706, |
| "token_weighted_savings": 0.0554001248583782, |
| "eligible_mean_token_savings": 0.11796541701020548, |
| "non_fallback_mean_token_savings": 0.19169380264158392 |
| } |
| }, |
| "by_difficulty": { |
| "easy": { |
| "tasks": 44, |
| "target_evidence_node_recall": 1.0, |
| "mean_task_node_recall": 1.0, |
| "perfect_task_recall": 1.0, |
| "mean_char_savings": 0.07079667106874572, |
| "p50_char_savings": 0.0, |
| "fallback_rate": 0.6363636363636364, |
| "pruning_coverage": 0.36363636363636365, |
| "eligible_tasks": 44, |
| "non_fallback_tasks": 16, |
| "non_fallback_target_evidence_node_recall": 1.0, |
| "non_fallback_perfect_task_recall": 1.0, |
| "mean_token_savings": 0.0627814245589974, |
| "p50_token_savings": 0.0, |
| "original_tokens": 293507, |
| "retained_tokens": 286055, |
| "token_weighted_savings": 0.02538951370836129, |
| "eligible_mean_token_savings": 0.0627814245589974, |
| "non_fallback_mean_token_savings": 0.17264891753724287 |
| }, |
| "medium": { |
| "tasks": 50, |
| "target_evidence_node_recall": 1.0, |
| "mean_task_node_recall": 1.0, |
| "perfect_task_recall": 1.0, |
| "mean_char_savings": 0.048085784729173534, |
| "p50_char_savings": 0.0, |
| "fallback_rate": 0.72, |
| "pruning_coverage": 0.28, |
| "eligible_tasks": 50, |
| "non_fallback_tasks": 14, |
| "non_fallback_target_evidence_node_recall": 1.0, |
| "non_fallback_perfect_task_recall": 1.0, |
| "mean_token_savings": 0.03925835808931578, |
| "p50_token_savings": 0.0, |
| "original_tokens": 353824, |
| "retained_tokens": 347346, |
| "token_weighted_savings": 0.01830853757800488, |
| "eligible_mean_token_savings": 0.03925835808931578, |
| "non_fallback_mean_token_savings": 0.14020842174755638 |
| }, |
| "hard": { |
| "tasks": 67, |
| "target_evidence_node_recall": 1.0, |
| "mean_task_node_recall": 1.0, |
| "perfect_task_recall": 1.0, |
| "mean_char_savings": 0.07382972100632573, |
| "p50_char_savings": 0.0, |
| "fallback_rate": 0.6417910447761194, |
| "pruning_coverage": 0.35820895522388063, |
| "eligible_tasks": 67, |
| "non_fallback_tasks": 24, |
| "non_fallback_target_evidence_node_recall": 1.0, |
| "non_fallback_perfect_task_recall": 1.0, |
| "mean_token_savings": 0.06467517894660949, |
| "p50_token_savings": 0.0, |
| "original_tokens": 432186, |
| "retained_tokens": 419950, |
| "token_weighted_savings": 0.028311884235028395, |
| "eligible_mean_token_savings": 0.06467517894660949, |
| "non_fallback_mean_token_savings": 0.1805515412259515 |
| } |
| } |
| }, |
| "inference": { |
| "device": "cuda", |
| "scored_chunks": 2294, |
| "score_seconds": 12.088709216041025, |
| "chunks_per_second": 189.7638498042449 |
| }, |
| "detail": [ |
| { |
| "host": "www.lowes.com", |
| "task": "Navigate to the top-level Appliances department from the main category navigation, rather than the similarly named Popular Categories link.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5546875, |
| "peak_confidence": 0.5078125, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.lowes.com", |
| "task": "Search the site for \u201cceiling fans\u201d using the search field and submit the search.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.43359375, |
| "peak_confidence": 0.6015625, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.lowes.com", |
| "task": "Open the selector for the currently selected Lowe\u2019s store.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.57421875, |
| "peak_confidence": 0.5703125, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.lowes.com", |
| "task": "Open the top-level Deals section from the main navigation, not the separate Shop Deals of the Day promotional link.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.54296875, |
| "peak_confidence": 0.515625, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.lowes.com", |
| "task": "Open the Home Care and Home maintenance subscription information, rather than the similarly priced $99 Maintenance navigation item.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.400390625, |
| "peak_confidence": 0.6328125, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.lowes.com", |
| "task": "Determine the sponsored Samsung appliance financing offer, including the payment duration, qualifying purchase amount, and where to get more information.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.4765625, |
| "peak_confidence": 0.58984375, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.lowes.com", |
| "task": "Report the currently selected Lowe\u2019s store name and ZIP code.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.439453125, |
| "peak_confidence": 0.62109375, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.lowes.com", |
| "task": "Identify the Popular Categories entries for bathroom products, building supplies, and doors and windows.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.58203125, |
| "peak_confidence": 0.51171875, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.lowes.com", |
| "task": "State the cutoff time, delivery promise, and product type in the vanity promotion.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.2333984375, |
| "peak_confidence": 0.6796875, |
| "original_tokens": 2606, |
| "retained_tokens": 2606, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "Open the top-level Building Materials department.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.482421875, |
| "peak_confidence": 0.6015625, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "Update the store ZIP code while keeping the currently displayed Port Chester store context in mind.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.546875, |
| "peak_confidence": 0.55859375, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "Use the main search field to search for an 18V cordless drill/driver kit; do not open the already-listed product card.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.54296875, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "Navigate to the page for scheduling a bathroom remodel, rather than opening a bathroom product or the Bath & Faucets department.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.58203125, |
| "peak_confidence": 0.5703125, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "Open the shopping cart.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.494140625, |
| "peak_confidence": 0.58984375, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "Navigate to the Store Locator from the footer.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.546875, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "Identify which savings tab is currently selected.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5859375, |
| "peak_confidence": 0.52734375, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "What maximum discount and product area are advertised for the online bath-vanity deal?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.4921875, |
| "peak_confidence": 0.640625, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "Which of these two visible products has more reviews: the Frigidaire side-by-side refrigerator or the GE freestanding gas range, and what are their displayed review counts?", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6171875, |
| "peak_confidence": 0.57421875, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.homedepot.com", |
| "task": "What action is offered for the Free In-Store Kids Workshops?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.53515625, |
| "peak_confidence": 0.55859375, |
| "original_tokens": 11254, |
| "retained_tokens": 11254, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "developer.mozilla.org", |
| "task": "Select the main documentation navigation buttons for HTML, CSS, JavaScript, Web APIs, and Learn, while excluding the Tools and About buttons.", |
| "recall": 1.0, |
| "hits": 7, |
| "required_count": 7, |
| "savings": 0.22504456327985745, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.6171875, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 3570, |
| "retained_tokens": 2907, |
| "token_savings": 0.18571428571428572, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "developer.mozilla.org", |
| "task": "Open the Featured articles entry titled \"Trusted Types API\" that is categorized under Web APIs and discusses protection against client-side XSS.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.22504456327985745, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.5546875, |
| "peak_confidence": 0.5234375, |
| "original_tokens": 3570, |
| "retained_tokens": 2907, |
| "token_savings": 0.18571428571428572, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "developer.mozilla.org", |
| "task": "Open the Featured articles entry titled \"Navigation API\" whose Web APIs description says it can examine an application's history entries.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.22504456327985745, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.59375, |
| "peak_confidence": 0.498046875, |
| "original_tokens": 3570, |
| "retained_tokens": 2907, |
| "token_savings": 0.18571428571428572, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "developer.mozilla.org", |
| "task": "Open the CSS article about bringing smooth, animated navigation to multi-page apps with one line of CSS, rather than the similarly worded Navigation API article.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.22504456327985745, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.6015625, |
| "peak_confidence": 0.52734375, |
| "original_tokens": 3570, |
| "retained_tokens": 2907, |
| "token_savings": 0.18571428571428572, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "developer.mozilla.org", |
| "task": "What three technologies does MDN say it has been documenting since 2005?", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.22504456327985745, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.515625, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 3570, |
| "retained_tokens": 2907, |
| "token_savings": 0.18571428571428572, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "developer.mozilla.org", |
| "task": "Which Latest news item is from 2 months ago, and what is its title?", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.22504456327985745, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.4453125, |
| "peak_confidence": 0.56640625, |
| "original_tokens": 3570, |
| "retained_tokens": 2907, |
| "token_savings": 0.18571428571428572, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "developer.mozilla.org", |
| "task": "Identify the recent contribution made to correct information related to HTML table manipulation methods, and report how long ago it was made.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.22504456327985745, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.5625, |
| "peak_confidence": 0.53125, |
| "original_tokens": 3570, |
| "retained_tokens": 2907, |
| "token_savings": 0.18571428571428572, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "developer.mozilla.org", |
| "task": "Who is quoted in the Contributor Spotlight, and what does the quote say MDN closely follows?", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.22504456327985745, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.5078125, |
| "peak_confidence": 0.5625, |
| "original_tokens": 3570, |
| "retained_tokens": 2907, |
| "token_savings": 0.18571428571428572, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.wikipedia.org", |
| "task": "Choose the English Wikipedia edition from the top-language links, rather than selecting the similarly labeled option in the search-language dropdown.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.384671267707955, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.6953125, |
| "peak_confidence": 0.478515625, |
| "original_tokens": 1956, |
| "retained_tokens": 1197, |
| "token_savings": 0.3880368098159509, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.wikipedia.org", |
| "task": "Search Wikipedia for \u201cquantum mechanics\u201d using English, keeping the search-language selector set to English, and activate the Search button.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.384671267707955, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.6953125, |
| "peak_confidence": 0.484375, |
| "original_tokens": 1956, |
| "retained_tokens": 1197, |
| "token_savings": 0.3880368098159509, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.wikipedia.org", |
| "task": "Open the official Wikipedia app download information and then choose the Apple App Store link.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.384671267707955, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.5703125, |
| "peak_confidence": 0.5234375, |
| "original_tokens": 1956, |
| "retained_tokens": 1197, |
| "token_savings": 0.3880368098159509, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.wikipedia.org", |
| "task": "Among the listed top-language editions, which link shows the highest stated article count? Report its language and displayed count.", |
| "recall": 1.0, |
| "hits": 10, |
| "required_count": 10, |
| "savings": 0.384671267707955, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.703125, |
| "peak_confidence": 0.46875, |
| "original_tokens": 1956, |
| "retained_tokens": 1197, |
| "token_savings": 0.3880368098159509, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.wikipedia.org", |
| "task": "According to the footer, which organization hosts Wikipedia and also hosts a range of other projects?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.384671267707955, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.58203125, |
| "peak_confidence": 0.53125, |
| "original_tokens": 1956, |
| "retained_tokens": 1197, |
| "token_savings": 0.3880368098159509, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.wikipedia.org", |
| "task": "Which other project is described as a \u201cFree travel guide\u201d?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.384671267707955, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.52734375, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 1956, |
| "retained_tokens": 1197, |
| "token_savings": 0.3880368098159509, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.wikipedia.org", |
| "task": "What license is this page available under?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.384671267707955, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.62890625, |
| "peak_confidence": 0.515625, |
| "original_tokens": 1956, |
| "retained_tokens": 1197, |
| "token_savings": 0.3880368098159509, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "Open the Mission Overview for the featured Roman Space Telescope topic titled \u201cUnveiling the Universe\u2019s Dark Side.\u201d", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.63671875, |
| "peak_confidence": 0.578125, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "Open the Featured News item that is a news release about the Commercial Swift Boost Mission.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.4765625, |
| "peak_confidence": 0.625, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "Under NASA Missions, open the James Webb Space Telescope mission card, not the similarly named gallery in More NASA Images.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.59765625, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "In the Image Of The Day section, open the first \u201cView full image page\u201d link associated with the Ancient Milky Way Merger image, rather than the later duplicate link.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.55078125, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "Open the Technology topic from More Topics from NASA.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.52734375, |
| "peak_confidence": 0.6015625, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "Use the site search box to search for \u201cArtemis\u201d instead of opening the Artemis mission card.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.484375, |
| "peak_confidence": 0.609375, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "Report the featured Roman Space Telescope\u2019s stated launch date and the kind of view it will provide.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.55859375, |
| "peak_confidence": 0.58203125, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "Among the Hubble, James Webb, and Chandra gallery cards, report the image count for each gallery and identify which has the most.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.64453125, |
| "peak_confidence": 0.51953125, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "List the four named cards shown in the NASA Missions section.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.640625, |
| "peak_confidence": 0.53515625, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.nasa.gov", |
| "task": "Report the page\u2019s last-updated date, page editor, and responsible NASA official from the footer.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.17232627341151763, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.4765625, |
| "peak_confidence": 0.62109375, |
| "original_tokens": 3434, |
| "retained_tokens": 2955, |
| "token_savings": 0.13948747815958062, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Open the Python Tutorial from the documentation sections.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.67578125, |
| "peak_confidence": 0.51953125, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Change the documentation language to Spanish | espa\u00f1ol while leaving the Python version set to 3.14.7.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.65625, |
| "peak_confidence": 0.51953125, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Change the selected Python documentation version to 3.12.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.63671875, |
| "peak_confidence": 0.5234375, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Open the download page for the documentation, not the page for the stable Python 3.14 version.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.69140625, |
| "peak_confidence": 0.51171875, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Open the resource about Python type safety and guides, rather than the Python C API reference.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.65234375, |
| "peak_confidence": 0.52734375, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Open the resource for third-party modules and PyPI.org, rather than the page about installing, configuring, and using Python.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.625, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Which documentation section covers the standard library and builtins?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.66015625, |
| "peak_confidence": 0.53515625, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "What is the currently selected theme?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.671875, |
| "peak_confidence": 0.52734375, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Which link provides an index of all functions, classes, and terms?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.6171875, |
| "peak_confidence": 0.5390625, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "docs.python.org", |
| "task": "Which footer link should be followed for more information about the documentation license?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.16570836082571405, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.62109375, |
| "peak_confidence": 0.55078125, |
| "original_tokens": 3474, |
| "retained_tokens": 2963, |
| "token_savings": 0.1470926885434658, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "Activate the featured health topic link for preventing skin cancer.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.671875, |
| "peak_confidence": 0.578125, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "Open the carousel story about two travelers on a hike, rather than a featured topic link.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5859375, |
| "peak_confidence": 0.59765625, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "Use the upper header navigation to open the About CDC menu, not the footer About CDC link.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.64453125, |
| "peak_confidence": 0.578125, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "Open the site search from the first header navigation, not the duplicate search link lower on the page.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6015625, |
| "peak_confidence": 0.58984375, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "Activate the featured topic about COVID-19 testing.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.66015625, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "Pause the rotating homepage carousel.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.66015625, |
| "peak_confidence": 0.58203125, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "List every health topic currently shown under Featured Topics.", |
| "recall": 1.0, |
| "hits": 7, |
| "required_count": 7, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.63671875, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "What does the official-government banner say about the website, and which control explains it?", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.64453125, |
| "peak_confidence": 0.56640625, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "Which carousel story is associated with an image description of a person holding their stomach in pain?", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.60546875, |
| "peak_confidence": 0.59375, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.cdc.gov", |
| "task": "What description does the CDC provide about its role and operating schedule in the footer?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.609375, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 1739, |
| "retained_tokens": 1739, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "Navigate to the World news section using the primary navigation.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.1416015625, |
| "peak_confidence": 0.734375, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "Open the site's search interface without selecting a news category.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.1845703125, |
| "peak_confidence": 0.6484375, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "Open the Politics section from the main navigation, rather than opening the article about the $40 trillion debt.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5859375, |
| "peak_confidence": 0.5703125, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "Open the article about the new CDC director acknowledging agency problems, not the separate link about CDC zombie programs.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.490234375, |
| "peak_confidence": 0.53515625, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "Advance the Short Stories carousel to the next item.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.2197265625, |
| "peak_confidence": 0.6875, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "Open the AP News quizzes section from the page navigation.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.32421875, |
| "peak_confidence": 0.64453125, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "What is the lead headline in the Russia-Ukraine war coverage, and what does its teaser say was set ablaze?", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.474609375, |
| "peak_confidence": 0.55078125, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "In the Today in History section, what event is listed for 2010?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5625, |
| "peak_confidence": 0.5078125, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "Which answer option is indicated as the answer to the quiz question about the Bavarian immigrant who popularized blue jeans?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.57421875, |
| "peak_confidence": 0.52734375, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "apnews.com", |
| "task": "What is the lead story headline in the Science section?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.50390625, |
| "peak_confidence": 0.5390625, |
| "original_tokens": 15515, |
| "retained_tokens": 15515, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "Use the flight booking form to prepare a search by reviewing the origin, destination, trip type, travel dates, passenger count, and then activating the flight search control.", |
| "recall": 1.0, |
| "hits": 6, |
| "required_count": 6, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.6015625, |
| "peak_confidence": 0.58984375, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "Change the trip type from the currently selected round-trip setting to a one-way trip.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.640625, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "Enable the option to shop for flights using miles.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.65234375, |
| "peak_confidence": 0.58203125, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "Navigate to Delta's Travel Info section, not the separate travel-products content in the page body.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.58203125, |
| "peak_confidence": 0.58984375, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "Open the page for searching Delta flight deals.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.50390625, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "Open the Help Center specifically from the Customer Service section of the footer, rather than the similarly named Popular Topics link.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.578125, |
| "peak_confidence": 0.546875, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "Report the current flight-search state: origin, destination placeholder, trip type, departure/return date field, passenger count, and whether Basic is included.", |
| "recall": 1.0, |
| "hits": 6, |
| "required_count": 6, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.609375, |
| "peak_confidence": 0.58203125, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "What maximum welcome-offer mileage is advertised, and what does the linked offer description say about earning miles on Delta purchases?", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.46484375, |
| "peak_confidence": 0.62109375, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "List the three topics shown under Popular Topics.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.53515625, |
| "peak_confidence": 0.546875, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.delta.com", |
| "task": "Identify the selected booking product tab and the currently checked fare option.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.17862858549796878, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.54296875, |
| "peak_confidence": 0.6015625, |
| "original_tokens": 2544, |
| "retained_tokens": 2205, |
| "token_savings": 0.13325471698113212, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "Open the featured marketplace category for Gold Sculptural Decor.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.67578125, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "Navigate to Etsy's New Arrivals section from the top navigation.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.578125, |
| "peak_confidence": 0.55078125, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "Open the Halloween product listing priced at $6.00, rather than the similarly themed $6.00\u2013$12.00 pattern listing.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.41015625, |
| "peak_confidence": 0.63671875, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "Open the Help Center link in the footer, not the similarly named Help Center link in the main content.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.59765625, |
| "peak_confidence": 0.5390625, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "Open Gift cards from the footer Shop links, not the Gift Cards link in the upper navigation.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.59765625, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "Which four items are listed under \u201cJump into featured interests\u201d?", |
| "recall": 1.0, |
| "hits": 5, |
| "required_count": 5, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.59375, |
| "peak_confidence": 0.5390625, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "List the five visible topics in the \u201cDiscover our best of summer 2026\u201d section.", |
| "recall": 1.0, |
| "hits": 6, |
| "required_count": 6, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.515625, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "Which Halloween product listing is shown at exactly $29.95?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.5703125, |
| "peak_confidence": 0.56640625, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.etsy.com", |
| "task": "Which two app marketplaces are offered in the Etsy footer?", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.08958294370675146, |
| "fallback": false, |
| "eligible": true, |
| "peak": 0.498046875, |
| "peak_confidence": 0.578125, |
| "original_tokens": 3432, |
| "retained_tokens": 3181, |
| "token_savings": 0.07313519813519809, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Locate the model-list search and filtering controls, using the model-name search rather than the global site search.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.62109375, |
| "peak_confidence": 0.53515625, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Activate the Text Generation task filter in the model filters, not the similarly named Text Generation label on a model card.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.671875, |
| "peak_confidence": 0.5234375, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Apply the Transformers library filter to the model list.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6328125, |
| "peak_confidence": 0.53125, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Use the model-name filter to search for models named Qwen3.8-27B.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.68359375, |
| "peak_confidence": 0.45703125, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Restrict the model list to base models that have inference available by enabling Base only and selecting Inference Available.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.65625, |
| "peak_confidence": 0.46875, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Identify the newest visible model and report its displayed update time.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6484375, |
| "peak_confidence": 0.47265625, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Which visible model has the largest displayed parameter count, and what is that count?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.68359375, |
| "peak_confidence": 0.490234375, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Name a visible Text Generation model whose card indicates that inference providers are available.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.625, |
| "peak_confidence": 0.4765625, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "huggingface.co", |
| "task": "Between Qwen/Qwen3.8-27B and unsloth/Qwen3.8-27B-GGUF, which visible entry has more parameters, and what are the two displayed counts?", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6875, |
| "peak_confidence": 0.48828125, |
| "original_tokens": 7432, |
| "retained_tokens": 7432, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Find the main Questions section of the page rather than the similarly named primary-navigation link.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.65234375, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "mixed", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Open the newest question about chaining programmatic Polars expression functions in with_columns.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.56640625, |
| "peak_confidence": 0.51171875, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Open the question tagged python-polars, not the question whose title merely mentions another data-processing library.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.578125, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Use the page-size controls to show 50 questions per page.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.447265625, |
| "peak_confidence": 0.6171875, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Go to the next page of newest questions.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.54296875, |
| "peak_confidence": 0.5390625, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Open the Unanswered question listing instead of the similarly named Bountied listing.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6015625, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Identify the exact title of the first question shown in the newest-questions list.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5234375, |
| "peak_confidence": 0.4921875, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Which listed question is associated with the angular tag? Provide its exact title.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6796875, |
| "peak_confidence": 0.55078125, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Identify the question that is explicitly marked closed, and provide its exact title.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6328125, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "stackoverflow.com", |
| "task": "Which collective is described as having 69k Members?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5703125, |
| "peak_confidence": 0.546875, |
| "original_tokens": 7417, |
| "retained_tokens": 7417, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "Open the shop category for hiking and camping, using the category labeled \"Camp & Hike\" rather than the separate \"Backpacking\" link.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.578125, |
| "peak_confidence": 0.55078125, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "Open REI's general outdoor gift ideas page, not the separate gift-card page.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.578125, |
| "peak_confidence": 0.5703125, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "Use the header link to sign in to REI, rather than the similarly named footer account link.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.62109375, |
| "peak_confidence": 0.5390625, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "Search the site for \"trail running\" using the main Search field.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.306640625, |
| "peak_confidence": 0.62109375, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "mixed", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "Open the \"Outlet deals\" item under Shop top categories, not the top-navigation \"REI Outlet\" destination.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.61328125, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "In the \"Gear we think you\u2019d love\" product carousel, switch from the currently selected \"Crowd favorites\" tab to \"Best Sellers\".", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.55859375, |
| "peak_confidence": 0.51953125, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "What does promotional message 1 say members can earn, and by what date must they join?", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.490234375, |
| "peak_confidence": 0.52734375, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "Among GU Energy Gel, Clif Bloks Energy Chews, and Dr. Bronner's 18-in-1 Pure-Castile Magic Soap, which listed product has the most reviews?", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.609375, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.rei.com", |
| "task": "What rating and current price comparison are shown for the HOKA Clifton 10 Road-Running Shoes - Men's?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.58984375, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 10235, |
| "retained_tokens": 10235, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "Find the complete restaurant search interface: the date control, time selector, party-size selector, location field, and button used to submit the search.", |
| "recall": 1.0, |
| "hits": 5, |
| "required_count": 5, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.63671875, |
| "peak_confidence": 0.5625, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "Change the reservation time to 11:00 PM while keeping the party size at 2 people.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5859375, |
| "peak_confidence": 0.51171875, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "Open the location picker to change the search area; do not request the device's current location.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.63671875, |
| "peak_confidence": 0.498046875, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "Navigate to restaurants near Rockefeller Center in New York City, rather than the general New York City restaurant listing.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.64453125, |
| "peak_confidence": 0.515625, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "Switch the city-directory panel from U.S. cities to the global-cities list.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.59765625, |
| "peak_confidence": 0.578125, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "Navigate to the Restaurants Near Me page, not the Restaurants Open Now page.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.66015625, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "Report the current reservation date, time, and party size shown in the search controls.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.59765625, |
| "peak_confidence": 0.5078125, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "According to the FAQ, what search dimensions and filters can users use to narrow restaurant choices?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6328125, |
| "peak_confidence": 0.4921875, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "List the cuisine-oriented New York City links shown under the Cuisines section, distinguishing them from the landmark links.", |
| "recall": 1.0, |
| "hits": 7, |
| "required_count": 7, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.65625, |
| "peak_confidence": 0.55078125, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.opentable.com", |
| "task": "Identify the four FAQ feature tabs and indicate which one is currently selected.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.64453125, |
| "peak_confidence": 0.57421875, |
| "original_tokens": 20103, |
| "retained_tokens": 20103, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Set up a round-trip flight search using the currently selected trip type, departing from LGA, entering an arrival airport, keeping the displayed departure and return dates, and submit the search.", |
| "recall": 1.0, |
| "hits": 6, |
| "required_count": 6, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.60546875, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Reverse the departure and arrival airports using the airport-swap control.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.55859375, |
| "peak_confidence": 0.5703125, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Change the fare display from dollars to points.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.63671875, |
| "peak_confidence": 0.53515625, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Enable the low fare calendar option and then submit the flight search.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.64453125, |
| "peak_confidence": 0.54296875, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Open the flight-status page, not the check-in page.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6484375, |
| "peak_confidence": 0.5390625, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Open the general Baggage Policies page, rather than the promotional link about updated checked-baggage policies.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.61328125, |
| "peak_confidence": 0.55859375, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Report the reservation-through date and the currently displayed departure and return dates.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.61328125, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Identify the baggage-related fees link shown beneath the flight search form.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6015625, |
| "peak_confidence": 0.5546875, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "Summarize the four Rapid Rewards benefits listed in the membership section.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5546875, |
| "peak_confidence": 0.56640625, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.southwest.com", |
| "task": "For the international vacation-package promotion, identify the displayed currency marker for its maximum discount and the link provided for the promotion's terms.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6328125, |
| "peak_confidence": 0.5390625, |
| "original_tokens": 4341, |
| "retained_tokens": 4341, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "Select \u201cConsumer Electronics\u201d as the search category, enter \u201cwireless headphones\u201d in the main search field, and submit the search.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.5, |
| "peak_confidence": 0.609375, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "Open the featured Depop shopping destination for the pre-loved fashion community, rather than one of the general fashion deal links.", |
| "recall": 1.0, |
| "hits": 4, |
| "required_count": 4, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.546875, |
| "peak_confidence": 0.60546875, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "Expand the Electronics category in the main shopping-category navigation.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.65234375, |
| "peak_confidence": 0.6015625, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "Open the currently live eBay Live event with the largest displayed audience.", |
| "recall": 1.0, |
| "hits": 26, |
| "required_count": 26, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.41015625, |
| "peak_confidence": 0.66015625, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "Open the \u201cPortable audio\u201d featured shopping category in the back-to-school deals section.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.578125, |
| "peak_confidence": 0.546875, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "Open the information page about recycled parts backed by eBay.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.578125, |
| "peak_confidence": 0.5625, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "What is the current shopping-cart item count?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6171875, |
| "peak_confidence": 0.52734375, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "Which category is currently selected in the search-category selector?", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.61328125, |
| "peak_confidence": 0.60546875, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "Identify the most-watched visible live event and give its host.", |
| "recall": 1.0, |
| "hits": 27, |
| "required_count": 27, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.60546875, |
| "peak_confidence": 0.53515625, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.ebay.com", |
| "task": "List the seven featured shopping categories shown under \u201cUp to 50% off back-to-school deals,\u201d in page order.", |
| "recall": 1.0, |
| "hits": 7, |
| "required_count": 7, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.349609375, |
| "peak_confidence": 0.5859375, |
| "original_tokens": 6604, |
| "retained_tokens": 6604, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Use the main property search controls to search for a home location.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.62109375, |
| "peak_confidence": 0.51171875, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Switch the main search mode to Rent.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.59765625, |
| "peak_confidence": 0.52734375, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "In the Contact Redfin form, submit a property search using that form rather than the similarly labeled hero search controls.", |
| "recall": 1.0, |
| "hits": 3, |
| "required_count": 3, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.640625, |
| "peak_confidence": 0.5, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Open the full list of homes-for-sale-by-state destinations, not the apartment or house-rental lists.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.625, |
| "peak_confidence": 0.5078125, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Expand the list of newest-home destination cities to reveal all available content.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.625, |
| "peak_confidence": 0.5078125, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Start the mortgage prequalification flow by selecting Get prequalified.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.62109375, |
| "peak_confidence": 0.51171875, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "action", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Identify the two app platforms for which Redfin provides downloads.", |
| "recall": 1.0, |
| "hits": 2, |
| "required_count": 2, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.49609375, |
| "peak_confidence": 0.56640625, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Report the cities shown in the newest-homes destination row that are in Florida.", |
| "recall": 1.0, |
| "hits": 5, |
| "required_count": 5, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.59375, |
| "peak_confidence": 0.51953125, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "hard" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Identify the states currently shown with a Homes for sale link in the state-browsing section, including the first and last visible entries.", |
| "recall": 1.0, |
| "hits": 40, |
| "required_count": 40, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.6640625, |
| "peak_confidence": 0.48828125, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "medium" |
| }, |
| { |
| "host": "www.redfin.com", |
| "task": "Provide the Redfin customer-support phone number listed for screen-reader or reading assistance.", |
| "recall": 1.0, |
| "hits": 1, |
| "required_count": 1, |
| "savings": 0.0, |
| "fallback": true, |
| "eligible": true, |
| "peak": 0.349609375, |
| "peak_confidence": 0.63671875, |
| "original_tokens": 5967, |
| "retained_tokens": 5967, |
| "token_savings": 0.0, |
| "kind": "evidence", |
| "difficulty": "easy" |
| } |
| ] |
| } |
