{
  "schema": "betterwright-frozen-reserve-eval-v1",
  "policy_tuned_on_reserve": false,
  "config": {
    "schema": "betterwright-relevance-config-v1",
    "relevance_threshold": 0.25,
    "confidence_threshold": 0.45,
    "max_retained_fraction": 1.0,
    "max_ranked_lines": 0,
    "strategy": "coarse-ref-context",
    "max_ranked_windows": 8,
    "ref_context_lines": 3,
    "window_chars": 1800,
    "minimum_validation_recall": 0.999,
    "minimum_validation_perfect_task_recall": 0.995,
    "validated_max_chars": 10000,
    "fallback_on_error": true,
    "model": "ProCreations/betterwright-encoder-350m"
  },
  "metrics": {
    "tasks": 161,
    "target_evidence_node_recall": 1.0,
    "mean_task_node_recall": 1.0,
    "perfect_task_recall": 1.0,
    "mean_char_savings": 0.0650058016826541,
    "p50_char_savings": 0.0,
    "fallback_rate": 0.6645962732919255,
    "pruning_coverage": 0.3354037267080745,
    "eligible_tasks": 161,
    "non_fallback_tasks": 54,
    "non_fallback_target_evidence_node_recall": 1.0,
    "non_fallback_perfect_task_recall": 1.0,
    "mean_token_savings": 0.05626420853717087,
    "p50_token_savings": 0.0,
    "original_tokens": 1079517,
    "retained_tokens": 1053351,
    "token_weighted_savings": 0.024238617826305653,
    "eligible_mean_token_savings": 0.05626420853717087,
    "non_fallback_mean_token_savings": 0.16775069582378718,
    "domains": 17,
    "by_kind": {
      "action": {
        "tasks": 81,
        "target_evidence_node_recall": 1.0,
        "mean_task_node_recall": 1.0,
        "perfect_task_recall": 1.0,
        "mean_char_savings": 0.047894415954788536,
        "p50_char_savings": 0.0,
        "fallback_rate": 0.7283950617283951,
        "pruning_coverage": 0.2716049382716049,
        "eligible_tasks": 81,
        "non_fallback_tasks": 22,
        "non_fallback_target_evidence_node_recall": 1.0,
        "non_fallback_perfect_task_recall": 1.0,
        "mean_token_savings": 0.040223486962387485,
        "p50_token_savings": 0.0,
        "original_tokens": 556762,
        "retained_tokens": 547396,
        "token_weighted_savings": 0.01682226876115822,
        "eligible_mean_token_savings": 0.040223486962387485,
        "non_fallback_mean_token_savings": 0.14809556563424484
      },
      "evidence": {
        "tasks": 67,
        "target_evidence_node_recall": 1.0,
        "mean_task_node_recall": 1.0,
        "perfect_task_recall": 1.0,
        "mean_char_savings": 0.07259474593610533,
        "p50_char_savings": 0.0,
        "fallback_rate": 0.6417910447761194,
        "pruning_coverage": 0.35820895522388063,
        "eligible_tasks": 67,
        "non_fallback_tasks": 24,
        "non_fallback_target_evidence_node_recall": 1.0,
        "non_fallback_perfect_task_recall": 1.0,
        "mean_token_savings": 0.06368484640893213,
        "p50_token_savings": 0.0,
        "original_tokens": 436257,
        "retained_tokens": 424249,
        "token_weighted_savings": 0.027525059769814564,
        "eligible_mean_token_savings": 0.06368484640893213,
        "non_fallback_mean_token_savings": 0.1777868628916022
      },
      "mixed": {
        "tasks": 13,
        "target_evidence_node_recall": 1.0,
        "mean_task_node_recall": 1.0,
        "perfect_task_recall": 1.0,
        "mean_char_savings": 0.13251064621926026,
        "p50_char_savings": 0.16570836082571405,
        "fallback_rate": 0.38461538461538464,
        "pruning_coverage": 0.6153846153846154,
        "eligible_tasks": 13,
        "non_fallback_tasks": 8,
        "non_fallback_target_evidence_node_recall": 1.0,
        "non_fallback_perfect_task_recall": 1.0,
        "mean_token_savings": 0.11796541701020548,
        "p50_token_savings": 0.1470926885434658,
        "original_tokens": 86498,
        "retained_tokens": 81706,
        "token_weighted_savings": 0.0554001248583782,
        "eligible_mean_token_savings": 0.11796541701020548,
        "non_fallback_mean_token_savings": 0.19169380264158392
      }
    },
    "by_difficulty": {
      "easy": {
        "tasks": 44,
        "target_evidence_node_recall": 1.0,
        "mean_task_node_recall": 1.0,
        "perfect_task_recall": 1.0,
        "mean_char_savings": 0.07079667106874572,
        "p50_char_savings": 0.0,
        "fallback_rate": 0.6363636363636364,
        "pruning_coverage": 0.36363636363636365,
        "eligible_tasks": 44,
        "non_fallback_tasks": 16,
        "non_fallback_target_evidence_node_recall": 1.0,
        "non_fallback_perfect_task_recall": 1.0,
        "mean_token_savings": 0.0627814245589974,
        "p50_token_savings": 0.0,
        "original_tokens": 293507,
        "retained_tokens": 286055,
        "token_weighted_savings": 0.02538951370836129,
        "eligible_mean_token_savings": 0.0627814245589974,
        "non_fallback_mean_token_savings": 0.17264891753724287
      },
      "medium": {
        "tasks": 50,
        "target_evidence_node_recall": 1.0,
        "mean_task_node_recall": 1.0,
        "perfect_task_recall": 1.0,
        "mean_char_savings": 0.048085784729173534,
        "p50_char_savings": 0.0,
        "fallback_rate": 0.72,
        "pruning_coverage": 0.28,
        "eligible_tasks": 50,
        "non_fallback_tasks": 14,
        "non_fallback_target_evidence_node_recall": 1.0,
        "non_fallback_perfect_task_recall": 1.0,
        "mean_token_savings": 0.03925835808931578,
        "p50_token_savings": 0.0,
        "original_tokens": 353824,
        "retained_tokens": 347346,
        "token_weighted_savings": 0.01830853757800488,
        "eligible_mean_token_savings": 0.03925835808931578,
        "non_fallback_mean_token_savings": 0.14020842174755638
      },
      "hard": {
        "tasks": 67,
        "target_evidence_node_recall": 1.0,
        "mean_task_node_recall": 1.0,
        "perfect_task_recall": 1.0,
        "mean_char_savings": 0.07382972100632573,
        "p50_char_savings": 0.0,
        "fallback_rate": 0.6417910447761194,
        "pruning_coverage": 0.35820895522388063,
        "eligible_tasks": 67,
        "non_fallback_tasks": 24,
        "non_fallback_target_evidence_node_recall": 1.0,
        "non_fallback_perfect_task_recall": 1.0,
        "mean_token_savings": 0.06467517894660949,
        "p50_token_savings": 0.0,
        "original_tokens": 432186,
        "retained_tokens": 419950,
        "token_weighted_savings": 0.028311884235028395,
        "eligible_mean_token_savings": 0.06467517894660949,
        "non_fallback_mean_token_savings": 0.1805515412259515
      }
    }
  },
  "inference": {
    "device": "cuda",
    "scored_chunks": 2294,
    "score_seconds": 12.088709216041025,
    "chunks_per_second": 189.7638498042449
  },
  "detail": [
    {
      "host": "www.lowes.com",
      "task": "Navigate to the top-level Appliances department from the main category navigation, rather than the similarly named Popular Categories link.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5546875,
      "peak_confidence": 0.5078125,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.lowes.com",
      "task": "Search the site for \u201cceiling fans\u201d using the search field and submit the search.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.43359375,
      "peak_confidence": 0.6015625,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.lowes.com",
      "task": "Open the selector for the currently selected Lowe\u2019s store.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.57421875,
      "peak_confidence": 0.5703125,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.lowes.com",
      "task": "Open the top-level Deals section from the main navigation, not the separate Shop Deals of the Day promotional link.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.54296875,
      "peak_confidence": 0.515625,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.lowes.com",
      "task": "Open the Home Care and Home maintenance subscription information, rather than the similarly priced $99 Maintenance navigation item.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.400390625,
      "peak_confidence": 0.6328125,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.lowes.com",
      "task": "Determine the sponsored Samsung appliance financing offer, including the payment duration, qualifying purchase amount, and where to get more information.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.4765625,
      "peak_confidence": 0.58984375,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.lowes.com",
      "task": "Report the currently selected Lowe\u2019s store name and ZIP code.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.439453125,
      "peak_confidence": 0.62109375,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.lowes.com",
      "task": "Identify the Popular Categories entries for bathroom products, building supplies, and doors and windows.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.58203125,
      "peak_confidence": 0.51171875,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.lowes.com",
      "task": "State the cutoff time, delivery promise, and product type in the vanity promotion.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.2333984375,
      "peak_confidence": 0.6796875,
      "original_tokens": 2606,
      "retained_tokens": 2606,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.homedepot.com",
      "task": "Open the top-level Building Materials department.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.482421875,
      "peak_confidence": 0.6015625,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.homedepot.com",
      "task": "Update the store ZIP code while keeping the currently displayed Port Chester store context in mind.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.546875,
      "peak_confidence": 0.55859375,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "www.homedepot.com",
      "task": "Use the main search field to search for an 18V cordless drill/driver kit; do not open the already-listed product card.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.54296875,
      "peak_confidence": 0.54296875,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.homedepot.com",
      "task": "Navigate to the page for scheduling a bathroom remodel, rather than opening a bathroom product or the Bath & Faucets department.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.58203125,
      "peak_confidence": 0.5703125,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.homedepot.com",
      "task": "Open the shopping cart.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.494140625,
      "peak_confidence": 0.58984375,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.homedepot.com",
      "task": "Navigate to the Store Locator from the footer.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.546875,
      "peak_confidence": 0.54296875,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.homedepot.com",
      "task": "Identify which savings tab is currently selected.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5859375,
      "peak_confidence": 0.52734375,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.homedepot.com",
      "task": "What maximum discount and product area are advertised for the online bath-vanity deal?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.4921875,
      "peak_confidence": 0.640625,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.homedepot.com",
      "task": "Which of these two visible products has more reviews: the Frigidaire side-by-side refrigerator or the GE freestanding gas range, and what are their displayed review counts?",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6171875,
      "peak_confidence": 0.57421875,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.homedepot.com",
      "task": "What action is offered for the Free In-Store Kids Workshops?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.53515625,
      "peak_confidence": 0.55859375,
      "original_tokens": 11254,
      "retained_tokens": 11254,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "developer.mozilla.org",
      "task": "Select the main documentation navigation buttons for HTML, CSS, JavaScript, Web APIs, and Learn, while excluding the Tools and About buttons.",
      "recall": 1.0,
      "hits": 7,
      "required_count": 7,
      "savings": 0.22504456327985745,
      "fallback": false,
      "eligible": true,
      "peak": 0.6171875,
      "peak_confidence": 0.5546875,
      "original_tokens": 3570,
      "retained_tokens": 2907,
      "token_savings": 0.18571428571428572,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "developer.mozilla.org",
      "task": "Open the Featured articles entry titled \"Trusted Types API\" that is categorized under Web APIs and discusses protection against client-side XSS.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.22504456327985745,
      "fallback": false,
      "eligible": true,
      "peak": 0.5546875,
      "peak_confidence": 0.5234375,
      "original_tokens": 3570,
      "retained_tokens": 2907,
      "token_savings": 0.18571428571428572,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "developer.mozilla.org",
      "task": "Open the Featured articles entry titled \"Navigation API\" whose Web APIs description says it can examine an application's history entries.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.22504456327985745,
      "fallback": false,
      "eligible": true,
      "peak": 0.59375,
      "peak_confidence": 0.498046875,
      "original_tokens": 3570,
      "retained_tokens": 2907,
      "token_savings": 0.18571428571428572,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "developer.mozilla.org",
      "task": "Open the CSS article about bringing smooth, animated navigation to multi-page apps with one line of CSS, rather than the similarly worded Navigation API article.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.22504456327985745,
      "fallback": false,
      "eligible": true,
      "peak": 0.6015625,
      "peak_confidence": 0.52734375,
      "original_tokens": 3570,
      "retained_tokens": 2907,
      "token_savings": 0.18571428571428572,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "developer.mozilla.org",
      "task": "What three technologies does MDN say it has been documenting since 2005?",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.22504456327985745,
      "fallback": false,
      "eligible": true,
      "peak": 0.515625,
      "peak_confidence": 0.5859375,
      "original_tokens": 3570,
      "retained_tokens": 2907,
      "token_savings": 0.18571428571428572,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "developer.mozilla.org",
      "task": "Which Latest news item is from 2 months ago, and what is its title?",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.22504456327985745,
      "fallback": false,
      "eligible": true,
      "peak": 0.4453125,
      "peak_confidence": 0.56640625,
      "original_tokens": 3570,
      "retained_tokens": 2907,
      "token_savings": 0.18571428571428572,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "developer.mozilla.org",
      "task": "Identify the recent contribution made to correct information related to HTML table manipulation methods, and report how long ago it was made.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.22504456327985745,
      "fallback": false,
      "eligible": true,
      "peak": 0.5625,
      "peak_confidence": 0.53125,
      "original_tokens": 3570,
      "retained_tokens": 2907,
      "token_savings": 0.18571428571428572,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "developer.mozilla.org",
      "task": "Who is quoted in the Contributor Spotlight, and what does the quote say MDN closely follows?",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.22504456327985745,
      "fallback": false,
      "eligible": true,
      "peak": 0.5078125,
      "peak_confidence": 0.5625,
      "original_tokens": 3570,
      "retained_tokens": 2907,
      "token_savings": 0.18571428571428572,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.wikipedia.org",
      "task": "Choose the English Wikipedia edition from the top-language links, rather than selecting the similarly labeled option in the search-language dropdown.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.384671267707955,
      "fallback": false,
      "eligible": true,
      "peak": 0.6953125,
      "peak_confidence": 0.478515625,
      "original_tokens": 1956,
      "retained_tokens": 1197,
      "token_savings": 0.3880368098159509,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.wikipedia.org",
      "task": "Search Wikipedia for \u201cquantum mechanics\u201d using English, keeping the search-language selector set to English, and activate the Search button.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.384671267707955,
      "fallback": false,
      "eligible": true,
      "peak": 0.6953125,
      "peak_confidence": 0.484375,
      "original_tokens": 1956,
      "retained_tokens": 1197,
      "token_savings": 0.3880368098159509,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.wikipedia.org",
      "task": "Open the official Wikipedia app download information and then choose the Apple App Store link.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.384671267707955,
      "fallback": false,
      "eligible": true,
      "peak": 0.5703125,
      "peak_confidence": 0.5234375,
      "original_tokens": 1956,
      "retained_tokens": 1197,
      "token_savings": 0.3880368098159509,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "www.wikipedia.org",
      "task": "Among the listed top-language editions, which link shows the highest stated article count? Report its language and displayed count.",
      "recall": 1.0,
      "hits": 10,
      "required_count": 10,
      "savings": 0.384671267707955,
      "fallback": false,
      "eligible": true,
      "peak": 0.703125,
      "peak_confidence": 0.46875,
      "original_tokens": 1956,
      "retained_tokens": 1197,
      "token_savings": 0.3880368098159509,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.wikipedia.org",
      "task": "According to the footer, which organization hosts Wikipedia and also hosts a range of other projects?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.384671267707955,
      "fallback": false,
      "eligible": true,
      "peak": 0.58203125,
      "peak_confidence": 0.53125,
      "original_tokens": 1956,
      "retained_tokens": 1197,
      "token_savings": 0.3880368098159509,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.wikipedia.org",
      "task": "Which other project is described as a \u201cFree travel guide\u201d?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.384671267707955,
      "fallback": false,
      "eligible": true,
      "peak": 0.52734375,
      "peak_confidence": 0.54296875,
      "original_tokens": 1956,
      "retained_tokens": 1197,
      "token_savings": 0.3880368098159509,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.wikipedia.org",
      "task": "What license is this page available under?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.384671267707955,
      "fallback": false,
      "eligible": true,
      "peak": 0.62890625,
      "peak_confidence": 0.515625,
      "original_tokens": 1956,
      "retained_tokens": 1197,
      "token_savings": 0.3880368098159509,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.nasa.gov",
      "task": "Open the Mission Overview for the featured Roman Space Telescope topic titled \u201cUnveiling the Universe\u2019s Dark Side.\u201d",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.63671875,
      "peak_confidence": 0.578125,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.nasa.gov",
      "task": "Open the Featured News item that is a news release about the Commercial Swift Boost Mission.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.4765625,
      "peak_confidence": 0.625,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.nasa.gov",
      "task": "Under NASA Missions, open the James Webb Space Telescope mission card, not the similarly named gallery in More NASA Images.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.59765625,
      "peak_confidence": 0.5546875,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.nasa.gov",
      "task": "In the Image Of The Day section, open the first \u201cView full image page\u201d link associated with the Ancient Milky Way Merger image, rather than the later duplicate link.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.55078125,
      "peak_confidence": 0.5546875,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.nasa.gov",
      "task": "Open the Technology topic from More Topics from NASA.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.52734375,
      "peak_confidence": 0.6015625,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.nasa.gov",
      "task": "Use the site search box to search for \u201cArtemis\u201d instead of opening the Artemis mission card.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.484375,
      "peak_confidence": 0.609375,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.nasa.gov",
      "task": "Report the featured Roman Space Telescope\u2019s stated launch date and the kind of view it will provide.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.55859375,
      "peak_confidence": 0.58203125,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.nasa.gov",
      "task": "Among the Hubble, James Webb, and Chandra gallery cards, report the image count for each gallery and identify which has the most.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.64453125,
      "peak_confidence": 0.51953125,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.nasa.gov",
      "task": "List the four named cards shown in the NASA Missions section.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.640625,
      "peak_confidence": 0.53515625,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.nasa.gov",
      "task": "Report the page\u2019s last-updated date, page editor, and responsible NASA official from the footer.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.17232627341151763,
      "fallback": false,
      "eligible": true,
      "peak": 0.4765625,
      "peak_confidence": 0.62109375,
      "original_tokens": 3434,
      "retained_tokens": 2955,
      "token_savings": 0.13948747815958062,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "docs.python.org",
      "task": "Open the Python Tutorial from the documentation sections.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.67578125,
      "peak_confidence": 0.51953125,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "docs.python.org",
      "task": "Change the documentation language to Spanish | espa\u00f1ol while leaving the Python version set to 3.14.7.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.65625,
      "peak_confidence": 0.51953125,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "docs.python.org",
      "task": "Change the selected Python documentation version to 3.12.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.63671875,
      "peak_confidence": 0.5234375,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "docs.python.org",
      "task": "Open the download page for the documentation, not the page for the stable Python 3.14 version.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.69140625,
      "peak_confidence": 0.51171875,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "docs.python.org",
      "task": "Open the resource about Python type safety and guides, rather than the Python C API reference.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.65234375,
      "peak_confidence": 0.52734375,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "docs.python.org",
      "task": "Open the resource for third-party modules and PyPI.org, rather than the page about installing, configuring, and using Python.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.625,
      "peak_confidence": 0.54296875,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "docs.python.org",
      "task": "Which documentation section covers the standard library and builtins?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.66015625,
      "peak_confidence": 0.53515625,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "docs.python.org",
      "task": "What is the currently selected theme?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.671875,
      "peak_confidence": 0.52734375,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "docs.python.org",
      "task": "Which link provides an index of all functions, classes, and terms?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.6171875,
      "peak_confidence": 0.5390625,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "docs.python.org",
      "task": "Which footer link should be followed for more information about the documentation license?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.16570836082571405,
      "fallback": false,
      "eligible": true,
      "peak": 0.62109375,
      "peak_confidence": 0.55078125,
      "original_tokens": 3474,
      "retained_tokens": 2963,
      "token_savings": 0.1470926885434658,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.cdc.gov",
      "task": "Activate the featured health topic link for preventing skin cancer.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.671875,
      "peak_confidence": 0.578125,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.cdc.gov",
      "task": "Open the carousel story about two travelers on a hike, rather than a featured topic link.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5859375,
      "peak_confidence": 0.59765625,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.cdc.gov",
      "task": "Use the upper header navigation to open the About CDC menu, not the footer About CDC link.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.64453125,
      "peak_confidence": 0.578125,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.cdc.gov",
      "task": "Open the site search from the first header navigation, not the duplicate search link lower on the page.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6015625,
      "peak_confidence": 0.58984375,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.cdc.gov",
      "task": "Activate the featured topic about COVID-19 testing.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.66015625,
      "peak_confidence": 0.5859375,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.cdc.gov",
      "task": "Pause the rotating homepage carousel.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.66015625,
      "peak_confidence": 0.58203125,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.cdc.gov",
      "task": "List every health topic currently shown under Featured Topics.",
      "recall": 1.0,
      "hits": 7,
      "required_count": 7,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.63671875,
      "peak_confidence": 0.5859375,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.cdc.gov",
      "task": "What does the official-government banner say about the website, and which control explains it?",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.64453125,
      "peak_confidence": 0.56640625,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.cdc.gov",
      "task": "Which carousel story is associated with an image description of a person holding their stomach in pain?",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.60546875,
      "peak_confidence": 0.59375,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.cdc.gov",
      "task": "What description does the CDC provide about its role and operating schedule in the footer?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.609375,
      "peak_confidence": 0.5859375,
      "original_tokens": 1739,
      "retained_tokens": 1739,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "apnews.com",
      "task": "Navigate to the World news section using the primary navigation.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.1416015625,
      "peak_confidence": 0.734375,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "apnews.com",
      "task": "Open the site's search interface without selecting a news category.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.1845703125,
      "peak_confidence": 0.6484375,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "apnews.com",
      "task": "Open the Politics section from the main navigation, rather than opening the article about the $40 trillion debt.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5859375,
      "peak_confidence": 0.5703125,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "apnews.com",
      "task": "Open the article about the new CDC director acknowledging agency problems, not the separate link about CDC zombie programs.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.490234375,
      "peak_confidence": 0.53515625,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "apnews.com",
      "task": "Advance the Short Stories carousel to the next item.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.2197265625,
      "peak_confidence": 0.6875,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "apnews.com",
      "task": "Open the AP News quizzes section from the page navigation.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.32421875,
      "peak_confidence": 0.64453125,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "apnews.com",
      "task": "What is the lead headline in the Russia-Ukraine war coverage, and what does its teaser say was set ablaze?",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.474609375,
      "peak_confidence": 0.55078125,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "apnews.com",
      "task": "In the Today in History section, what event is listed for 2010?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5625,
      "peak_confidence": 0.5078125,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "apnews.com",
      "task": "Which answer option is indicated as the answer to the quiz question about the Bavarian immigrant who popularized blue jeans?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.57421875,
      "peak_confidence": 0.52734375,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "apnews.com",
      "task": "What is the lead story headline in the Science section?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.50390625,
      "peak_confidence": 0.5390625,
      "original_tokens": 15515,
      "retained_tokens": 15515,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.delta.com",
      "task": "Use the flight booking form to prepare a search by reviewing the origin, destination, trip type, travel dates, passenger count, and then activating the flight search control.",
      "recall": 1.0,
      "hits": 6,
      "required_count": 6,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.6015625,
      "peak_confidence": 0.58984375,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.delta.com",
      "task": "Change the trip type from the currently selected round-trip setting to a one-way trip.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.640625,
      "peak_confidence": 0.5859375,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.delta.com",
      "task": "Enable the option to shop for flights using miles.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.65234375,
      "peak_confidence": 0.58203125,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.delta.com",
      "task": "Navigate to Delta's Travel Info section, not the separate travel-products content in the page body.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.58203125,
      "peak_confidence": 0.58984375,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.delta.com",
      "task": "Open the page for searching Delta flight deals.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.50390625,
      "peak_confidence": 0.5859375,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.delta.com",
      "task": "Open the Help Center specifically from the Customer Service section of the footer, rather than the similarly named Popular Topics link.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.578125,
      "peak_confidence": 0.546875,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.delta.com",
      "task": "Report the current flight-search state: origin, destination placeholder, trip type, departure/return date field, passenger count, and whether Basic is included.",
      "recall": 1.0,
      "hits": 6,
      "required_count": 6,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.609375,
      "peak_confidence": 0.58203125,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.delta.com",
      "task": "What maximum welcome-offer mileage is advertised, and what does the linked offer description say about earning miles on Delta purchases?",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.46484375,
      "peak_confidence": 0.62109375,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.delta.com",
      "task": "List the three topics shown under Popular Topics.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.53515625,
      "peak_confidence": 0.546875,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.delta.com",
      "task": "Identify the selected booking product tab and the currently checked fare option.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.17862858549796878,
      "fallback": false,
      "eligible": true,
      "peak": 0.54296875,
      "peak_confidence": 0.6015625,
      "original_tokens": 2544,
      "retained_tokens": 2205,
      "token_savings": 0.13325471698113212,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.etsy.com",
      "task": "Open the featured marketplace category for Gold Sculptural Decor.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.67578125,
      "peak_confidence": 0.5859375,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.etsy.com",
      "task": "Navigate to Etsy's New Arrivals section from the top navigation.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.578125,
      "peak_confidence": 0.55078125,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.etsy.com",
      "task": "Open the Halloween product listing priced at $6.00, rather than the similarly themed $6.00\u2013$12.00 pattern listing.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.41015625,
      "peak_confidence": 0.63671875,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.etsy.com",
      "task": "Open the Help Center link in the footer, not the similarly named Help Center link in the main content.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.59765625,
      "peak_confidence": 0.5390625,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.etsy.com",
      "task": "Open Gift cards from the footer Shop links, not the Gift Cards link in the upper navigation.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.59765625,
      "peak_confidence": 0.54296875,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.etsy.com",
      "task": "Which four items are listed under \u201cJump into featured interests\u201d?",
      "recall": 1.0,
      "hits": 5,
      "required_count": 5,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.59375,
      "peak_confidence": 0.5390625,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.etsy.com",
      "task": "List the five visible topics in the \u201cDiscover our best of summer 2026\u201d section.",
      "recall": 1.0,
      "hits": 6,
      "required_count": 6,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.515625,
      "peak_confidence": 0.5859375,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.etsy.com",
      "task": "Which Halloween product listing is shown at exactly $29.95?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.5703125,
      "peak_confidence": 0.56640625,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.etsy.com",
      "task": "Which two app marketplaces are offered in the Etsy footer?",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.08958294370675146,
      "fallback": false,
      "eligible": true,
      "peak": 0.498046875,
      "peak_confidence": 0.578125,
      "original_tokens": 3432,
      "retained_tokens": 3181,
      "token_savings": 0.07313519813519809,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "huggingface.co",
      "task": "Locate the model-list search and filtering controls, using the model-name search rather than the global site search.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.62109375,
      "peak_confidence": 0.53515625,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "huggingface.co",
      "task": "Activate the Text Generation task filter in the model filters, not the similarly named Text Generation label on a model card.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.671875,
      "peak_confidence": 0.5234375,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "huggingface.co",
      "task": "Apply the Transformers library filter to the model list.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6328125,
      "peak_confidence": 0.53125,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "huggingface.co",
      "task": "Use the model-name filter to search for models named Qwen3.8-27B.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.68359375,
      "peak_confidence": 0.45703125,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "huggingface.co",
      "task": "Restrict the model list to base models that have inference available by enabling Base only and selecting Inference Available.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.65625,
      "peak_confidence": 0.46875,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "huggingface.co",
      "task": "Identify the newest visible model and report its displayed update time.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6484375,
      "peak_confidence": 0.47265625,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "huggingface.co",
      "task": "Which visible model has the largest displayed parameter count, and what is that count?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.68359375,
      "peak_confidence": 0.490234375,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "huggingface.co",
      "task": "Name a visible Text Generation model whose card indicates that inference providers are available.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.625,
      "peak_confidence": 0.4765625,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "huggingface.co",
      "task": "Between Qwen/Qwen3.8-27B and unsloth/Qwen3.8-27B-GGUF, which visible entry has more parameters, and what are the two displayed counts?",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6875,
      "peak_confidence": 0.48828125,
      "original_tokens": 7432,
      "retained_tokens": 7432,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "stackoverflow.com",
      "task": "Find the main Questions section of the page rather than the similarly named primary-navigation link.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.65234375,
      "peak_confidence": 0.5546875,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "mixed",
      "difficulty": "hard"
    },
    {
      "host": "stackoverflow.com",
      "task": "Open the newest question about chaining programmatic Polars expression functions in with_columns.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.56640625,
      "peak_confidence": 0.51171875,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "stackoverflow.com",
      "task": "Open the question tagged python-polars, not the question whose title merely mentions another data-processing library.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.578125,
      "peak_confidence": 0.54296875,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "stackoverflow.com",
      "task": "Use the page-size controls to show 50 questions per page.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.447265625,
      "peak_confidence": 0.6171875,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "stackoverflow.com",
      "task": "Go to the next page of newest questions.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.54296875,
      "peak_confidence": 0.5390625,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "stackoverflow.com",
      "task": "Open the Unanswered question listing instead of the similarly named Bountied listing.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6015625,
      "peak_confidence": 0.5546875,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "stackoverflow.com",
      "task": "Identify the exact title of the first question shown in the newest-questions list.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5234375,
      "peak_confidence": 0.4921875,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "stackoverflow.com",
      "task": "Which listed question is associated with the angular tag? Provide its exact title.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6796875,
      "peak_confidence": 0.55078125,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "stackoverflow.com",
      "task": "Identify the question that is explicitly marked closed, and provide its exact title.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6328125,
      "peak_confidence": 0.5546875,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "stackoverflow.com",
      "task": "Which collective is described as having 69k Members?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5703125,
      "peak_confidence": 0.546875,
      "original_tokens": 7417,
      "retained_tokens": 7417,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.rei.com",
      "task": "Open the shop category for hiking and camping, using the category labeled \"Camp & Hike\" rather than the separate \"Backpacking\" link.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.578125,
      "peak_confidence": 0.55078125,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.rei.com",
      "task": "Open REI's general outdoor gift ideas page, not the separate gift-card page.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.578125,
      "peak_confidence": 0.5703125,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.rei.com",
      "task": "Use the header link to sign in to REI, rather than the similarly named footer account link.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.62109375,
      "peak_confidence": 0.5390625,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.rei.com",
      "task": "Search the site for \"trail running\" using the main Search field.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.306640625,
      "peak_confidence": 0.62109375,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "mixed",
      "difficulty": "medium"
    },
    {
      "host": "www.rei.com",
      "task": "Open the \"Outlet deals\" item under Shop top categories, not the top-navigation \"REI Outlet\" destination.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.61328125,
      "peak_confidence": 0.54296875,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.rei.com",
      "task": "In the \"Gear we think you\u2019d love\" product carousel, switch from the currently selected \"Crowd favorites\" tab to \"Best Sellers\".",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.55859375,
      "peak_confidence": 0.51953125,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.rei.com",
      "task": "What does promotional message 1 say members can earn, and by what date must they join?",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.490234375,
      "peak_confidence": 0.52734375,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.rei.com",
      "task": "Among GU Energy Gel, Clif Bloks Energy Chews, and Dr. Bronner's 18-in-1 Pure-Castile Magic Soap, which listed product has the most reviews?",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.609375,
      "peak_confidence": 0.5546875,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.rei.com",
      "task": "What rating and current price comparison are shown for the HOKA Clifton 10 Road-Running Shoes - Men's?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.58984375,
      "peak_confidence": 0.5546875,
      "original_tokens": 10235,
      "retained_tokens": 10235,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.opentable.com",
      "task": "Find the complete restaurant search interface: the date control, time selector, party-size selector, location field, and button used to submit the search.",
      "recall": 1.0,
      "hits": 5,
      "required_count": 5,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.63671875,
      "peak_confidence": 0.5625,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.opentable.com",
      "task": "Change the reservation time to 11:00 PM while keeping the party size at 2 people.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5859375,
      "peak_confidence": 0.51171875,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.opentable.com",
      "task": "Open the location picker to change the search area; do not request the device's current location.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.63671875,
      "peak_confidence": 0.498046875,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.opentable.com",
      "task": "Navigate to restaurants near Rockefeller Center in New York City, rather than the general New York City restaurant listing.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.64453125,
      "peak_confidence": 0.515625,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.opentable.com",
      "task": "Switch the city-directory panel from U.S. cities to the global-cities list.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.59765625,
      "peak_confidence": 0.578125,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.opentable.com",
      "task": "Navigate to the Restaurants Near Me page, not the Restaurants Open Now page.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.66015625,
      "peak_confidence": 0.54296875,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.opentable.com",
      "task": "Report the current reservation date, time, and party size shown in the search controls.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.59765625,
      "peak_confidence": 0.5078125,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.opentable.com",
      "task": "According to the FAQ, what search dimensions and filters can users use to narrow restaurant choices?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6328125,
      "peak_confidence": 0.4921875,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.opentable.com",
      "task": "List the cuisine-oriented New York City links shown under the Cuisines section, distinguishing them from the landmark links.",
      "recall": 1.0,
      "hits": 7,
      "required_count": 7,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.65625,
      "peak_confidence": 0.55078125,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.opentable.com",
      "task": "Identify the four FAQ feature tabs and indicate which one is currently selected.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.64453125,
      "peak_confidence": 0.57421875,
      "original_tokens": 20103,
      "retained_tokens": 20103,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.southwest.com",
      "task": "Set up a round-trip flight search using the currently selected trip type, departing from LGA, entering an arrival airport, keeping the displayed departure and return dates, and submit the search.",
      "recall": 1.0,
      "hits": 6,
      "required_count": 6,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.60546875,
      "peak_confidence": 0.5546875,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.southwest.com",
      "task": "Reverse the departure and arrival airports using the airport-swap control.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.55859375,
      "peak_confidence": 0.5703125,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.southwest.com",
      "task": "Change the fare display from dollars to points.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.63671875,
      "peak_confidence": 0.53515625,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.southwest.com",
      "task": "Enable the low fare calendar option and then submit the flight search.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.64453125,
      "peak_confidence": 0.54296875,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.southwest.com",
      "task": "Open the flight-status page, not the check-in page.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6484375,
      "peak_confidence": 0.5390625,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.southwest.com",
      "task": "Open the general Baggage Policies page, rather than the promotional link about updated checked-baggage policies.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.61328125,
      "peak_confidence": 0.55859375,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.southwest.com",
      "task": "Report the reservation-through date and the currently displayed departure and return dates.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.61328125,
      "peak_confidence": 0.5546875,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.southwest.com",
      "task": "Identify the baggage-related fees link shown beneath the flight search form.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6015625,
      "peak_confidence": 0.5546875,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.southwest.com",
      "task": "Summarize the four Rapid Rewards benefits listed in the membership section.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5546875,
      "peak_confidence": 0.56640625,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.southwest.com",
      "task": "For the international vacation-package promotion, identify the displayed currency marker for its maximum discount and the link provided for the promotion's terms.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6328125,
      "peak_confidence": 0.5390625,
      "original_tokens": 4341,
      "retained_tokens": 4341,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.ebay.com",
      "task": "Select \u201cConsumer Electronics\u201d as the search category, enter \u201cwireless headphones\u201d in the main search field, and submit the search.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.5,
      "peak_confidence": 0.609375,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.ebay.com",
      "task": "Open the featured Depop shopping destination for the pre-loved fashion community, rather than one of the general fashion deal links.",
      "recall": 1.0,
      "hits": 4,
      "required_count": 4,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.546875,
      "peak_confidence": 0.60546875,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.ebay.com",
      "task": "Expand the Electronics category in the main shopping-category navigation.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.65234375,
      "peak_confidence": 0.6015625,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.ebay.com",
      "task": "Open the currently live eBay Live event with the largest displayed audience.",
      "recall": 1.0,
      "hits": 26,
      "required_count": 26,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.41015625,
      "peak_confidence": 0.66015625,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.ebay.com",
      "task": "Open the \u201cPortable audio\u201d featured shopping category in the back-to-school deals section.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.578125,
      "peak_confidence": 0.546875,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.ebay.com",
      "task": "Open the information page about recycled parts backed by eBay.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.578125,
      "peak_confidence": 0.5625,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.ebay.com",
      "task": "What is the current shopping-cart item count?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6171875,
      "peak_confidence": 0.52734375,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.ebay.com",
      "task": "Which category is currently selected in the search-category selector?",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.61328125,
      "peak_confidence": 0.60546875,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.ebay.com",
      "task": "Identify the most-watched visible live event and give its host.",
      "recall": 1.0,
      "hits": 27,
      "required_count": 27,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.60546875,
      "peak_confidence": 0.53515625,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.ebay.com",
      "task": "List the seven featured shopping categories shown under \u201cUp to 50% off back-to-school deals,\u201d in page order.",
      "recall": 1.0,
      "hits": 7,
      "required_count": 7,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.349609375,
      "peak_confidence": 0.5859375,
      "original_tokens": 6604,
      "retained_tokens": 6604,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.redfin.com",
      "task": "Use the main property search controls to search for a home location.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.62109375,
      "peak_confidence": 0.51171875,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.redfin.com",
      "task": "Switch the main search mode to Rent.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.59765625,
      "peak_confidence": 0.52734375,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.redfin.com",
      "task": "In the Contact Redfin form, submit a property search using that form rather than the similarly labeled hero search controls.",
      "recall": 1.0,
      "hits": 3,
      "required_count": 3,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.640625,
      "peak_confidence": 0.5,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.redfin.com",
      "task": "Open the full list of homes-for-sale-by-state destinations, not the apartment or house-rental lists.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.625,
      "peak_confidence": 0.5078125,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "hard"
    },
    {
      "host": "www.redfin.com",
      "task": "Expand the list of newest-home destination cities to reveal all available content.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.625,
      "peak_confidence": 0.5078125,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "medium"
    },
    {
      "host": "www.redfin.com",
      "task": "Start the mortgage prequalification flow by selecting Get prequalified.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.62109375,
      "peak_confidence": 0.51171875,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "action",
      "difficulty": "easy"
    },
    {
      "host": "www.redfin.com",
      "task": "Identify the two app platforms for which Redfin provides downloads.",
      "recall": 1.0,
      "hits": 2,
      "required_count": 2,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.49609375,
      "peak_confidence": 0.56640625,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    },
    {
      "host": "www.redfin.com",
      "task": "Report the cities shown in the newest-homes destination row that are in Florida.",
      "recall": 1.0,
      "hits": 5,
      "required_count": 5,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.59375,
      "peak_confidence": 0.51953125,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "hard"
    },
    {
      "host": "www.redfin.com",
      "task": "Identify the states currently shown with a Homes for sale link in the state-browsing section, including the first and last visible entries.",
      "recall": 1.0,
      "hits": 40,
      "required_count": 40,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.6640625,
      "peak_confidence": 0.48828125,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "medium"
    },
    {
      "host": "www.redfin.com",
      "task": "Provide the Redfin customer-support phone number listed for screen-reader or reading assistance.",
      "recall": 1.0,
      "hits": 1,
      "required_count": 1,
      "savings": 0.0,
      "fallback": true,
      "eligible": true,
      "peak": 0.349609375,
      "peak_confidence": 0.63671875,
      "original_tokens": 5967,
      "retained_tokens": 5967,
      "token_savings": 0.0,
      "kind": "evidence",
      "difficulty": "easy"
    }
  ]
}