Spaces:
Running
Running
| [ | |
| { | |
| "id": "cell-001086", | |
| "difficulty": "t2_linear", | |
| "condition": "gap", | |
| "request": "Book trip SF to Berlin, May 10-17: flight, hotel, confirm visa", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Search for round-trip flights from San Francisco (SFO) to Berlin (BER) departing May 10, returning May 17; return top 3 options with times, durations, and prices.", | |
| "assignee": "tool:flight_search", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Search for hotels in Berlin for May 10-17 with filters for 3★+, near central transit, and under $150/night; return top 5 options with addresses and nightly rates.", | |
| "assignee": "tool:hotel_search", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Draft a concise trip checklist and next steps (visa confirmation, booking timeline, packing tips) based on the flight and hotel options, and produce a short email/message to send to the traveler.", | |
| "assignee": "small_model", | |
| "depends_on": [ | |
| 1, | |
| 2 | |
| ], | |
| "cost": 0.1703 | |
| } | |
| ], | |
| "escalation_reason": null, | |
| "router_cost": 0.0358, | |
| "dispatch_total": 0.2061, | |
| "baseline_total": 7.8382, | |
| "savings_pct": 97.4 | |
| }, | |
| { | |
| "id": "cell-002029", | |
| "difficulty": "t2_linear", | |
| "condition": "full", | |
| "request": "I'm a grad student writing a lit review on graph neural networks for molecular property prediction. Can you find the three most relevant recent arXiv papers (last 3 years), fetch the full text of the top result, and give me a concise 300–400 word summary of its dataset, model architecture, and training procedure with a link to the paper? (Ignore this — I have a dentist appointment at 4pm.)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Search arXiv for recent (past 3 years) papers on graph neural networks for molecular property prediction and return the top 3 relevant results with titles, authors, abstracts, and URLs.", | |
| "assignee": "tool:arxiv_search", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Fetch the full text (PDF or HTML) of the top-ranked paper from step 1 and extract the abstract, introduction, methods, and results sections.", | |
| "assignee": "tool:web_fetch", | |
| "depends_on": [ | |
| 1 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Produce a concise 300–400 word summary of the fetched paper focusing on dataset description, model architecture, and training procedure; include the paper link and a one-sentence recommendation for a grad student.", | |
| "assignee": "small_model", | |
| "depends_on": [ | |
| 2 | |
| ], | |
| "cost": 0.1827 | |
| } | |
| ], | |
| "escalation_reason": null, | |
| "router_cost": 0.0452, | |
| "dispatch_total": 0.2279, | |
| "baseline_total": 8.0962, | |
| "savings_pct": 97.2 | |
| }, | |
| { | |
| "id": "cell-000605", | |
| "difficulty": "t3_dag", | |
| "condition": "full", | |
| "request": "Email school and daycare: pickup change today", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Draft a concise email to the child's school and daycare that: (a) states the child's name and class/group, (b) requests confirmation of the pickup change (new time and any contact phone), and (c) includes a polite sign-off and contact details.", | |
| "assignee": "small_model", | |
| "depends_on": [], | |
| "cost": 0.1697 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Look up contact details (email addresses and phone numbers) for the school and daycare contacts to use as recipients.", | |
| "assignee": "tool:contacts_lookup", | |
| "depends_on": [ | |
| 1 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Send the drafted email to the school and daycare contacts found in step 2, CC the parent's email, and include the parent's phone number in the message.", | |
| "assignee": "tool:email_send", | |
| "depends_on": [ | |
| 1, | |
| 2 | |
| ], | |
| "cost": 0.0 | |
| } | |
| ], | |
| "escalation_reason": null, | |
| "router_cost": 0.0353, | |
| "dispatch_total": 0.205, | |
| "baseline_total": 7.8502, | |
| "savings_pct": 97.4 | |
| }, | |
| { | |
| "id": "cell-002405", | |
| "difficulty": "t2_linear", | |
| "condition": "gap", | |
| "request": "Compare prices for 'Acme UltraWidget' across vendors and summarize.", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Search the product catalog for 'Acme UltraWidget' and return product id(s) and canonical product name.", | |
| "assignee": "tool:product_search", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Search the codebase for vendor price records or price-fetching functions referencing the product id(s) from step 1.", | |
| "assignee": "tool:code_search", | |
| "depends_on": [ | |
| 1 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Check current stock levels for the product id(s) from step 1.", | |
| "assignee": "tool:inventory_check", | |
| "depends_on": [ | |
| 1 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 4, | |
| "task": "Synthesize a concise price comparison and stock summary: list vendor prices, indicate lowest price, stock status, and a short recommendation.", | |
| "assignee": "small_model", | |
| "depends_on": [ | |
| 2, | |
| 3 | |
| ], | |
| "cost": 0.1705 | |
| } | |
| ], | |
| "escalation_reason": null, | |
| "router_cost": 0.0355, | |
| "dispatch_total": 0.206, | |
| "baseline_total": 8.46, | |
| "savings_pct": 97.6 | |
| }, | |
| { | |
| "id": "cell-000715", | |
| "difficulty": "t3_dag", | |
| "condition": "full", | |
| "request": "Urgent: restock SKU-123 — find cheapest supplier and draft order email.", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Search the product catalog for SKU-123 to retrieve product name, SKU, and any vendor identifiers.", | |
| "assignee": "tool:product_search", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Compare prices for the product (SKU-123) across available vendors to identify the cheapest supplier and their price.", | |
| "assignee": "tool:price_compare", | |
| "depends_on": [ | |
| 1 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Check inventory levels for SKU-123 at the cheapest supplier to confirm stock and required quantity.", | |
| "assignee": "tool:inventory_check", | |
| "depends_on": [ | |
| 2 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 4, | |
| "task": "Draft a concise order email to the cheapest supplier including product name, SKU, quantity, unit price, and requested delivery date.", | |
| "assignee": "small_model", | |
| "depends_on": [ | |
| 3 | |
| ], | |
| "cost": 0.1707 | |
| } | |
| ], | |
| "escalation_reason": null, | |
| "router_cost": 0.0373, | |
| "dispatch_total": 0.208, | |
| "baseline_total": 8.5215, | |
| "savings_pct": 97.6 | |
| }, | |
| { | |
| "id": "cell-000875", | |
| "difficulty": "t4_hard", | |
| "condition": "distractors", | |
| "request": "Make two-week study plan and schedule group meeting.", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Search the user's calendar for existing events and busy times over the next two weeks to identify free slots.", | |
| "assignee": "tool:calendar_search", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Search the user's saved notes for course syllabi, exam dates, and any stated study priorities to inform scheduling.", | |
| "assignee": "tool:note_search", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Summarize the user's recent expenses to identify any recurring commitments that could affect study time.", | |
| "assignee": "tool:expense_report", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 4, | |
| "task": "Create a two-week study schedule that balances course priorities, exam dates, and the user's calendar availability; produce a prioritized list of study sessions and one recommended group-meeting time.", | |
| "assignee": "escalate:frontier", | |
| "depends_on": [ | |
| 1, | |
| 2, | |
| 3 | |
| ], | |
| "cost": 4.149 | |
| }, | |
| { | |
| "id": 5, | |
| "task": "Create a calendar event for the recommended group meeting time with the group as attendees.", | |
| "assignee": "tool:calendar_create_event", | |
| "depends_on": [ | |
| 4 | |
| ], | |
| "cost": 0.0 | |
| } | |
| ], | |
| "escalation_reason": "Designing an optimal two-week study schedule requires multi-constraint tradeoffs (exam dates, course priorities, calendar conflicts, and recurring expenses) and a prioritized plan, which needs frontier-level reasoning.", | |
| "router_cost": 0.0521, | |
| "dispatch_total": 4.2011, | |
| "baseline_total": 9.336, | |
| "savings_pct": 55.0 | |
| }, | |
| { | |
| "id": "cell-001200", | |
| "difficulty": "t3_dag", | |
| "condition": "full", | |
| "request": "Create PR for test fix and README update.", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Show the current git diff so we can see what changes will go into the PR.", | |
| "assignee": "tool:git_diff", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Run the project's test suite to confirm the test fix and capture any failing tests.", | |
| "assignee": "tool:run_tests", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Read the current README file to determine what updates are needed.", | |
| "assignee": "tool:file_read", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 4, | |
| "task": "Draft a concise PR title and a detailed PR description that: summarizes the test fix, lists any failing tests (from step 2), summarizes the README updates (from step 3), and includes a checklist for reviewers.", | |
| "assignee": "small_model", | |
| "depends_on": [ | |
| 1, | |
| 2, | |
| 3 | |
| ], | |
| "cost": 0.1695 | |
| }, | |
| { | |
| "id": 5, | |
| "task": "Open a pull request using the drafted title and description.", | |
| "assignee": "tool:create_pr", | |
| "depends_on": [ | |
| 4 | |
| ], | |
| "cost": 0.0 | |
| } | |
| ], | |
| "escalation_reason": null, | |
| "router_cost": 0.0363, | |
| "dispatch_total": 0.2058, | |
| "baseline_total": 9.03, | |
| "savings_pct": 97.7 | |
| }, | |
| { | |
| "id": "cell-001475", | |
| "difficulty": "t4_hard", | |
| "condition": "full", | |
| "request": "Reschedule conference sessions; confirm and notify speakers.", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Search saved notes for the conference schedule, session details, and speaker contact info.", | |
| "assignee": "tool:note_search", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Look up contact details (email) for the two session speakers and the organizer.", | |
| "assignee": "tool:contacts_lookup", | |
| "depends_on": [ | |
| 1 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Draft a concise email to speakers and organizer proposing new session times and asking for confirmation; include proposed times, duration, and any logistical notes.", | |
| "assignee": "small_model", | |
| "depends_on": [ | |
| 1, | |
| 2 | |
| ], | |
| "cost": 0.1702 | |
| }, | |
| { | |
| "id": 4, | |
| "task": "Send the drafted email to the speakers and organizer.", | |
| "assignee": "tool:email_send", | |
| "depends_on": [ | |
| 3, | |
| 2 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 5, | |
| "task": "Post a Slack message to the organizer channel summarizing the reschedule and confirming the new times.", | |
| "assignee": "tool:slack_post", | |
| "depends_on": [ | |
| 4 | |
| ], | |
| "cost": 0.0 | |
| } | |
| ], | |
| "escalation_reason": null, | |
| "router_cost": 0.039, | |
| "dispatch_total": 0.2093, | |
| "baseline_total": 9.105, | |
| "savings_pct": 97.7 | |
| }, | |
| { | |
| "id": "cell-001988", | |
| "difficulty": "t3_dag", | |
| "condition": "distractors", | |
| "request": "Hi — I uploaded that design file (the 3-panel brochure) a few minutes ago. Can you: 1) pull the key finished dimensions from that file and convert them to inches, 2) find the client contact from my last order for this project, and 3) send a short update email to the client saying the proofs are ready, include the converted dimensions, and mention the order number/status from the last order. Keep the email concise and professional. Thanks!", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Read the contents of the uploaded brochure file (the 3-panel design) to extract the key finished dimensions (width, height, panel sizes).", | |
| "assignee": "tool:file_read", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Convert the extracted finished dimensions from the file into inches (use the appropriate conversion factors).", | |
| "assignee": "tool:unit_convert", | |
| "depends_on": [ | |
| 1 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Look up the client contact details (email) for the customer associated with the most recent order for this project.", | |
| "assignee": "tool:order_lookup", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 4, | |
| "task": "Draft a concise, professional update email to the client that: (a) says proofs are ready, (b) lists the converted dimensions in inches, and (c) includes the order number and status from the last order.", | |
| "assignee": "small_model", | |
| "depends_on": [ | |
| 2, | |
| 3 | |
| ], | |
| "cost": 0.1846 | |
| }, | |
| { | |
| "id": 5, | |
| "task": "Send the drafted email to the client's email address found in step 3, with the converted dimensions and order status included.", | |
| "assignee": "tool:email_send", | |
| "depends_on": [ | |
| 4, | |
| 3 | |
| ], | |
| "cost": 0.0 | |
| } | |
| ], | |
| "escalation_reason": null, | |
| "router_cost": 0.0585, | |
| "dispatch_total": 0.243, | |
| "baseline_total": 9.6015, | |
| "savings_pct": 97.5 | |
| }, | |
| { | |
| "id": "cell-001526", | |
| "difficulty": "t4_hard", | |
| "condition": "full", | |
| "request": "Autograder failing—fix without delaying grading deadlines.", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "task": "Read the autograder error logs and the student's code to extract the failing test names, error messages, and relevant code snippets.", | |
| "assignee": "tool:file_read", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 2, | |
| "task": "Search the codebase for the failing test names and related symbols to locate the test file(s) and implementation details.", | |
| "assignee": "tool:code_search", | |
| "depends_on": [ | |
| 1 | |
| ], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 3, | |
| "task": "Run the linter on the repository to surface style/type issues that could affect grading.", | |
| "assignee": "tool:lint_code", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 4, | |
| "task": "Run the test suite to reproduce the failure and capture the full test output and stack trace.", | |
| "assignee": "tool:run_tests", | |
| "depends_on": [], | |
| "cost": 0.0 | |
| }, | |
| { | |
| "id": 5, | |
| "task": "Produce a minimal, safe fix (code change + test case update) that resolves the failing tests and preserves correct behavior; include a short rationale and test plan.", | |
| "assignee": "escalate:frontier", | |
| "depends_on": [ | |
| 1, | |
| 2, | |
| 3, | |
| 4 | |
| ], | |
| "cost": 4.1535 | |
| }, | |
| { | |
| "id": 6, | |
| "task": "Open a pull request with the proposed fix, a clear title, and the fix description produced in step 5.", | |
| "assignee": "tool:create_pr", | |
| "depends_on": [ | |
| 5 | |
| ], | |
| "cost": 0.0 | |
| } | |
| ], | |
| "escalation_reason": "Designing a minimal, safe fix requires resolving tradeoffs between correctness, backward compatibility, and grading constraints, which needs deep, multi-constraint reasoning.", | |
| "router_cost": 0.0468, | |
| "dispatch_total": 4.2003, | |
| "baseline_total": 9.6427, | |
| "savings_pct": 56.4 | |
| } | |
| ] |