| window.DEMO_DATA = {"meta":{"benchmark":"GTA","tool_execution":"real","seed":42,"tau_pre":0.3,"tau_post":0.4,"exploration_constant":1.4,"max_rollouts":40,"trunc_chars":300},"cases":[{"id":"case1","title":"Case 1 \u2014 Wholesale catalogue: total price (OCR \u2192 Calculator)","query":"I want to buy all the items in the picture, how much should it cost?","steps":[{"type":"rollout","candidates":[{"tool":"OCR","r_pre":0.8,"status":"kept","args":{"image":"image/image_407.jpg"}},{"tool":"Calculator","r_pre":0.3,"status":"kept","args":{"expression":"sum(float(item) for item in ocr_results.split())"}}],"backprop":[{"node":"OCR","reward":0.7,"Q":0.7,"N":1},{"node":"root","reward":0.7,"Q":0.7,"N":1}],"selected":{"action":"root","Q":0.0,"N":0,"r_pre":0.0},"executed":{"tool":"OCR","args":{"image":"image/image_407.jpg"},"output":"(204, 66, 398, 98) C 0 MPANY\n(691, 42, 1186, 120) VARIOUS ITEMS\n(1193, 32, 1231, 177) L\n(242, 92, 400, 146) LOGO\n(913, 137, 1184, 170) Summer 2020 Collection\n(364, 256, 574, 280) DUMMY PRODUCT #1\n(954, 256, 1166, 280) DUMMY PRODUCT #2\n(362, 286, 480, 310) ID : 202256\n(954, 286, 1070, 310) ID: 202250","error":false,"truncated":true},"r_post":0.7,"tau_post":0.4,"post_pruned":false,"rollout":1,"best_q":0.7},{"type":"rollout","candidates":[],"backprop":[{"node":"Calculator","reward":0.0,"Q":0.0,"N":1},{"node":"root","reward":0.0,"Q":0.35,"N":2}],"selected":{"action":"Calculator","Q":0.0,"N":0,"r_pre":0.3},"realize":"Calculator","executed":{"tool":"Calculator","args":{"expression":"sum(float(item) for item in ocr_results.split())"},"output":"invocation_failed: Sympify of expression 'could not parse 'sum(float(item) for item in ocr_results.split())'' failed, because of exception being raised:\nSyntaxError: cannot assign to function call (<string>, line 1)","error":true,"truncated":false},"r_post":0.0,"tau_post":0.4,"post_pruned":true,"rollout":2,"best_q":0.7},{"type":"rollout","candidates":[{"tool":"OCR","r_pre":null,"status":"dedup","args":{"image":"image/image_407.jpg"}},{"tool":"Calculator","r_pre":0.9,"status":"kept","args":{"expression":"45.99 + 32.99 + 869.99 + 29.99 + 125.99 + 4.99 + 12.99 + 22.99"}}],"backprop":[{"node":"Calculator","reward":0.9,"Q":0.9,"N":1},{"node":"OCR","reward":0.9,"Q":0.8,"N":2},{"node":"root","reward":0.9,"Q":0.533333,"N":3}],"selected":{"action":"OCR","Q":0.7,"N":1,"r_pre":0.8},"executed":{"tool":"Calculator","args":{"expression":"45.99 + 32.99 + 869.99 + 29.99 + 125.99 + 4.99 + 12.99 + 22.99"},"output":"1145.92","error":false,"truncated":false},"r_post":0.9,"tau_post":0.4,"post_pruned":false,"rollout":3,"best_q":0.8},{"type":"terminal","event":"search_exhausted","rollouts_used":3,"exhausted":[{"node":"Calculator","depth":2},{"node":"OCR","depth":1},{"node":"root","depth":0}],"dedups":[{"tool":"Calculator","args":{"expression":"45.99 + 32.99 + 869.99 + 29.99 + 125.99 + 4.99 + 12.99 + 22.99"}},{"tool":"OCR","args":{"image":"image/image_407.jpg"}}]},{"type":"final","plan":[{"tool":"OCR","args":{"image":"image/image_407.jpg"}},{"tool":"Calculator","args":{"expression":"45.99 + 32.99 + 869.99 + 29.99 + 125.99 + 4.99 + 12.99 + 22.99"}}],"answer":"The total cost for all the items in the picture is $1,145.92.","reward":0.9,"rollouts_used":3}]},{"id":"case2","title":"Case 2 \u2014 Counting sheep: equal numbers? (CountGivenObject \u00d7 2)","query":"Are there equal numbers of small and big sheep?","steps":[{"type":"rollout","candidates":[{"tool":"Calculator","r_pre":0.0,"status":"pre_pruned","args":{"expression":"len([1, 2]) == len([3, 4])"}},{"tool":"TextToBbox","r_pre":0.8,"status":"kept","args":{"image":"image/image_260.jpg","text":"sheep","top1":false}},{"tool":"CountGivenObject","r_pre":0.5,"status":"kept","args":{"image":"image/image_260.jpg","text":"small sheep"}}],"backprop":[{"node":"TextToBbox","reward":0.0,"Q":0.0,"N":1},{"node":"root","reward":0.0,"Q":0.0,"N":1}],"selected":{"action":"root","Q":0.0,"N":0,"r_pre":0.0},"executed":{"tool":"TextToBbox","args":{"image":"image/image_260.jpg","text":"sheep","top1":false},"output":"invocation_failed: No real backend for tool: TextToBbox","error":true,"truncated":false},"r_post":0.0,"tau_post":0.4,"post_pruned":true,"rollout":1,"best_q":0.0},{"type":"rollout","candidates":[],"backprop":[{"node":"CountGivenObject","reward":0.5,"Q":0.5,"N":1},{"node":"root","reward":0.5,"Q":0.25,"N":2}],"selected":{"action":"CountGivenObject","Q":0.0,"N":0,"r_pre":0.5},"realize":"CountGivenObject","executed":{"tool":"CountGivenObject","args":{"image":"image/image_260.jpg","text":"small sheep"},"output":4,"error":false,"truncated":false},"r_post":0.5,"tau_post":0.4,"post_pruned":false,"rollout":2,"best_q":0.5},{"type":"rollout","candidates":[{"tool":"Calculator","r_pre":0.0,"status":"pre_pruned","args":{"expression":"4 == 4"}},{"tool":"CountGivenObject","r_pre":0.9,"status":"kept","args":{"image":"image/image_260.jpg","text":"big sheep"}},{"tool":"TextToBbox","r_pre":0.7,"status":"kept","args":{"image":"image/image_260.jpg","text":"big sheep"}}],"backprop":[{"node":"CountGivenObject","reward":1.0,"Q":1.0,"N":1},{"node":"CountGivenObject","reward":1.0,"Q":0.75,"N":2},{"node":"root","reward":1.0,"Q":0.5,"N":3}],"selected":{"action":"CountGivenObject","Q":0.5,"N":1,"r_pre":0.5},"executed":{"tool":"CountGivenObject","args":{"image":"image/image_260.jpg","text":"big sheep"},"output":4,"error":false,"truncated":false},"r_post":1.0,"tau_post":0.4,"post_pruned":false,"rollout":3,"best_q":0.75},{"type":"rollout","candidates":[],"backprop":[{"node":"TextToBbox","reward":0.0,"Q":0.0,"N":1},{"node":"CountGivenObject","reward":0.0,"Q":0.5,"N":3},{"node":"root","reward":0.0,"Q":0.375,"N":4}],"selected":{"action":"TextToBbox","Q":0.0,"N":0,"r_pre":0.7},"realize":"TextToBbox","executed":{"tool":"TextToBbox","args":{"image":"image/image_260.jpg","text":"big sheep"},"output":"invocation_failed: No real backend for tool: TextToBbox","error":true,"truncated":false},"r_post":0.0,"tau_post":0.4,"post_pruned":true,"rollout":4,"best_q":0.5},{"type":"rollout","candidates":[{"tool":"Calculator","r_pre":0.0,"status":"pre_pruned","args":{"expression":"4 == 4"}},{"tool":"CountGivenObject","r_pre":0.2,"status":"pre_pruned","args":{"image":"image/image_260.jpg","text":"small sheep","bbox":"(0, 0, 100, 100)"}},{"tool":"TextToBbox","r_pre":0.3,"status":"kept","args":{"image":"image/image_260.jpg","text":"small sheep and big sheep","top1":false}}],"backprop":[{"node":"TextToBbox","reward":0.0,"Q":0.0,"N":1},{"node":"CountGivenObject","reward":0.0,"Q":0.5,"N":2},{"node":"CountGivenObject","reward":0.0,"Q":0.375,"N":4},{"node":"root","reward":0.0,"Q":0.3,"N":5}],"selected":{"action":"CountGivenObject","Q":1.0,"N":1,"r_pre":0.9},"executed":{"tool":"TextToBbox","args":{"image":"image/image_260.jpg","text":"small sheep and big sheep","top1":false},"output":"invocation_failed: No real backend for tool: TextToBbox","error":true,"truncated":false},"r_post":0.0,"tau_post":0.4,"post_pruned":true,"rollout":5,"best_q":0.375},{"type":"terminal","event":"search_exhausted","rollouts_used":5,"exhausted":[{"node":"CountGivenObject","depth":2},{"node":"CountGivenObject","depth":1},{"node":"root","depth":0}],"dedups":[]},{"type":"final","plan":[{"tool":"CountGivenObject","args":{"image":"image/image_260.jpg","text":"small sheep"}},{"tool":"CountGivenObject","args":{"image":"image/image_260.jpg","text":"big sheep"}}],"answer":"Based on the tool outputs, there are equal numbers of small and big sheep, with 4 small sheep and 4 big sheep.","reward":0.5,"rollouts_used":5}]},{"id":"case3","title":"Case 3 \u2014 How far can the animal fly in 2.5 hours? (ImageDescription \u2192 GoogleSearch \u2192 Calculator)","query":"How far can the animal in the picture fly in 2.5hours at most?","steps":[{"type":"rollout","candidates":[{"tool":"ImageDescription","r_pre":0.8,"status":"kept","args":{"image":"image/image_395.jpg"}},{"tool":"GoogleSearch","r_pre":0.5,"status":"kept","args":{"query":"How far can the animal in the picture fly in 2.5 hours at most?","k":5}},{"tool":"Calculator","r_pre":0.3,"status":"kept","args":{"expression":"2.5 * 100"}}],"backprop":[{"node":"ImageDescription","reward":0.8,"Q":0.8,"N":1},{"node":"root","reward":0.8,"Q":0.8,"N":1}],"selected":{"action":"root","Q":0.0,"N":0,"r_pre":0.0},"executed":{"tool":"ImageDescription","args":{"image":"image/image_395.jpg"},"output":"a bald eagle flying over a lake","error":false,"truncated":false},"r_post":0.8,"tau_post":0.4,"post_pruned":false,"rollout":1,"best_q":0.8},{"type":"rollout","candidates":[],"backprop":[{"node":"Calculator","reward":0.5,"Q":0.5,"N":1},{"node":"root","reward":0.5,"Q":0.65,"N":2}],"selected":{"action":"Calculator","Q":0.0,"N":0,"r_pre":0.3},"realize":"Calculator","executed":{"tool":"Calculator","args":{"expression":"2.5 * 100"},"output":"250","error":false,"truncated":false},"r_post":0.5,"tau_post":0.4,"post_pruned":false,"rollout":2,"best_q":0.8},{"type":"rollout","candidates":[],"backprop":[{"node":"GoogleSearch","reward":0.5,"Q":0.5,"N":1},{"node":"root","reward":0.5,"Q":0.6,"N":3}],"selected":{"action":"GoogleSearch","Q":0.0,"N":0,"r_pre":0.5},"realize":"GoogleSearch","executed":{"tool":"GoogleSearch","args":{"query":"How far can the animal in the picture fly in 2.5 hours at most?","k":5},"output":"1. How Far Can a Drone Fly? Maximum Ranges and Legal Limits - JOUAV \u2014 March 26, 2026 - Depending on the setup, FPV drones can fly anywhere from 0.5 miles (0.8 kilometers) to 4 miles (6.5 kilometers), though many racers prefer closer ranges to minimize latency in video feedback.\n2. Bird - Flight, Fea","error":false,"truncated":true},"r_post":0.5,"tau_post":0.4,"post_pruned":false,"rollout":3,"best_q":0.8},{"type":"rollout","candidates":[{"tool":"ImageDescription","r_pre":null,"status":"dedup","args":{"image":"image/image_395.jpg"}},{"tool":"GoogleSearch","r_pre":0.8,"status":"kept","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?","k":5}},{"tool":"Calculator","r_pre":0.5,"status":"kept","args":{"expression":"2.5 * 150"}}],"backprop":[{"node":"GoogleSearch","reward":0.7,"Q":0.7,"N":1},{"node":"ImageDescription","reward":0.7,"Q":0.75,"N":2},{"node":"root","reward":0.7,"Q":0.625,"N":4}],"selected":{"action":"ImageDescription","Q":0.8,"N":1,"r_pre":0.8},"executed":{"tool":"GoogleSearch","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?","k":5},"output":"1. Bald eagle - Wikipedia \u2014 The bald eagle (Haliaeetus leucocephalus) is a bird of prey found in North America. A sea eagle, it has two known subspecies and forms a species pair with the white-tailed eagle (Haliaeetus albicilla), which occupies the same niche as the bald eagle in the Palearctic. Its","error":false,"truncated":true},"r_post":0.7,"tau_post":0.4,"post_pruned":false,"rollout":4,"best_q":0.75},{"type":"rollout","candidates":[],"backprop":[{"node":"Calculator","reward":0.5,"Q":0.5,"N":1},{"node":"ImageDescription","reward":0.5,"Q":0.666667,"N":3},{"node":"root","reward":0.5,"Q":0.6,"N":5}],"selected":{"action":"Calculator","Q":0.0,"N":0,"r_pre":0.5},"realize":"Calculator","executed":{"tool":"Calculator","args":{"expression":"2.5 * 150"},"output":"375","error":false,"truncated":false},"r_post":0.5,"tau_post":0.4,"post_pruned":false,"rollout":5,"best_q":0.666667},{"type":"rollout","candidates":[{"tool":"Calculator","r_pre":0.8,"status":"kept","args":{"expression":"2.5 * 45"}},{"tool":"ImageDescription","r_pre":0.8,"status":"kept","args":{"expression":"2.5 * 45"}},{"tool":"GoogleSearch","r_pre":0.5,"status":"kept","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?"}}],"backprop":[{"node":"Calculator","reward":0.9,"Q":0.9,"N":1},{"node":"GoogleSearch","reward":0.9,"Q":0.8,"N":2},{"node":"ImageDescription","reward":0.9,"Q":0.725,"N":4},{"node":"root","reward":0.9,"Q":0.65,"N":6}],"selected":{"action":"GoogleSearch","Q":0.7,"N":1,"r_pre":0.8},"executed":{"tool":"Calculator","args":{"expression":"2.5 * 45"},"output":"112.5","error":false,"truncated":false},"r_post":0.9,"tau_post":0.4,"post_pruned":false,"rollout":6,"best_q":0.725},{"type":"rollout","candidates":[],"backprop":[{"node":"GoogleSearch","reward":0.7,"Q":0.7,"N":1},{"node":"GoogleSearch","reward":0.7,"Q":0.766667,"N":3},{"node":"ImageDescription","reward":0.7,"Q":0.72,"N":5},{"node":"root","reward":0.7,"Q":0.657143,"N":7}],"selected":{"action":"GoogleSearch","Q":0.0,"N":0,"r_pre":0.5},"realize":"GoogleSearch","executed":{"tool":"GoogleSearch","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?"},"output":"1. Bald Eagle Behavior | American Eagle Foundation \u2014 How fast can a bald eagle fly? Average migratory flight / soaring speed is 50 kilometers/hour (31.06 mph). Normal flight speed going from perch to perch is much less than this.\n2. Bald Eagle | Audubon Field Guide \u2014 The Bald Eagle nest is one of th","error":false,"truncated":true},"r_post":0.7,"tau_post":0.4,"post_pruned":false,"rollout":7,"best_q":0.72},{"type":"rollout","candidates":[{"tool":"GoogleSearch","r_pre":null,"status":"dedup","args":{"query":"How far can the animal in the picture fly in 2.5 hours at most?","k":5}},{"tool":"ImageDescription","r_pre":0.2,"status":"pre_pruned","args":{"image":"image/image_395.jpg"}},{"tool":"Calculator","r_pre":0.3,"status":"kept","args":{"expression":"2.5 * 30"}}],"backprop":[{"node":"Calculator","reward":0.5,"Q":0.5,"N":1},{"node":"GoogleSearch","reward":0.5,"Q":0.5,"N":2},{"node":"root","reward":0.5,"Q":0.6375,"N":8}],"selected":{"action":"GoogleSearch","Q":0.5,"N":1,"r_pre":0.5},"executed":{"tool":"Calculator","args":{"expression":"2.5 * 30"},"output":"75","error":false,"truncated":false},"r_post":0.5,"tau_post":0.4,"post_pruned":false,"rollout":8,"best_q":0.72},{"type":"rollout","candidates":[],"backprop":[{"node":"ImageDescription","reward":0.0,"Q":0.0,"N":1},{"node":"GoogleSearch","reward":0.0,"Q":0.575,"N":4},{"node":"ImageDescription","reward":0.0,"Q":0.6,"N":6},{"node":"root","reward":0.0,"Q":0.566667,"N":9}],"selected":{"action":"ImageDescription","Q":0.0,"N":0,"r_pre":0.8},"realize":"ImageDescription","executed":{"tool":"ImageDescription","args":{"expression":"2.5 * 45"},"output":"invocation_failed: Incorrect image source. Must be a valid URL starting with `http://` or `https://`, a valid path to an image file, or a base64 encoded string. Got <local dataset dir>. Failed with Incorrect padding","error":true,"truncated":false,"redacted":true},"r_post":0.0,"tau_post":0.4,"post_pruned":true,"rollout":9,"best_q":0.6},{"type":"rollout","candidates":[{"tool":"Calculator","r_pre":null,"status":"dedup","args":{"expression":"2.5 * 150"}},{"tool":"ImageDescription","r_pre":null,"status":"dedup","args":{"image":"image/image_395.jpg"}},{"tool":"GoogleSearch","r_pre":0.7,"status":"kept","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?","k":5}}],"backprop":[{"node":"GoogleSearch","reward":0.7,"Q":0.7,"N":1},{"node":"Calculator","reward":0.7,"Q":0.6,"N":2},{"node":"ImageDescription","reward":0.7,"Q":0.614286,"N":7},{"node":"root","reward":0.7,"Q":0.58,"N":10}],"selected":{"action":"Calculator","Q":0.5,"N":1,"r_pre":0.5},"executed":{"tool":"GoogleSearch","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?","k":5},"output":"1. Bald eagle - Wikipedia \u2014 The bald eagle (Haliaeetus leucocephalus) is a bird of prey found in North America. A sea eagle, it has two known subspecies and forms a species pair with the white-tailed eagle (Haliaeetus albicilla), which occupies the same niche as the bald eagle in the Palearctic. Its","error":false,"truncated":true},"r_post":0.7,"tau_post":0.4,"post_pruned":false,"rollout":10,"best_q":0.614286},{"type":"rollout","candidates":[{"tool":"GoogleSearch","r_pre":null,"status":"dedup","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?"}},{"tool":"ImageDescription","r_pre":null,"status":"dedup","args":{"image":"image/image_395.jpg"}},{"tool":"Calculator","r_pre":0.8,"status":"kept","args":{"expression":"2.5 * 45"}}],"backprop":[{"node":"Calculator","reward":0.8,"Q":0.8,"N":1},{"node":"GoogleSearch","reward":0.8,"Q":0.75,"N":2},{"node":"GoogleSearch","reward":0.8,"Q":0.62,"N":5},{"node":"ImageDescription","reward":0.8,"Q":0.6375,"N":8},{"node":"root","reward":0.8,"Q":0.6,"N":11}],"selected":{"action":"GoogleSearch","Q":0.7,"N":1,"r_pre":0.5},"executed":{"tool":"Calculator","args":{"expression":"2.5 * 45"},"output":"112.5","error":false,"truncated":false},"r_post":0.8,"tau_post":0.4,"post_pruned":false,"rollout":11,"best_q":0.6375,"discovery":[{"candidates":[{"tool":"Calculator","r_pre":null,"status":"dedup","args":{"expression":"2.5 * 45"}},{"tool":"ImageDescription","r_pre":null,"status":"dedup","args":{"image":"image/image_395.jpg"}},{"tool":"GoogleSearch","r_pre":0.2,"status":"pre_pruned","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?"}}],"selected":{"action":"Calculator","Q":0.9,"N":1,"r_pre":0.8},"exhausted":{"node":"Calculator","depth":3}}]},{"type":"terminal","event":"early_stop","rollouts_used":11,"exhausted":[{"node":"Calculator","depth":3}],"dedups":[],"baseline":0.8,"best_recent":0.8,"improvement":0.0,"delta":0.001,"patience":10},{"type":"final","plan":[{"tool":"ImageDescription","args":{"image":"image/image_395.jpg"}},{"tool":"GoogleSearch","args":{"query":"How far can a bald eagle fly in 2.5 hours at most?","k":5}},{"tool":"Calculator","args":{"expression":"2.5 * 45"}}],"answer":"The animal in the picture is a bald eagle. Based on the information provided, a bald eagle can fly up to 45 miles per hour while gliding. Therefore, in 2.5 hours, the maximum distance it can fly is 112.5 miles.","reward":0.9,"rollouts_used":11}]}]}; | |