{ "ID_train": { "episodes": [ 0, 209, 315, 458, 595, 721, 822, 974 ], "summary": { "S": { "n": 45, "emit_rate": 1.0, "iou_mean": 0.11796170461997797, "iou_median": 0.030150753768844653, "iou>0.25": 0.2222222222222222, "iou>0.5": 0.044444444444444446, "cdist_median_px": 9.179921386246992, "cdist_mean_px": 11.049622307725615 }, "A_kf-8": { "n": 39, "emit_rate": 0.9743589743589743, "iou_mean": 0.10797293058707275, "iou_median": 0.0, "iou>0.25": 0.18421052631578946, "iou>0.5": 0.05263157894736842, "cdist_median_px": 15.063482709525058, "cdist_mean_px": 18.698789657927637 }, "A_kf": { "n": 45, "emit_rate": 0.9777777777777777, "iou_mean": 0.07213438167319552, "iou_median": 0.0, "iou>0.25": 0.09090909090909091, "iou>0.5": 0.022727272727272728, "cdist_median_px": 13.80580224681427, "cdist_mean_px": 16.87872355008344 } }, "records": { "S": [ { "ep": 0, "frame_abs": 0, "t_rel": 0, "emitted": true, "iou": 0.012234910277324618, "cdist_px": 18.317168247646524, "box": [ 0.12218963831867058, 0.44868035190615835, 0.18572825024437928, 0.4750733137829912 ], "gt_box": [ 0.14173998044965788, 0.426197458455523, 0.22189638318670576, 0.4496578690127077 ], "gen_text": "The orange cup sits on the table directly below the gripper; the gripper is open and positioned low over the table surface, with the orange cup visible just below the pads. @grip_now is open and converging." }, { "ep": 0, "frame_abs": 40, "t_rel": 40, "emitted": true, "iou": 0.030281690140845058, "cdist_px": 9.179921386246992, "box": [ 0.34408602150537637, 0.581622678396872, 0.386119257086999, 0.5992179863147605 ], "gt_box": [ 0.34310850439882695, 0.5698924731182796, 0.3949169110459433, 0.5826001955034213 ], "gen_text": "The orange cup sits on the table directly in front of the white cup; the gripper is open and positioned directly above the cup, with the fingers converging around its rim; @grip_now shows the fingers wide apart in a pre-grasp configuration." }, { "ep": 0, "frame_abs": 80, "t_rel": 80, "emitted": true, "iou": 0.2794561933534743, "cdist_px": 3.4811139588058007, "box": [ 0.4780058651026393, 0.6207233626588465, 0.5141739980449658, 0.6314760508308895 ], "gt_box": [ 0.4760508308895406, 0.6265884652981427, 0.5190615835777126, 0.6363636363636364 ], "gen_text": "The gripper is open and positioned directly above the orange cup, which is now directly below the open fingers; the white cup sits on the table to the right of the gripper, and the orange cup sits to the left. The gripper is open and converging on the orange cup, at @grip_now ." }, { "ep": 0, "frame_abs": 120, "t_rel": 120, "emitted": true, "iou": 0.6896551724137915, "cdist_px": 1.0557184750733484, "box": [ 0.5679374389051809, 0.6089931573802542, 0.5992179863147605, 0.6207233626588465 ], "gt_box": [ 0.5610948191593352, 0.6099706744868035, 0.6001955034213099, 0.6197458455522972 ], "gen_text": "The gripper is holding the orange cup and moving it laterally over the white cup; the wrist view shows the cup's rim and rim just above the cup's rim. The gripper is closed around the cup at @grip_now ." }, { "ep": 0, "frame_abs": 122, "t_rel": 122, "emitted": true, "iou": 0.4490566037735849, "cdist_px": 2.2999712484618104, "box": [ 0.5591397849462365, 0.6109481915933529, 0.592375366568915, 0.6226783968719453 ], "gt_box": [ 0.5581622678396871, 0.6089931573802542, 0.5972629521016618, 0.6177908113391984 ], "gen_text": "The gripper is holding the orange cup and moving it laterally over the white cup on the table; the wrist view shows the cup's rim and rim just above the cup's rim. The gripper is closed around the cup at @grip_now ." }, { "ep": 0, "frame_abs": 162, "t_rel": 162, "emitted": true, "iou": 0.0, "cdist_px": 10.502464900709478, "box": [ 0.4780058651026393, 0.5425219941348973, 0.5063538611925709, 0.5532746823069403 ], "gt_box": [ 0.46920821114369504, 0.5268817204301075, 0.5043988269794721, 0.5366568914956011 ], "gen_text": "The gripper is holding the orange cup and moving it laterally over the white cup; the wrist view shows the cup's rim and rim just above the cup's rim. The gripper is closed around the cup at @grip_now ." }, { "ep": 209, "frame_abs": 58316, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 22.372229667879044, "box": [ 0.2883675464320626, 0.6295210166177908, 0.31671554252199413, 0.64613880742913 ], "gt_box": [ 0.34017595307917886, 0.6432062561094819, 0.37927663734115347, 0.6598240469208211 ], "gen_text": "The white cloth bag has been deposited on the black table; the gripper is open and moving downward toward the white cloth bag on the table, still open at @grip_now ." }, { "ep": 209, "frame_abs": 58356, "t_rel": 40, "emitted": true, "iou": 0.3052516411378555, "cdist_px": 4.809384164222871, "box": [ 0.5131964809384164, 0.5004887585532747, 0.5434995112414467, 0.5161290322580645 ], "gt_box": [ 0.5112414467253177, 0.49266862170087977, 0.5513196480938416, 0.509286412512219 ], "gen_text": "The white cloth bag has been placed on the table; the gripper is open and descending toward the white cloth bag which sits on the table surface to the right of the bag; the wrist camera shows the bag's white fabric filling the field of view. @grip_now is wide open." }, { "ep": 209, "frame_abs": 58396, "t_rel": 80, "emitted": true, "iou": 0.04034582132564831, "cdist_px": 14.182516737632799, "box": [ 0.7047898338220919, 0.27370478983382207, 0.7341153470185728, 0.29423264907135877 ], "gt_box": [ 0.6725317693059628, 0.2541544477028348, 0.718475073313783, 0.2785923753665689 ], "gen_text": "The gripper is open and moving downward toward the white cloth bag on the black table; the white bag with pink cloth and the green cloth are visible to the right. The gripper is open at @grip_now ." }, { "ep": 209, "frame_abs": 58436, "t_rel": 120, "emitted": true, "iou": 0.0, "cdist_px": 28.294856993452836, "box": [ 0.7888563049853372, 0.23069403714565004, 0.8162267839687195, 0.25219941348973607 ], "gt_box": [ 0.8084066471163245, 0.187683284457478, 0.8504398826979472, 0.21212121212121213 ], "gen_text": "The gripper is open and descending toward the white cloth bag on the black table, with the white cloth bag visible below and the pink cloth nearby; the gripper is open at @grip_now ." }, { "ep": 209, "frame_abs": 58460, "t_rel": 144, "emitted": true, "iou": 0.0, "cdist_px": 28.742281085271756, "box": [ 0.7888563049853372, 0.23362658846529813, 0.8113391984359726, 0.2561094819159335 ], "gt_box": [ 0.8054740957966764, 0.1906158357771261, 0.8406647116324536, 0.2130987292277615 ], "gen_text": "The gripper is descending toward the white cloth bag on the table, with the wrist camera showing the cloth fabric and the bag edge; the gripper is open and approaching the cloth. @grip_now shows a wide-open approach configuration." }, { "ep": 209, "frame_abs": 58500, "t_rel": 184, "emitted": true, "iou": 0.0, "cdist_px": 24.24325196455312, "box": [ 0.260019550342131, 0.36852394916911047, 0.3020527859237537, 0.38807429130009774 ], "gt_box": [ 0.2678396871945259, 0.3313782991202346, 0.3206256109481916, 0.3509286412512219 ], "gen_text": "The gripper is open and descending toward the white cloth bag on the table; the wrist camera shows the cloth fabric filling the gripper's field of view. The pile of white and pink cloths is visible below. @grip_now is open and descending." }, { "ep": 315, "frame_abs": 111797, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 19.928042634671698, "box": [ 0.11925708699902249, 0.38025415444770283, 0.16911045943304007, 0.3998044965786901 ], "gt_box": [ 0.08797653958944282, 0.35288367546432065, 0.14760508308895406, 0.3724340175953079 ], "gen_text": "The robot arm has swung forward and downward, positioning the open gripper above the table surface near the yellow-green mug; the wrist camera shows the mug's rim and the mug body directly below the open gripper. The gripper is open and converging on the mug at @grip_now ." }, { "ep": 315, "frame_abs": 111837, "t_rel": 40, "emitted": true, "iou": 0.20192307692307718, "cdist_px": 6.803547151097615, "box": [ 0.3313782991202346, 0.447702834799609, 0.36070381231671556, 0.4633431085043988 ], "gt_box": [ 0.34017595307917886, 0.4418377321603128, 0.38220918866080156, 0.45650048875855326 ], "gen_text": "The gripper is open and positioned above the table, with the yellow-green mug visible below the open fingers; the arm is repositioning to align with the mug's cylindrical body in preparation for release. @grip_now is open." }, { "ep": 315, "frame_abs": 111862, "t_rel": 65, "emitted": true, "iou": 0.34146341463414637, "cdist_px": 3.0591131955532522, "box": [ 0.4868035190615836, 0.4496578690127077, 0.5180840664711632, 0.45943304007820135 ], "gt_box": [ 0.49071358748778104, 0.45356793743890517, 0.5239491691104594, 0.4633431085043988 ], "gen_text": "The gripper is holding the yellow-green marker pen firmly at the table surface; the fingers are beginning to close around the marker barrel at the table surface, @grip_now in a partially closed hold on the marker barrel." }, { "ep": 315, "frame_abs": 111900, "t_rel": 103, "emitted": true, "iou": 0.34977578475336324, "cdist_px": 3.041822144872222, "box": [ 0.4173998044965787, 0.5024437927663734, 0.4467253176930596, 0.5131964809384164 ], "gt_box": [ 0.42130987292277616, 0.5073313782991202, 0.45454545454545453, 0.5151515151515151 ], "gen_text": "The gripper is descending over the table surface near the centre of the black tray, with the orange-labeled plastic water bottle visible in the wrist view below; the gripper is nearly closed around the bottle body, @grip_now at its configuration." }, { "ep": 458, "frame_abs": 160966, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 10.993899837053345, "box": [ 0.6217008797653959, 0.4173998044965787, 0.6549364613880743, 0.4252199413489736 ], "gt_box": [ 0.6314760508308895, 0.4310850439882698, 0.6842619745845552, 0.43792766373411535 ], "gen_text": "The black bag sits on the table center-right; the gripper is open and repositioning above the bag, with the wrist camera showing the bag's black body and the table surface below, at @grip_now ." }, { "ep": 458, "frame_abs": 161006, "t_rel": 40, "emitted": true, "iou": 0.32019704433497387, "cdist_px": 3.850970792491152, "box": [ 0.7204301075268817, 0.33822091886608013, 0.7517106549364614, 0.3460410557184751 ], "gt_box": [ 0.7067448680351907, 0.3362658846529814, 0.7458455522971652, 0.34310850439882695 ], "gen_text": "The gripper is now low and close to the black bag, with the fingers nearly closed around the bag's body; the wrist camera shows the bag's flat face and the table surface below. @grip_now shows the pads converging on the bag." }, { "ep": 458, "frame_abs": 161046, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 10.549524754459588, "box": [ 0.8172043010752689, 0.3069403714565005, 0.8406647116324536, 0.3176930596285435 ], "gt_box": [ 0.8406647116324536, 0.30498533724340177, 0.8748778103616813, 0.31378299120234604 ], "gen_text": "The gripper is open and descending toward the black bag; the wrist view shows the bag's black body and the table surface below, with the white cloth and red mug visible in the background; the gripper is open and converging on the black bag. @grip_now shows the open converging configuration." }, { "ep": 458, "frame_abs": 161068, "t_rel": 102, "emitted": true, "iou": 0.13986013986013987, "cdist_px": 6.750843825943232, "box": [ 0.8787878787878788, 0.3235581622678397, 0.9071358748778103, 0.3343108504398827 ], "gt_box": [ 0.8914956011730205, 0.3196480938416422, 0.927663734115347, 0.3284457478005865 ], "gen_text": "The gripper is closing around the black bag's body, with the bag's black fabric visible between the pads; the wrist camera shows the bag's flat face and the bag's white interior. The gripper is nearly fully closed, @grip_now in a closed hold on the bag." }, { "ep": 458, "frame_abs": 161108, "t_rel": 142, "emitted": true, "iou": 0.33644859813084116, "cdist_px": 4.462662691033364, "box": [ 0.8172043010752689, 0.2903225806451613, 0.8514173998044966, 0.30303030303030304 ], "gt_box": [ 0.8240469208211144, 0.28641251221896386, 0.8651026392961877, 0.2991202346041056 ], "gen_text": "The gripper is descending toward the black bag with fingers partially closed, approaching the bag's body from above; the wrist view shows the bag's interior filling the frame. The gripper is open and converging on the bag at @grip_now ." }, { "ep": 458, "frame_abs": 161148, "t_rel": 182, "emitted": true, "iou": 0.031105990783410038, "cdist_px": 12.144791494955532, "box": [ 0.6940371456500489, 0.44868035190615835, 0.7282502443792767, 0.4662756598240469 ], "gt_box": [ 0.6578690127077224, 0.44281524926686217, 0.696969696969697, 0.47116324535679377 ], "gen_text": "The gripper is descending over the black bag, with the fingers beginning to close around the bag's body; the wrist view shows the bag's interior filling the field of view. The gripper is nearly closed, @grip_now at its configuration." }, { "ep": 595, "frame_abs": 203818, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 26.10757695069303, "box": [ 0.41837732160312807, 0.34017595307917886, 0.4398826979472141, 0.3597262952101662 ], "gt_box": [ 0.4496578690127077, 0.3069403714565005, 0.48484848484848486, 0.3235581622678397 ], "gen_text": "The gripper is open and repositioning above the table surface, now positioned above the near-left area near the pink cloth; the red cup and white bowl remain in the background; the gripper is open at @grip_now ." }, { "ep": 595, "frame_abs": 203858, "t_rel": 40, "emitted": true, "iou": 0.1862068965517241, "cdist_px": 8.497812080544039, "box": [ 0.6168132942326491, 0.520039100684262, 0.6441837732160313, 0.5395894428152492 ], "gt_box": [ 0.6207233626588465, 0.530791788856305, 0.6539589442815249, 0.5542521994134897 ], "gen_text": "The gripper is open and repositioning above the table surface near the white cloth and black bag; the wrist view shows the open gripper above the bag, with the wrist camera showing the bag's contents. @grip_now is open." }, { "ep": 595, "frame_abs": 203898, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 14.475929342079798, "box": [ 0.7605083088954057, 0.6656891495601173, 0.7859237536656891, 0.6862170087976539 ], "gt_box": [ 0.7634408602150538, 0.6871945259042033, 0.7917888563049853, 0.7096774193548387 ], "gen_text": "The gripper is open and repositioning above the table surface near the toy food items and the small toy box; the wrist camera shows the stuffed toy box and the small green block directly below the open pads; the gripper is open and repositioning. @grip_now is open." }, { "ep": 595, "frame_abs": 203917, "t_rel": 99, "emitted": true, "iou": 0.0, "cdist_px": 18.18447725434211, "box": [ 0.7605083088954057, 0.6891495601173021, 0.7859237536656891, 0.7096774193548387 ], "gt_box": [ 0.7634408602150538, 0.7174975562072337, 0.7898338220918866, 0.7380254154447703 ], "gen_text": "The gripper is descending toward the small green-and-white decorative garland on the table; the wrist camera shows the toy bag and white cloth visible below; the gripper is open and approaching the cloth. @grip_now is open converging on the grasp." }, { "ep": 595, "frame_abs": 203957, "t_rel": 139, "emitted": true, "iou": 0.0, "cdist_px": 13.321402860350297, "box": [ 0.4144672531769306, 0.5738025415444771, 0.436950146627566, 0.5904203323558163 ], "gt_box": [ 0.37732160312805474, 0.5777126099706745, 0.40175953079178883, 0.5953079178885631 ], "gen_text": "The gripper is holding the small orange-labeled candy and has moved it to a position above and near the white mug on the table; the wrist view shows the candy near the mug rim and the mug body below. The gripper is closed around the candy at @grip_now ." }, { "ep": 595, "frame_abs": 203984, "t_rel": 166, "emitted": true, "iou": 0.0, "cdist_px": 14.108846793184835, "box": [ 0.3939393939393939, 0.5826001955034213, 0.4125122189638319, 0.5972629521016618 ], "gt_box": [ 0.3587487781036168, 0.5728250244379277, 0.37927663734115347, 0.5855327468230694 ], "gen_text": "The gripper is open and repositioning above the clear plastic container on the table, with the green marker visible inside; the wrist view shows the container lid and the marker inside; the gripper is open at @grip_now ." }, { "ep": 721, "frame_abs": 246200, "t_rel": 0, "emitted": true, "iou": 0.13852813852813844, "cdist_px": 14.14949174158794, "box": [ 0.19452590420332355, 0.26881720430107525, 0.24144672531769307, 0.29423264907135877 ], "gt_box": [ 0.20234604105571846, 0.25024437927663734, 0.2873900293255132, 0.2805474095796676 ], "gen_text": "The gripper is open and positioned above the table, hovering just above the blue notebook/drawer area with the white paper towel roll visible below the open gripper at @grip_now ." }, { "ep": 721, "frame_abs": 246240, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 8.317906676941812, "box": [ 0.4682306940371456, 0.5004887585532747, 0.5004887585532747, 0.509286412512219 ], "gt_box": [ 0.4780058651026393, 0.4887585532746823, 0.5190615835777126, 0.5004887585532747 ], "gen_text": "The gripper is now very close to the blue notebook on the white surface, with fingers nearly touching the notebook's cover; the wrist view shows the notebook's edge filling the field with the gripper pads, at @grip_now ." }, { "ep": 721, "frame_abs": 246280, "t_rel": 80, "emitted": true, "iou": 0.030150753768844653, "cdist_px": 9.004052221702503, "box": [ 0.6871945259042033, 0.5180840664711632, 0.7116324535679375, 0.5288367546432062 ], "gt_box": [ 0.6735092864125122, 0.5043988269794721, 0.6930596285434996, 0.5210166177908113 ], "gen_text": "The gripper is now very close to the blue book on the table, with the open fingers nearly touching the book's cover; the gripper is open and nearly touching the book, at @grip_now ." }, { "ep": 721, "frame_abs": 246320, "t_rel": 120, "emitted": true, "iou": 0.10843373493975901, "cdist_px": 5.72154048057566, "box": [ 0.6959921798631477, 0.5112414467253177, 0.7194525904203324, 0.5190615835777126 ], "gt_box": [ 0.6901270772238515, 0.4995112414467253, 0.7135874877810362, 0.5141739980449658 ], "gen_text": "The gripper is now very close to the blue book on the table, with the open fingers nearly touching the book's cover; the gripper is open and nearly touching the book, at @grip_now ." }, { "ep": 721, "frame_abs": 246360, "t_rel": 160, "emitted": true, "iou": 0.0, "cdist_px": 12.200682316150465, "box": [ 0.6529814271749755, 0.5180840664711632, 0.6774193548387096, 0.5278592375366569 ], "gt_box": [ 0.6529814271749755, 0.4965786901270772, 0.6764418377321603, 0.5112414467253177 ], "gen_text": "The gripper is now very close to the book's cover, with the fingers nearly touching the spine; the gripper is open and nearly in contact with the book's spine, at @grip_now ." }, { "ep": 721, "frame_abs": 246400, "t_rel": 200, "emitted": true, "iou": 0.15584415584415556, "cdist_px": 8.070882288355085, "box": [ 0.5454545454545454, 0.4232649071358749, 0.5786901270772239, 0.4310850439882698 ], "gt_box": [ 0.5650048875855328, 0.4252199413489736, 0.603128054740958, 0.4340175953079179 ], "gen_text": "The gripper is now very close to the white paper-towel on the table surface, with the fingers nearly touching the paper; the gripper is open and nearly in contact with the paper, @grip_now open and converging on the paper-towel." }, { "ep": 822, "frame_abs": 269794, "t_rel": 0, "emitted": true, "iou": 0.652777777777778, "cdist_px": 2.7124294322280136, "box": [ 0.386119257086999, 0.3675464320625611, 0.43206256109481916, 0.3831867057673509 ], "gt_box": [ 0.38514173998044965, 0.3675464320625611, 0.447702834799609, 0.38514173998044965 ], "gen_text": "The gripper is now low and close to the toaster, with the toaster's lever and lever visible between the pads; the red mug and red bowl of coffee beans are to the right of the toaster. The gripper is open and converging on the toaster, at @grip_now ." }, { "ep": 822, "frame_abs": 269834, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 14.382794869799701, "box": [ 0.4750733137829912, 0.6441837732160313, 0.5141739980449658, 0.6510263929618768 ], "gt_box": [ 0.45454545454545453, 0.6647116324535679, 0.49853372434017595, 0.6705767350928641 ], "gen_text": "The gripper is now directly over the toaster's side lever, fingers now open and converging on the lever at @grip_now ; the toaster remains stationary on the table." }, { "ep": 822, "frame_abs": 269843, "t_rel": 49, "emitted": true, "iou": 0.0, "cdist_px": 11.108052963239516, "box": [ 0.4946236559139785, 0.64613880742913, 0.533724340175953, 0.656891495601173 ], "gt_box": [ 0.47214076246334313, 0.6598240469208211, 0.5141739980449658, 0.6686217008797654 ], "gen_text": "The gripper is now very close to the toaster's side lever, with the pads nearly seated on the toaster's side panel; the toaster's silver body is directly below the gripper, and the red bowl of coffee beans is visible to the right. The gripper is nearly fully closed, @grip_now showing a partially closed contact configuration." }, { "ep": 822, "frame_abs": 269883, "t_rel": 89, "emitted": true, "iou": 0.0, "cdist_px": 9.636236512687766, "box": [ 0.603128054740958, 0.6236559139784946, 0.6402737047898338, 0.6314760508308895 ], "gt_box": [ 0.5874877810361682, 0.6363636363636364, 0.624633431085044, 0.6432062561094819 ], "gen_text": "The gripper is now very close to the toaster's side lever, with the finger pads nearly seated on the toaster's side panel, at @grip_now ; the toaster remains on the table surface directly below the gripper." }, { "ep": 822, "frame_abs": 269892, "t_rel": 98, "emitted": true, "iou": 0.0, "cdist_px": 8.88747457762684, "box": [ 0.5738025415444771, 0.6089931573802542, 0.6080156402737048, 0.6177908113391984 ], "gt_box": [ 0.5610948191593352, 0.6226783968719453, 0.5972629521016618, 0.6285434995112414 ], "gen_text": "The gripper is now very close to the toaster's side lever, with the finger pads nearly seated on the toaster's side lever; the toaster remains on the table surface and the red bowl of coffee beans is visible to the right. The gripper is nearly fully closed, @grip_now in a partially closed, seated configuration around the toaster's side lever." }, { "ep": 974, "frame_abs": 303932, "t_rel": 0, "emitted": true, "iou": 0.266106442577031, "cdist_px": 7.5148880589928755, "box": [ 0.36656891495601174, 0.29521016617790813, 0.4125122189638319, 0.31085043988269795 ], "gt_box": [ 0.375366568914956, 0.3010752688172043, 0.4398826979472141, 0.31671554252199413 ], "gen_text": "The blue chip bag has been set down on the tray; the gripper is open and positioned above the tray, with the blue chip bag visible to the left. The gripper is open and converging on the blue chip bag at @grip_now ." }, { "ep": 974, "frame_abs": 303972, "t_rel": 40, "emitted": true, "iou": 0.14224137931034475, "cdist_px": 9.013195407530125, "box": [ 0.4173998044965787, 0.34701857282502446, 0.4613880742913001, 0.3616813294232649 ], "gt_box": [ 0.4291300097751711, 0.35581622678396874, 0.4887585532746823, 0.3704789833822092 ], "gen_text": "The blue chip bag has been placed on the tray; the gripper is open and retracting upward away from the tray, with the blue chip bag still in place on the tray. @grip_now at its configuration." }, { "ep": 974, "frame_abs": 304012, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 10.854243606883378, "box": [ 0.5992179863147605, 0.36070381231671556, 0.6324535679374389, 0.36852394916911047 ], "gt_box": [ 0.6080156402737048, 0.375366568914956, 0.6539589442815249, 0.3831867057673509 ], "gen_text": "The gripper is open and positioned above the tray surface, hovering just above the blue chip bag; the wrist camera shows the bag directly below the gripper fingers. The gripper is open and converging on the blue chip bag. @grip_now at its configuration." }, { "ep": 974, "frame_abs": 304052, "t_rel": 120, "emitted": true, "iou": 0.04042179261862926, "cdist_px": 6.591684226904857, "box": [ 0.6705767350928641, 0.386119257086999, 0.7038123167155426, 0.3939393939393939 ], "gt_box": [ 0.6539589442815249, 0.39296187683284456, 0.6930596285434996, 0.40078201368523947 ], "gen_text": "The gripper is now very close to the tray surface, hovering just above the blue chip bag; the gripper is open and nearly touching the bag, @grip_now in a converging approach configuration around the bag's body." }, { "ep": 974, "frame_abs": 304058, "t_rel": 126, "emitted": true, "iou": 0.06050955414012757, "cdist_px": 7.700750470984123, "box": [ 0.6842619745845552, 0.37732160312805474, 0.7165200391006843, 0.386119257086999 ], "gt_box": [ 0.6627565982404692, 0.3841642228739003, 0.7028347996089932, 0.39296187683284456 ], "gen_text": "The gripper is closing around the blue chip bag, which is now seated between the pads; the gripper is mid-close, @grip_now in a partially closed configuration around the bag." }, { "ep": 974, "frame_abs": 304098, "t_rel": 166, "emitted": true, "iou": 0.0, "cdist_px": 7.6045553621805775, "box": [ 0.581622678396872, 0.34408602150537637, 0.6158357771260997, 0.35288367546432065 ], "gt_box": [ 0.5874877810361682, 0.3333333333333333, 0.6236559139784946, 0.3411534701857282 ], "gen_text": "The gripper is closing around the blue chip bag, which is now seated on the tray surface; the gripper is mid-close, @grip_now at its configuration." } ], "A_kf-8": [ { "ep": 0, "frame_abs": 32, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 20.24178073629091, "box": [ 0.3548387096774194, 0.5992179863147605, 0.3998044965786901, 0.6158357771260997 ], "gt_box": [ 0.34310850439882695, 0.5698924731182796, 0.3949169110459433, 0.5826001955034213 ], "gen_text": "Descend further and swing the nose toward the orange cup's rim, keeping the gripper open @grip and aligning the pads to straddle the cup rim." }, { "ep": 0, "frame_abs": 72, "t_rel": 72, "emitted": true, "iou": 0.35520000000000007, "cdist_px": 2.949511075819388, "box": [ 0.48191593352883677, 0.6217008797653959, 0.5180840664711632, 0.6324535679374389 ], "gt_box": [ 0.4760508308895406, 0.6265884652981427, 0.5190615835777126, 0.6363636363636364 ], "gen_text": "Descend the last small distance to seat the pads around the orange cup's rim, then close the gripper @grip firmly around the cup's rim." }, { "ep": 0, "frame_abs": 112, "t_rel": 112, "emitted": true, "iou": 0.27027027027026973, "cdist_px": 3.4408602150537604, "box": [ 0.5650048875855328, 0.6148582600195504, 0.5962854349951124, 0.6256109481915934 ], "gt_box": [ 0.5610948191593352, 0.6099706744868035, 0.6001955034213099, 0.6197458455522972 ], "gen_text": "Close the gripper firmly around the orange cup's rim to secure a stable grasp, then begin lifting upward @grip ." }, { "ep": 0, "frame_abs": 114, "t_rel": 114, "emitted": true, "iou": 0.15584415584415556, "cdist_px": 4.504731442047713, "box": [ 0.5650048875855328, 0.6011730205278593, 0.5962854349951124, 0.6119257086999023 ], "gt_box": [ 0.5581622678396871, 0.6089931573802542, 0.5972629521016618, 0.6177908113391984 ], "gen_text": "Close the gripper firmly around the orange cup's rim to secure a stable grasp, then begin lifting upward @grip ." }, { "ep": 0, "frame_abs": 154, "t_rel": 154, "emitted": true, "iou": 0.0, "cdist_px": 17.002021099246505, "box": [ 0.4780058651026393, 0.552297165200391, 0.5063538611925709, 0.5640273704789834 ], "gt_box": [ 0.46920821114369504, 0.5268817204301075, 0.5043988269794721, 0.5366568914956011 ], "gen_text": "Close the gripper firmly around the orange cup's rim to secure a stable grasp @grip , then begin lifting upward." }, { "ep": 209, "frame_abs": 58348, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 33.734646448191285, "box": [ 0.4613880742913001, 0.4506353861192571, 0.4858260019550342, 0.4682306940371456 ], "gt_box": [ 0.5112414467253177, 0.49266862170087977, 0.5513196480938416, 0.509286412512219 ], "gen_text": "Continue descending and swinging the gripper toward the white cloth bag on the table, maintaining @grip closed on the white cloth bag." }, { "ep": 209, "frame_abs": 58388, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 55.34736040925989, "box": [ 0.6471163245356794, 0.3411534701857282, 0.6793743890518084, 0.36070381231671556 ], "gt_box": [ 0.6725317693059628, 0.2541544477028348, 0.718475073313783, 0.2785923753665689 ], "gen_text": "Descend the open gripper toward the white cloth bag on the table, aligning the fingers around its body at @grip , ready to close." }, { "ep": 209, "frame_abs": 58428, "t_rel": 112, "emitted": true, "iou": 0.0, "cdist_px": 31.28252681909266, "box": [ 0.8172043010752689, 0.2375366568914956, 0.8435972629521017, 0.260019550342131 ], "gt_box": [ 0.8084066471163245, 0.187683284457478, 0.8504398826979472, 0.21212121212121213 ], "gen_text": "Carry the white cloth bag across toward the pile area on the table, maintaining the closed grip @grip , then descend to deposit it." }, { "ep": 209, "frame_abs": 58452, "t_rel": 136, "emitted": true, "iou": 0.0, "cdist_px": 42.22396428602006, "box": [ 0.7888563049853372, 0.25806451612903225, 0.8172043010752689, 0.2756598240469208 ], "gt_box": [ 0.8054740957966764, 0.1906158357771261, 0.8406647116324536, 0.2130987292277615 ], "gen_text": "Close the gripper firmly onto the white cloth bag, seating the pads at @grip , then begin lifting upward." }, { "ep": 209, "frame_abs": 58492, "t_rel": 176, "emitted": true, "iou": 0.0, "cdist_px": 52.9969100821839, "box": [ 0.41348973607038125, 0.3020527859237537, 0.45161290322580644, 0.3235581622678397 ], "gt_box": [ 0.2678396871945259, 0.3313782991202346, 0.3206256109481916, 0.3509286412512219 ], "gen_text": "Open the gripper to release the white cloth bag onto the table surface, then lift and retract the arm to a neutral raised position clear of the table surface. @grip " }, { "ep": 315, "frame_abs": 111829, "t_rel": 32, "emitted": true, "iou": 0.3357400722021661, "cdist_px": 4.145594615148918, "box": [ 0.3411534701857282, 0.447702834799609, 0.37145650048875856, 0.46236559139784944 ], "gt_box": [ 0.34017595307917886, 0.4418377321603128, 0.38220918866080156, 0.45650048875855326 ], "gen_text": "Continue descending toward the yellow-green mug, keeping the gripper open to straddle the yellow-green mug's rim and descend slightly to align pads with the mug's rim @grip ." }, { "ep": 315, "frame_abs": 111854, "t_rel": 57, "emitted": true, "iou": 0.5593220338983051, "cdist_px": 1.885062601758143, "box": [ 0.49071358748778104, 0.4496578690127077, 0.52297165200391, 0.4613880742913001 ], "gt_box": [ 0.49071358748778104, 0.45356793743890517, 0.5239491691104594, 0.4633431085043988 ], "gen_text": "Close the gripper firmly around the yellow-green mug to secure the grasp, then begin lifting upward away from the table surface. @grip " }, { "ep": 315, "frame_abs": 111892, "t_rel": 95, "emitted": true, "iou": 0.09473684210526316, "cdist_px": 5.145269724213798, "box": [ 0.41642228739002934, 0.5131964809384164, 0.447702834799609, 0.5239491691104594 ], "gt_box": [ 0.42130987292277616, 0.5073313782991202, 0.45454545454545453, 0.5151515151515151 ], "gen_text": "Open the gripper fully to release the green round object and retract upward away from the table; open @grip to complete the release." }, { "ep": 315, "frame_abs": 111899, "t_rel": 102, "emitted": true, "iou": 0.11111111111111101, "cdist_px": 5.96285434995113, "box": [ 0.41642228739002934, 0.5034213098729228, 0.447702834799609, 0.5131964809384164 ], "gt_box": [ 0.4027370478983382, 0.4965786901270772, 0.4389051808406647, 0.5063538611925709 ], "gen_text": "Open the gripper fully to release the orange-labeled object onto the table surface, then retract upward and away; open @grip fully open." }, { "ep": 458, "frame_abs": 160998, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 31.807829864737425, "box": [ 0.6725317693059628, 0.3812316715542522, 0.7047898338220919, 0.38807429130009774 ], "gt_box": [ 0.7067448680351907, 0.3362658846529814, 0.7458455522971652, 0.34310850439882695 ], "gen_text": "Continue descending and swinging the nose toward the black bag to align the finger pads squarely on the bag's body, keeping @grip Sub open." }, { "ep": 458, "frame_abs": 161038, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 29.003332846260143, "box": [ 0.8162267839687195, 0.3479960899315738, 0.8435972629521017, 0.35581622678396874 ], "gt_box": [ 0.8406647116324536, 0.30498533724340177, 0.8748778103616813, 0.31378299120234604 ], "gen_text": "Continue closing the gripper firmly around the black bag body, then begin lifting upward \u2014 maintain @grip closed on the bag." }, { "ep": 458, "frame_abs": 161060, "t_rel": 94, "emitted": true, "iou": 0.0, "cdist_px": 41.33075496022969, "box": [ 0.8719452590420332, 0.3831867057673509, 0.906158357771261, 0.3919843597262952 ], "gt_box": [ 0.8914956011730205, 0.3196480938416422, 0.927663734115347, 0.3284457478005865 ], "gen_text": "Close the gripper firmly around the black bag body, seating the pads against its sides, then begin lifting upward \u2014 @grip closing onto the black bag." }, { "ep": 458, "frame_abs": 161100, "t_rel": 134, "emitted": true, "iou": 0.0, "cdist_px": 30.99441469098542, "box": [ 0.8172043010752689, 0.33528836754643204, 0.8465298142717498, 0.3460410557184751 ], "gt_box": [ 0.8240469208211144, 0.28641251221896386, 0.8651026392961877, 0.2991202346041056 ], "gen_text": "Close the gripper firmly around the black bag's body, seating the pads against it, then lift upward to extract the bag from the table surface. @grip " }, { "ep": 458, "frame_abs": 161140, "t_rel": 174, "emitted": true, "iou": 0.2285714285714289, "cdist_px": 7.700750470984155, "box": [ 0.6764418377321603, 0.4418377321603128, 0.7135874877810362, 0.458455522971652 ], "gt_box": [ 0.6578690127077224, 0.44281524926686217, 0.696969696969697, 0.47116324535679377 ], "gen_text": "Close the gripper firmly around the black bag's body, seating the pads against it, then begin lifting upward to extract the bag's handle from the table surface. @grip " }, { "ep": 595, "frame_abs": 203850, "t_rel": 32, "emitted": true, "iou": 0.07531380753138076, "cdist_px": 12.104693439477975, "box": [ 0.6148582600195504, 0.5131964809384164, 0.6471163245356794, 0.5347018572825024 ], "gt_box": [ 0.6207233626588465, 0.530791788856305, 0.6539589442815249, 0.5542521994134897 ], "gen_text": "Descend toward the white paper towel roll on the table and align the open gripper @grip over it to grasp it." }, { "ep": 595, "frame_abs": 203890, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 41.42042083817192, "box": [ 0.6764418377321603, 0.6471163245356794, 0.7038123167155426, 0.6656891495601173 ], "gt_box": [ 0.7634408602150538, 0.6871945259042033, 0.7917888563049853, 0.7096774193548387 ], "gen_text": "Close the fingers firmly around the white pill bottle on the table and begin lifting \u2014 @grip closing to a firm hold on the bottle." }, { "ep": 595, "frame_abs": 203909, "t_rel": 91, "emitted": true, "iou": 0.0, "cdist_px": 31.45868752372138, "box": [ 0.7536656891495601, 0.6686217008797654, 0.7810361681329423, 0.6891495601173021 ], "gt_box": [ 0.7634408602150538, 0.7174975562072337, 0.7898338220918866, 0.7380254154447703 ], "gen_text": "Close the fingers firmly around the white pill bottle to complete the grasp; hold @grip ." }, { "ep": 595, "frame_abs": 203949, "t_rel": 131, "emitted": false, "iou": null, "cdist_px": null, "box": null, "gt_box": [ 0.37732160312805474, 0.5777126099706745, 0.40175953079178883, 0.5953079178885631 ], "gen_text": " " }, { "ep": 595, "frame_abs": 203976, "t_rel": 158, "emitted": true, "iou": 0.0, "cdist_px": 16.325753140312408, "box": [ 0.3812316715542522, 0.5933528836754643, 0.4027370478983382, 0.6089931573802542 ], "gt_box": [ 0.3587487781036168, 0.5728250244379277, 0.37927663734115347, 0.5855327468230694 ], "gen_text": "Open the gripper fully to release the candy into the container, then retract upward and move back toward the candy pile on the table; @grip should be fully open." }, { "ep": 721, "frame_abs": 246232, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 29.547727650192467, "box": [ 0.40078201368523947, 0.47409579667644186, 0.4418377321603128, 0.4838709677419355 ], "gt_box": [ 0.4780058651026393, 0.4887585532746823, 0.5190615835777126, 0.5004887585532747 ], "gen_text": "Hold the retracted open gripper @grip and retract upward away from the placed book." }, { "ep": 721, "frame_abs": 246272, "t_rel": 72, "emitted": true, "iou": 0.22222222222222301, "cdist_px": 5.179180463996687, "box": [ 0.656891495601173, 0.5131964809384164, 0.6881720430107527, 0.52297165200391 ], "gt_box": [ 0.6735092864125122, 0.5043988269794721, 0.6930596285434996, 0.5210166177908113 ], "gen_text": "Lift the book upward and retract away from the placed placement; maintain @grip closed on the book." }, { "ep": 721, "frame_abs": 246312, "t_rel": 112, "emitted": true, "iou": 0.5333333333333321, "cdist_px": 0.6498834853800477, "box": [ 0.6871945259042033, 0.5034213098729228, 0.7155425219941349, 0.512218963831867 ], "gt_box": [ 0.6901270772238515, 0.4995112414467253, 0.7135874877810362, 0.5141739980449658 ], "gen_text": "Close the gripper firmly around the blue book to complete the grasp, achieve @grip fully seated on the book's cover, then begin lifting upward." }, { "ep": 721, "frame_abs": 246352, "t_rel": 152, "emitted": true, "iou": 0.4347826086956522, "cdist_px": 2.5726348621068915, "box": [ 0.6471163245356794, 0.5024437927663734, 0.6725317693059628, 0.5112414467253177 ], "gt_box": [ 0.6529814271749755, 0.4965786901270772, 0.6764418377321603, 0.5112414467253177 ], "gen_text": "Open the gripper fully to release the book and retract the arm upward and away from the book now. @grip " }, { "ep": 721, "frame_abs": 246392, "t_rel": 192, "emitted": true, "iou": 0.0, "cdist_px": 13.80121227873771, "box": [ 0.5405669599217986, 0.44086021505376344, 0.5747800586510264, 0.4496578690127077 ], "gt_box": [ 0.5650048875855328, 0.4252199413489736, 0.603128054740958, 0.4340175953079179 ], "gen_text": "Move the gripper down and laterally toward the open book on the table, keeping the gripper open @grip as you approach from above." }, { "ep": 822, "frame_abs": 269826, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 31.522676043236828, "box": [ 0.41837732160312807, 0.6187683284457478, 0.46725317693059626, 0.6256109481915934 ], "gt_box": [ 0.45454545454545453, 0.6647116324535679, 0.49853372434017595, 0.6705767350928641 ], "gen_text": "Continue lifting and retracting the open gripper @grip to clear the toaster, then begin lifting upward and moving away from the toaster." }, { "ep": 822, "frame_abs": 269835, "t_rel": 41, "emitted": true, "iou": 0.0, "cdist_px": 8.535084693701975, "box": [ 0.47116324535679377, 0.6471163245356794, 0.5083088954056696, 0.6549364613880743 ], "gt_box": [ 0.47214076246334313, 0.6598240469208211, 0.5141739980449658, 0.6686217008797654 ], "gen_text": "Close @grip firmly onto the toaster's side lever, seating the pads against it, then begin lifting straight up to disengage once the lever is in place." }, { "ep": 822, "frame_abs": 269875, "t_rel": 81, "emitted": true, "iou": 0.4415584415584416, "cdist_px": 2.3541729381803043, "box": [ 0.581622678396872, 0.6324535679374389, 0.6207233626588465, 0.6422287390029325 ], "gt_box": [ 0.5874877810361682, 0.6363636363636364, 0.624633431085044, 0.6432062561094819 ], "gen_text": "Open @grip fully and begin lifting the arm upward and away from the toaster, releasing the toaster's side lever." }, { "ep": 822, "frame_abs": 269884, "t_rel": 90, "emitted": true, "iou": 0.057692307692307605, "cdist_px": 4.911301211837757, "box": [ 0.5679374389051809, 0.6275659824046921, 0.6080156402737048, 0.635386119257087 ], "gt_box": [ 0.5610948191593352, 0.6226783968719453, 0.5972629521016618, 0.6285434995112414 ], "gen_text": "Open the gripper fully at @grip and lift the arm upward and away from the toaster to clear the workspace, completing the release." }, { "ep": 822, "frame_abs": 269912, "t_rel": 118, "emitted": true, "iou": 0.0, "cdist_px": 16.806973282711375, "box": [ 0.45943304007820135, 0.5659824046920822, 0.5210166177908113, 0.5738025415444771 ], "gt_box": [ 0.501466275659824, 0.5738025415444771, 0.5669599217986315, 0.5835777126099707 ], "gen_text": "Continue lifting and retracting the arm upward away from the toaster, keeping the gripper open @grip as the arm withdraws to a clear position." }, { "ep": 974, "frame_abs": 303964, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 26.323335988451564, "box": [ 0.3812316715542522, 0.38220918866080156, 0.4203323558162268, 0.3939393939393939 ], "gt_box": [ 0.4291300097751711, 0.35581622678396874, 0.4887585532746823, 0.3704789833822092 ], "gen_text": "Continue descending and moving horizontally toward the blue chip bag, keeping the gripper open @grip and centering the finger line over the bag's body." }, { "ep": 974, "frame_abs": 304004, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 21.39147835329474, "box": [ 0.5503421309872922, 0.3812316715542522, 0.5953079178885631, 0.39100684261974583 ], "gt_box": [ 0.6080156402737048, 0.375366568914956, 0.6539589442815249, 0.3831867057673509 ], "gen_text": "Retract the open gripper upward and away from the blue chip bag, keeping it open at @grip to complete the release." }, { "ep": 974, "frame_abs": 304044, "t_rel": 112, "emitted": true, "iou": 0.0, "cdist_px": 11.224433553668401, "box": [ 0.6744868035190615, 0.37732160312805474, 0.7067448680351907, 0.3870967741935484 ], "gt_box": [ 0.6539589442815249, 0.39296187683284456, 0.6930596285434996, 0.40078201368523947 ], "gen_text": "Lower the gripper the remaining distance to seat the pads around the blue chip bag, then begin closing @grip to grasp it." }, { "ep": 974, "frame_abs": 304050, "t_rel": 118, "emitted": true, "iou": 0.2272727272727277, "cdist_px": 5.06082046028969, "box": [ 0.678396871945259, 0.38025415444770283, 0.7116324535679375, 0.3890518084066471 ], "gt_box": [ 0.6627565982404692, 0.3841642228739003, 0.7028347996089932, 0.39296187683284456 ], "gen_text": "Lower the gripper the remaining distance to seat the pads around the blue chip bag, centering the pads around the bag's body at @grip ." }, { "ep": 974, "frame_abs": 304090, "t_rel": 158, "emitted": true, "iou": 0.0, "cdist_px": 7.663360056305131, "box": [ 0.5962854349951124, 0.34408602150537637, 0.6295210166177908, 0.35288367546432065 ], "gt_box": [ 0.5874877810361682, 0.3333333333333333, 0.6236559139784946, 0.3411534701857282 ], "gen_text": "Open the gripper fully to release the blue chip bag onto the tray @grip , then retract the arm upward and away from the tray." } ], "A_kf": [ { "ep": 0, "frame_abs": 0, "t_rel": 0, "emitted": true, "iou": 0.08121360711002142, "cdist_px": 13.904472165652612, "box": [ 0.1300097751710655, 0.4447702834799609, 0.1935483870967742, 0.4682306940371456 ], "gt_box": [ 0.14173998044965788, 0.426197458455523, 0.22189638318670576, 0.4496578690127077 ], "gen_text": "Descend and move horizontally toward the orange cup on the table, keeping the gripper open @grip as you approach the target." }, { "ep": 0, "frame_abs": 40, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 24.272204456258105, "box": [ 0.3841642228739003, 0.5992179863147605, 0.43010752688172044, 0.6158357771260997 ], "gt_box": [ 0.34310850439882695, 0.5698924731182796, 0.3949169110459433, 0.5826001955034213 ], "gen_text": "Continue descending toward the orange cup, swinging the nose to align the finger-line across the cup's rim, and begin converging the fingers @grip to align the pads around the cup's rim." }, { "ep": 0, "frame_abs": 80, "t_rel": 80, "emitted": true, "iou": 0.21186440677966104, "cdist_px": 6.295088734310252, "box": [ 0.4946236559139785, 0.6217008797653959, 0.5317693059628543, 0.6324535679374389 ], "gt_box": [ 0.4760508308895406, 0.6265884652981427, 0.5190615835777126, 0.6363636363636364 ], "gen_text": "Descend the last few centimetres to contact the orange cup's rim, then close the gripper @grip firmly around the cup's rim to secure a stable grasp." }, { "ep": 0, "frame_abs": 120, "t_rel": 120, "emitted": true, "iou": 0.0867052023121388, "cdist_px": 5.72154048057566, "box": [ 0.5591397849462365, 0.6011730205278593, 0.5904203323558163, 0.6119257086999023 ], "gt_box": [ 0.5610948191593352, 0.6099706744868035, 0.6001955034213099, 0.6197458455522972 ], "gen_text": "Close the gripper firmly around the orange cup's rim to secure a stable grasp @grip , then begin lifting upward." }, { "ep": 0, "frame_abs": 122, "t_rel": 122, "emitted": true, "iou": 0.3170731707317078, "cdist_px": 4.295072679778156, "box": [ 0.5513196480938416, 0.6041055718475073, 0.5835777126099707, 0.6158357771260997 ], "gt_box": [ 0.5581622678396871, 0.6089931573802542, 0.5972629521016618, 0.6177908113391984 ], "gen_text": "Close the gripper firmly around the orange cup's rim to secure a stable grasp @grip , then begin lifting upward." }, { "ep": 0, "frame_abs": 162, "t_rel": 162, "emitted": true, "iou": 0.0, "cdist_px": 11.005644531798673, "box": [ 0.48289345063538613, 0.541544477028348, 0.5161290322580645, 0.5532746823069403 ], "gt_box": [ 0.46920821114369504, 0.5268817204301075, 0.5043988269794721, 0.5366568914956011 ], "gen_text": "Close the gripper firmly around the orange cup's rim to secure a stable grasp @grip , then begin lifting upward." }, { "ep": 209, "frame_abs": 58316, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 26.01958846152436, "box": [ 0.29423264907135877, 0.613880742913001, 0.32160312805474095, 0.6324535679374389 ], "gt_box": [ 0.34017595307917886, 0.6432062561094819, 0.37927663734115347, 0.6598240469208211 ], "gen_text": "Continue moving the open gripper forward and downward toward the white cloth bag on the table, keeping the gripper open at @grip as it converges on the target." }, { "ep": 209, "frame_abs": 58356, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 20.390360152783504, "box": [ 0.5043988269794721, 0.4613880742913001, 0.5366568914956011, 0.4780058651026393 ], "gt_box": [ 0.5112414467253177, 0.49266862170087977, 0.5513196480938416, 0.509286412512219 ], "gen_text": "Continue descending toward the white cloth bag on the table, keeping the gripper open and aligning fingers over the bag's body at @grip ." }, { "ep": 209, "frame_abs": 58396, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 21.184715009891203, "box": [ 0.6930596285434996, 0.2883675464320626, 0.7243401759530792, 0.3088954056695992 ], "gt_box": [ 0.6725317693059628, 0.2541544477028348, 0.718475073313783, 0.2785923753665689 ], "gen_text": "Carry the white cloth bag across toward the pile area on the table, maintaining the closed grip, then descend to deposit it @grip ." }, { "ep": 209, "frame_abs": 58436, "t_rel": 120, "emitted": true, "iou": 0.0, "cdist_px": 30.34672219902267, "box": [ 0.8181818181818182, 0.2375366568914956, 0.8435972629521017, 0.2570869990224829 ], "gt_box": [ 0.8084066471163245, 0.187683284457478, 0.8504398826979472, 0.21212121212121213 ], "gen_text": "Close the gripper firmly around the white cloth bag, seating the pads around it at @grip , then begin lifting upward." }, { "ep": 209, "frame_abs": 58460, "t_rel": 144, "emitted": true, "iou": 0.0, "cdist_px": 49.81765151175357, "box": [ 0.7820136852394917, 0.26881720430107525, 0.8084066471163245, 0.2873900293255132 ], "gt_box": [ 0.8054740957966764, 0.1906158357771261, 0.8406647116324536, 0.2130987292277615 ], "gen_text": "Close the gripper firmly onto the white cloth bag and begin lifting upward off the table surface \u2014 @grip ." }, { "ep": 209, "frame_abs": 58500, "t_rel": 184, "emitted": true, "iou": 0.0, "cdist_px": 19.364946954126253, "box": [ 0.21896383186705767, 0.32649071358748777, 0.2619745845552297, 0.35386119257087 ], "gt_box": [ 0.2678396871945259, 0.3313782991202346, 0.3206256109481916, 0.3509286412512219 ], "gen_text": "Open the gripper to release the white cloth bag at @grip onto the table surface and retract the arm to a neutral position." }, { "ep": 315, "frame_abs": 111797, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 34.031800047710064, "box": [ 0.16715542521994134, 0.38807429130009774, 0.2101661779081134, 0.40762463343108507 ], "gt_box": [ 0.08797653958944282, 0.35288367546432065, 0.14760508308895406, 0.3724340175953079 ], "gen_text": "Continue descending and converging toward the yellow-green mug, keeping the gripper open @grip as the fingers converge toward the target object." }, { "ep": 315, "frame_abs": 111837, "t_rel": 40, "emitted": true, "iou": 0.5261669024045255, "cdist_px": 1.8869878415661652, "box": [ 0.3489736070381232, 0.4398826979472141, 0.37927663734115347, 0.45356793743890517 ], "gt_box": [ 0.34017595307917886, 0.4418377321603128, 0.38220918866080156, 0.45650048875855326 ], "gen_text": "Continue descending toward the yellow-green mug, keeping the gripper open to straddle the mug handle, maintaining the open @grip as the fingers converge toward the mug's cylindrical body." }, { "ep": 315, "frame_abs": 111862, "t_rel": 65, "emitted": true, "iou": 0.23489932885906045, "cdist_px": 4.231734666747882, "box": [ 0.48289345063538613, 0.447702834799609, 0.5180840664711632, 0.458455522971652 ], "gt_box": [ 0.49071358748778104, 0.45356793743890517, 0.5239491691104594, 0.4633431085043988 ], "gen_text": "Open the gripper fully to release the green mug onto the table surface and retract upward; let the mug rest flat on the table @grip ." }, { "ep": 315, "frame_abs": 111900, "t_rel": 103, "emitted": true, "iou": 0.2825112107623319, "cdist_px": 4.301963828948331, "box": [ 0.4125122189638319, 0.5034213098729228, 0.4418377321603128, 0.5131964809384164 ], "gt_box": [ 0.42130987292277616, 0.5073313782991202, 0.45454545454545453, 0.5151515151515151 ], "gen_text": "Open the gripper fully to release the yellow-green marker onto the table surface, then lift the gripper upward and away from the marker. @grip " }, { "ep": 458, "frame_abs": 160966, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 32.18255328576448, "box": [ 0.6187683284457478, 0.3812316715542522, 0.6549364613880743, 0.39002932551319647 ], "gt_box": [ 0.6314760508308895, 0.4310850439882698, 0.6842619745845552, 0.43792766373411535 ], "gen_text": "Continue descending toward the black bag, swinging the nose to align the finger-line across the bag's body, gripper open at @grip ." }, { "ep": 458, "frame_abs": 161006, "t_rel": 40, "emitted": true, "iou": 0.17270788912579899, "cdist_px": 3.7877747281768444, "box": [ 0.7174975562072337, 0.34017595307917886, 0.7438905180840665, 0.34995112414467255 ], "gt_box": [ 0.7067448680351907, 0.3362658846529814, 0.7458455522971652, 0.34310850439882695 ], "gen_text": "Continue descending and swinging the nose toward the black bag to align the finger pads squarely on the bag's body, maintaining @grip open as you close the remaining vertical gap." }, { "ep": 458, "frame_abs": 161046, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 17.811134235722957, "box": [ 0.8279569892473119, 0.3304007820136852, 0.8514173998044966, 0.34017595307917886 ], "gt_box": [ 0.8406647116324536, 0.30498533724340177, 0.8748778103616813, 0.31378299120234604 ], "gen_text": "Continue closing the gripper firmly around the black bag body, then begin lifting upward \u2014 maintain @grip closed on the bag." }, { "ep": 458, "frame_abs": 161068, "t_rel": 102, "emitted": true, "iou": 0.0, "cdist_px": 42.07613351228001, "box": [ 0.8719452590420332, 0.3831867057673509, 0.9022482893450635, 0.3939393939393939 ], "gt_box": [ 0.8914956011730205, 0.3196480938416422, 0.927663734115347, 0.3284457478005865 ], "gen_text": "Close the gripper firmly around the black bag body, seating the pads against its sides, then begin lifting upward \u2014 @grip closing onto the black bag." }, { "ep": 458, "frame_abs": 161108, "t_rel": 142, "emitted": true, "iou": 0.0, "cdist_px": 20.766255247767436, "box": [ 0.8172043010752689, 0.3176930596285435, 0.8484848484848485, 0.3313782991202346 ], "gt_box": [ 0.8240469208211144, 0.28641251221896386, 0.8651026392961877, 0.2991202346041056 ], "gen_text": "Close the gripper firmly around the black bag's body, seating the pads against it, then begin lifting upward to extract the bag's contents. @grip " }, { "ep": 458, "frame_abs": 161148, "t_rel": 182, "emitted": true, "iou": 0.17245508982036012, "cdist_px": 9.599436979502515, "box": [ 0.6735092864125122, 0.45943304007820135, 0.7145650048875856, 0.4780058651026393 ], "gt_box": [ 0.6578690127077224, 0.44281524926686217, 0.696969696969697, 0.47116324535679377 ], "gen_text": "Open the gripper fully to release the black bag and retract upward away from the table; let the bag settle open @grip ." }, { "ep": 595, "frame_abs": 203818, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 44.92164946707415, "box": [ 0.41642228739002934, 0.3734115347018573, 0.43792766373411535, 0.39002932551319647 ], "gt_box": [ 0.4496578690127077, 0.3069403714565005, 0.48484848484848486, 0.3235581622678397 ], "gen_text": "Continue descending and converging the approach toward the red-and-white plush toy, aligning the open gripper @grip over the target spot for grasping." }, { "ep": 595, "frame_abs": 203858, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 15.187898170603978, "box": [ 0.6148582600195504, 0.509286412512219, 0.6471163245356794, 0.5288367546432062 ], "gt_box": [ 0.6207233626588465, 0.530791788856305, 0.6539589442815249, 0.5542521994134897 ], "gen_text": "Close the fingers firmly around the white pill bottle on the table, seating both pads against it at @grip , then begin lifting upward." }, { "ep": 595, "frame_abs": 203898, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 39.32227046626855, "box": [ 0.6852394916911045, 0.6471163245356794, 0.7106549364613881, 0.6656891495601173 ], "gt_box": [ 0.7634408602150538, 0.6871945259042033, 0.7917888563049853, 0.7096774193548387 ], "gen_text": "Close the fingers firmly around the white pill bottle on the table and begin lifting upward, then begin transporting it to the home/rest area, keeping @grip closed on the pill bottle." }, { "ep": 595, "frame_abs": 203917, "t_rel": 99, "emitted": true, "iou": 0.0, "cdist_px": 29.555468558208318, "box": [ 0.7536656891495601, 0.6715542521994134, 0.782991202346041, 0.6920821114369502 ], "gt_box": [ 0.7634408602150538, 0.7174975562072337, 0.7898338220918866, 0.7380254154447703 ], "gen_text": "Close the gripper around the white pill bottle on the christmas lights, seating the pads against its sides at @grip , then begin lifting upward." }, { "ep": 595, "frame_abs": 203957, "t_rel": 139, "emitted": true, "iou": 0.01724137931034481, "cdist_px": 12.020026689434136, "box": [ 0.3939393939393939, 0.5933528836754643, 0.41935483870967744, 0.6119257086999023 ], "gt_box": [ 0.37732160312805474, 0.5777126099706745, 0.40175953079178883, 0.5953079178885631 ], "gen_text": "Open the gripper fully to release the pill onto the table surface near the white cup, then retract upward and toward the remaining empty white bowl or trashcan. @grip " }, { "ep": 595, "frame_abs": 203984, "t_rel": 166, "emitted": false, "iou": null, "cdist_px": null, "box": null, "gt_box": [ 0.3587487781036168, 0.5728250244379277, 0.37927663734115347, 0.5855327468230694 ], "gen_text": " " }, { "ep": 721, "frame_abs": 246200, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 29.446326572123656, "box": [ 0.21896383186705767, 0.2991202346041056, 0.2619745845552297, 0.3235581622678397 ], "gt_box": [ 0.20234604105571846, 0.25024437927663734, 0.2873900293255132, 0.2805474095796676 ], "gen_text": "Continue descending the open gripper toward the blue book on the white tray, aligning the finger pads with the book's cover faces. @grip " }, { "ep": 721, "frame_abs": 246240, "t_rel": 40, "emitted": true, "iou": 0.054054054054054015, "cdist_px": 11.64467022388501, "box": [ 0.4496578690127077, 0.49364613880742914, 0.4838709677419355, 0.5024437927663734 ], "gt_box": [ 0.4780058651026393, 0.4887585532746823, 0.5190615835777126, 0.5004887585532747 ], "gen_text": "Continue descending toward the blue notebook and align the open gripper pads around its edges; keep @grip open and converging on the notebook cover." }, { "ep": 721, "frame_abs": 246280, "t_rel": 80, "emitted": true, "iou": 0.24528301886792575, "cdist_px": 4.6700361847259435, "box": [ 0.6803519061583577, 0.509286412512219, 0.7116324535679375, 0.5190615835777126 ], "gt_box": [ 0.6735092864125122, 0.5043988269794721, 0.6930596285434996, 0.5210166177908113 ], "gen_text": "Lift the book upward and retract away from the placed placement; maintain the closed grip at @grip until the arm is fully lower." }, { "ep": 721, "frame_abs": 246320, "t_rel": 120, "emitted": true, "iou": 0.3962264150943387, "cdist_px": 2.8207424413867543, "box": [ 0.6871945259042033, 0.5073313782991202, 0.7155425219941349, 0.5151515151515151 ], "gt_box": [ 0.6901270772238515, 0.4995112414467253, 0.7135874877810362, 0.5141739980449658 ], "gen_text": "Close the gripper firmly around the blue book to complete the grasp, achieve @grip fully closed on the book and lift it off the table surface." }, { "ep": 721, "frame_abs": 246360, "t_rel": 160, "emitted": true, "iou": 0.0, "cdist_px": 9.797355081675581, "box": [ 0.6422287390029325, 0.5131964809384164, 0.6715542521994134, 0.5239491691104594 ], "gt_box": [ 0.6529814271749755, 0.4965786901270772, 0.6764418377321603, 0.5112414467253177 ], "gen_text": "Open the gripper fully to release the blue book onto the table and begin retracting the arm upward to clear the workspace; open @grip completely." }, { "ep": 721, "frame_abs": 246400, "t_rel": 200, "emitted": true, "iou": 0.187155963302752, "cdist_px": 7.553721933428121, "box": [ 0.5454545454545454, 0.4232649071358749, 0.581622678396872, 0.4310850439882698 ], "gt_box": [ 0.5650048875855328, 0.4252199413489736, 0.603128054740958, 0.4340175953079179 ], "gen_text": "Move the gripper down toward the white paper towel roll on the table surface, keeping fingers open and converging the fingers to align with the cylindrical roll; keep @grip open and converging toward the roll." }, { "ep": 822, "frame_abs": 269794, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 13.707132327975929, "box": [ 0.386119257086999, 0.34701857282502446, 0.43206256109481916, 0.36363636363636365 ], "gt_box": [ 0.38514173998044965, 0.3675464320625611, 0.447702834799609, 0.38514173998044965 ], "gen_text": "Descend and close the gripper to seat the pads firmly against the toaster's side lever, beginning to close @grip around the lever." }, { "ep": 822, "frame_abs": 269834, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 11.258977363138818, "box": [ 0.46725317693059626, 0.6471163245356794, 0.5063538611925709, 0.6549364613880743 ], "gt_box": [ 0.45454545454545453, 0.6647116324535679, 0.49853372434017595, 0.6705767350928641 ], "gen_text": "Close @grip firmly onto the toaster's side lever, seating the pads against it, then begin lifting straight up to disengage." }, { "ep": 822, "frame_abs": 269843, "t_rel": 49, "emitted": true, "iou": 0.0, "cdist_px": 8.372683671628025, "box": [ 0.48191593352883677, 0.6471163245356794, 0.5210166177908113, 0.656891495601173 ], "gt_box": [ 0.47214076246334313, 0.6598240469208211, 0.5141739980449658, 0.6686217008797654 ], "gen_text": "Close @grip firmly onto the toaster's side lever, seating the pads against it, then begin lifting straight up to clear the toaster surface." }, { "ep": 822, "frame_abs": 269883, "t_rel": 89, "emitted": true, "iou": 0.0, "cdist_px": 8.406033304104655, "box": [ 0.5679374389051809, 0.6275659824046921, 0.6080156402737048, 0.635386119257087 ], "gt_box": [ 0.5874877810361682, 0.6363636363636364, 0.624633431085044, 0.6432062561094819 ], "gen_text": "Open the gripper fully at @grip and lift the arm upward and away from the toaster to clear the workspace, completing the release." }, { "ep": 822, "frame_abs": 269892, "t_rel": 98, "emitted": true, "iou": 0.0, "cdist_px": 14.822009078177553, "box": [ 0.5386119257086999, 0.6001955034213099, 0.581622678396872, 0.6099706744868035 ], "gt_box": [ 0.5610948191593352, 0.6226783968719453, 0.5972629521016618, 0.6285434995112414 ], "gen_text": "Open the gripper fully at @grip and lift the arm upward and away from the toaster, withdrawing contact with the toaster's side lever." }, { "ep": 974, "frame_abs": 303932, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 20.157189665622994, "box": [ 0.34701857282502446, 0.3235581622678397, 0.38807429130009774, 0.33822091886608013 ], "gt_box": [ 0.375366568914956, 0.3010752688172043, 0.4398826979472141, 0.31671554252199413 ], "gen_text": "Continue descending and moving horizontally toward the blue chip bag, keeping the gripper open @grip and centering the finger line over the bag's body." }, { "ep": 974, "frame_abs": 303972, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 17.80993246996377, "box": [ 0.4173998044965787, 0.38220918866080156, 0.45650048875855326, 0.3939393939393939 ], "gt_box": [ 0.4291300097751711, 0.35581622678396874, 0.4887585532746823, 0.3704789833822092 ], "gen_text": "Continue descending and moving horizontally toward the blue chip bag, keeping the gripper open @grip as the fingers converge around the bag's body." }, { "ep": 974, "frame_abs": 304012, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 18.710606905740995, "box": [ 0.5972629521016618, 0.3460410557184751, 0.6392961876832844, 0.35581622678396874 ], "gt_box": [ 0.6080156402737048, 0.375366568914956, 0.6539589442815249, 0.3831867057673509 ], "gen_text": "Retract the open gripper upward and away from the blue chip bag, keeping it open at @grip to complete the release." }, { "ep": 974, "frame_abs": 304052, "t_rel": 120, "emitted": true, "iou": 0.03870967741935491, "cdist_px": 6.803519061583562, "box": [ 0.6695992179863147, 0.38514173998044965, 0.7047898338220919, 0.3939393939393939 ], "gt_box": [ 0.6539589442815249, 0.39296187683284456, 0.6930596285434996, 0.40078201368523947 ], "gen_text": "Close the gripper firmly around the blue chip bag at @grip , securing a stable grasp before lifting." }, { "ep": 974, "frame_abs": 304058, "t_rel": 126, "emitted": true, "iou": 0.10264385692068455, "cdist_px": 6.803547151097604, "box": [ 0.6813294232649071, 0.37732160312805474, 0.7145650048875856, 0.3870967741935484 ], "gt_box": [ 0.6627565982404692, 0.3841642228739003, 0.7028347996089932, 0.39296187683284456 ], "gen_text": "Close the gripper firmly around the blue chip bag at @grip to secure a stable grasp, then begin lifting upward away from the tray." }, { "ep": 974, "frame_abs": 304098, "t_rel": 166, "emitted": true, "iou": 0.047001620745542795, "cdist_px": 5.5782577041614285, "box": [ 0.581622678396872, 0.34017595307917886, 0.6158357771260997, 0.34995112414467255 ], "gt_box": [ 0.5874877810361682, 0.3333333333333333, 0.6236559139784946, 0.3411534701857282 ], "gen_text": "Open the gripper fully to release the blue chip bag onto the tray @grip , then retract the arm upward away from the tray." } ] }, "episode_meta": { "0": { "prompt": "Move object into or out of container (ex: drawer, clothes hamper, plate, trashcan, washer)", "plan_text": "Plan:1. Move the open gripper down toward the orange cup on the table with fingers open. 2. Align the fingers around the orange cup's rim and descend to seat the pads around the cup's rim. 3. Close the gripper firmly around the orange cup's rim to secure a stable grasp. 4. Lift the orange cup off the table surface. 5. Transport the orange cup laterally over the white cup, then transport it to the white cup. 6. Descend and lower the orange cup onto the white cup. 7. Open the gripper to release the orange cup into the white cup. 8. Retract the arm upward and away from the placed cup.", "lo": 0, "hi": 443, "kfs": [ 0, 40, 80, 120, 122, 162 ] }, "209": { "prompt": "Clump objects (cloth, rope, plush toy)", "plan_text": "Plan:1. Move the open gripper down toward the white cloth with pink cloth and pink cloth lying on the black table. 2. Lift the white cloth and pink cloth clear of the table and transport it to the pile area and release it. 3. Move the open gripper to the white cloth bag on the table. 4. Move the open gripper to the white cloth bag on the table and release it. 5. Align the open gripper above the white cloth bag. 6. Close the gripper to grasp the white cloth bag. 7. Lift the white cloth bag and carry it to the pile area on the table. 8. Retract the open gripper to the home position, releasing the white cloth bag. 9. Retract the open gripper away from the white cloth bag.", "lo": 58316, "hi": 59312, "kfs": [ 0, 40, 80, 120, 144, 184 ] }, "315": { "prompt": "Move object to a new position and orientation (ex: grasping, relocating, flipping)", "plan_text": "Plan:1", "lo": 111797, "hi": 111907, "kfs": [ 0, 40, 65, 103, 110 ] }, "458": { "prompt": "Do any two tasks consecutively.\n\nSuggested tasks:\n* Fold, spread out, or clump object (ex: cloth, charging cord, clothes)\n* Fold, spread out, or clump object (e", "plan_text": "Plan:1. Move the open gripper down toward the black bag in the centre of the table. 2. Close the gripper firmly around the black bag's body. ", "lo": 160966, "hi": 161383, "kfs": [ 0, 40, 80, 102, 142, 182 ] }, "595": { "prompt": "Do any three tasks consecutively.\n\nSuggested tasks:\t\t\n* Move object into or out of container (ex: drawer, clothes hamper, plate, trashcan, washer)\n* Move lid on", "plan_text": "Plan:1", "lo": 203818, "hi": 204319, "kfs": [ 0, 40, 80, 99, 139, 166 ] }, "721": { "prompt": "open book", "plan_text": "Plan:1", "lo": 246200, "hi": 246491, "kfs": [ 0, 40, 80, 120, 160, 200 ] }, "822": { "prompt": "Open or close slidable objects (ex: toaster, drawers, sliding doors, dressers)", "plan_text": "Plan:1. Move the arm forward and downward toward the toaster's side lever. 2. Descend and align the gripper pads with the toaster's side lever. 3. Close the gripper firmly onto the toaster's side lever. 4. Lift the toaster straight up clear of the table surface. 5. Lower the toaster onto the table surface. 6. Open the gripper and lift the arm upward away from the toaster.", "lo": 269794, "hi": 269920, "kfs": [ 0, 40, 49, 89, 98, 126 ] }, "974": { "prompt": "pnp-table-to-tray-allgreen", "plan_text": "Plan:1. Move the open gripper down toward the blue chip bag on the tray. 2. Align the gripper fingers around the blue chip bag. 3. Close the gripper firmly around the blue chip bag to secure a stable grasp. 4. Lift the blue chip bag clear of the tray surface. 5. Transport the blue chip bag laterally across the tray to the left side away from the tray. 6. Descend and align the gripper fingers around the blue chip bag's body. 7. Open the gripper to release the blue chip bag and retract the arm upward away from the tray.", "lo": 303932, "hi": 304905, "kfs": [ 0, 40, 80, 120, 126, 166 ] } } }, "OOD_heldout": { "episodes": [ 48, 50, 52, 320, 401, 551, 553, 691 ], "summary": { "S": { "n": 40, "emit_rate": 1.0, "iou_mean": 0.08711565588449274, "iou_median": 0.0, "iou>0.25": 0.15, "iou>0.5": 0.05, "cdist_median_px": 13.029323686687976, "cdist_mean_px": 18.658044406941706 }, "A_kf-8": { "n": 36, "emit_rate": 0.9722222222222222, "iou_mean": 0.022230993548962318, "iou_median": 0.0, "iou>0.25": 0.02857142857142857, "iou>0.5": 0.0, "cdist_median_px": 28.47744066583376, "cdist_mean_px": 32.32004703183897 }, "A_kf": { "n": 40, "emit_rate": 0.975, "iou_mean": 0.06527735594134083, "iou_median": 0.0, "iou>0.25": 0.1282051282051282, "iou>0.5": 0.02564102564102564, "cdist_median_px": 17.69706645491634, "cdist_mean_px": 27.166004677976407 } }, "records": { "S": [ { "ep": 48, "frame_abs": 10623, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 69.23830329213996, "box": [ 0.0, 0.6471163245356794, 0.0009775171065493646, 0.6715542521994134 ], "gt_box": [ 0.0, 0.7546432062561095, 0.022482893450635387, 0.7800586510263929 ], "gen_text": "The robot arm has swung forward and descended noticeably, bringing the open gripper closer to the table surface; the white rope and red-and-white board are both visible in the background, and the gripper is open at @grip_now ." }, { "ep": 48, "frame_abs": 10663, "t_rel": 40, "emitted": true, "iou": 0.01173512154233025, "cdist_px": 12.905373203224624, "box": [ 0.386119257086999, 0.6158357771260997, 0.41935483870967744, 0.6304985337243402 ], "gt_box": [ 0.41837732160312807, 0.6168132942326491, 0.458455522971652, 0.6334310850439883 ], "gen_text": "The robot arm has swung forward and descended noticeably, positioning the open gripper directly above the white rope coiled in the bag; the wrist camera shows the rope held between the pads as the arm closes in. The gripper is open at @grip_now ." }, { "ep": 48, "frame_abs": 10703, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 20.5018221506938, "box": [ 0.447702834799609, 0.5718475073313783, 0.4760508308895406, 0.5845552297165201 ], "gt_box": [ 0.4858260019550342, 0.5483870967741935, 0.5190615835777126, 0.5630498533724341 ], "gen_text": "The gripper is now very close to the table surface, hovering just above the white rope which is being lowered toward the table; the wrist view shows the gripper fingers nearly touching the rope's body, at @grip_now ." }, { "ep": 48, "frame_abs": 10709, "t_rel": 86, "emitted": true, "iou": 0.0, "cdist_px": 21.380111495110928, "box": [ 0.4418377321603128, 0.5591397849462365, 0.4682306940371456, 0.5738025415444771 ], "gt_box": [ 0.48778103616813295, 0.5405669599217986, 0.5210166177908113, 0.555229716520039 ], "gen_text": "The gripper is closed and holding the white rope, which has been lifted off the table and is now elevated above the table; the arm is moving horizontally and the gripper is descending toward the table surface to the right. @grip_now is closed on the rope." }, { "ep": 48, "frame_abs": 10749, "t_rel": 126, "emitted": true, "iou": 0.0918114143920596, "cdist_px": 11.36332418453035, "box": [ 0.1652003910068426, 0.70772238514174, 0.20527859237536658, 0.7272727272727273 ], "gt_box": [ 0.16911045943304007, 0.7223851417399805, 0.22776148582600195, 0.7448680351906158 ], "gen_text": "The robot arm is raised high and extended across the table, carrying the white rope in a closed hold; the red-and-white board is visible below and to the left, at @grip_now ." }, { "ep": 50, "frame_abs": 11063, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 38.23331707220203, "box": [ 0.10557184750733138, 0.6451612903225806, 0.1495601173020528, 0.6656891495601173 ], "gt_box": [ 0.02932551319648094, 0.6920821114369502, 0.08797653958944282, 0.7096774193548387 ], "gen_text": "The robot arm has descended and moved forward; the green paddle-shaped object is now directly below the open gripper at @grip_now ." }, { "ep": 50, "frame_abs": 11103, "t_rel": 40, "emitted": true, "iou": 0.06198347107438019, "cdist_px": 7.901144454685701, "box": [ 0.38025415444770283, 0.6324535679374389, 0.4125122189638319, 0.6471163245356794 ], "gt_box": [ 0.3831867057673509, 0.6451612903225806, 0.4232649071358749, 0.6578690127077224 ], "gen_text": "The robot arm has swung forward and descended noticeably, positioning the gripper at the target drop zone near the table surface; the green paddle-shaped object is visible in the background and the green-and-white striped cloth is to the left; the gripper is open and descending, @grip_now converging toward the target drop zone." }, { "ep": 50, "frame_abs": 11143, "t_rel": 80, "emitted": true, "iou": 0.37168141592920356, "cdist_px": 2.5270660617097174, "box": [ 0.45650048875855326, 0.5738025415444771, 0.4838709677419355, 0.5835777126099707 ], "gt_box": [ 0.45454545454545453, 0.5698924731182796, 0.48778103616813295, 0.5796676441837733 ], "gen_text": "The gripper is now very close to the green paddle-shaped object, with the pads nearly touching the paddle's body; the arm is still low, with the gripper still open at @grip_now ." }, { "ep": 50, "frame_abs": 11148, "t_rel": 85, "emitted": true, "iou": 0.0, "cdist_px": 9.365266896542876, "box": [ 0.4398826979472141, 0.5738025415444771, 0.4682306940371456, 0.5835777126099707 ], "gt_box": [ 0.44868035190615835, 0.5610948191593352, 0.48191593352883677, 0.5698924731182796 ], "gen_text": "The gripper is inside or at the rim of the green paddle-shaped object, with the pads beginning to close around the paddle handle; the arm is positioned low over the table, with the green paddle visible in the wrist camera directly below the pads at @grip_now ." }, { "ep": 50, "frame_abs": 11188, "t_rel": 125, "emitted": true, "iou": 0.0, "cdist_px": 10.648477509970208, "box": [ 0.3675464320625611, 0.667644183773216, 0.39882697947214074, 0.6832844574780058 ], "gt_box": [ 0.3626588465298143, 0.6852394916911045, 0.4066471163245357, 0.6989247311827957 ], "gen_text": "The gripper is holding the green paddle-shaped object and has moved laterally and downward toward the target drop zone; the arm is positioned above and to the side of the table, with the paddle's tip pointing downward, at @grip_now ." }, { "ep": 52, "frame_abs": 11645, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 30.322070138699953, "box": [ 0.18084066471163246, 0.7448680351906158, 0.2238514173998045, 0.7634408602150538 ], "gt_box": [ 0.21407624633431085, 0.7869012707722385, 0.2649071358748778, 0.8064516129032258 ], "gen_text": "The robot arm has descended and swung forward, now hovering above the table surface near the centre-right of the wooden tabletop, with the white cable loop and the orange-labeled pieces visible below; the gripper is open and descending toward the table surface. @grip_now shows an open converging configuration." }, { "ep": 52, "frame_abs": 11685, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 15.403883169964569, "box": [ 0.41837732160312807, 0.5933528836754643, 0.4506353861192571, 0.6080156402737048 ], "gt_box": [ 0.4525904203323558, 0.5845552297165201, 0.4946236559139785, 0.5972629521016618 ], "gen_text": "The gripper is now very close to the table surface, nearly touching the white rectangular object; the fingers are open and converging toward the object, at @grip_now ." }, { "ep": 52, "frame_abs": 11714, "t_rel": 69, "emitted": true, "iou": 0.0, "cdist_px": 13.638069805006124, "box": [ 0.4555229716520039, 0.5630498533724341, 0.4838709677419355, 0.5738025415444771 ], "gt_box": [ 0.47116324535679377, 0.544477028347996, 0.5053763440860215, 0.555229716520039 ], "gen_text": "The gripper is positioned at the top of the black basket, with the finger pads converging around the top's rim; the fingers are beginning to close on the basket's metal face. @grip_now shows the pads converging on the target." }, { "ep": 52, "frame_abs": 11754, "t_rel": 109, "emitted": true, "iou": 0.1568627450980391, "cdist_px": 12.632864711462107, "box": [ 0.1466275659824047, 0.7106549364613881, 0.1935483870967742, 0.7321603128054741 ], "gt_box": [ 0.10850439882697947, 0.7106549364613881, 0.16226783968719452, 0.7380254154447703 ], "gen_text": "The robot arm has descended and moved forward; the gripper is now very close to the table surface, with the wrist camera showing the toaster and the red bowl visible to the right; the gripper is open and nearly at contact with the table surface at @grip_now ." }, { "ep": 52, "frame_abs": 11788, "t_rel": 143, "emitted": true, "iou": 0.11304347826086947, "cdist_px": 9.685058746026032, "box": [ 0.32160312805474095, 0.6392961876832844, 0.35288367546432065, 0.6549364613880743 ], "gt_box": [ 0.34408602150537637, 0.6363636363636364, 0.3831867057673509, 0.6520039100684262 ], "gen_text": "The gripper is holding the white plastic bag aloft and moving laterally toward the right side of the table, with the bag held in the closed gripper; the arm is descending toward the table surface at @grip_now ." }, { "ep": 320, "frame_abs": 112497, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 79.13998054600873, "box": [ 0.18572825024437928, 0.7849462365591398, 0.23264907135874877, 0.8123167155425219 ], "gt_box": [ 0.18181818181818182, 0.9081133919843597, 0.2375366568914956, 0.9364613880742912 ], "gen_text": "A white cardboard box sits on the white table to the right of the green marker; the gripper is open and positioned high above the table, well clear of the box, at @grip_now ." }, { "ep": 320, "frame_abs": 112532, "t_rel": 75, "emitted": true, "iou": 0.0, "cdist_px": 37.58652296242149, "box": [ 0.3519061583577713, 0.7126099706744868, 0.3831867057673509, 0.7282502443792767 ], "gt_box": [ 0.34408602150537637, 0.7712609970674487, 0.38025415444770283, 0.7869012707722385 ], "gen_text": "The gripper is closing around the green marker on the table surface, with the pads engaging the marker's body; the cardboard box is visible to the right in the background; @grip_now shows the pads seated against the marker." }, { "ep": 320, "frame_abs": 112564, "t_rel": 107, "emitted": true, "iou": 0.0, "cdist_px": 25.659703020707965, "box": [ 0.3235581622678397, 0.6608015640273704, 0.36070381231671556, 0.6754643206256109 ], "gt_box": [ 0.3176930596285435, 0.6989247311827957, 0.3626588465298143, 0.7174975562072337 ], "gen_text": "The gripper is descending toward the interior of the white cardboard box, now holding the green marker; the wrist view shows the marker gripped between the pads and the box interior. The gripper is nearly closed, @grip_now at its configuration." }, { "ep": 401, "frame_abs": 137967, "t_rel": 0, "emitted": true, "iou": 0.08058608058608054, "cdist_px": 13.049088177433385, "box": [ 0.41642228739002934, 0.6578690127077224, 0.4555229716520039, 0.6735092864125122 ], "gt_box": [ 0.4447702834799609, 0.6656891495601173, 0.4975562072336266, 0.6754643206256109 ], "gen_text": "The robot arm has descended and moved forward; the white cloth has been deposited in the teal bowl and is now visible in the wrist view above the counter near the right edge of the counter, with the open teal bowl visible to the left. The gripper is open and converging on the cloth at @grip_now ." }, { "ep": 401, "frame_abs": 138007, "t_rel": 40, "emitted": true, "iou": 0.37271214642262895, "cdist_px": 3.5060583044645237, "box": [ 0.41837732160312807, 0.4946236559139785, 0.447702834799609, 0.5112414467253177 ], "gt_box": [ 0.4115347018572825, 0.49364613880742914, 0.44574780058651026, 0.5024437927663734 ], "gen_text": "The gripper is now low and close to the counter surface, hovering just above the white cloth near the right edge of the counter; the teal bowl sits in the background at @grip_now ." }, { "ep": 401, "frame_abs": 138047, "t_rel": 80, "emitted": true, "iou": 0.5053995680345578, "cdist_px": 2.3714008949890633, "box": [ 0.6852394916911045, 0.4780058651026393, 0.7126099706744868, 0.49071358748778104 ], "gt_box": [ 0.6871945259042033, 0.4809384164222874, 0.7233626588465298, 0.4897360703812317 ], "gen_text": "The gripper is now very close to the cloth surface at the counter surface; the wrist camera shows the white cloth filling the gripper gap, and the gripper is open and nearly at contact height, @grip_now at its configuration." }, { "ep": 401, "frame_abs": 138056, "t_rel": 89, "emitted": true, "iou": 0.1365248226950352, "cdist_px": 7.795292892161548, "box": [ 0.7057673509286413, 0.4750733137829912, 0.7331378299120235, 0.4858260019550342 ], "gt_box": [ 0.6803519061583577, 0.4731182795698925, 0.7165200391006843, 0.48191593352883677 ], "gen_text": "The gripper is closing around the white cloth at the counter surface; the wrist view shows the cloth fabric filling the finger gap between the pads. @grip_now is in a partially closed, partially closed configuration on the cloth." }, { "ep": 401, "frame_abs": 138087, "t_rel": 120, "emitted": true, "iou": 0.38571428571428434, "cdist_px": 2.244886710733674, "box": [ 0.5347018572825024, 0.5034213098729228, 0.5640273704789834, 0.5141739980449658 ], "gt_box": [ 0.530791788856305, 0.5034213098729228, 0.5610948191593352, 0.5083088954056696 ], "gen_text": "The gripper is at the level of the cloth near the teal bowl, fingers nearly closed around the cloth fabric; the wrist view shows the cloth fabric filling the gap between the pads. @grip_now shows the pads seated against the cloth fabric." }, { "ep": 401, "frame_abs": 138109, "t_rel": 142, "emitted": true, "iou": 0.05585585585585578, "cdist_px": 5.054132123419438, "box": [ 0.6881720430107527, 0.4868035190615836, 0.718475073313783, 0.4975562072336266 ], "gt_box": [ 0.6842619745845552, 0.4809384164222874, 0.718475073313783, 0.48778103616813295 ], "gen_text": "The gripper is closing around the white cloth at the counter surface; the wrist view shows the cloth fabric filling the finger gap between the pads. The gripper is actively closing on the cloth at @grip_now ." }, { "ep": 551, "frame_abs": 196830, "t_rel": 0, "emitted": true, "iou": 0.09438775510204077, "cdist_px": 15.197294696790024, "box": [ 0.24437927663734116, 0.41348973607038125, 0.2991202346041056, 0.4389051808406647 ], "gt_box": [ 0.26295210166177907, 0.39589442815249265, 0.33724340175953077, 0.42130987292277616 ], "gen_text": "The cardboard box has been deposited in the coffee machine tray on the table; the gripper is open and retracting upward away from the tray, with the cardboard box still resting on the tray surface. @grip_now at its current open configuration." }, { "ep": 551, "frame_abs": 196870, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 16.683316273862534, "box": [ 0.5962854349951124, 0.624633431085044, 0.6471163245356794, 0.6344086021505376 ], "gt_box": [ 0.603128054740958, 0.6441837732160313, 0.6725317693059628, 0.6637341153470185 ], "gen_text": "The gripper has descended and moved forward over the white mug, now hovering just above the open top of the white mug; the wrist camera shows the open finger pads with the green marker visible inside the mug rim. The gripper is open at @grip_now ." }, { "ep": 551, "frame_abs": 196910, "t_rel": 80, "emitted": true, "iou": 0.5531914893617021, "cdist_px": 2.34604105571849, "box": [ 0.7047898338220919, 0.6119257086999023, 0.7429130009775171, 0.6236559139784946 ], "gt_box": [ 0.696969696969697, 0.6119257086999023, 0.7429130009775171, 0.6295210166177908 ], "gen_text": "The gripper is open and repositioning above the cardboard box on the table; the wrist view shows the open finger pads with the green cap visible inside the box; the wrist camera shows the box's open face. The gripper is open at @grip_now ." }, { "ep": 551, "frame_abs": 196948, "t_rel": 118, "emitted": true, "iou": 0.3076923076923077, "cdist_px": 4.7445744084825, "box": [ 0.7370478983382209, 0.5845552297165201, 0.7741935483870968, 0.5962854349951124 ], "gt_box": [ 0.7321603128054741, 0.5884652981427175, 0.7751710654936461, 0.6070381231671554 ], "gen_text": "The gripper is at table level directly over the cardboard box, fingers beginning to close around its sides; the wrist view shows the pads converging on the box face. The gripper is actively closing on the box face. @grip_now is in a partially closed, seated configuration around the box face." }, { "ep": 551, "frame_abs": 196988, "t_rel": 158, "emitted": true, "iou": 0.0, "cdist_px": 7.406561387952283, "box": [ 0.5454545454545454, 0.5395894428152492, 0.5845552297165201, 0.5464320625610948 ], "gt_box": [ 0.5454545454545454, 0.5503421309872922, 0.5943304007820137, 0.5581622678396871 ], "gen_text": "The gripper is holding the cardboard box upright and has moved it over toward the white mug; the wrist view shows the box held in the pads with the gripper beginning to open, at @grip_now with fingers opening wide." }, { "ep": 551, "frame_abs": 197014, "t_rel": 184, "emitted": true, "iou": 0.0, "cdist_px": 12.800816984253723, "box": [ 0.6168132942326491, 0.541544477028348, 0.6480938416422287, 0.5483870967741935 ], "gt_box": [ 0.624633431085044, 0.5610948191593352, 0.6666666666666666, 0.5659824046920822 ], "gen_text": "The gripper is holding the green marker and has moved to a position above and near the white mug; the wrist view shows the green marker resting on the mug rim with the gripper nearly closed at @grip_now ." }, { "ep": 553, "frame_abs": 197381, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 14.226968133523467, "box": [ 0.10068426197458455, 0.43010752688172044, 0.15347018572825025, 0.4496578690127077 ], "gt_box": [ 0.07820136852394917, 0.41055718475073316, 0.14173998044965788, 0.4291300097751711 ], "gen_text": "The gripper is open and positioned above the table, hovering just above the white plastic water bottle near the center of the table; the green marker and white plastic water bottle are visible to the right; the gripper is open at @grip_now ." }, { "ep": 553, "frame_abs": 197421, "t_rel": 40, "emitted": true, "iou": 0.006121313299944674, "cdist_px": 18.559042063870024, "box": [ 0.22776148582600195, 0.4682306940371456, 0.27468230694037143, 0.4838709677419355 ], "gt_box": [ 0.26392961876832843, 0.45356793743890517, 0.32746823069403713, 0.46920821114369504 ], "gen_text": "The gripper is open and moving toward the green marker lying on the white plate; the white bowl and red mug are visible to the right; the gripper is open at @grip_now ." }, { "ep": 553, "frame_abs": 197461, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 13.009559195942566, "box": [ 0.4780058651026393, 0.4858260019550342, 0.5180840664711632, 0.49560117302052786 ], "gt_box": [ 0.4946236559139785, 0.5034213098729228, 0.5454545454545454, 0.5102639296187683 ], "gen_text": "The gripper is open and moving toward the green marker on the white table surface; the green marker is visible between the open fingers and the gripper is descending toward it, @grip_now in a wide-open converging configuration above the table." }, { "ep": 553, "frame_abs": 197501, "t_rel": 120, "emitted": true, "iou": 0.0, "cdist_px": 13.154809118421529, "box": [ 0.6177908113391984, 0.5004887585532747, 0.6471163245356794, 0.5083088954056696 ], "gt_box": [ 0.6344086021505376, 0.5180840664711632, 0.6735092864125122, 0.5239491691104594 ], "gen_text": "The gripper is open and positioned above the white paper cup at the centre of the table; the green marker is visible between the finger pads and the paper cup is to the right; the gripper is open at @grip_now ." }, { "ep": 553, "frame_abs": 197511, "t_rel": 130, "emitted": true, "iou": 0.0, "cdist_px": 10.69959343201537, "box": [ 0.6480938416422287, 0.4868035190615836, 0.6813294232649071, 0.49266862170087977 ], "gt_box": [ 0.6529814271749755, 0.5024437927663734, 0.6920821114369502, 0.509286412512219 ], "gen_text": "The gripper is now low and close above the white paper cup on the table surface, with the green Expo marker visible between the pads; the gripper is nearly closed, @grip_now at its configuration." }, { "ep": 553, "frame_abs": 197551, "t_rel": 170, "emitted": true, "iou": 0.17932296431838962, "cdist_px": 7.825706716344606, "box": [ 0.4144672531769306, 0.5738025415444771, 0.45650048875855326, 0.5845552297165201 ], "gt_box": [ 0.4291300097751711, 0.5777126099706745, 0.4789833822091887, 0.5933528836754643 ], "gen_text": "The gripper is holding the white marker pen and has moved it to a position above the table near the cardboard box; the wrist view shows the pen body and the cardboard box below on the table surface. The gripper is closed and holding the marker, at @grip_now at its configuration." }, { "ep": 691, "frame_abs": 233450, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 36.49055516286062, "box": [ 0.6490713587487781, 0.3519061583577713, 0.6871945259042033, 0.3782991202346041 ], "gt_box": [ 0.6236559139784946, 0.2932551319648094, 0.6695992179863147, 0.3255131964809384 ], "gen_text": "The arm has descended and moved forward; the green marker pen is visible in the wrist view with the open gripper positioned above and to the right of the pen; the gripper is open and converging on the pen at @grip_now ." }, { "ep": 691, "frame_abs": 233490, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 40.25921974704357, "box": [ 0.7204301075268817, 0.5933528836754643, 0.7448680351906158, 0.6109481915933529 ], "gt_box": [ 0.6334310850439883, 0.5581622678396871, 0.6539589442815249, 0.5698924731182796 ], "gen_text": "The gripper is closing around the green marker pen on the table; the fingers are nearly fully closed around the pen's barrel, at @grip_now ." }, { "ep": 691, "frame_abs": 233498, "t_rel": 48, "emitted": true, "iou": 0.0, "cdist_px": 43.8686850668466, "box": [ 0.7057673509286413, 0.5933528836754643, 0.7302052785923754, 0.6089931573802542 ], "gt_box": [ 0.6070381231671554, 0.5562072336265884, 0.6275659824046921, 0.5689149560117303 ], "gen_text": "The gripper is closing around the green marker pen at the table surface, with the pads seated on the pen's body; the fingers are beginning to close around the pen at @grip_now ." }, { "ep": 691, "frame_abs": 233525, "t_rel": 75, "emitted": true, "iou": 0.0, "cdist_px": 26.89581430943152, "box": [ 0.5962854349951124, 0.6295210166177908, 0.6158357771260997, 0.6422287390029325 ], "gt_box": [ 0.5347018572825024, 0.6060606060606061, 0.552297165200391, 0.6197458455522972 ], "gen_text": "The gripper is closing around the green marker pen at the table surface; the wrist view shows the pen's orange cap and the green cap visible between the pads. The grip is secure on the pen at @grip_now ." } ], "A_kf-8": [ { "ep": 48, "frame_abs": 10655, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 15.910867908919625, "box": [ 0.37732160312805474, 0.6119257086999023, 0.4125122189638319, 0.6295210166177908 ], "gt_box": [ 0.41837732160312807, 0.6168132942326491, 0.458455522971652, 0.6334310850439883 ], "gen_text": "Continue descending and moving horizontally toward the white rope, yawing the wrist to square the finger line across the table surface; keep the gripper open at @grip as the arm converge toward the rope." }, { "ep": 48, "frame_abs": 10695, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 41.23459612238728, "box": [ 0.41642228739002934, 0.5982404692082112, 0.447702834799609, 0.6148582600195504 ], "gt_box": [ 0.4858260019550342, 0.5483870967741935, 0.5190615835777126, 0.5630498533724341 ], "gen_text": "Close the gripper firmly around the white rope to secure the grasp, then begin lifting upward and swinging across toward the red-and-white board. @grip " }, { "ep": 48, "frame_abs": 10701, "t_rel": 78, "emitted": true, "iou": 0.0, "cdist_px": 57.6526164585662, "box": [ 0.41642228739002934, 0.6217008797653959, 0.4496578690127077, 0.635386119257087 ], "gt_box": [ 0.48778103616813295, 0.5405669599217986, 0.5210166177908113, 0.555229716520039 ], "gen_text": "Continue lifting and retracting the open gripper at @grip upward and away from the table, completing the task." }, { "ep": 48, "frame_abs": 10741, "t_rel": 118, "emitted": true, "iou": 0.0, "cdist_px": 90.3730102582385, "box": [ 0.34506353861192574, 0.6197458455522972, 0.37927663734115347, 0.6334310850439883 ], "gt_box": [ 0.16911045943304007, 0.7223851417399805, 0.22776148582600195, 0.7448680351906158 ], "gen_text": "Continue descending and moving horizontally toward the white rope, swinging the nose to align the finger pads with the rope; keep @grip open and converging until the pads bracket the rope." }, { "ep": 50, "frame_abs": 11095, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 19.181436371981302, "box": [ 0.33235581622678395, 0.6432062561094819, 0.3675464320625611, 0.6608015640273704 ], "gt_box": [ 0.3831867057673509, 0.6451612903225806, 0.4232649071358749, 0.6578690127077224 ], "gen_text": "Retract the arm upward and away from the placed green paddle-shaped object, keeping the gripper fully open at @grip to complete the placement." }, { "ep": 50, "frame_abs": 11135, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 21.23805235996808, "box": [ 0.42717497556207235, 0.5982404692082112, 0.4633431085043988, 0.6109481915933529 ], "gt_box": [ 0.45454545454545453, 0.5698924731182796, 0.48778103616813295, 0.5796676441837733 ], "gen_text": "Hold the gripper open at @grip and retract upward away from the placed object, completing the placement of the green paddle-shaped object in the bin." }, { "ep": 50, "frame_abs": 11140, "t_rel": 77, "emitted": true, "iou": 0.0, "cdist_px": 27.09229529984454, "box": [ 0.41837732160312807, 0.5982404692082112, 0.45454545454545453, 0.6109481915933529 ], "gt_box": [ 0.44868035190615835, 0.5610948191593352, 0.48191593352883677, 0.5698924731182796 ], "gen_text": "Hold the arm in the raised retracted position with the green paddle-shaped object held at @grip , then sweep laterally toward the target drop zone on the table." }, { "ep": 50, "frame_abs": 11180, "t_rel": 117, "emitted": true, "iou": 0.0, "cdist_px": 38.43989542525569, "box": [ 0.39882697947214074, 0.6285434995112414, 0.43010752688172044, 0.6402737047898338 ], "gt_box": [ 0.3626588465298143, 0.6852394916911045, 0.4066471163245357, 0.6989247311827957 ], "gen_text": "Continue moving the gripper forward and downward toward the target drop zone on the table surface, keeping the green paddle-shaped object elevated and the arm elevated. @grip " }, { "ep": 52, "frame_abs": 11677, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 34.86983698734097, "box": [ 0.37732160312805474, 0.6158357771260997, 0.4125122189638319, 0.6295210166177908 ], "gt_box": [ 0.4525904203323558, 0.5845552297165201, 0.4946236559139785, 0.5972629521016618 ], "gen_text": "Continue descending and swinging the final horizontal axis toward the target placement spot while keeping @grip closed on the cord." }, { "ep": 52, "frame_abs": 11706, "t_rel": 61, "emitted": true, "iou": 0.0, "cdist_px": 32.71893882918861, "box": [ 0.4398826979472141, 0.5913978494623656, 0.4731182795698925, 0.6041055718475073 ], "gt_box": [ 0.47116324535679377, 0.544477028347996, 0.5053763440860215, 0.555229716520039 ], "gen_text": "Open the fingers to release the white braided rope onto the green/mint cloth and retract upward away from the cloth; hold @grip ." }, { "ep": 52, "frame_abs": 11746, "t_rel": 101, "emitted": true, "iou": 0.0, "cdist_px": 34.72319088707492, "box": [ 0.20821114369501467, 0.6989247311827957, 0.2482893450635386, 0.7204301075268817 ], "gt_box": [ 0.10850439882697947, 0.7106549364613881, 0.16226783968719452, 0.7380254154447703 ], "gen_text": "Continue descending and converging toward the target slot on the cable loop, keeping the gripper open at @grip to align the cable loop with the target target." }, { "ep": 52, "frame_abs": 11780, "t_rel": 135, "emitted": true, "iou": 0.0, "cdist_px": 23.55523359056558, "box": [ 0.3411534701857282, 0.5992179863147605, 0.37438905180840665, 0.6158357771260997 ], "gt_box": [ 0.34408602150537637, 0.6363636363636364, 0.3831867057673509, 0.6520039100684262 ], "gen_text": "Continue transporting the gripped power cord away from the table and descend toward the target drop zone at the final rest position on the table, maintaining the closed grip. @grip " }, { "ep": 52, "frame_abs": 11794, "t_rel": 149, "emitted": true, "iou": 0.0, "cdist_px": 33.802626349209255, "box": [ 0.26588465298142716, 0.6217008797653959, 0.3020527859237537, 0.6383186705767351 ], "gt_box": [ 0.19452590420332355, 0.6598240469208211, 0.23655913978494625, 0.6725317693059628 ], "gen_text": "Continue descending and swinging the open gripper laterally to align with the target placement spot on the table; keep @grip open and converging as the fingers bracket the target spot." }, { "ep": 320, "frame_abs": 112489, "t_rel": 32, "emitted": false, "iou": null, "cdist_px": null, "box": null, "gt_box": [ 0.18181818181818182, 0.9081133919843597, 0.2375366568914956, 0.9364613880742912 ], "gen_text": "Move the gripper down and forward toward the green marker on the table, keeping @grip open and converging on the marker body." }, { "ep": 320, "frame_abs": 112524, "t_rel": 67, "emitted": true, "iou": 0.0, "cdist_px": 41.028301602894985, "box": [ 0.30791788856304986, 0.7096774193548387, 0.3421309872922776, 0.7272727272727273 ], "gt_box": [ 0.34408602150537637, 0.7712609970674487, 0.38025415444770283, 0.7869012707722385 ], "gen_text": "Retract the arm upward and away from the box, keeping the gripper open @grip as the arm withdraws to a clear home position." }, { "ep": 320, "frame_abs": 112556, "t_rel": 99, "emitted": true, "iou": 0.0, "cdist_px": 37.40552199808424, "box": [ 0.26588465298142716, 0.6480938416422287, 0.3088954056695992, 0.667644183773216 ], "gt_box": [ 0.3176930596285435, 0.6989247311827957, 0.3626588465298143, 0.7174975562072337 ], "gen_text": "Retract the arm upward and away from the box, keeping the gripper open @grip as the arm withdraws to a clear position." }, { "ep": 320, "frame_abs": 112564, "t_rel": 107, "emitted": true, "iou": 0.0, "cdist_px": 64.20795498861006, "box": [ 0.3391984359726295, 0.6432062561094819, 0.37927663734115347, 0.6617790811339198 ], "gt_box": [ 0.30009775171065495, 0.7409579667644184, 0.3460410557184751, 0.7605083088954057 ], "gen_text": "Retract the arm upward and away from the box, keeping the gripper open @grip as the arm withdraws to a clear position." }, { "ep": 401, "frame_abs": 137999, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 75.41913111702746, "box": [ 0.3870967741935484, 0.6060606060606061, 0.43010752688172044, 0.624633431085044 ], "gt_box": [ 0.4115347018572825, 0.49364613880742914, 0.44574780058651026, 0.5024437927663734 ], "gen_text": "Retract the gripper upward and away from the bowl, keeping @grip fully open to clear the bowl without disturbing the cloth." }, { "ep": 401, "frame_abs": 138039, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 17.14416754721026, "box": [ 0.6471163245356794, 0.46920821114369504, 0.6754643206256109, 0.4809384164222874 ], "gt_box": [ 0.6871945259042033, 0.4809384164222874, 0.7233626588465298, 0.4897360703812317 ], "gen_text": "Close the gripper firmly around the white cloth at @grip to secure a stable grasp, then begin lifting upward." }, { "ep": 401, "frame_abs": 138048, "t_rel": 81, "emitted": true, "iou": 0.08047690014903118, "cdist_px": 9.372610189235335, "box": [ 0.7106549364613881, 0.46920821114369504, 0.7380254154447703, 0.48289345063538613 ], "gt_box": [ 0.6803519061583577, 0.4731182795698925, 0.7165200391006843, 0.48191593352883677 ], "gen_text": "Close the gripper firmly around the cloth at @grip to secure a stable grasp, then begin lifting upward." }, { "ep": 401, "frame_abs": 138079, "t_rel": 112, "emitted": true, "iou": 0.0, "cdist_px": 18.373504730393076, "box": [ 0.4946236559139785, 0.4789833822091887, 0.5259042033235581, 0.4916911045943304 ], "gt_box": [ 0.530791788856305, 0.5034213098729228, 0.5610948191593352, 0.5083088954056696 ], "gen_text": "Close the gripper firmly around the white cloth fabric at @grip , securing a stable grasp on the fabric before lifting." }, { "ep": 401, "frame_abs": 138101, "t_rel": 134, "emitted": true, "iou": 0.34924078091106225, "cdist_px": 3.16715542521996, "box": [ 0.6959921798631477, 0.4780058651026393, 0.7243401759530792, 0.49071358748778104 ], "gt_box": [ 0.6842619745845552, 0.4809384164222874, 0.718475073313783, 0.48778103616813295 ], "gen_text": "Close the gripper firmly to secure the cloth between the pads at @grip , completing the grasp before lifting." }, { "ep": 551, "frame_abs": 196862, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 85.39369453874592, "box": [ 0.3812316715542522, 0.6217008797653959, 0.43010752688172044, 0.6314760508308895 ], "gt_box": [ 0.603128054740958, 0.6441837732160313, 0.6725317693059628, 0.6637341153470185 ], "gen_text": "Continue descending and swinging the gripper nose toward the cardboard box, keeping fingers open and converging @grip as the pads converge on the box's sides." }, { "ep": 551, "frame_abs": 196902, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 18.136238183027558, "box": [ 0.6549364613880743, 0.6011730205278593, 0.6940371456500489, 0.6158357771260997 ], "gt_box": [ 0.696969696969697, 0.6119257086999023, 0.7429130009775171, 0.6295210166177908 ], "gen_text": "Close the gripper fingers firmly around the cardboard box's sides to secure a stable grasp, then begin lifting upward off the table. @grip " }, { "ep": 551, "frame_abs": 196940, "t_rel": 110, "emitted": true, "iou": 0.13926325247079963, "cdist_px": 7.292565884982784, "box": [ 0.7448680351906158, 0.581622678396872, 0.7800586510263929, 0.5933528836754643 ], "gt_box": [ 0.7321603128054741, 0.5884652981427175, 0.7751710654936461, 0.6070381231671554 ], "gen_text": "Close the gripper fingers firmly around the cardboard box's sides to secure a stable grip @grip ." }, { "ep": 551, "frame_abs": 196980, "t_rel": 150, "emitted": true, "iou": 0.20910384068278803, "cdist_px": 3.458808604683111, "box": [ 0.544477028347996, 0.544477028347996, 0.5933528836754643, 0.5532746823069403 ], "gt_box": [ 0.5454545454545454, 0.5503421309872922, 0.5943304007820137, 0.5581622678396871 ], "gen_text": "Open the gripper fully @grip to release the cardboard box, then lift the arm upward and retract to clear the workspace." }, { "ep": 551, "frame_abs": 197006, "t_rel": 176, "emitted": true, "iou": 0.0, "cdist_px": 20.149802704924284, "box": [ 0.6217008797653959, 0.5288367546432062, 0.656891495601173, 0.5356793743890518 ], "gt_box": [ 0.624633431085044, 0.5610948191593352, 0.6666666666666666, 0.5659824046920822 ], "gen_text": "Close the gripper @grip firmly around the box's sides, securing a grip on the box face before lifting." }, { "ep": 553, "frame_abs": 197413, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 28.47744066583376, "box": [ 0.19452590420332355, 0.4613880742913001, 0.24144672531769307, 0.4780058651026393 ], "gt_box": [ 0.26392961876832843, 0.45356793743890517, 0.32746823069403713, 0.46920821114369504 ], "gen_text": "Continue descending and converging toward the green marker on the white mug, keeping the fingers open and converging on the marker's barrel at @grip ." }, { "ep": 553, "frame_abs": 197453, "t_rel": 72, "emitted": true, "iou": 0.0, "cdist_px": 36.3473670497067, "box": [ 0.4144672531769306, 0.47409579667644186, 0.4506353861192571, 0.48289345063538613 ], "gt_box": [ 0.4946236559139785, 0.5034213098729228, 0.5454545454545454, 0.5102639296187683 ], "gen_text": "Continue descending toward the green marker on the white table, swinging the nose forward and down to align the finger pads with the marker's barrel \u2014 @grip open and converging on the marker." }, { "ep": 553, "frame_abs": 197493, "t_rel": 112, "emitted": true, "iou": 0.0, "cdist_px": 18.94059908504513, "box": [ 0.603128054740958, 0.4946236559139785, 0.6363636363636364, 0.5024437927663734 ], "gt_box": [ 0.6344086021505376, 0.5180840664711632, 0.6735092864125122, 0.5239491691104594 ], "gen_text": "Close the fingers firmly around the green marker pen to secure the grip, then begin lifting upward out of the white bowl \u2014 @grip ." }, { "ep": 553, "frame_abs": 197503, "t_rel": 122, "emitted": true, "iou": 0.0, "cdist_px": 13.415637936073894, "box": [ 0.6217008797653959, 0.4946236559139785, 0.6549364613880743, 0.5004887585532747 ], "gt_box": [ 0.6529814271749755, 0.5024437927663734, 0.6920821114369502, 0.509286412512219 ], "gen_text": "Close the fingers firmly around the green marker handle to secure the grasp, then begin lifting upward out of the white plastic container; achieve @grip fully seated on the marker barrel." }, { "ep": 553, "frame_abs": 197543, "t_rel": 162, "emitted": true, "iou": 0.0, "cdist_px": 24.255231063638995, "box": [ 0.4173998044965787, 0.544477028347996, 0.45650048875855326, 0.5532746823069403 ], "gt_box": [ 0.4291300097751711, 0.5777126099706745, 0.4789833822091887, 0.5933528836754643 ], "gen_text": "Continue moving the open gripper laterally and downward toward the white plastic food container on the table, descending to align the finger pads with the container's sides; maintain @grip open as you close the remaining vertical distance." }, { "ep": 691, "frame_abs": 233482, "t_rel": 32, "emitted": true, "iou": 0.0, "cdist_px": 30.45943304007823, "box": [ 0.6471163245356794, 0.5992179863147605, 0.6774193548387096, 0.6217008797653959 ], "gt_box": [ 0.6334310850439883, 0.5581622678396871, 0.6539589442815249, 0.5698924731182796 ], "gen_text": "Retract the arm upward and away from the placed green marker pen, keeping @grip fully open as the robot returns to a neutral position." }, { "ep": 691, "frame_abs": 233490, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 48.229647959663936, "box": [ 0.7174975562072337, 0.5943304007820137, 0.7429130009775171, 0.6119257086999023 ], "gt_box": [ 0.6070381231671554, 0.5562072336265884, 0.6275659824046921, 0.5689149560117303 ], "gen_text": "Close the gripper firmly around the green marker pen at @grip , securing a stable hold before lifting." }, { "ep": 691, "frame_abs": 233517, "t_rel": 67, "emitted": true, "iou": 0.0, "cdist_px": 26.222781417523144, "box": [ 0.5992179863147605, 0.6226783968719453, 0.6217008797653959, 0.635386119257087 ], "gt_box": [ 0.5347018572825024, 0.6060606060606061, 0.552297165200391, 0.6197458455522972 ], "gen_text": "Open the gripper fully to release the green marker pen onto the table and retract the arm upward and away. @grip " }, { "ep": 691, "frame_abs": 233536, "t_rel": 86, "emitted": true, "iou": 0.0, "cdist_px": 33.5114535332206, "box": [ 0.5962854349951124, 0.6236559139784946, 0.6187683284457478, 0.6383186705767351 ], "gt_box": [ 0.5317693059628543, 0.5855327468230694, 0.5503421309872922, 0.603128054740958 ], "gen_text": "Close the gripper firmly around the green marker pen to secure a stable grasp, then begin lifting upward away from the table surface. @grip " } ], "A_kf": [ { "ep": 48, "frame_abs": 10623, "t_rel": 0, "emitted": false, "iou": null, "cdist_px": null, "box": null, "gt_box": [ 0.0, 0.7546432062561095, 0.022482893450635387, 0.7800586510263929 ], "gen_text": "Continue descending and swinging the arm leftward and slightly upward to position the open gripper above the table, aligning the finger pads with the white rope at @grip ." }, { "ep": 48, "frame_abs": 10663, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 16.46330352487609, "box": [ 0.3812316715542522, 0.6060606060606061, 0.4125122189638319, 0.6226783968719453 ], "gt_box": [ 0.41837732160312807, 0.6168132942326491, 0.458455522971652, 0.6334310850439883 ], "gen_text": "Continue descending toward the white rope, swinging the nose to align the finger pads with the rope body, and begin converging @grip toward the grasp target." }, { "ep": 48, "frame_abs": 10703, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 33.15028442673555, "box": [ 0.4173998044965787, 0.5826001955034213, 0.447702834799609, 0.5962854349951124 ], "gt_box": [ 0.4858260019550342, 0.5483870967741935, 0.5190615835777126, 0.5630498533724341 ], "gen_text": "Close the gripper firmly around the white rope to secure the grasp, then begin lifting upward and swinging across toward the red-and-white board. @grip " }, { "ep": 48, "frame_abs": 10709, "t_rel": 86, "emitted": true, "iou": 0.0, "cdist_px": 24.340890433008042, "box": [ 0.43792766373411535, 0.5659824046920822, 0.46725317693059626, 0.5786901270772239 ], "gt_box": [ 0.48778103616813295, 0.5405669599217986, 0.5210166177908113, 0.555229716520039 ], "gen_text": "Open the gripper fully at @grip to release the white rope, then retract the arm upward and away from the placed rope." }, { "ep": 48, "frame_abs": 10749, "t_rel": 126, "emitted": true, "iou": 0.0, "cdist_px": 29.395810172766293, "box": [ 0.1681329423264907, 0.6774193548387096, 0.20527859237536658, 0.6989247311827957 ], "gt_box": [ 0.16911045943304007, 0.7223851417399805, 0.22776148582600195, 0.7448680351906158 ], "gen_text": "Continue descending and moving horizontally toward the white rope, letting the gripper @grip open and converging the fingers to bracket the rope." }, { "ep": 50, "frame_abs": 11063, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 51.681106629496796, "box": [ 0.10850439882697947, 0.6207233626588465, 0.1495601173020528, 0.6402737047898338 ], "gt_box": [ 0.02932551319648094, 0.6920821114369502, 0.08797653958944282, 0.7096774193548387 ], "gen_text": "Continue descending toward the target drop zone, keeping @grip open and converging the fingers to bracket the green paddle-shaped object." }, { "ep": 50, "frame_abs": 11103, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 9.798603390922116, "box": [ 0.3812316715542522, 0.6285434995112414, 0.4173998044965787, 0.6441837732160313 ], "gt_box": [ 0.3831867057673509, 0.6451612903225806, 0.4232649071358749, 0.6578690127077224 ], "gen_text": "Retract the arm upward and away from the placed green paddle-shaped object, keeping the gripper fully open at @grip to complete the placement." }, { "ep": 50, "frame_abs": 11143, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 17.69706645491634, "box": [ 0.4398826979472141, 0.5953079178885631, 0.4731182795698925, 0.6070381231671554 ], "gt_box": [ 0.45454545454545453, 0.5698924731182796, 0.48778103616813295, 0.5796676441837733 ], "gen_text": "Close the gripper firmly around the green paddle-shaped object @grip to secure the grasp before lifting." }, { "ep": 50, "frame_abs": 11148, "t_rel": 85, "emitted": true, "iou": 0.0, "cdist_px": 20.268529038601166, "box": [ 0.4398826979472141, 0.5913978494623656, 0.4731182795698925, 0.6021505376344086 ], "gt_box": [ 0.44868035190615835, 0.5610948191593352, 0.48191593352883677, 0.5698924731182796 ], "gen_text": "Continue transporting the green paddle-shaped object laterally toward the target drop zone on the table surface, maintaining the closed @grip hold on the paddle." }, { "ep": 50, "frame_abs": 11188, "t_rel": 125, "emitted": true, "iou": 0.0, "cdist_px": 39.419773273732716, "box": [ 0.36656891495601174, 0.6236559139784946, 0.39882697947214074, 0.6373411534701857 ], "gt_box": [ 0.3626588465298143, 0.6852394916911045, 0.4066471163245357, 0.6989247311827957 ], "gen_text": "Continue descending toward the target drop zone, keeping the gripper open @grip and moving the arm forward toward the target drop zone." }, { "ep": 52, "frame_abs": 11645, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 51.65474547310158, "box": [ 0.1739980449657869, 0.7096774193548387, 0.21603128054740958, 0.7302052785923754 ], "gt_box": [ 0.21407624633431085, 0.7869012707722385, 0.2649071358748778, 0.8064516129032258 ], "gen_text": "Continue descending and converging toward the target object on the tabletop, keeping the gripper open at @grip as the arm angles down to approach the target object." }, { "ep": 52, "frame_abs": 11685, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 18.28793176266142, "box": [ 0.40860215053763443, 0.5933528836754643, 0.44379276637341153, 0.6089931573802542 ], "gt_box": [ 0.4525904203323558, 0.5845552297165201, 0.4946236559139785, 0.5972629521016618 ], "gen_text": "Descend the final distance onto the power cable, converging the fingers to bracket the power cable body at @grip before closing." }, { "ep": 52, "frame_abs": 11714, "t_rel": 69, "emitted": true, "iou": 0.0, "cdist_px": 21.978489058711745, "box": [ 0.4398826979472141, 0.5738025415444771, 0.4731182795698925, 0.5845552297165201 ], "gt_box": [ 0.47116324535679377, 0.544477028347996, 0.5053763440860215, 0.555229716520039 ], "gen_text": "Open the fingers to release the white braided rope and retract the gripper upward away from the closed bundle; hold @grip ." }, { "ep": 52, "frame_abs": 11754, "t_rel": 109, "emitted": true, "iou": 0.4717573221757323, "cdist_px": 4.976858217945802, "box": [ 0.12218963831867058, 0.7165200391006843, 0.17008797653958943, 0.7419354838709677 ], "gt_box": [ 0.10850439882697947, 0.7106549364613881, 0.16226783968719452, 0.7380254154447703 ], "gen_text": "Open the fingers to release the cable segment onto the tabletop at @grip , then retract the gripper upward and away." }, { "ep": 52, "frame_abs": 11788, "t_rel": 143, "emitted": true, "iou": 0.0, "cdist_px": 10.398135766080717, "box": [ 0.3411534701857282, 0.6217008797653959, 0.3704789833822092, 0.635386119257087 ], "gt_box": [ 0.34408602150537637, 0.6363636363636364, 0.3831867057673509, 0.6520039100684262 ], "gen_text": "Continue descending and swinging the white cord toward the power cable plug on the table; keep @grip closed until the cord is fully seated." }, { "ep": 320, "frame_abs": 112497, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 102.6870623852784, "box": [ 0.19745845552297164, 0.7497556207233627, 0.24535679374389052, 0.7741935483870968 ], "gt_box": [ 0.18181818181818182, 0.9081133919843597, 0.2375366568914956, 0.9364613880742912 ], "gen_text": "Continue descending toward the green marker on the table surface, swinging the nose to align the finger-line across the marker's body while keeping @grip open and converging." }, { "ep": 320, "frame_abs": 112532, "t_rel": 75, "emitted": true, "iou": 0.0, "cdist_px": 53.84395641703548, "box": [ 0.3411534701857282, 0.6871945259042033, 0.37145650048875856, 0.7028347996089932 ], "gt_box": [ 0.34408602150537637, 0.7712609970674487, 0.38025415444770283, 0.7869012707722385 ], "gen_text": "Close the gripper firmly around the green marker at @grip , securing the pads against the marker body, then begin lifting upward." }, { "ep": 320, "frame_abs": 112564, "t_rel": 107, "emitted": true, "iou": 0.0, "cdist_px": 36.31408031843398, "box": [ 0.3391984359726295, 0.6432062561094819, 0.37927663734115347, 0.6617790811339198 ], "gt_box": [ 0.3176930596285435, 0.6989247311827957, 0.3626588465298143, 0.7174975562072337 ], "gen_text": "Retract the arm upward and away from the box, keeping the gripper open @grip as the arm withdraws to a clear position." }, { "ep": 401, "frame_abs": 137967, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 15.765675182556201, "box": [ 0.41642228739002934, 0.6471163245356794, 0.4555229716520039, 0.6647116324535679 ], "gt_box": [ 0.4447702834799609, 0.6656891495601173, 0.4975562072336266, 0.6754643206256109 ], "gen_text": "Descend the final distance onto the white cloth and begin closing the fingers @grip around the cloth fabric, seating the pads against the cloth fabric." }, { "ep": 401, "frame_abs": 138007, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 13.448418740262486, "box": [ 0.4398826979472141, 0.5034213098729228, 0.47409579667644186, 0.520039100684262 ], "gt_box": [ 0.4115347018572825, 0.49364613880742914, 0.44574780058651026, 0.5024437927663734 ], "gen_text": "Retract the gripper upward and away from the bowl with @grip fully open, leaving the cloth in the bowl." }, { "ep": 401, "frame_abs": 138047, "t_rel": 80, "emitted": true, "iou": 0.5662921348314611, "cdist_px": 1.7026765489581837, "box": [ 0.6871945259042033, 0.4780058651026393, 0.7145650048875856, 0.49071358748778104 ], "gt_box": [ 0.6871945259042033, 0.4809384164222874, 0.7233626588465298, 0.4897360703812317 ], "gen_text": "Close the gripper firmly around the cloth at @grip to secure a stable grasp, then begin lifting upward." }, { "ep": 401, "frame_abs": 138056, "t_rel": 89, "emitted": true, "iou": 0.0, "cdist_px": 12.144791494955536, "box": [ 0.718475073313783, 0.4701857282502444, 0.7458455522971652, 0.4838709677419355 ], "gt_box": [ 0.6803519061583577, 0.4731182795698925, 0.7165200391006843, 0.48191593352883677 ], "gen_text": "Close the gripper firmly around the white cloth at @grip to secure a stable grasp, then begin lifting upward." }, { "ep": 401, "frame_abs": 138087, "t_rel": 120, "emitted": true, "iou": 0.36982248520709926, "cdist_px": 2.455650605918851, "box": [ 0.5366568914956011, 0.5034213098729228, 0.5640273704789834, 0.5141739980449658 ], "gt_box": [ 0.530791788856305, 0.5034213098729228, 0.5610948191593352, 0.5083088954056696 ], "gen_text": "Close the gripper firmly around the cloth fabric to secure a stable grasp @grip , then begin lifting upward." }, { "ep": 401, "frame_abs": 138109, "t_rel": 142, "emitted": true, "iou": 0.49411764705882366, "cdist_px": 0.52785923753666, "box": [ 0.6852394916911045, 0.4780058651026393, 0.7145650048875856, 0.49071358748778104 ], "gt_box": [ 0.6842619745845552, 0.4809384164222874, 0.718475073313783, 0.48778103616813295 ], "gen_text": "Close the gripper firmly around the white cloth fabric to secure a stable grasp, keeping a firm closed hold at @grip ." }, { "ep": 551, "frame_abs": 196830, "t_rel": 0, "emitted": true, "iou": 0.3133913043478267, "cdist_px": 8.673039463961707, "box": [ 0.2561094819159335, 0.3831867057673509, 0.3147605083088954, 0.4125122189638319 ], "gt_box": [ 0.26295210166177907, 0.39589442815249265, 0.33724340175953077, 0.42130987292277616 ], "gen_text": "Retract the arm upward and clear of the workspace with the gripper fully open at @grip to clear the workspace." }, { "ep": 551, "frame_abs": 196870, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 21.510920817303372, "box": [ 0.5738025415444771, 0.6236559139784946, 0.6236559139784946, 0.6334310850439883 ], "gt_box": [ 0.603128054740958, 0.6441837732160313, 0.6725317693059628, 0.6637341153470185 ], "gen_text": "Descend the last few centimetres to seat the pads against the box's sides, then close the fingers to grasp it @grip ." }, { "ep": 551, "frame_abs": 196910, "t_rel": 80, "emitted": true, "iou": 0.021582733812949638, "cdist_px": 10.59881008617151, "box": [ 0.7135874877810362, 0.5992179863147605, 0.7536656891495601, 0.6129032258064516 ], "gt_box": [ 0.696969696969697, 0.6119257086999023, 0.7429130009775171, 0.6295210166177908 ], "gen_text": "Close the gripper fingers firmly around the cardboard box's sides to secure a stable grasp, then begin lifting upward off the table. @grip " }, { "ep": 551, "frame_abs": 196948, "t_rel": 118, "emitted": true, "iou": 0.0, "cdist_px": 13.322320968356246, "box": [ 0.7223851417399805, 0.5718475073313783, 0.7595307917888563, 0.5845552297165201 ], "gt_box": [ 0.7321603128054741, 0.5884652981427175, 0.7751710654936461, 0.6070381231671554 ], "gen_text": "Close the gripper fingers firmly around the cardboard box to complete the grasp @grip , then begin lifting upward." }, { "ep": 551, "frame_abs": 196988, "t_rel": 158, "emitted": true, "iou": 0.0, "cdist_px": 9.399872808674003, "box": [ 0.541544477028348, 0.5366568914956011, 0.5845552297165201, 0.5434995112414467 ], "gt_box": [ 0.5454545454545454, 0.5503421309872922, 0.5943304007820137, 0.5581622678396871 ], "gen_text": "Open the gripper fully @grip to release the cardboard box, then lift the arm clear of the workspace." }, { "ep": 551, "frame_abs": 197014, "t_rel": 184, "emitted": true, "iou": 0.0, "cdist_px": 14.093952695544877, "box": [ 0.6148582600195504, 0.5395894428152492, 0.6480938416422287, 0.5464320625610948 ], "gt_box": [ 0.624633431085044, 0.5610948191593352, 0.6666666666666666, 0.5659824046920822 ], "gen_text": "Close the gripper firmly around the cardboard box lid and lift it upward off the table, seating @grip firmly onto the box walls to secure a stable grasp." }, { "ep": 553, "frame_abs": 197381, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 30.44344851167773, "box": [ 0.12121212121212122, 0.45161290322580644, 0.17790811339198437, 0.47214076246334313 ], "gt_box": [ 0.07820136852394917, 0.41055718475073316, 0.14173998044965788, 0.4291300097751711 ], "gen_text": "Continue descending and converging toward the green marker on the table, aligning the open fingers around the marker's shaft at @grip , ready to close." }, { "ep": 553, "frame_abs": 197421, "t_rel": 40, "emitted": true, "iou": 0.17073170731707363, "cdist_px": 13.329276450147713, "box": [ 0.23362658846529813, 0.45650048875855326, 0.2844574780058651, 0.47214076246334313 ], "gt_box": [ 0.26392961876832843, 0.45356793743890517, 0.32746823069403713, 0.46920821114369504 ], "gen_text": "Continue descending and converging toward the green marker on the white mug, keeping the gripper open @grip to align the fingers on either side of the marker." }, { "ep": 553, "frame_abs": 197461, "t_rel": 80, "emitted": true, "iou": 0.0, "cdist_px": 5.5008423523187355, "box": [ 0.49560117302052786, 0.49364613880742914, 0.5366568914956011, 0.5034213098729228 ], "gt_box": [ 0.4946236559139785, 0.5034213098729228, 0.5454545454545454, 0.5102639296187683 ], "gen_text": "Continue descending toward the green marker on the white paper towel roll, swinging the nose to align the finger pads with the marker's shaft, keeping @grip open and converging onto the marker." }, { "ep": 553, "frame_abs": 197501, "t_rel": 120, "emitted": true, "iou": 0.0, "cdist_px": 21.915959688957777, "box": [ 0.6217008797653959, 0.4838709677419355, 0.656891495601173, 0.4916911045943304 ], "gt_box": [ 0.6344086021505376, 0.5180840664711632, 0.6735092864125122, 0.5239491691104594 ], "gen_text": "Close the fingers firmly around the green marker handle to secure the grasp; hold @grip closed on the marker barrel." }, { "ep": 553, "frame_abs": 197511, "t_rel": 130, "emitted": true, "iou": 0.0, "cdist_px": 14.390552537660291, "box": [ 0.6471163245356794, 0.4809384164222874, 0.6813294232649071, 0.4868035190615836 ], "gt_box": [ 0.6529814271749755, 0.5024437927663734, 0.6920821114369502, 0.509286412512219 ], "gen_text": "Close the fingers firmly around the green marker pen to secure the grip, then begin lifting upward out of the white plastic cup. @grip " }, { "ep": 553, "frame_abs": 197551, "t_rel": 170, "emitted": true, "iou": 0.13812154696132584, "cdist_px": 7.665753718768058, "box": [ 0.4173998044965787, 0.5728250244379277, 0.458455522971652, 0.5826001955034213 ], "gt_box": [ 0.4291300097751711, 0.5777126099706745, 0.4789833822091887, 0.5933528836754643 ], "gen_text": "Continue descending toward the white marker on the table surface, keeping the gripper open and aligning the pads around the marker's barrel \u2014 @grip opening to release the marker onto the table surface." }, { "ep": 691, "frame_abs": 233450, "t_rel": 0, "emitted": true, "iou": 0.0, "cdist_px": 193.000359636585, "box": [ 0.5718475073313783, 0.5953079178885631, 0.6158357771260997, 0.6236559139784946 ], "gt_box": [ 0.6236559139784946, 0.2932551319648094, 0.6695992179863147, 0.3255131964809384 ], "gen_text": "Descend and swing the arm forward and downward toward the green marker pen on the left side of the table, keeping the gripper open @grip as the arm angles down toward it." }, { "ep": 691, "frame_abs": 233490, "t_rel": 40, "emitted": true, "iou": 0.0, "cdist_px": 39.95187395807395, "box": [ 0.7174975562072337, 0.5943304007820137, 0.7429130009775171, 0.6119257086999023 ], "gt_box": [ 0.6334310850439883, 0.5581622678396871, 0.6539589442815249, 0.5698924731182796 ], "gen_text": "Close the gripper firmly around the green marker pen at @grip , securing a stable hold before lifting." }, { "ep": 691, "frame_abs": 233498, "t_rel": 48, "emitted": true, "iou": 0.0, "cdist_px": 42.50059573266758, "box": [ 0.6959921798631477, 0.5962854349951124, 0.7214076246334311, 0.6129032258064516 ], "gt_box": [ 0.6070381231671554, 0.5562072336265884, 0.6275659824046921, 0.5689149560117303 ], "gen_text": "Close the gripper firmly around the green marker pen at @grip , securing a stable hold before lifting." }, { "ep": 691, "frame_abs": 233525, "t_rel": 75, "emitted": true, "iou": 0.0, "cdist_px": 24.774904459719107, "box": [ 0.5962854349951124, 0.6226783968719453, 0.6158357771260997, 0.635386119257087 ], "gt_box": [ 0.5347018572825024, 0.6060606060606061, 0.552297165200391, 0.6197458455522972 ], "gen_text": "Close the gripper firmly around the green marker pen to secure a stable grasp @grip , then begin lifting upward." } ] }, "episode_meta": { "48": { "prompt": "wire harness", "plan_text": "Plan:1. Move the open gripper down toward the remaining rope on the table. 2. Close the gripper firmly around the white rope to secure the grasp. 3. Lift the white rope upward and swing the arm across toward the red-and-white board. 4. Lower the gripper to seat the rope between the pads. 5. Open the gripper to release the white rope and retract the arm upward and away.", "lo": 10623, "hi": 10861, "kfs": [ 0, 40, 80, 86, 126, 166 ] }, "50": { "prompt": "wire harness", "plan_text": "Plan:1. Move the arm forward and downward toward the target drop zone on the table surface. 2. Descend onto the target drop zone and close the gripper to grasp the green paddle-shaped object. 3. Lift the green paddle-shaped object clear of the table and transport it laterally toward the target drop zone on the table. 4. Lower the green paddle-shaped object into the target drop zone and release it. 5. Retract the arm upward and away from the placed object.", "lo": 11063, "hi": 11392, "kfs": [ 0, 40, 80, 85, 125, 165 ] }, "52": { "prompt": "wire harness", "plan_text": "Plan:1", "lo": 11645, "hi": 11802, "kfs": [ 0, 40, 69, 109, 143, 157 ] }, "320": { "prompt": "Move object into or out of container (ex: drawer, clothes hamper, plate, trashcan, washer)", "plan_text": "Plan:1. Move the open gripper down toward the green marker lying on the white table surface. 2. Close the gripper firmly around the green marker. 3. Lift the green marker upward and swing the arm laterally toward the white cardboard box. 4. Lower the green marker into the open white cardboard box and open the gripper to release it. 5. Retract the arm upward and away from the box.", "lo": 112457, "hi": 112572, "kfs": [ 0, 40, 75, 107, 115 ] }, "401": { "prompt": "Move object into or out of container (ex: drawer, clothes hamper, plate, trashcan, washer)", "plan_text": "Plan:1. Move the gripper down toward the white cloth near the right edge of the counter and align the finger pads with the cloth fabric. 2. Close the gripper firmly around the cloth to secure a stable grasp. 3. Lift the cloth upward off the counter and transport it toward the open teal bowl. 4. Transport the cloth toward the teal bowl while keeping the gripper closed around it. 5. Descend and lower the cloth into the teal bowl. 6. Open the gripper to release the cloth inside the teal bowl. 7. Retract the gripper upward and away from the bowl.", "lo": 137967, "hi": 138374, "kfs": [ 0, 40, 80, 89, 120, 142 ] }, "551": { "prompt": "Move object into or out of container (ex: drawer, clothes hamper, plate, trashcan, washer)", "plan_text": "Plan:1. Move the open gripper down toward the cardboard box on the table with fingers wide open. 2. Align the finger pads with the box's sides and close the fingers to grasp it. 3. Lift the cardboard box clear of the table. 4. Transport the cardboard box laterally toward the coffee machine tray. 5. Descend and lower the cardboard box over the open top of the white mug. 6. Open the gripper to release the cardboard box into the coffee machine tray. 7. Retract the arm upward and away to clear the workspace.", "lo": 196830, "hi": 197045, "kfs": [ 0, 40, 80, 118, 158, 184 ] }, "553": { "prompt": "Move object into or out of container (ex: drawer, clothes hamper, plate, trashcan, washer)", "plan_text": "Plan:1", "lo": 197381, "hi": 197656, "kfs": [ 0, 40, 80, 120, 130, 170 ] }, "691": { "prompt": "Move object to a new position and orientation (ex: grasping, relocating, flipping)", "plan_text": "Plan:1. Move the arm forward and downward toward the green marker pen on the left side of the table. 2. Descend and align the open gripper over the green marker pen on the left side of the table. 3. Close the gripper firmly around the green marker pen to secure a stable grasp. 4. Lift the green marker pen clear of the table surface. 5. Transport the green marker pen horizontally across the table toward the target placement area. 6. Lower the green marker pen onto the table surface and open the gripper to release it. 7. Retract the arm upward and away.", "lo": 233450, "hi": 233544, "kfs": [ 0, 40, 48, 75, 94 ] } } } }