{ "f0f120e4d4b0441da90ec53b16ee169d": { "scene_description": "The ego vehicle proceeds through the intersection, continuing along the current roadway.", "key_frames": { "4a0798f849ca477ab18009c3a20b7df2": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Brown SUV.", "2d_bbox": [ 966.6, 403.3, 1224.1, 591.7 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Black sedan.", "2d_bbox": [ 816.7, 431.6, 917.2, 505.0 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "Green light.", "2d_bbox": [ 676.4, 0.0, 1452.6, 171.5 ] } }, "QA": { "perception": [ { "Q": "What are objects to the front right of the ego car?", "A": "There are many barriers and one construction vehicle to the front right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front left of the ego car?", "A": "There is one truck and one barrier to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the front left of the ego car?", "A": "One truck is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back left of the ego car?", "A": "There are two barriers, many trucks, two trailers, and one car to the back left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the back left of the ego car?", "A": "Many trucks are parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the construction vehicle that is to the front right of the ego car?", "A": "The construction vehicle to the front right of the ego car is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front of the ego car?", "A": "There are many obstacles in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trailers that are to the back left of the ego car?", "A": "Two trailers are parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There are two cars, one truck, and one barrier to the back of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the back of the ego car?", "A": "Two cars are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the back left of the ego car?", "A": "One car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the back of the ego car?", "A": "One truck is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back right of the ego car?", "A": "There are two trailers to the back right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trailers that are to the back right of the ego car?", "A": "One of the trailers is parked, and one is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there standing pedestrians to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trailers to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trucks to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trailers to the back right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked construction vehicles to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There are two moving cars behind the ego car and two barriers in front of it.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the relative positioning of the important objects in the current scene?", "A": " is at the back of .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are some traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There are two traffic elements in the front view. The information of these traffic elements is [(traffic light, green, 674.86, 0.14, 723.33, 109.18), (traffic light, green, 1018.98, 7.19, 1071.77, 125.6)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which lanes are each important object on in the scene?", "A": " is in the left lane, and is in the ego lane.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a brown SUV to the back of the ego vehicle, a black sedan to the back of the ego vehicle, and a green light to the front of the ego vehicle. The IDs of these objects are , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Turn left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Brown SUV.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Black sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Green light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Turn left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Turn left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "Please proceed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly notice that . The object is a traffic sign, so the ego vehicle should keep going ahead at the same speed. Secondly notice that . The object is turning left, so the ego vehicle should keep going ahead at the same speed. Thirdly notice that . The object is going ahead, so the ego vehicle should keep going ahead at the same speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Neither is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will change its motion state based on ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observations of , what are possible actions to be taken by ? What is the reason?", "A": "The action is to turn left, the reason is there is no safety issue.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed. The reason is to follow the traffic rules, which has a high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "No such action will lead to a collision.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed. The reason is to follow the traffic rules, which has a high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle accelerates and goes straight?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Back up.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Keep going at the same speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "Green light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Go straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Keep going at the same speed, decelerate gradually without braking.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Back up, turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": ", , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observation of , what actions may take?", "A": "Turn left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What will affect driving judgment in this scene?", "A": "Water droplets on the glass will affect visibility.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is driving fast.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n008-2018-09-18-13-10-39-0400__CAM_FRONT__1537291010612404.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n008-2018-09-18-13-10-39-0400__CAM_FRONT_LEFT__1537291010604799.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n008-2018-09-18-13-10-39-0400__CAM_FRONT_RIGHT__1537291010620482.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n008-2018-09-18-13-10-39-0400__CAM_BACK__1537291010637558.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n008-2018-09-18-13-10-39-0400__CAM_BACK_LEFT__1537291010647405.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n008-2018-09-18-13-10-39-0400__CAM_BACK_RIGHT__1537291010628113.jpg" } }, "ffd1bdf020d145759224c629b501d2b2": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Stationary", "Visual_description": "Champagne-colored sedan.", "2d_bbox": [ 769.2, 409.7, 1240.1, 796.3 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "Green light.", "2d_bbox": [ 710.5, 303.0, 976.8, 493.6 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "Red light.", "2d_bbox": [ 923.5, 257.2, 1245.2, 508.4 ] } }, "QA": { "perception": [ { "Q": "What are objects to the back left of the ego car?", "A": "There are many trucks, one car, two barriers, and one traffic cone to the back left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the back left of the ego car?", "A": "Many trucks are parked, and one is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front left of the ego car?", "A": "There is one car, three barriers, one trailer, and three trucks to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the front left of the ego car?", "A": "One car is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the back left of the ego car?", "A": "One car is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There is one pedestrian, one bus, and two cars behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the pedestrian that is to the back of the ego car?", "A": "The pedestrian to the back of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front of the ego car?", "A": "There is one barrier in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trailer that is to the front left of the ego car?", "A": "The trailer to the front left of the ego car is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the bus that is to the back of the ego car?", "A": "One bus is stopped.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the back of the ego car?", "A": "One of the cars is parked, and the other is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the front left of the ego car?", "A": "Three trucks are parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the back right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there motorcycles without riders to the back of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the front left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there bicycles with riders to the front left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked cars to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trailers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trucks to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trucks to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There is one moving car behind the ego car, one parked car behind the ego car, and one stopped bus behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There are many traffic elements in the front view. The information of these traffic elements is [(traffic light, green, 13.19, 382.76, 39.21, 427.96), (traffic light, unknown, 255.86, 274.14, 279.25, 324.67), (traffic light, unknown, 388.89, 266.58, 413.72, 318.86), (traffic light, green, 713.29, 306.48, 731.57, 345.14), (traffic light, green, 826.91, 313.56, 844.49, 351.52), (traffic light, unknown, 399.02, 400.97, 412.73, 429.78)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which lanes are each important object on in the scene?", "A": " is on the ego lane.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a champagne-colored sedan to the back of the ego vehicle, a green light to the front of the ego vehicle, and a red light to the front left of the ego vehicle. The IDs of these objects are , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Champagne-colored sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Green light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "Please proceed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "Please wait.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly, notice . The object is a traffic sign, so the ego vehicle should continue at the same speed. Secondly, notice . The object is a traffic sign, so the ego vehicle should accelerate and continue ahead. Thirdly, notice . The object is stationary, so the ego vehicle should continue ahead at the same speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Both are traffic signs.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep stationary, the reason is to follow the traffic rules, high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Back up.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Keep going at the same speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Decelerate gradually without braking.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "Green light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Go straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Keep going at the same speed, decelerate gradually without braking.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Backing up, turning left, turning right, and changing to the right lane are dangerous actions for the ego vehicle in this scenario.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": ", , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will water droplets on the glass affect the driver's line of sight?", "A": "Yes, water droplets on the glass can affect the driver's line of sight. Slow down when the road is slippery on rainy days.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is driving slowly.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n008-2018-09-18-13-10-39-0400__CAM_FRONT__1537291002262404.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n008-2018-09-18-13-10-39-0400__CAM_FRONT_LEFT__1537291002254799.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n008-2018-09-18-13-10-39-0400__CAM_FRONT_RIGHT__1537291002270482.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n008-2018-09-18-13-10-39-0400__CAM_BACK__1537291002287558.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n008-2018-09-18-13-10-39-0400__CAM_BACK_LEFT__1537291002297405.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n008-2018-09-18-13-10-39-0400__CAM_BACK_RIGHT__1537291002278113.jpg" } }, "d9075c2a5f864a2b8abf41e703f4cf1c": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White sedan.", "2d_bbox": [ 206.6, 454.6, 259.3, 489.6 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Gray sedan.", "2d_bbox": [ 835.0, 463.3, 1113.7, 696.7 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "Green light.", "2d_bbox": [ 721.6, 257.1, 1024.0, 330.1 ] } }, "QA": { "perception": [ { "Q": "What are objects to the back left of the ego car?", "A": "There are many trucks, two cars, and many barriers to the back left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the back left of the ego car?", "A": "Many trucks are parked, and one is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front of the ego car?", "A": "There are three barriers in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the back left of the ego car?", "A": "Two cars are parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There is one pedestrian, one bus, and two cars behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the pedestrian that is to the back of the ego car?", "A": "The pedestrian to the back of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front left of the ego car?", "A": "There is one barrier and two trailers to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trailers that are to the front left of the ego car?", "A": "Two trailers are parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the bus that is to the back of the ego car?", "A": "One bus is stopped.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the back of the ego car?", "A": "Two cars are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front right of the ego car?", "A": "There are two trailers to the front right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trailers that are to the front right of the ego car?", "A": "One of the trailers is parked and one is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trailers to the front of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there stopped buses to the front left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there stopped buses to the front of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked cars to the front left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked cars to the front of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there bicycles without riders to the front left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the back of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked cars to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There are two moving cars behind the ego car and one stopped bus behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the relative positioning of the important objects in the current scene?", "A": " is on the back right of .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There are many traffic elements in the front view. The information of these traffic elements is [(traffic light, unknown, 158.81, 207.08, 188.41, 267.63), (traffic light, unknown, 320.87, 197.09, 349.75, 258.41), (traffic light, green, 722.29, 256.69, 742.39, 300.29), (traffic light, green, 855.73, 262.25, 876.6, 307.73), (traffic light, unknown, 365.1, 367.54, 380.95, 400.24)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a white sedan to the front left of the ego vehicle, a gray sedan to the back of the ego vehicle, and a green light to the front of the ego vehicle. The IDs of these objects are , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Gray sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Green light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "Please proceed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly notice that ; the object is a traffic sign, so the ego vehicle should continue going ahead at the same speed. Secondly, notice that ; the object is going ahead, so the ego vehicle should continue going ahead at the same speed. Thirdly, notice that ; the object is going ahead, so the ego vehicle should continue going ahead at the same speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Neither is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will change its motion state based on ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observations of , what are possible actions to be taken by ? What is the reason?", "A": "The action is to keep going at the same speed, the reason is there is no safety issue.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed, the reason is that there is no safety issue. The probability of this action is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "No such action will lead to a collision.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed. The reason is that there is no safety issue, and the probability is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Back up.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Keep going at the same speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "Green light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Go straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Keep going at the same speed and brake suddenly.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Back up, turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": ", , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observation of , what actions may take?", "A": "Keep going at the same speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is driving slowly.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n008-2018-09-18-13-10-39-0400__CAM_FRONT__1537291005162404.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n008-2018-09-18-13-10-39-0400__CAM_FRONT_LEFT__1537291005154799.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n008-2018-09-18-13-10-39-0400__CAM_FRONT_RIGHT__1537291005170482.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n008-2018-09-18-13-10-39-0400__CAM_BACK__1537291005187558.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n008-2018-09-18-13-10-39-0400__CAM_BACK_LEFT__1537291005197405.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n008-2018-09-18-13-10-39-0400__CAM_BACK_RIGHT__1537291005178113.jpg" } }, "dfb8d8959b9944d69dcec6d05e419f04": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Golden sedan.", "2d_bbox": [ 1448.3, 470.0, 1600.0, 540.0 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Black sedan car.", "2d_bbox": [ 906.8, 470.8, 1065.5, 527.4 ] }, "": { "Category": "Vehicle", "Status": "Stationary", "Visual_description": "White truck.", "2d_bbox": [ 1218.3, 435.0, 1370.0, 510.0 ] } }, "QA": { "perception": [ { "Q": "What are objects to the front right of the ego car?", "A": "There are many barriers and one construction vehicle to the front right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front left of the ego car?", "A": "There are many barriers to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There are three barriers, many trucks, and one car to the back of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back left of the ego car?", "A": "There are many trucks, one barrier, two trailers, and two cars to the back left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the back left of the ego car?", "A": "Many trucks are parked, and two are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the construction vehicle that is to the front right of the ego car?", "A": "The construction vehicle to the front right of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front of the ego car?", "A": "There are many obstacles in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trailers that are to the back left of the ego car?", "A": "One of the trailers is parked, and one is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the back of the ego car?", "A": "Many trucks are parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the back left of the ego car?", "A": "One of the cars is moving, and one is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the back of the ego car?", "A": "One car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back right of the ego car?", "A": "There are two trailers and one barrier to the back right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trailers that are to the back right of the ego car?", "A": "Two trailers are parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there motorcycles with riders to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trailers to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the front of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there bicycles without riders to the front of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the back right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trucks to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trucks to the back of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked cars to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the back left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trailers to the back right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There is one moving car behind the ego car and two barriers in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the relative positioning of the important objects in the current scene?", "A": " is at the back of , is at the back right of , and is at the back left of .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There is one traffic element in the front view. The information of this traffic element is [(road sign, go straight, 1037.73, 409.81, 1049.48, 426.83)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a golden sedan to the back of the ego vehicle, a black sedan car to the back of the ego vehicle, and a white truck to the back of the ego vehicle. The IDs of these objects are , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Turn left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Golden sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Black sedan car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White truck.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly notice that , the object is turning left, so the ego vehicle should keep going ahead at the same speed. Secondly notice that , the object is going ahead, so the ego vehicle should keep going ahead at the same speed. Thirdly notice that , it is stationary, so the ego vehicle should keep going ahead at the same speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Neither is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will change its motion state based on ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observations of , what are possible actions to be taken by ? What is the reason?", "A": "The action is none, the reason is that there is no safety issue.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Neither is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will change its motion state based on ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observations of , what are possible actions to be taken by ? What is the reason?", "A": "The action is to keep going at the same speed; the reason is there is no safety issue.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed because there is no safety issue. The probability of taking this action is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "No such action will lead to a collision.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed. The reason is that there is no safety issue, and the probability is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "No action taken by the ego vehicle will lead to a collision with .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed. The reason is that there is no safety issue. The probability of this action is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "No such action will lead to a collision.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Go straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Keep going at the same speed, accelerate, and proceed ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Turning right, changing to the right lane.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": ", , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What's your comment on this scene?", "A": "There is rainwater on the glass, so the ego vehicle should slow down.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is driving fast.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n008-2018-09-18-13-10-39-0400__CAM_FRONT__1537291013612404.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n008-2018-09-18-13-10-39-0400__CAM_FRONT_LEFT__1537291013604799.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n008-2018-09-18-13-10-39-0400__CAM_FRONT_RIGHT__1537291013620482.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n008-2018-09-18-13-10-39-0400__CAM_BACK__1537291013637558.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n008-2018-09-18-13-10-39-0400__CAM_BACK_LEFT__1537291013647405.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n008-2018-09-18-13-10-39-0400__CAM_BACK_RIGHT__1537291013628113.jpg" } } } }, "54cdaaae372d421fa4734d66f51a8c48": { "scene_description": "The ego vehicle proceeds along the current road, coming to a halt at the traffic light intersection.", "key_frames": { "542eaf1fc9b34895a9e55fab57cb4cf4": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Red car.", "2d_bbox": [ 624.3, 469.4, 814.8, 537.8 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White SUV.", "2d_bbox": [ 905.0, 471.6, 1091.9, 535.0 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White sedan.", "2d_bbox": [ 1146.6, 480.0, 1472.9, 583.4 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Black sedan.", "2d_bbox": [ 786.7, 470.0, 1010.6, 545.0 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "Red light.", "2d_bbox": [ 687.3, 340.1, 740.8, 383.5 ] } }, "QA": { "perception": [ { "Q": "What are objects to the front left of the ego car?", "A": "There are many barriers, one pedestrian, one car, and one traffic cone to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front of the ego car?", "A": "There are many cars in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front of the ego car?", "A": "Many cars are moving, and one is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There are two trucks and two cars behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the back of the ego car?", "A": "Two trucks are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front right of the ego car?", "A": "There is one truck and three cars to the front right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the front right of the ego car?", "A": "The truck to the front right of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the back of the ego car?", "A": "One of the cars is parked, and one is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front right of the ego car?", "A": "Three cars are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back right of the ego car?", "A": "There are three traffic cones to the back right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the pedestrian that is to the front left of the ego car?", "A": "The pedestrian to the front left of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the front left of the ego car?", "A": "One car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trailers to the back right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the back of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the back of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the back right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked cars to the back of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There are three moving cars in front of the ego car, one parked car in front of the ego car, one parked car behind the ego car, and one moving truck behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the relative positioning of the important objects in the current scene?", "A": " is on the front right of , is on the back right of , is on the back right of , is at the back of , is at the back of , and is on the front left of .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There are many traffic elements in the front view. The information of these traffic elements is [(traffic light, unknown, 1558.87, 394.64, 1581.4, 440.1), (traffic light, unknown, 1517.53, 392.57, 1544.05, 454.21), (traffic light, unknown, 1352.3, 294.75, 1387.07, 316.96), (traffic light, red, 1277.86, 431.93, 1292.52, 458.56), (traffic light, red, 1153.6, 434.17, 1163.9, 460.68), (traffic light, red, 1135.69, 206.88, 1176.67, 341.86), (traffic light, unknown, 791.36, 424.31, 801.51, 456.68), (traffic light, red, 684.35, 344.66, 717.71, 358.66), (traffic light, red, 406.01, 433.21, 416.42, 454.42), (traffic light, unknown, 1111.65, 222.31, 1138.91, 389.36), (traffic light, green, 418.75, 431.51, 428.8, 456.11), (traffic light, unknown, 539.47, 435.74, 555.41, 459.12), (traffic light, green, 109.87, 362.66, 135.18, 419.98)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a red car to the front of the ego vehicle, a white SUV to the front of the ego vehicle, a white sedan to the front of the ego vehicle, a black sedan to the front of the ego vehicle, and a red light to the front of the ego vehicle. The IDs of these objects are , , , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Red car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White SUV.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Black sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": ", no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "Please wait.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly, notice that . The object is going ahead, so the ego vehicle should decelerate gradually until it comes to a stop. Secondly, notice that . The object is going ahead, so the ego vehicle should slow down and go ahead. Thirdly, notice that . The object is going ahead, so the ego vehicle should slow down and go ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Neither is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will change its motion state based on ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observations of , what are possible actions to be taken by ? What is the reason?", "A": "The action is to keep going at the same speed, the reason is to maintain a safe distance.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to brake gently to a stop. The reason is to follow the traffic rules, and the probability is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerate and go straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to brake gently to a stop. The reason is to follow the traffic rules. The probability is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerate and go straight can lead to a collision with .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to brake gently to a stop. The reason is to follow the traffic rules, and the probability is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to brake gently to a stop. The reason is to follow the traffic rules, which has a high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerating and going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Brake gently to a stop.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Brake gently to a stop, brake suddenly, and slightly offset to the left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Accelerate and go ahead, change to the right lane, and turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": "The priority of the objects that the ego vehicle should consider (in descending order) is , , , , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Driving towards the front right can lead to a collision with .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is driving slowly.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n015-2018-08-03-15-00-36+0800__CAM_FRONT__1533280039162460.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_LEFT__1533280039154844.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_RIGHT__1533280039170339.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n015-2018-08-03-15-00-36+0800__CAM_BACK__1533280039187525.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n015-2018-08-03-15-00-36+0800__CAM_BACK_LEFT__1533280039197423.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_BACK_RIGHT__1533280039177893.jpg" } }, "1b45a97a0e5e49fe9cd345dd4bd729c3": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White sedan car.", "2d_bbox": [ 653.3, 468.3, 894.9, 558.3 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Blue sedan.", "2d_bbox": [ 276.7, 483.3, 580.0, 596.7 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White sedan.", "2d_bbox": [ 701.7, 483.4, 883.3, 575.0 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White van.", "2d_bbox": [ 0.0, 418.3, 208.6, 531.7 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "Red light.", "2d_bbox": [ 673.8, 301.3, 751.5, 380.5 ] } }, "QA": { "perception": [ { "Q": "What are objects to the front left of the ego car?", "A": "There are many barriers, three cars, and one pedestrian to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front left of the ego car?", "A": "Three cars are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There are two trucks and one car behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the back of the ego car?", "A": "Two trucks are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front right of the ego car?", "A": "There is one truck and many cars to the front right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the front right of the ego car?", "A": "The truck to the front right of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front right of the ego car?", "A": "Many cars are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front of the ego car?", "A": "There is one car in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the front of the ego car?", "A": "The car in front is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back right of the ego car?", "A": "There are three traffic cones to the back right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the pedestrian that is to the front left of the ego car?", "A": "The pedestrian to the front left of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the back of the ego car?", "A": "One car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back left of the ego car?", "A": "There is one traffic cone to the back left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trailers to the back left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked cars to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the back left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the back right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the back of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the back of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There is one moving car in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the relative positioning of the important objects in the current scene?", "A": " is on the back left of , is on the back right of , is on the back right of , is on the front left of , is on the back right of , and is on the front left of .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There are many traffic elements in the front view. The information of these traffic elements is [(traffic light, red, 1349.39, 427.87, 1364.41, 458.51), (traffic light, red, 1201.96, 430.33, 1215.54, 459.26), (traffic light, unknown, 1530.22, -0.5, 1600.0, 178.2), (traffic light, red, 796.33, 417.88, 806.52, 452.78), (traffic light, red, 675.57, 330.72, 712.54, 344.54), (traffic light, red, 358.07, 421.3, 372.21, 451.48), (traffic light, unknown, 1415.28, 0.0, 1511.0, 303.81), (traffic light, unknown, 1188.92, 432.49, 1204.51, 458.7), (traffic light, green, 374.27, 421.68, 386.66, 454.39), (traffic light, unknown, 515.37, 428.53, 532.27, 456.6)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a white sedan car to the front of the ego vehicle, a blue sedan to the front right of the ego vehicle, a white sedan to the front right of the ego vehicle, a white van to the front of the ego vehicle, and a red light to the front of the ego vehicle. The IDs of these objects are , , , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White sedan car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Blue sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White van.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "No entry.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly, notice . It is turning right, so the ego vehicle should remain stationary. Secondly, notice . It is going ahead, so the ego vehicle should remain stationary. Thirdly, notice . It is going ahead, so the ego vehicle should remain stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Neither is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will change its motion state based on ?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observations of , what are possible actions to be taken by ? What is the reason?", "A": "The action is to keep going at the same speed, and the reason is to maintain a safe distance.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Neither is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will change its motion state based on ?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observations of , what are possible actions to be taken by ? What is the reason?", "A": "The action is to keep going at the same speed, the reason is there is no safety issue.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason for this action is to follow the traffic rules. The probability of this action is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerating and going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason is to follow the traffic rules, which has a high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Sharp right turn.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason for this action is to follow the traffic rules. The probability of taking this action is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Sharp right turn.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary, the reason is to follow the traffic rules, high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Sharp left turn.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Keep stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Brake gently to a stop, slightly offset to the right, and slightly offset to the left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Changing to the right lane, accelerating, and going ahead, and backing up are dangerous actions to take for the ego vehicle in this scenario.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": "The priority of the objects that the ego vehicle should consider (in descending order) is , , , , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is not moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n015-2018-08-03-15-00-36+0800__CAM_FRONT__1533280042612460.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_LEFT__1533280042604844.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_RIGHT__1533280042620339.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n015-2018-08-03-15-00-36+0800__CAM_BACK__1533280042637525.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n015-2018-08-03-15-00-36+0800__CAM_BACK_LEFT__1533280042647423.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_BACK_RIGHT__1533280042627893.jpg" } }, "d5e16062410f4e329d31a881b28e5c1c": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White truck.", "2d_bbox": [ 700.0, 375.0, 1017.4, 593.4 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White sedan.", "2d_bbox": [ 645.0, 475.0, 727.9, 525.0 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Gray sedan.", "2d_bbox": [ 1026.6, 483.4, 1352.9, 600.0 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White sedan.", "2d_bbox": [ 213.3, 483.3, 477.8, 596.7 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "No entry.", "2d_bbox": [ 691.0, 271.5, 738.0, 372.3 ] } }, "QA": { "perception": [ { "Q": "What are objects to the front left of the ego car?", "A": "There are many barriers, two cars, and one pedestrian to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front left of the ego car?", "A": "One of the cars is moving, and one is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back left of the ego car?", "A": "There is one truck, one car, and one traffic cone to the back left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the back left of the ego car?", "A": "One truck is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front right of the ego car?", "A": "There is one truck and one car to the front right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the front right of the ego car?", "A": "One truck is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front of the ego car?", "A": "There are three cars in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front of the ego car?", "A": "Three cars are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There is one truck at the back of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the back of the ego car?", "A": "One truck is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the front right of the ego car?", "A": "One car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back right of the ego car?", "A": "There are three traffic cones to the back right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the pedestrian that is to the front left of the ego car?", "A": "The pedestrian to the front left of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the back left of the ego car?", "A": "One car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving buses to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked construction vehicles to the back right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there motorcycles without riders to the back right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trailers to the front right of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked trailers to the back of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the back right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There are two moving cars in front of the ego car, and one moving truck to the back left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the relative positioning of the important objects in the current scene?", "A": " is on the front left of , is on the front left of , is on the front left of , is at the back of , is at the back right of and is at the back right of .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There are many traffic elements in the front view. The information of these traffic elements is [(traffic light, unknown, 1528.62, -0.49, 1600.0, 180.83), (traffic light, unknown, 1413.7, 0.0, 1507.84, 309.35), (traffic light, red, 1349.39, 433.01, 1364.18, 460.82), (traffic light, red, 1204.64, 433.88, 1214.48, 462.47), (traffic light, unknown, 1188.18, 432.14, 1206.92, 462.81), (traffic light, red, 796.46, 420.99, 806.52, 456.22), (traffic light, red, 675.4, 333.3, 714.09, 345.23), (traffic light, red, 358.3, 424.28, 373.88, 455.37), (traffic light, green, 374.27, 421.68, 385.09, 453.91), (traffic light, unknown, 513.99, 430.99, 532.27, 456.6)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a white truck to the front right of the ego vehicle, a white sedan to the front of the ego vehicle, a gray sedan to the front of the ego vehicle, a white sedan to the front right of the ego vehicle, and a no entry to the front of the ego vehicle. The IDs of these objects are , , , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White truck.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Gray sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "No entry.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "No entry.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly, notice that . The object is turning right, so the ego vehicle should remain stationary. Secondly, notice that . The object is a traffic sign, so the ego vehicle should remain stationary. Thirdly, notice that . The object is going ahead, so the ego vehicle should remain stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason for taking this action is to follow traffic rules. The probability of this action is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle turns left at a constant speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Turning right and accelerating.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason is to follow the traffic rules, with a high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerate and go straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason for this action is to follow the traffic rules, which has a high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerate and go straight actions can lead to a collision with .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason for taking this action is to follow the traffic rules, which has a high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Brake gently to a stop.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Brake gently to a stop, slightly offset to the left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Accelerate and go ahead, and turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": "The priority of the objects that the ego vehicle should consider (in descending order) is , , , , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerating towards the right front can lead to a collision with .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is not moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n015-2018-08-03-15-00-36+0800__CAM_FRONT__1533280045112460.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_LEFT__1533280045104844.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_RIGHT__1533280045120339.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n015-2018-08-03-15-00-36+0800__CAM_BACK__1533280045137525.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n015-2018-08-03-15-00-36+0800__CAM_BACK_LEFT__1533280045147423.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_BACK_RIGHT__1533280045127893.jpg" } }, "7abe79e5d3d745bcb237a4b7e03d1577": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White truck.", "2d_bbox": [ 1065.0, 301.7, 1600.0, 683.3 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White sedan.", "2d_bbox": [ 790.0, 475.0, 999.9, 556.6 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White van.", "2d_bbox": [ 0.0, 425.0, 298.2, 536.6 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "Red light.", "2d_bbox": [ 675.5, 313.0, 813.5, 425.6 ] } }, "QA": { "perception": [ { "Q": "What are objects to the front left of the ego car?", "A": "There are many barriers and one car to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back left of the ego car?", "A": "There is one truck to the back left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the back left of the ego car?", "A": "One truck is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front of the ego car?", "A": "There is one truck and many cars in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the front of the ego car?", "A": "One truck is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front of the ego car?", "A": "Many cars are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There is one truck behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the back of the ego car?", "A": "One truck is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back right of the ego car?", "A": "There are three traffic cones to the back right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the front left of the ego car?", "A": "One car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there stopped buses to the front of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trailers to the front left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the back right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There are two moving cars in front of the ego car, one parked truck to the back left of the ego car, and one moving truck in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the relative positioning of the important objects in the current scene?", "A": " is in front of , is on the front left of , and is on the back left of .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There are many traffic elements in the front view. The information of these traffic elements is [(traffic light, unknown, 1530.95, 0.05, 1600.0, 180.14), (traffic light, unknown, 1413.7, 0.0, 1507.84, 309.35), (traffic light, red, 796.46, 420.99, 806.52, 456.22), (traffic light, red, 675.4, 333.3, 714.09, 345.23), (traffic light, red, 358.3, 424.28, 373.88, 455.37), (traffic light, green, 374.27, 421.68, 385.09, 453.91), (traffic light, unknown, 513.99, 430.99, 532.27, 456.6)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a white truck to the front of the ego vehicle, a white sedan to the front of the ego vehicle, a white van to the front of the ego vehicle, and a red light to the front of the ego vehicle. The IDs of these objects are , , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White truck.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White van.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "The future state of is to decelerate gradually without braking.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "Please wait.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly, notice . It is a traffic sign, so the ego vehicle should remain stationary. Secondly, notice . It is going ahead, so the ego vehicle should remain stationary. Thirdly, notice . It is turning right, so the ego vehicle should remain stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary, the reason is to follow the traffic rules, high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Moderate right turn.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason for this action is to follow the traffic rules, with a high probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerating and going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to remain stationary. The reason is to maintain a safe distance, which is high in probability.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle steps on the brakes?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Moderate left turn.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Keep stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Brake gently to a stop, slightly offset to the right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Accelerate and go ahead, and turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": "The priority of the objects that the ego vehicle should consider in descending order is , , , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is not moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n015-2018-08-03-15-00-36+0800__CAM_FRONT__1533280048162460.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_LEFT__1533280048154844.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_RIGHT__1533280048170339.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n015-2018-08-03-15-00-36+0800__CAM_BACK__1533280048187525.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n015-2018-08-03-15-00-36+0800__CAM_BACK_LEFT__1533280048197442.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_BACK_RIGHT__1533280048177893.jpg" } }, "0dd14c14cea14dc1b0c0c9b0c9c7c4c3": { "key_object_infos": { "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White sedan.", "2d_bbox": [ 1220.0, 463.3, 1435.2, 526.7 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "White van.", "2d_bbox": [ 551.7, 441.7, 715.7, 508.3 ] }, "": { "Category": "Vehicle", "Status": "Moving", "Visual_description": "Black sedan.", "2d_bbox": [ 706.7, 461.7, 839.0, 508.3 ] }, "": { "Category": "Traffic element", "Status": null, "Visual_description": "No entry.", "2d_bbox": [ 668.5, 352.2, 711.5, 391.2 ] } }, "QA": { "perception": [ { "Q": "What are objects to the front of the ego car?", "A": "There are two barriers and many cars in front of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front left of the ego car?", "A": "There are many barriers, one pedestrian, one car, and one traffic cone to the front left of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the front right of the ego car?", "A": "There are three cars, one truck, and many traffic cones to the front right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front right of the ego car?", "A": "Two of the cars are moving, and one is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back of the ego car?", "A": "There are two trucks and two cars behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the trucks that are to the back of the ego car?", "A": "Two trucks are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the truck that is to the front right of the ego car?", "A": "One truck is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the back of the ego car?", "A": "Two cars are moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the cars that are to the front of the ego car?", "A": "Many cars are moving, and one is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are objects to the back right of the ego car?", "A": "There is one construction vehicle to the back right of the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the construction vehicle that is to the back right of the ego car?", "A": "One construction vehicle is parked.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the pedestrian that is to the front left of the ego car?", "A": "The pedestrian to the front left of the ego car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the status of the car that is to the front left of the ego car?", "A": "One car is moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there motorcycles without riders to the back of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the back left of the ego car?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front left of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the back of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there traffic cones to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving trucks to the front right of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there moving cars to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there barriers to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are there parked cars to the front of the ego car?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Please describe the current scene.", "A": "There are two moving cars in front of the ego car, one moving car behind the ego car, one parked car in front of the ego car, and two moving trucks behind the ego car.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the relative positioning of the important objects in the current scene?", "A": " is on the front left of , is on the front left of , and is on the front right of .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is there any traffic element in the front view?", "A": "Yes, there are traffic elements in the front view.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Identify all the traffic elements in the front view, categorize them, determine their status, and predict the bounding box around each one. The output should be a list formatted as (c, s, x1, y1, x2, y2), where c represents the category, s denotes the status, and x1, y1, x2, y2 are the offsets of the top-left and bottom-right corners of the box relative to the center point.", "A": "There are many traffic elements in the front view. The information of these traffic elements is as follows: [(traffic light, unknown, 1388.67, 404.5, 1416.79, 444.85), (traffic light, unknown, 1420.53, 405.04, 1441.0, 444.1), (traffic light, unknown, 1332.59, 403.21, 1364.32, 449.16), (traffic light, red, 1104.73, 430.06, 1114.02, 449.29), (traffic light, unknown, 1248.27, 409.43, 1260.88, 436.63), (traffic light, unknown, 1224.48, 409.13, 1237.94, 444.48), (traffic light, red, 1019.72, 432.13, 1025.95, 449.13), (traffic light, unknown, 894.59, 363.08, 913.09, 410.02), (traffic light, red, 746.8, 426.13, 753.74, 448.79), (traffic light, red, 661.25, 364.55, 689.01, 373.77), (traffic light, red, 455.67, 431.7, 463.98, 449.11), (traffic light, unknown, 1144.38, 335.32, 1166.51, 349.1), (traffic light, unknown, 465.64, 430.43, 473.32, 449.02), (traffic light, unknown, 553.13, 433.37, 565.46, 450.54), (traffic light, unknown, 1057.03, 210.4, 1121.88, 236.94), (traffic light, unknown, 245.12, 363.78, 272.54, 408.44), (traffic light, red, 224.69, 361.46, 243.81, 408.34), (traffic light, unknown, 176.09, 389.88, 193.24, 426.23), (traffic light, unknown, 305.88, 391.35, 321.12, 430.11), (traffic light, unknown, 253.83, 276.69, 272.58, 291.36), (road sign, go straight, 1.32, 642.97, 370.97, 825.53), (road sign, turn right, 769.18, 647.13, 882.13, 839.28)].", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What are the important objects in the current scene? Those objects will be considered for the future reasoning and driving decision.", "A": "There is a white sedan to the front of the ego vehicle, a white van to the front of the ego vehicle, a black sedan to the front of the ego vehicle, and a no entry to the front of the ego vehicle. The IDs of these objects are , , , and .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the observed status of object ?", "A": "Moving.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the moving status of object ?", "A": "Going ahead.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "White van.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "Black sedan.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the visual description of ?", "A": "No entry.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "prediction": [ { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Which object is most likely to be occluded by ? Would this object affect the ego vehicle? Based on this object, what action of the ego vehicle is dangerous?", "A": "None, no, none.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the future state of ?", "A": "Keep going straight.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is a traffic sign or a road barrier?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would be in the moving direction of the ego vehicle?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What does mean?", "A": "No entry.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what object is most likely to consider ?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object should the ego vehicle notice first when the ego vehicle is getting to the next possible location? What is the state of the object that is first noticed by the ego vehicle and what action should the ego vehicle take? What object should the ego vehicle notice second when the ego vehicle is getting to the next possible location? What is the state of the object perceived by the ego vehicle as second and what action should the ego vehicle take? What object should the ego vehicle notice third? What is the state of the object perceived by the ego vehicle as third and what action should the ego vehicle take?", "A": "Firstly, notice that . The object is going ahead, so the ego vehicle should decelerate gradually until it comes to a stop. Secondly, notice that . The object is going ahead, so the ego vehicle should decelerate gradually until it comes to a stop. Thirdly, notice that . The object is a traffic sign, so the ego vehicle should decelerate gradually until it comes to a stop.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Only one of the boxes is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Would take into account?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": "None.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Are and traffic signs?", "A": "Neither is a traffic sign.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will be in the moving direction of ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Will change its motion state based on ?", "A": "No.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on the observations of , what are possible actions to be taken by ? What is the reason?", "A": "The action is to keep going at the same speed; the reason is that there are no safety issues.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What object would consider to be most relevant to its decision?", "A": "The ego vehicle.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Except for the ego vehicle, what object would consider to be most relevant to its decision?", "A": ".", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What kind of traffic sign is ?", "A": "Traffic light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "planning": [ { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed, the reason being that there is no safety issue. The probability of this action is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed. The reason is that there is no safety issue, and the probability is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerating and going straight can lead to a collision with .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is an object that the ego vehicle should consider in the current scene?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions could the ego vehicle take based on ? Why take this action and what's the probability?", "A": "The action is to keep going at the same speed. The reason is that there is no safety issue. The probability is high.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the probability of colliding with after the ego vehicle goes straight and keeps the same speed?", "A": "Low.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Accelerate and go straight can lead to a collision with .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Is it necessary for the ego vehicle to take into account?", "A": "Yes.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "Based on in this scene, what is the most possible action of the ego vehicle?", "A": "Brake gently to a stop.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the traffic signal that the ego vehicle should pay attention to?", "A": "Red light.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the target action of the ego vehicle?", "A": "Stationary.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are safe actions to take for the ego vehicle?", "A": "Brake gently to a stop, slightly offset to the left.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "In this scenario, what are dangerous actions to take for the ego vehicle?", "A": "Accelerate and go ahead, and turn right.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What is the priority of the objects that the ego vehicle should consider?(in descending order)", "A": "The priority of the objects that the ego vehicle should consider, in descending order, is , , , .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What actions taken by the ego vehicle can lead to a collision with ?", "A": "Driving forward to the right can lead to a collision with .", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null }, { "Q": "What's your comment on this scene?", "A": "There is nothing in particular to pay attention to in this scene; the road is quite empty.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ], "behavior": [ { "Q": "Predict the behavior of the ego vehicle.", "A": "The ego vehicle is going straight. The ego vehicle is driving with normal speed.", "C": null, "con_up": null, "con_down": null, "cluster": null, "layer": null } ] }, "image_paths": { "CAM_FRONT": "../nuscenes/samples/CAM_FRONT/n015-2018-08-03-15-00-36+0800__CAM_FRONT__1533280036162460.jpg", "CAM_FRONT_LEFT": "../nuscenes/samples/CAM_FRONT_LEFT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_LEFT__1533280036154844.jpg", "CAM_FRONT_RIGHT": "../nuscenes/samples/CAM_FRONT_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_FRONT_RIGHT__1533280036170339.jpg", "CAM_BACK": "../nuscenes/samples/CAM_BACK/n015-2018-08-03-15-00-36+0800__CAM_BACK__1533280036187525.jpg", "CAM_BACK_LEFT": "../nuscenes/samples/CAM_BACK_LEFT/n015-2018-08-03-15-00-36+0800__CAM_BACK_LEFT__1533280036197423.jpg", "CAM_BACK_RIGHT": "../nuscenes/samples/CAM_BACK_RIGHT/n015-2018-08-03-15-00-36+0800__CAM_BACK_RIGHT__1533280036177893.jpg" } } } } }