| [ |
| { |
| "image": [ |
| "data/example/images/161_CAM_FRONT_LEFT.jpg", |
| "data/example/images/161_CAM_FRONT.jpg", |
| "data/example/images/161_CAM_FRONT_RIGHT.jpg" |
| ], |
| "conversations": [ |
| { |
| "from": "human", |
| "value": "You are an expert autonomous driving agent.\nTask 1: Critical Object Detection\nFor each class below, answer \"yes\" or \"no\" to indicate whether it affects the ego vehicle\u2019s behavior or future trajectory:\n[nearby_vehicle, pedestrian, cyclist, construction, traffic_element, weather_condition, road_hazard, emergency_vehicle, animal, special_vehicle, conflicting_vehicle, door_opening_vehicle]\nTask 2: Scene Reasoning\nPredict the future behavior of the identified critical objects and explain how the identified critical objects or conditions affect the ego vehicle\u2019s next 3-second trajectory.\nTask 3: Meta-Behavior Prediction\nPredict the ego vehicle\u2019s future meta-driving behavior:\n- speed \u2208 {keep, accelerate, decelerate, stop, other}\n- command \u2208 {straight, yield, left_turn, right_turn, lane_follow, lane_change_left, lane_change_right, reverse, overtake, other}\nTask 4: Trajectory Prediction\nPredict the optimal 5-second future trajectory (5 waypoints, 1 s intervals).\n\nInput:\n- <image>: three front-view frames from left front, center front, right front cameras. \n- High-level navigation command: GO_STRAIGHT\n- Historical ego state: Provided are the previous ego vehicle status recorded over the last 3.0 seconds (at 0.5-second intervals).(t-3.0s) [-43.35, 0.12], Acceleration: X 0.04, Y 0.03 m/s\u00b2, Velocity: X 14.28, Y -0.19 m/s,; (t-2.5s) [-36.21, 0.04], Acceleration: X 0.04, Y 0.04 m/s\u00b2, Velocity: X 14.36, Y -0.12 m/s,; (t-2.0s) [-29.02, -0.02], Acceleration: X 0.06, Y 0.04 m/s\u00b2, Velocity: X 14.43, Y -0.04 m/s,; (t-1.5s) [-21.81, -0.02], Acceleration: X 0.04, Y 0.04 m/s\u00b2, Velocity: X 14.49, Y 0.04 m/s,; (t-1.0s) [-14.57, -0.01], Acceleration: X 0.04, Y 0.00 m/s\u00b2, Velocity: X 14.55, Y 0.03 m/s,; (t-0.5s) [-7.30, 0.00], Acceleration: X 0.04, Y -0.02 m/s\u00b2, Velocity: X 14.61, Y 0.00 m/s,; (t+0.0s) [0.00, 0.00], Acceleration: X 0.03, Y -0.01 m/s\u00b2, Velocity: X 14.64, Y -0.01 m/s,\n\nOutput:" |
| }, |
| { |
| "from": "gpt", |
| "value": "{\"critical_objects\": {\"nearby_vehicle\": \"yes\", \"pedestrian\": \"no\", \"cyclist\": \"no\", \"construction\": \"no\", \"traffic_element\": \"no\", \"weather_condition\": \"no\", \"road_hazard\": \"no\", \"emergency_vehicle\": \"no\", \"animal\": \"no\", \"special_vehicle\": \"no\", \"conflicting_vehicle\": \"no\", \"door_opening_vehicle\": \"no\"}, \"explanation\": \"<|mdm_start|>There is a nearby vehicle visible ahead in the lane, roughly 20-30 meters away, which is relevant to the ego vehicle's forward path. No pedestrians, cyclists, or conflicting vehicles are detected that impact the ego's immediate trajectory. The nearby vehicle may influence the ego car's longitudinal behavior, potentially requiring speed adjustment or maintaining a safe gap to avoid collision. No other road conditions or traffic elements are present that demand lateral maneuvering or yielding.<|mdm_end|>\", \"future_meta_behavior\": {\"longitudinal\": \"<|mdm_start|>keep speed <|mdm_end|>\", \"lateral\": \"<|mdm_start|>go straight <|mdm_end|>\"}, \"trajectory\": \"<|mdm_start|>[[+14.70,-00.04], [+29.55,-00.21], [+44.51,-00.56], [+59.50,-01.06], [+74.39,-01.69]]<|mdm_end|>\"}" |
| } |
| ], |
| "velocity": [ |
| [ |
| 14.177639961242676, |
| -0.28636378049850464 |
| ], |
| [ |
| 14.206523895263672, |
| -0.2542072534561157 |
| ], |
| [ |
| 14.242470741271973, |
| -0.21632453799247742 |
| ], |
| [ |
| 14.280794143676758, |
| -0.18633034825325012 |
| ], |
| [ |
| 14.32573127746582, |
| -0.1581643521785736 |
| ], |
| [ |
| 14.364027976989746, |
| -0.1164020299911499 |
| ], |
| [ |
| 14.376612663269043, |
| -0.0786110982298851 |
| ], |
| [ |
| 14.432589530944824, |
| -0.03503814712166786 |
| ], |
| [ |
| 14.445201873779297, |
| 0.0017584636807441711 |
| ], |
| [ |
| 14.486393928527832, |
| 0.04326183348894119 |
| ], |
| [ |
| 14.508514404296875, |
| 0.02730157971382141 |
| ], |
| [ |
| 14.54558277130127, |
| 0.02939201146364212 |
| ], |
| [ |
| 14.568070411682129, |
| 0.01794784516096115 |
| ], |
| [ |
| 14.610406875610352, |
| 0.002713128924369812 |
| ], |
| [ |
| 14.636354446411133, |
| -0.0077445730566978455 |
| ], |
| [ |
| 14.636354446411133, |
| -0.0077445730566978455 |
| ] |
| ], |
| "acceleration": [ |
| [ |
| 0.0554962158203125, |
| 0.04731619358062744 |
| ], |
| [ |
| 0.028883934020996094, |
| 0.032156527042388916 |
| ], |
| [ |
| 0.03594684600830078, |
| 0.037882715463638306 |
| ], |
| [ |
| 0.038323402404785156, |
| 0.029994189739227295 |
| ], |
| [ |
| 0.0449371337890625, |
| 0.028165996074676514 |
| ], |
| [ |
| 0.03829669952392578, |
| 0.041762322187423706 |
| ], |
| [ |
| 0.012584686279296875, |
| 0.0377909317612648 |
| ], |
| [ |
| 0.05597686767578125, |
| 0.04357295110821724 |
| ], |
| [ |
| 0.012612342834472656, |
| 0.03679661080241203 |
| ], |
| [ |
| 0.041192054748535156, |
| 0.04150336980819702 |
| ], |
| [ |
| 0.02212047576904297, |
| -0.01596025377511978 |
| ], |
| [ |
| 0.03706836700439453, |
| 0.0020904317498207092 |
| ], |
| [ |
| 0.022487640380859375, |
| -0.01144416630268097 |
| ], |
| [ |
| 0.042336463928222656, |
| -0.015234716236591339 |
| ], |
| [ |
| 0.02594757080078125, |
| -0.010457701981067657 |
| ], |
| [ |
| 0.02594757080078125, |
| -0.010457701981067657 |
| ] |
| ], |
| "turn angle": [ |
| [ |
| -1.157118206648066 |
| ], |
| [ |
| -1.025123996188436 |
| ], |
| [ |
| -0.8701811833133231 |
| ], |
| [ |
| -0.7475310321840807 |
| ], |
| [ |
| -0.6325528164914213 |
| ], |
| [ |
| -0.4642986681159677 |
| ], |
| [ |
| -0.31328932404115056 |
| ], |
| [ |
| -0.13909728422231943 |
| ], |
| [ |
| 0.006974810578374877 |
| ], |
| [ |
| 0.171106288866873 |
| ], |
| [ |
| 0.10781692747344618 |
| ], |
| [ |
| 0.11577644860881336 |
| ], |
| [ |
| 0.07058829548055097 |
| ], |
| [ |
| 0.010639733456972271 |
| ], |
| [ |
| -0.030317064983087984 |
| ], |
| [ |
| -0.030317064983087984 |
| ] |
| ], |
| "history waypoints": [ |
| [ |
| -53.98876953125, |
| 0.314453125 |
| ], |
| [ |
| -50.45166015625, |
| 0.240234375 |
| ], |
| [ |
| -46.90283203125, |
| 0.17578125 |
| ], |
| [ |
| -43.34716796875, |
| 0.12109375 |
| ], |
| [ |
| -39.78076171875, |
| 0.07421875 |
| ], |
| [ |
| -36.2060546875, |
| 0.03515625 |
| ], |
| [ |
| -32.61962890625, |
| 0.005859375 |
| ], |
| [ |
| -29.02490234375, |
| -0.015625 |
| ], |
| [ |
| -25.42626953125, |
| -0.025390625 |
| ], |
| [ |
| -21.81396484375, |
| -0.0234375 |
| ], |
| [ |
| -18.19775390625, |
| -0.017578125 |
| ], |
| [ |
| -14.57421875, |
| -0.0078125 |
| ], |
| [ |
| -10.94287109375, |
| -0.001953125 |
| ], |
| [ |
| -7.302734375, |
| 0.00390625 |
| ], |
| [ |
| -3.65478515625, |
| 0.001953125 |
| ], |
| [ |
| 0.0, |
| 0.0 |
| ] |
| ], |
| "future waypoints": [ |
| [ |
| 3.66162109375, |
| -0.00390625 |
| ], |
| [ |
| 7.33056640625, |
| -0.01171875 |
| ], |
| [ |
| 11.0087890625, |
| -0.0234375 |
| ], |
| [ |
| 14.69677734375, |
| -0.041015625 |
| ], |
| [ |
| 18.3955078125, |
| -0.0703125 |
| ], |
| [ |
| 22.10400390625, |
| -0.107421875 |
| ], |
| [ |
| 25.82275390625, |
| -0.158203125 |
| ], |
| [ |
| 29.55029296875, |
| -0.21484375 |
| ], |
| [ |
| 33.28466796875, |
| -0.28515625 |
| ], |
| [ |
| 37.02197265625, |
| -0.365234375 |
| ], |
| [ |
| 40.76318359375, |
| -0.45703125 |
| ], |
| [ |
| 44.5078125, |
| -0.556640625 |
| ], |
| [ |
| 48.255859375, |
| -0.666015625 |
| ], |
| [ |
| 52.00439453125, |
| -0.7890625 |
| ], |
| [ |
| 55.7529296875, |
| -0.921875 |
| ], |
| [ |
| 59.50048828125, |
| -1.064453125 |
| ], |
| [ |
| 63.24609375, |
| -1.2109375 |
| ], |
| [ |
| 66.984375, |
| -1.369140625 |
| ], |
| [ |
| 70.70361328125, |
| -1.53125 |
| ], |
| [ |
| 74.39453125, |
| -1.69140625 |
| ] |
| ], |
| "navigation_command": "GO_STRAIGHT", |
| "meta_speed": "keep_speed", |
| "meta_direction": "keep_lane", |
| "meta_maneuver": "none", |
| "meta_pair": "keep_speed|keep_lane", |
| "meta_diag": { |
| "stop": false, |
| "dir": { |
| "cum_heading_max_deg": 1.3082489608625636, |
| "cum_window_idx": 6, |
| "cum_window_len": 8, |
| "net_heading_deg": -2.3626158041539758, |
| "lane_shift_abs": 1.6875, |
| "mode": "none" |
| }, |
| "spd": { |
| "dv": 0.10174981205892841, |
| "slope": 0.04766133458276179 |
| }, |
| "overtake": { |
| "overtake": false |
| } |
| }, |
| "sample_id": "104252_0ccbcc47f5d0920f6918be5b7148e05c_161" |
| }, |
| { |
| "image": [ |
| "data/example/images/227_CAM_FRONT_LEFT.jpg", |
| "data/example/images/227_CAM_FRONT.jpg", |
| "data/example/images/227_CAM_FRONT_RIGHT.jpg" |
| ], |
| "conversations": [ |
| { |
| "from": "human", |
| "value": "You are an expert autonomous driving agent.\nTask 1: Critical Object Detection\nFor each class below, answer \"yes\" or \"no\" to indicate whether it affects the ego vehicle\u2019s behavior or future trajectory:\n[nearby_vehicle, pedestrian, cyclist, construction, traffic_element, weather_condition, road_hazard, emergency_vehicle, animal, special_vehicle, conflicting_vehicle, door_opening_vehicle]\nTask 2: Scene Reasoning\nPredict the future behavior of the identified critical objects and explain how the identified critical objects or conditions affect the ego vehicle\u2019s next 3-second trajectory.\nTask 3: Meta-Behavior Prediction\nPredict the ego vehicle\u2019s future meta-driving behavior:\n- speed \u2208 {keep, accelerate, decelerate, stop, other}\n- command \u2208 {straight, yield, left_turn, right_turn, lane_follow, lane_change_left, lane_change_right, reverse, overtake, other}\nTask 4: Trajectory Prediction\nPredict the optimal 5-second future trajectory (5 waypoints, 1 s intervals).\n\nInput:\n- <image>: three front-view frames from left front, center front, right front cameras. \n- High-level navigation command: GO_STRAIGHT\n- Historical ego state: Provided are the previous ego vehicle status recorded over the last 3.0 seconds (at 0.5-second intervals).(t-3.0s) [-52.48, -0.25], Acceleration: X -0.03, Y 0.01 m/s\u00b2, Velocity: X 17.18, Y 0.03 m/s,; (t-2.5s) [-43.90, -0.23], Acceleration: X 0.01, Y 0.01 m/s\u00b2, Velocity: X 17.18, Y 0.06 m/s,; (t-2.0s) [-35.30, -0.19], Acceleration: X 0.10, Y 0.03 m/s\u00b2, Velocity: X 17.36, Y 0.13 m/s,; (t-1.5s) [-26.62, -0.13], Acceleration: X 0.09, Y 0.00 m/s\u00b2, Velocity: X 17.57, Y 0.14 m/s,; (t-1.0s) [-17.83, -0.06], Acceleration: X 0.09, Y -0.03 m/s\u00b2, Velocity: X 17.76, Y 0.09 m/s,; (t-0.5s) [-8.96, -0.02], Acceleration: X 0.09, Y -0.03 m/s\u00b2, Velocity: X 17.93, Y 0.05 m/s,; (t+0.0s) [0.00, 0.00], Acceleration: X 0.09, Y -0.05 m/s\u00b2, Velocity: X 18.01, Y 0.00 m/s,\n\nOutput:" |
| }, |
| { |
| "from": "gpt", |
| "value": "{\"critical_objects\": {\"nearby_vehicle\": \"no\", \"pedestrian\": \"no\", \"cyclist\": \"no\", \"construction\": \"no\", \"traffic_element\": \"yes\", \"weather_condition\": \"no\", \"road_hazard\": \"no\", \"emergency_vehicle\": \"no\", \"animal\": \"no\", \"special_vehicle\": \"no\", \"conflicting_vehicle\": \"no\", \"door_opening_vehicle\": \"no\"}, \"explanation\": \"<|mdm_start|>There are no nearby vehicles, pedestrians, cyclists, or road hazards that directly affect the ego vehicle's immediate path. However, there is a traffic sign on the right side of the road, visible in the front camera, which is relevant to the ego vehicle's navigation and must be considered. Since there is no visible obstruction or conflict from other road users, the ego car is expected to continue forward while observing the traffic sign for compliance.<|mdm_end|>\", \"future_meta_behavior\": {\"longitudinal\": \"<|mdm_start|>speed up <|mdm_end|>\", \"lateral\": \"<|mdm_start|>lane follow <|mdm_end|>\"}, \"trajectory\": \"<|mdm_start|>[[+18.21,-00.04], [+36.70,-00.04], [+55.41,+00.09], [+74.27,+00.32], [+93.21,+00.56]]<|mdm_end|>\"}" |
| } |
| ], |
| "velocity": [ |
| [ |
| 17.293224334716797, |
| 0.01984262466430664 |
| ], |
| [ |
| 17.25619125366211, |
| 0.02274799346923828 |
| ], |
| [ |
| 17.207965850830078, |
| 0.01689434051513672 |
| ], |
| [ |
| 17.17523765563965, |
| 0.028439760208129883 |
| ], |
| [ |
| 17.174583435058594, |
| 0.05048727989196777 |
| ], |
| [ |
| 17.18166732788086, |
| 0.0648794174194336 |
| ], |
| [ |
| 17.25986671447754, |
| 0.10085272789001465 |
| ], |
| [ |
| 17.364519119262695, |
| 0.12607264518737793 |
| ], |
| [ |
| 17.48014259338379, |
| 0.1374499797821045 |
| ], |
| [ |
| 17.574678421020508, |
| 0.1386864185333252 |
| ], |
| [ |
| 17.66480827331543, |
| 0.12388968467712402 |
| ], |
| [ |
| 17.759626388549805, |
| 0.09441614151000977 |
| ], |
| [ |
| 17.835474014282227, |
| 0.0806589126586914 |
| ], |
| [ |
| 17.927324295043945, |
| 0.049605607986450195 |
| ], |
| [ |
| 18.01459503173828, |
| 0.003991603851318359 |
| ], |
| [ |
| 18.01459503173828, |
| 0.003991603851318359 |
| ] |
| ], |
| "acceleration": [ |
| [ |
| -0.04537773132324219, |
| -0.00475001335144043 |
| ], |
| [ |
| -0.0370330810546875, |
| 0.0029053688049316406 |
| ], |
| [ |
| -0.04822540283203125, |
| -0.0058536529541015625 |
| ], |
| [ |
| -0.03272819519042969, |
| 0.011545419692993164 |
| ], |
| [ |
| -0.0006542205810546875, |
| 0.02204751968383789 |
| ], |
| [ |
| 0.007083892822265625, |
| 0.01439213752746582 |
| ], |
| [ |
| 0.07819938659667969, |
| 0.035973310470581055 |
| ], |
| [ |
| 0.10465240478515625, |
| 0.02521991729736328 |
| ], |
| [ |
| 0.11562347412109375, |
| 0.011377334594726562 |
| ], |
| [ |
| 0.09453582763671875, |
| 0.0012364387512207031 |
| ], |
| [ |
| 0.09012985229492188, |
| -0.014796733856201172 |
| ], |
| [ |
| 0.094818115234375, |
| -0.029473543167114258 |
| ], |
| [ |
| 0.07584762573242188, |
| -0.01375722885131836 |
| ], |
| [ |
| 0.09185028076171875, |
| -0.03105330467224121 |
| ], |
| [ |
| 0.08727073669433594, |
| -0.045614004135131836 |
| ], |
| [ |
| 0.08727073669433594, |
| -0.045614004135131836 |
| ] |
| ], |
| "turn angle": [ |
| [ |
| 0.06574240446917488 |
| ], |
| [ |
| 0.07553018183624063 |
| ], |
| [ |
| 0.05625151203584294 |
| ], |
| [ |
| 0.09487360662439473 |
| ], |
| [ |
| 0.16842910545081521 |
| ], |
| [ |
| 0.2163526424091161 |
| ], |
| [ |
| 0.3347864727595571 |
| ], |
| [ |
| 0.41598062769483385 |
| ], |
| [ |
| 0.45051929007722474 |
| ], |
| [ |
| 0.45212670938472665 |
| ], |
| [ |
| 0.4018294208445754 |
| ], |
| [ |
| 0.3046007468790821 |
| ], |
| [ |
| 0.2591119118248104 |
| ], |
| [ |
| 0.15853925980856878 |
| ], |
| [ |
| 0.012695375611996521 |
| ], |
| [ |
| 0.012695375611996521 |
| ] |
| ], |
| "history waypoints": [ |
| [ |
| -65.43408203125, |
| -0.259521484375 |
| ], |
| [ |
| -61.10791015625, |
| -0.25518798828125 |
| ], |
| [ |
| -56.79052734375, |
| -0.251220703125 |
| ], |
| [ |
| -52.48486328125, |
| -0.24609375 |
| ], |
| [ |
| -48.1875, |
| -0.2413330078125 |
| ], |
| [ |
| -43.89599609375, |
| -0.2301025390625 |
| ], |
| [ |
| -39.60302734375, |
| -0.214111328125 |
| ], |
| [ |
| -35.2998046875, |
| -0.1922607421875 |
| ], |
| [ |
| -30.9736328125, |
| -0.161865234375 |
| ], |
| [ |
| -26.619140625, |
| -0.1273193359375 |
| ], |
| [ |
| -22.240234375, |
| -0.09197998046875 |
| ], |
| [ |
| -17.8349609375, |
| -0.0606689453125 |
| ], |
| [ |
| -13.408203125, |
| -0.03753662109375 |
| ], |
| [ |
| -8.9599609375, |
| -0.01702880859375 |
| ], |
| [ |
| -4.4931640625, |
| -0.0032958984375 |
| ], |
| [ |
| 0.0, |
| 0.0 |
| ] |
| ], |
| "future waypoints": [ |
| [ |
| 4.51611328125, |
| -0.00628662109375 |
| ], |
| [ |
| 9.0576171875, |
| -0.017333984375 |
| ], |
| [ |
| 13.6220703125, |
| -0.03204345703125 |
| ], |
| [ |
| 18.20654296875, |
| -0.04229736328125 |
| ], |
| [ |
| 22.8076171875, |
| -0.0499267578125 |
| ], |
| [ |
| 27.42529296875, |
| -0.0509033203125 |
| ], |
| [ |
| 32.056640625, |
| -0.04705810546875 |
| ], |
| [ |
| 36.70263671875, |
| -0.0372314453125 |
| ], |
| [ |
| 41.3603515625, |
| -0.0177001953125 |
| ], |
| [ |
| 46.03076171875, |
| 0.01068115234375 |
| ], |
| [ |
| 50.71337890625, |
| 0.04681396484375 |
| ], |
| [ |
| 55.4072265625, |
| 0.0919189453125 |
| ], |
| [ |
| 60.1083984375, |
| 0.1434326171875 |
| ], |
| [ |
| 64.8212890625, |
| 0.198486328125 |
| ], |
| [ |
| 69.54248046875, |
| 0.25885009765625 |
| ], |
| [ |
| 74.271484375, |
| 0.320068359375 |
| ], |
| [ |
| 78.9990234375, |
| 0.37872314453125 |
| ], |
| [ |
| 83.73583984375, |
| 0.44134521484375 |
| ], |
| [ |
| 88.47265625, |
| 0.50323486328125 |
| ], |
| [ |
| 93.2138671875, |
| 0.56427001953125 |
| ] |
| ], |
| "navigation_command": "GO_STRAIGHT", |
| "meta_speed": "accelerate", |
| "meta_direction": "keep_lane", |
| "meta_maneuver": "none", |
| "meta_pair": "accelerate|keep_lane", |
| "meta_diag": { |
| "stop": false, |
| "dir": { |
| "cum_heading_max_deg": 0.7228069052223384, |
| "cum_window_idx": 3, |
| "cum_window_len": 8, |
| "net_heading_deg": 0.8769201986116071, |
| "lane_shift_abs": 0.570556640625, |
| "mode": "none" |
| }, |
| "spd": { |
| "dv": 0.8003457647869112, |
| "slope": 0.17529829260145602 |
| }, |
| "overtake": { |
| "overtake": false |
| } |
| }, |
| "sample_id": "114317_c086330168a335e2702e9bec118cb3a2_227" |
| } |
| ] |
