scripts/data/test_pkl_data.py

import mmcv
from typing import Dict, List
import pickle
import random
import os
import copy


datatomove = [
    "scene-0980",
    "scene-0001",
    "scene-0011",
    "scene-0023",
    "scene-0034",
    "scene-0052",
    "scene-0318",
    "scene-0200",
    "scene-0028",
    "scene-0297",
    "scene-0067",
    "scene-0159",
    "scene-0185",
    "scene-0021",
    "scene-0025",
    "scene-0072",
    "scene-0157",
    "scene-0208",
    "scene-0366",
    "scene-0400",
]


def remove_indices(original_list, indices_to_remove):
    indices_to_remove_set = set(indices_to_remove)
    return [
        item
        for idx, item in enumerate(original_list)
        if idx not in indices_to_remove_set
    ]


def sort_data(data_dict: Dict):
    return data_dict["token"]


predroot = "/mnt/hdd2/datasets/nuScenes/infos/e2e_av"
train_file = os.path.join(predroot, "nuscenes_infos_temporal_train.pkl")
val_file = os.path.join(predroot, "nuscenes_infos_temporal_val.pkl")
nuscenes_metadata = os.path.join(predroot, "../../v1.0-trainval/scene.json")
# predroot = '/mnt/hdd2/datasets/navsim/openscene-v1.1/infos/nuplan_navsim_val.pkl'
# predroot = '/mnt/hdd2/datasets/navsim/openscene-v1.1/meta_datas_paradrive/trainval/2021.10.11.08.31.07_veh-50_01184_01318.pkl'

outputs_train_ori: Dict = mmcv.load(train_file)
outputs_train: List[Dict] = outputs_train_ori["infos"]
print(f"len of train is {len(outputs_train)}")

outputs_val_ori: Dict = mmcv.load(val_file)
outputs_val: List[Dict] = outputs_val_ori["infos"]
print(f"len of val is {len(outputs_val)}")

# self.outputs = outputs
# outputs.sort(key=sort_data)

print(outputs_train_ori.keys())


nuscenes_metadata = mmcv.load(nuscenes_metadata)
scene_token2name = dict()
for metadata_seq in nuscenes_metadata:
    # print(metadata_seq["token"])
    # print(metadata_seq["name"])

    scene_token2name[metadata_seq["token"]] = metadata_seq["name"]


data_move_list = []
data_add_list = []
for index in range(len(outputs_train)):
    data = outputs_train[index]
    scene_token = data["scene_token"]
    scene_name = scene_token2name[scene_token]
    if scene_name in datatomove:
        data_move_list.append(index)
        data_add_list.append(copy.deepcopy(data))
    # print(index)

    # print('\nnew frame')
    # print("scene_token", data["scene_token"])
    # print("frame_idx", data["frame_idx"])
    # print("sample_token", data["token"])
    # print("log_name", data["log_name"])
    # print("log_token", data["log_token"])
    # zxc

outputs_train_new = remove_indices(outputs_train, data_move_list)
outputs_val.extend(data_add_list)

print(f"len of new train is {len(outputs_train_new)}")
print(f"len of new val is {len(outputs_val)}")
# print(len(outputs))
# zxc

output_path_train = os.path.join(predroot, "nuscenes_infos_temporal_train_new.pkl")
outputs_train_new = {
    'infos': outputs_train_new,
    "metadata": outputs_train_ori['metadata'],
}
mmcv.dump(outputs_train_new, output_path_train)
# with open(output_path_train, "wb") as f:
#     pickle.dump(outputs_train_new, f)

output_path_val = os.path.join(predroot, "nuscenes_infos_temporal_val_new.pkl")
outputs_val_new = {
    'infos': outputs_val,
    "metadata": outputs_val_ori['metadata'],
}
mmcv.dump(outputs_val_new, output_path_val)
# with open(output_path_val, "wb") as f:
#     pickle.dump(outputs_val_new, f)