import argparse import numpy as np from waymo_preprocess import WaymoProcessor if __name__ == "__main__": """ Waymo Dataset preprocessing script =========================== This script facilitates the preprocessing of the Waymo dataset Usage: ------ python preprocess.py \ --data_root \ --target_dir \ [additional_arguments] Example: -------- python preprocess.py --data_root data/waymo/raw/ --target_dir data/waymo/processed --split training --workers 3 --scene_ids 700 754 114 Arguments: ---------- --data_root (str): The root directory where the Waymo dataset is stored. This is a required argument. --split (str): Specifies the name of the data split. Default is set to "training". --target_dir (str): Designates the directory where the processed data will be saved. This is a mandatory argument. --workers (int): The number of processing threads. Default is set to 4. --scene_ids (list[int]): List of specific scene IDs for processing. Should be integers separated by spaces. --split_file (str): If provided, indicates the path to a file located in `data/waymo_splits` that contains the desired scene IDs. --start_idx (int): Used in conjunction with `num_scenes` to generate a list of scene IDs when neither `scene_ids` nor `split_file` are provided. --num_scenes (int): The total number of scenes to be processed. --process_keys (list[str]): Denotes the types of data components to be processed. Options include but aren't limited to "images", "lidar", "calib", "pose", etc. Notes: ------ The logic of the script ensures that if specific scene IDs (`scene_ids`) are provided, they are prioritized. If a split file (`split_file`) is indicated, it is utilized next. If neither is available, the script uses the `start_idx` and `num_scenes` parameters to determine the scene IDs. """ parser = argparse.ArgumentParser(description="Data converter arg parser") parser.add_argument( "--data_root", type=str, required=True, help="root path of waymo dataset" ) parser.add_argument("--split", type=str, default="training", help="split name") parser.add_argument( "--target_dir", type=str, required=True, help="output directory of processed data", ) parser.add_argument( "--workers", type=int, default=4, help="number of threads to be used" ) # priority: scene_ids > split_file > start_idx + num_scenes parser.add_argument( "--scene_ids", default=None, type=int, nargs="+", help="scene ids to be processed, a list of integers separated by space. Range: [0, 798] for training, [0, 202] for validation", ) parser.add_argument( "--split_file", type=str, default=None, help="Split file in data/waymo_splits" ) parser.add_argument( "--start_idx", type=int, default=0, help="If no scene id or split_file is given, use start_idx and num_scenes to generate scene_ids_list", ) parser.add_argument( "--num_scenes", type=int, default=200, help="number of scenes to be processed", ) parser.add_argument( "--process_keys", nargs="+", default=[ "images", "lidar", "calib", "pose", "dynamic_masks", ], ) args = parser.parse_args() if args.scene_ids is not None: scene_ids_list = args.scene_ids elif args.split_file is not None: # parse the split file split_file = open(args.split_file, "r").readlines()[1:] scene_ids_list = [int(line.strip().split(",")[0]) for line in split_file] else: scene_ids_list = np.arange(args.start_idx, args.start_idx + args.num_scenes) waymo_processor = WaymoProcessor( load_dir=args.data_root, save_dir=args.target_dir, prefix=args.split, process_keys=args.process_keys, process_id_list=scene_ids_list, workers=args.workers, ) if args.scene_ids is not None and args.workers == 1: for scene_id in args.scene_ids: waymo_processor.convert_one(scene_id) else: waymo_processor.convert()