Use dynamic batch size by default

onnx_to_engine.py

@@ -91,14 +91,24 @@ def main(onnx_path, engine_path, batch_size):
     # You don't have to use it with Polygraphy loaders if you don't want to.
     calibrator = Calibrator(data_loader=calib_data_from_video(), cache=f"{onnx_path}-calib.cache")
 
+    if batch_size < 1: # dynamic batch size
 
-    profiles = [
-        # The low-latency case. For best performance, min == opt == max.
-        Profile().add("input", 
-                      min=(1, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1]), 
-                      opt=(4, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1]), 
-                      max=(9, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1])),
-    ]
+        profiles = [
+            # The low-latency case. For best performance, min == opt == max.
+            Profile().add("input", 
+                        min=(1, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1]), 
+                        opt=(4, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1]), 
+                        max=(9, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1])),
+        ]
+    
+    else: # fixed
+        profiles = [
+            # The low-latency case. For best performance, min == opt == max.
+            Profile().add("input", 
+                        min=(batch_size, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1]), 
+                        opt=(batch_size, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1]), 
+                        max=(batch_size, 3, MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1])),
+        ]
 
     # We must enable int8 mode in addition to providing the calibrator.
     build_engine = EngineFromNetwork(
@@ -126,7 +136,7 @@ if __name__ == "__main__":
     parser.add_argument("video_path", type=str, help="The path to the video file used to calibrate int8 engine")
     parser.add_argument("onnx_path", type=str, help="The path to the input ONNX model file")
     parser.add_argument("engine_path", type=str, help="The path to the exported TensorRT Engine model file")
-    parser.add_argument("--batch_size", type=int, default=1, help="Input batch size")
+    parser.add_argument("--batch_size", type=int, default=-1, help="Input batch size (not specified if dynamic)")
     args = parser.parse_args()
     VIDEO_PATH = args.video_path
     MODEL_INPUT_SIZE=(416,416) if 'rtmo-t' in args.onnx_path else (640,640)