add model.eval()

inference.py

@@ -20,6 +20,7 @@ def load_model(device, dtype):
         torch_dtype=dtype,
         device_map=device,
     )
+    model.eval()
 
     models = {
         "tokenizer": tokenizer,