Update inference.py

inference.py

@@ -33,12 +33,18 @@ from transformers import (
 from deepseek_vl.models import MultiModalityCausalLM, VLChatProcessor
 from deepseek_vl.utils.conversation import Conversation
 
+from transformers import BitsAndBytesConfig
 
+nf8_config = BitsAndBytesConfig(
+    load_in_8bit=True,
+    bnb_8bit_use_double_quant=True,
+    bnb_8bit_quant_type="nf8",
+)
 def load_model(model_path):
     vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)
     tokenizer = vl_chat_processor.tokenizer
     vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(
-        model_path, trust_remote_code=True
+        model_path, trust_remote_code=True, quantization_config=nf8_config
     )
     vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()
     return tokenizer, vl_gpt, vl_chat_processor