Update app.py

app.py

@@ -1,10 +1,19 @@
 import gradio as gr
 from transformers import AutoModelForCausalLM, AutoTokenizer
+import bitsandbytes as bnb
 
 # 加载本地模型和tokenizer
 model_name = "ganchengguang/OIELLM-8B-Instruction"  # 替换为你的模型名称
 tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    device_map="auto",
+    load_in_8bit=True,
+    quantization_config=bnb.configs.BitsAndBytesConfig(
+        load_in_8bit=True,
+        load_in_8bit_fp32_cpu_offload=True
+    )
+)
 
 # 定义语言和选项的映射
 options = {
@@ -20,7 +29,7 @@ def respond(message, language, task, system_message, max_tokens, temperature, to
     messages.append({"role": "user", "content": message + " " + options[language][task]})
 
     # 编码输入
-    inputs = tokenizer(messages, return_tensors="pt", padding=True, truncation=True)
+    inputs = tokenizer(message + " " + options[language][task], return_tensors="pt", padding=True, truncation=True)
     # 生成回复
     outputs = model.generate(
         inputs["input_ids"],