Update app.py

app.py

@@ -28,7 +28,7 @@ class ChatClient:
         self.model = AutoModelForCausalLM.from_pretrained(model_path).to(self.device)
         self.model.eval()  # 设置为评估模式
 
-    async def chat_completion(self, messages, max_tokens, stream=False, temperature=1.0, top_p=1.0):
+    def chat_completion(self, messages, max_tokens, stream=False, temperature=1.0, top_p=1.0):
         """
         生成对话回复。
         """
@@ -47,17 +47,15 @@ class ChatClient:
         }
 
         # 使用生成器生成文本
-        output_sequences = self.model.generate(**inputs, **gen_kwargs)
+        # output_sequences = self.model.generate(**inputs, **gen_kwargs)
 
         # 解码生成的文本
         # result_text = self.tokenizer.decode(output_sequences[0], skip_special_tokens=True)
         # yield result_text
 
-        # 解码生成的文本
-        for sequence in output_sequences:
-            result_text = self.tokenizer.decode(sequence, skip_special_tokens=True)
-            await anyio.sleep(0)  # Yield control, simulating asynchronous operation
-            yield result_text
+        # claude 3.5
+        for token in self.model.generate(**inputs, **gen_kwargs, streamer=None):
+            yield self.tokenizer.decode(token, skip_special_tokens=True)
 
 # 创建客户端实例，指定模型路径
 model_path = 'model/v3/'
@@ -68,7 +66,7 @@ client = ChatClient(model_path)
 
 
 
-async def respond(
+def respond(
     message,
     history: list[tuple[str, str]],
     system_message,
@@ -91,21 +89,19 @@ async def respond(
 
     response = ""
 
-    async for message in client.chat_completion(
+    for message in client.chat_completion(
         messages,
         max_tokens=max_tokens,
         stream=True,
         temperature=temperature,
         top_p=top_p,
     ):
-        # print(message)
-        # token = message
-        # #token = message.choices[0].delta.content
-
-        # response += token
-        # yield response
+        print(message)
+        token = message
+        #token = message.choices[0].delta.content
 
-        yield message
+        response += token
+        yield response
 
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface