updated

app.py

@@ -1,117 +1,148 @@
+from openai import OpenAI
 import openai
+import base64
+import PIL.Image as Image
 import requests
 import json
 import os
+import pillow_heif
 import gradio as gr
+import tempfile
+import shutil
 
-# Set your ModelScope API credentials and base URL
-openai.api_key = 'cf707b08-a58a-4683-aba5-5fa0f0bea8fe'
-openai.api_base = 'https://api-inference.modelscope.cn/v1/'
-
-def chat_with_model(conv_history, user_message, image_paths=None):
+def load_image(image_path):
     """
-    与 ModelScope 的 Qwen2.5-VL-72B-Instruct 模型交互，处理用户消息和可选的图像。
-
-    参数：
-    - conv_history: 对话历史记录的列表。
-    - user_message: 用户的最新消息。
-    - image_paths: 上传的图像文件路径列表（如果有）。
-
-    返回：
-    - 更新后的对话显示内容（Markdown 格式）。
-    - 空字符串（用于清空文本输入框）。
-    - 更新后的对话历史记录。
-    - 清空后的图像组件（空列表，用于删除网页上的图片）。
+    Process a local image: if it's in HEIC format, convert it to JPG,
+    then return a base64 encoded data URL string.
+    """
+    if image_path.lower().endswith(".heic"):
+        heif_file = pillow_heif.open_heif(image_path)
+        image = Image.frombytes(heif_file.mode, heif_file.size, heif_file.data)
+        jpg_path = image_path.rsplit(".", 1)[0] + ".jpg"
+        image.save(jpg_path, "JPEG")
+        image_path = jpg_path
+
+    with open(image_path, "rb") as image_file:
+        image_data = image_file.read()
+        base64_encoded = base64.b64encode(image_data).decode("utf-8")
+    mime_type = Image.open(image_path).get_format_mimetype()
+    return f"data:{mime_type};base64,{base64_encoded}"
+
+def chat_with_model(conv_history, user_message, image_files):
+    """
+    Sends a conversation turn to the ModelScope Qwen2.5-VL-72B-Instruct model.
+    Supports multiple images by:
+      1. Copying each uploaded file to a temporary directory.
+      2. Processing it (converting HEIC to JPG if necessary).
+      3. Deleting the temporary file after inference.
+    
+    Returns:
+      - A Markdown-formatted conversation display.
+      - An empty string to clear the text input.
+      - An empty list to clear the file input.
+      - The updated conversation history.
     """
+    client = OpenAI(
+        base_url='https://api-inference.modelscope.cn/v1/',
+        api_key='cf707b08-a58a-4683-aba5-5fa0f0bea8fe'
+    )
 
-    # 系统提示，指导助手的行为
     system_prompt = {
         'role': 'system',
         'content': (
-            "你是一个乐于助人的助手，帮助辅导学生。老师上传了一张学生解答问题的图片。"
-            "如果学生的解答有错误，请纠正。如果解答正确，请说'解答正确'。"
-            "如果解答有误，请提供正确的解答，并仔细解释学生哪里出错了。"
+            "You are a helpful assistant helping a student with their homework. "
+            "The teacher has uploaded a picture of a student's answer to a question. "
+            "If the student's answer is incorrect, please correct it. "
+            "If the answer is correct, say 'correct answer'. "
+            "if the question is not answered, solve the question with explaining your step\
+                clearly and easy to understand."
         )
     }
-
-    # 构建消息历史记录
-    messages = [system_prompt]
-    if conv_history:
-        for user, assistant in conv_history:
-            messages.append({'role': 'user', 'content': user})
-            messages.append({'role': 'assistant', 'content': assistant})
-    messages.append({'role': 'user', 'content': user_message})
-
-    # 准备 API 请求的有效载荷
-    payload = {
-        'model': 'Qwen/Qwen2.5-VL-72B-Instruct',
-        'messages': messages
+    
+    # Build a new user message with text and attached images.
+    new_message = {
+        'role': 'user',
+        'content': [{'type': 'text', 'text': user_message}]
     }
-
-    # 如果提供了图像，处理图像上传（支持多个图像）
-    files = None
-    if image_paths:
-        if isinstance(image_paths, list):
-            files = []
-            for path in image_paths:
-                with open(path, "rb") as f:
-                    image_bytes = f.read()
-                files.append(("file", (os.path.basename(path), image_bytes, "application/octet-stream")))
-        else:
-            with open(image_paths, "rb") as f:
-                image_bytes = f.read()
-            files = {
-                "file": (os.path.basename(image_paths), image_bytes, "application/octet-stream")
-            }
-
-    # 向 ModelScope API 发送请求
-    response = requests.post(
-        openai.api_base + 'chat/completions',
-        headers={
-            'Authorization': f'Bearer {openai.api_key}',
-            'Content-Type': 'application/json'
-        },
-        json=payload,
-        files=files
+    
+    temp_files = []
+    imgs = []
+    if image_files is not None:
+        for file_path in image_files:
+            # Copy the file to a temporary directory.
+            temp_dir = tempfile.gettempdir()
+            filename = os.path.basename(file_path)
+            temp_path = os.path.join(temp_dir, filename)
+            shutil.copy(file_path, temp_path)
+            temp_files.append(temp_path)
+            # Process the image (HEIC conversion if needed).
+            data_url = load_image(temp_path)
+            imgs.append(data_url)
+    
+    # For each processed image, add a separate content block.
+    if imgs:
+        for img in imgs:
+            new_message['content'].append({
+                'type': 'image_url',
+                'image_url': img
+            })
+
+    # Combine conversation history with the new message.
+    # conv_history should be a list of prior messages (each a dict).
+    all_messages = [system_prompt] + conv_history + [new_message]
+    
+    response = client.chat.completions.create(
+        model='Qwen/Qwen2.5-VL-72B-Instruct',
+        messages=all_messages,
+        stream=False
     )
 
-    # 处理 API 响应
-    if response.status_code != 200:
-        assistant_response = f"**错误：** {response.status_code} - {response.text}"
-    else:
-        assistant_response = response.json().get("choices", [{}])[0].get("message", {}).get("content", "API 未返回输出。")
+    assistant_response = response.choices[0].message.content
 
-    # 更新对话历史记录
-    conv_history.append((user_message, assistant_response))
+    # Update the conversation history with the new turn.
+    conv_history.append(new_message)
+    conv_history.append({'role': 'assistant', 'content': assistant_response})
 
-    # 构建整个对话的 Markdown 显示内容
+    # Build Markdown conversation display.
     conversation_display = ""
-    for user, assistant in conv_history:
-        conversation_display += f"**用户：** {user}\n\n**助手：**\n{assistant}\n\n---\n\n"
-
-    # 返回对话显示、清空文本输入框、更新状态以及清空文件上传组件（图片）
-    return conversation_display, "", conv_history, []
-
-# 定义 Gradio 界面
+    for msg in conv_history:
+        if msg['role'] == 'user':
+            conversation_display += f"**User:** {msg['content'][0]['text']}\n"
+            # Indicate image uploads if any.
+            if len(msg['content']) > 1:
+                conversation_display += "**User Uploaded Images:** " + " ".join("[Image]" for _ in range(len(msg['content']) - 1)) + "\n"
+        elif msg['role'] == 'assistant':
+            conversation_display += f"**Assistant:** {msg['content']}\n"
+        conversation_display += "\n---\n\n"
+
+    # Delete all temporary files.
+    for path in temp_files:
+        if os.path.exists(path):
+            os.remove(path)
+
+    # Return outputs:
+    # - The conversation display.
+    # - An empty string to clear the text box.
+    # - An empty list to clear the file upload widget.
+    # - The updated conversation history.
+    return conversation_display, "", [], conv_history
+
+# Build the Gradio UI.
 with gr.Blocks() as demo:
-    gr.Markdown("# 与 Qwen2.5-VL-72B-Instruct 模型的多轮对话")
-    
-    # 显示对话的 Markdown 组件
-    conversation_md = gr.Markdown(value="*对话内容将显示在这里...*")
-    # 保存对话历史记录的状态
-    state = gr.State([])
+    gr.Markdown("# Multi-turn, Multi-image Conversation with Qwen2.5-VL-72B-Instruct")
+    conversation_md = gr.Markdown(value="Conversation will be displayed here...")
+    state = gr.State([])  # This state stores conversation history.
 
     with gr.Row():
-        txt = gr.Textbox(label="您的消息", placeholder="输入您的消息...", lines=2)
-        # 使用 gr.File 组件接受多个文件上传
-        img = gr.File(label="上传图片（可选）", file_count="multiple", type="filepath")
-    
-    btn = gr.Button("发送")
-    btn.click(
+        text_input = gr.Textbox(label="Your Message", placeholder="Enter your message...", lines=2)
+        # Use type 'filepath' for file uploads.
+        file_input = gr.File(label="Upload Images (Optional)", file_count="multiple", type="filepath")
+    send_button = gr.Button("Send")
+
+    send_button.click(
         chat_with_model,
-        inputs=[state, txt, img],
-        outputs=[conversation_md, txt, state, img]
+        inputs=[state, text_input, file_input],
+        outputs=[conversation_md, text_input, file_input, state]
     )
 
-# 启动 Gradio 界面
 demo.launch(share=True)

requirements.txt

@@ -1,3 +1,5 @@
 gradio
 openai
 requests
+pillow
+pillow-heif