Update app.py

app.py

@@ -64,42 +64,48 @@ MODEL_CONFIGS = {
         "default_max_tokens": 8192,
         "has_max_tokens_limit": True,
         "supports_vision": True,
-        "supports_files": True
+        "supports_files": True,
+        "image_format": "url"  # Claude 4 Sonnet 需要 URL 格式
     },
     "anthropic/claude-3.5-sonnet": {
         "min_max_tokens": 1,
         "default_max_tokens": 8192,
         "has_max_tokens_limit": False,
         "supports_vision": True,
-        "supports_files": True
+        "supports_files": True,
+        "image_format": "data_url"  # Claude 3.5 支持 data URL
     },
     "anthropic/claude-3-sonnet": {
         "min_max_tokens": 1,
         "default_max_tokens": 4096,
         "has_max_tokens_limit": False,
         "supports_vision": True,
-        "supports_files": True
+        "supports_files": True,
+        "image_format": "data_url"
     },
     "anthropic/claude-3.5-haiku": {
         "min_max_tokens": 1,
         "default_max_tokens": 4096,
         "has_max_tokens_limit": False,
         "supports_vision": True,
-        "supports_files": True
+        "supports_files": True,
+        "image_format": "data_url"
     },
     "anthropic/claude-3-haiku": {
         "min_max_tokens": 1,
         "default_max_tokens": 4096,
         "has_max_tokens_limit": False,
         "supports_vision": True,
-        "supports_files": True
+        "supports_files": True,
+        "image_format": "data_url"
     },
     "google/gemini-2.5-pro": {
         "min_max_tokens": 1,
         "default_max_tokens": 8192,
         "has_max_tokens_limit": False,
         "supports_vision": True,
-        "supports_files": True
+        "supports_files": True,
+        "image_format": "data_url"
     }
 }
 
@@ -172,38 +178,74 @@ def decode_base64_file(data_url: str) -> tuple[str, str, str]:
         logger.error(f"Failed to parse data URL: {e}")
         return None, None, None
 
-def format_image_for_replicate(base64_data: str) -> str:
+async def upload_image_to_temp_service(session: aiohttp.ClientSession, base64_data: str) -> str:
     """
-    将 base64 图片数据格式化为 Replicate 期望的格式
+    将 base64 图片上传到临时图片托管服务
+    这里使用 imgbb 作为示例，你也可以使用其他服务
     """
-    # 检查 base64 数据是否已经包含 data URL 前缀
-    if base64_data.startswith("data:"):
-        return base64_data
-    
-    # 如果没有前缀，添加默认的 JPEG data URL 前缀
-    # 但首先尝试检测实际的图片格式
     try:
-        # 解码 base64 数据的前几个字节来检测格式
-        decoded_bytes = base64.b64decode(base64_data[:100])
-        
-        if decoded_bytes.startswith(b'\xff\xd8\xff'):
-            # JPEG
-            return f"data:image/jpeg;base64,{base64_data}"
-        elif decoded_bytes.startswith(b'\x89PNG\r\n\x1a\n'):
-            # PNG
-            return f"data:image/png;base64,{base64_data}"
-        elif decoded_bytes.startswith(b'GIF87a') or decoded_bytes.startswith(b'GIF89a'):
-            # GIF
-            return f"data:image/gif;base64,{base64_data}"
-        elif decoded_bytes.startswith(b'RIFF') and b'WEBP' in decoded_bytes[:20]:
-            # WebP
-            return f"data:image/webp;base64,{base64_data}"
+        # 从 base64 data URL 中提取纯 base64 数据
+        if base64_data.startswith("data:"):
+            base64_content = base64_data.split(",")[1]
         else:
-            # 默认使用 JPEG
-            return f"data:image/jpeg;base64,{base64_data}"
+            base64_content = base64_data
+        
+        # 使用 imgbb API（需要免费注册获取 API key）
+        # 这里暂时返回原始 data URL，你需要根据实际情况实现图片上传
+        logger.warning("Image upload to external service not implemented, using workaround")
+        
+        # 临时解决方案：对于 Claude 4，我们需要找到另一种方式
+        # 可以考虑：
+        # 1. 使用临时文件服务（如 imgbb, imgur 等）
+        # 2. 使用自己的文件服务器
+        # 3. 修改为使用 claude-3.5-sonnet 作为替代
+        
+        return None  # 返回 None 表示上传失败
+        
     except Exception as e:
-        logger.warning(f"Failed to detect image format: {e}, using JPEG as default")
-        return f"data:image/jpeg;base64,{base64_data}"
+        logger.error(f"Failed to upload image: {e}")
+        return None
+
+def format_image_for_model(base64_data: str, model_config: Dict[str, Any]) -> str:
+    """
+    根据模型配置格式化图片数据
+    """
+    image_format = model_config.get("image_format", "data_url")
+    
+    if image_format == "data_url":
+        # 检查 base64 数据是否已经包含 data URL 前缀
+        if base64_data.startswith("data:"):
+            return base64_data
+        
+        # 如果没有前缀，添加默认的 JPEG data URL 前缀
+        try:
+            # 解码 base64 数据的前几个字节来检测格式
+            decoded_bytes = base64.b64decode(base64_data[:100])
+            
+            if decoded_bytes.startswith(b'\xff\xd8\xff'):
+                # JPEG
+                return f"data:image/jpeg;base64,{base64_data}"
+            elif decoded_bytes.startswith(b'\x89PNG\r\n\x1a\n'):
+                # PNG
+                return f"data:image/png;base64,{base64_data}"
+            elif decoded_bytes.startswith(b'GIF87a') or decoded_bytes.startswith(b'GIF89a'):
+                # GIF
+                return f"data:image/gif;base64,{base64_data}"
+            elif decoded_bytes.startswith(b'RIFF') and b'WEBP' in decoded_bytes[:20]:
+                # WebP
+                return f"data:image/webp;base64,{base64_data}"
+            else:
+                # 默认使用 JPEG
+                return f"data:image/jpeg;base64,{base64_data}"
+        except Exception as e:
+            logger.warning(f"Failed to detect image format: {e}, using JPEG as default")
+            return f"data:image/jpeg;base64,{base64_data}"
+    
+    elif image_format == "url":
+        # 对于需要 URL 的模型，返回 None 表示需要上传
+        return None
+    
+    return base64_data
 
 def extract_content_from_message(message: Dict[str, Any]) -> tuple[str, List[str], List[Dict[str, str]]]:
     """
@@ -237,9 +279,8 @@ def extract_content_from_message(message: Dict[str, Any]) -> tuple[str, List[str
                         try:
                             if ";base64," in url:
                                 base64_data = url.split(";base64,")[1]
-                                # 格式化为 Replicate 期望的格式
-                                formatted_image = format_image_for_replicate(base64_data)
-                                images.append(formatted_image)
+                                # 先存储原始的 base64 数据，稍后根据模型需求格式化
+                                images.append(url)  # 保存完整的 data URL
                                 logger.info(f"Found base64 image, size: {len(base64_data)} chars")
                             else:
                                 logger.warning(f"Image URL format not supported: {url[:100]}...")
@@ -260,8 +301,7 @@ def extract_content_from_message(message: Dict[str, Any]) -> tuple[str, List[str
                             
                             if file_ext in SUPPORTED_IMAGE_EXTENSIONS and mime_type.startswith("image/"):
                                 # 图片文件
-                                formatted_image = format_image_for_replicate(file_content)
-                                images.append(formatted_image)
+                                images.append(file_url)  # 保存完整的 data URL
                                 logger.info(f"Found image file: {filename}")
                             elif file_ext in SUPPORTED_TEXT_EXTENSIONS or mime_type.startswith("text/"):
                                 # 文本文件
@@ -307,7 +347,7 @@ def format_files_for_prompt(files: List[Dict[str, str]]) -> str:
     
     return "\n".join(file_sections)
 
-def transform_openai_to_replicate(openai_request: Dict[str, Any], model_override: str = None) -> Dict[str, Any]:
+async def transform_openai_to_replicate(openai_request: Dict[str, Any], model_override: str = None) -> Dict[str, Any]:
     """将OpenAI格式的请求转换为Replicate格式"""
     try:
         messages = openai_request.get("messages", [])
@@ -373,6 +413,18 @@ def transform_openai_to_replicate(openai_request: Dict[str, Any], model_override
         if has_files and not model_config.get("supports_files", False):
             logger.warning(f"Model {model} may not support file processing")
         
+        # 处理图片格式
+        formatted_image = None
+        if has_images and primary_image:
+            if model_config.get("image_format") == "url":
+                # Claude 4 需要 URL 格式，暂时降级到 Claude 3.5
+                logger.warning(f"Model {model} requires URL format for images, falling back to claude-3.5-sonnet")
+                model = "anthropic/claude-3.5-sonnet"
+                model_config = MODEL_CONFIGS[model]
+                formatted_image = format_image_for_model(primary_image, model_config)
+            else:
+                formatted_image = format_image_for_model(primary_image, model_config)
+        
         # 构建 Replicate 格式的输入
         replicate_input = {}
         
@@ -404,10 +456,10 @@ def transform_openai_to_replicate(openai_request: Dict[str, Any], model_override
         
         replicate_input["prompt"] = prompt
         
-        # 处理图片 - 使用正确的 data URL 格式
-        if has_images and primary_image:
-            replicate_input["image"] = primary_image
-            logger.info(f"Added primary image to request: {primary_image[:100]}...")
+        # 处理图片
+        if formatted_image:
+            replicate_input["image"] = formatted_image
+            logger.info(f"Added image to request for model {model}: {formatted_image[:100]}...")
         
         # 只在有 system_prompt 时才添加
         if system_prompt:
@@ -563,12 +615,13 @@ async def root():
         "message": "Replicate API Proxy for LobeChat with Vision and File Support",
         "status": "running",
         "replicate_token_configured": bool(REPLICATE_API_TOKEN),
-        "version": "1.1.1",
+        "version": "1.1.2",
         "supported_models": list(MODEL_CONFIGS.keys()),
         "vision_support": True,
         "file_support": True,
         "supported_text_files": list(SUPPORTED_TEXT_EXTENSIONS),
-        "supported_image_files": list(SUPPORTED_IMAGE_EXTENSIONS)
+        "supported_image_files": list(SUPPORTED_IMAGE_EXTENSIONS),
+        "notes": "Claude 4 Sonnet image support temporarily falls back to Claude 3.5 Sonnet"
     }
 
 @app.get("/health")
@@ -613,7 +666,7 @@ async def chat_completions(request: Request):
         logger.info(f"Message count: {len(body.get('messages', []))}")
         
         # 转换请求格式
-        replicate_data, model = transform_openai_to_replicate(body)
+        replicate_data, model = await transform_openai_to_replicate(body)
         
         if body.get("stream", False):
             # 流式响应