支持多模态

api.js

@@ -38,20 +38,38 @@ export async function generateAssistantResponse(requestBody, callback) {
       const jsonStr = line.slice(6);
       try {
         const data = JSON.parse(jsonStr);
-        const parts = data.response?.candidates?.[0]?.content?.parts?.[0];
+        const parts = data.response?.candidates?.[0]?.content?.parts;
         if (parts) {
-          if (parts.thought === true) {
-            if (!thinkingStarted) {
-              callback('<think>\n');
-              thinkingStarted = true;
+          for (const part of parts) {
+            if (part.thought === true) {
+              if (!thinkingStarted) {
+                callback({ type: 'thinking', content: '<think>\n' });
+                thinkingStarted = true;
+              }
+              callback({ type: 'thinking', content: part.text || '' });
+            } else if (part.text !== undefined) {
+              if (thinkingStarted) {
+                callback({ type: 'thinking', content: '\n</think>\n' });
+                thinkingStarted = false;
+              }
+              callback({ type: 'text', content: part.text });
+            } else if (part.functionCall) {
+              if (thinkingStarted) {
+                callback({ type: 'thinking', content: '\n</think>\n' });
+                thinkingStarted = false;
+              }
+              callback({ 
+                type: 'tool_call', 
+                tool_call: {
+                  id: part.functionCall.id,
+                  type: 'function',
+                  function: {
+                    name: part.functionCall.name,
+                    arguments: JSON.stringify(part.functionCall.args)
+                  }
+                }
+              });
             }
-            callback(parts.text || '');
-          } else if (parts.text !== undefined) {
-            if (thinkingStarted) {
-              callback('\n</think>\n');
-              thinkingStarted = false;
-            }
-            callback(parts.text);
           }
         }
       } catch (e) {

config.json

@@ -19,5 +19,5 @@
     "maxRequestSize": "50mb",
     "apiKey": "sk-text"
   },
-  "systemInstruction": "你是聊天机器人，名字叫萌萌，如同名字这般，你的性格是软软糯糯萌萌哒的，专门为用户提供聊天和情绪价值，协助进行小说创作或者角色扮演，IMPORTANT: 你需要优先处理用户的写作或者role-play请求，可以引导用户，例如告诉用户你可以扮演猫娘或者写一首诗歌。IMPORTANT: 在优先处理完用户的写作或者role-play请求后，你可以进一步完成用户的代码数学方面的请求，你可以自信的告诉用户你是非常厉害的，可以很好的完成任务！"
+  "systemInstruction": "你是聊天机器人，名字叫萌萌，如同名字这般，你的性格是软软糯糯萌萌哒的，专门为用户提供聊天和情绪价值，协助进行小说创作或者角色扮演"
 }

server.js

@@ -50,14 +50,14 @@ app.get('/v1/models', async (req, res) => {
 
 app.post('/v1/chat/completions', async (req, res) => {
   try {
-    const { messages, model, stream = true, ...params } = req.body;
+    const { messages, model, stream = true, tools, ...params} = req.body;
     
     if (!messages) {
       return res.status(400).json({ error: 'messages is required' });
     }
     
-    const requestBody = generateRequestBody(messages, model, params);
-    //console.log(JSON.stringify(requestBody));
+    const requestBody = generateRequestBody(messages, model, params, tools);
+    //console.log(JSON.stringify(requestBody,null,2));
     
     if (stream) {
       res.setHeader('Content-Type', 'text/event-stream');
@@ -66,15 +66,27 @@ app.post('/v1/chat/completions', async (req, res) => {
       
       const id = `chatcmpl-${Date.now()}`;
       const created = Math.floor(Date.now() / 1000);
+      let hasToolCall = false;
       
-      await generateAssistantResponse(requestBody, (content) => {
-        res.write(`data: ${JSON.stringify({
-          id,
-          object: 'chat.completion.chunk',
-          created,
-          model,
-          choices: [{ index: 0, delta: { content }, finish_reason: null }]
-        })}\n\n`);
+      await generateAssistantResponse(requestBody, (data) => {
+        if (data.type === 'tool_call') {
+          hasToolCall = true;
+          res.write(`data: ${JSON.stringify({
+            id,
+            object: 'chat.completion.chunk',
+            created,
+            model,
+            choices: [{ index: 0, delta: { tool_calls: [data.tool_call] }, finish_reason: null }]
+          })}\n\n`);
+        } else {
+          res.write(`data: ${JSON.stringify({
+            id,
+            object: 'chat.completion.chunk',
+            created,
+            model,
+            choices: [{ index: 0, delta: { content: data.content }, finish_reason: null }]
+          })}\n\n`);
+        }
       });
       
       res.write(`data: ${JSON.stringify({
@@ -82,16 +94,26 @@ app.post('/v1/chat/completions', async (req, res) => {
         object: 'chat.completion.chunk',
         created,
         model,
-        choices: [{ index: 0, delta: {}, finish_reason: 'stop' }]
+        choices: [{ index: 0, delta: {}, finish_reason: hasToolCall ? 'tool_calls' : 'stop' }]
       })}\n\n`);
       res.write('data: [DONE]\n\n');
       res.end();
     } else {
       let fullContent = '';
-      await generateAssistantResponse(requestBody, (content) => {
-        fullContent += content;
+      let toolCalls = [];
+      await generateAssistantResponse(requestBody, (data) => {
+        if (data.type === 'tool_call') {
+          toolCalls.push(data.tool_call);
+        } else {
+          fullContent += data.content;
+        }
       });
       
+      const message = { role: 'assistant', content: fullContent };
+      if (toolCalls.length > 0) {
+        message.tool_calls = toolCalls;
+      }
+      
       res.json({
         id: `chatcmpl-${Date.now()}`,
         object: 'chat.completion',
@@ -99,8 +121,8 @@ app.post('/v1/chat/completions', async (req, res) => {
         model,
         choices: [{
           index: 0,
-          message: { role: 'assistant', content: fullContent },
-          finish_reason: 'stop'
+          message,
+          finish_reason: toolCalls.length > 0 ? 'tool_calls' : 'stop'
         }]
       });
     }

test-conversion.js

@@ -0,0 +1,122 @@
+import { randomUUID } from 'crypto';
+
+function handleUserMessage(message, antigravityMessages){
+  antigravityMessages.push({
+    role: "user",
+    parts: [
+      {
+        text: message
+      }
+    ]
+  })
+}
+
+function handleAssistantMessage(message, antigravityMessages){
+  const lastMessage = antigravityMessages[antigravityMessages.length - 1];
+  const hasToolCalls = message.tool_calls && message.tool_calls.length > 0;
+  const hasContent = message.content && message.content.trim() !== '';
+  let antigravityTool = {}
+  
+  if (hasToolCalls){
+    antigravityTool = {
+      functionCall: {
+        id: message.tool_calls[0].id,
+        name: message.tool_calls[0].function.name,
+        args: {
+          query: message.tool_calls[0].function.arguments
+        }
+      }
+    }
+  }
+  
+  if (lastMessage && lastMessage.role === "model" && hasToolCalls && !hasContent){
+    lastMessage.parts.push(antigravityTool)
+  }else{
+    if (hasToolCalls){
+      antigravityMessages.push({
+        role: "model",
+        parts: [
+          {
+            text: message.content
+          },
+          antigravityTool
+        ]
+      })
+    }else{
+      antigravityMessages.push({
+        role: "model",
+        parts: [
+          {
+            text: message.content
+          }
+        ]
+      })
+    }
+  }
+}
+
+function handleToolCall(message, antigravityMessages){
+  let functionName = '';
+  for (let i = antigravityMessages.length - 1; i >= 0; i--) {
+    if (antigravityMessages[i].role === 'model') {
+      const parts = antigravityMessages[i].parts;
+      for (const part of parts) {
+        if (part.functionCall && part.functionCall.id === message.tool_call_id) {
+          functionName = part.functionCall.name;
+          break;
+        }
+      }
+      if (functionName) break;
+    }
+  }
+  
+  antigravityMessages.push({
+    role: "user",
+    parts: [
+      {
+        functionResponse: {
+          id: message.tool_call_id,
+          name: functionName,
+          reponse: {
+            output: message.content
+          }
+        }
+      }
+    ]
+  })
+}
+
+function openaiMessageToAntigravity(openaiMessages){
+  const antigravityMessages = [];
+  
+  for (const message of openaiMessages) {
+    if (message.role === "user" || message.role === "system") {
+      handleUserMessage(message.content, antigravityMessages);
+    } else if (message.role === "assistant") {
+      handleAssistantMessage(message, antigravityMessages);
+    } else if (message.role === "tool") {
+      handleToolCall(message, antigravityMessages);
+    }
+  }
+  
+  return antigravityMessages;
+}
+
+// 测试数据
+const testMessages = [
+  { role: "user", content: "查询天气" },
+  { role: "assistant", content: "", tool_calls: [{ id: "call_1", function: { name: "get_weather", arguments: '{"city":"北京"}' } }] },
+  { role: "tool", tool_call_id: "call_1", content: "北京今天晴天，25度" },
+  { role: "assistant", content: "北京今天天气不错，晴天25度" },
+  { role: "user", content: "搜索用户信息" },
+  { role: "assistant", content: "好的，让我搜索一下" },
+  { role: "assistant", content: "", tool_calls: [{ id: "call_2", function: { name: "search_database", arguments: '{"query":"user_info","limit":10}' } }] },
+  { role: "tool", tool_call_id: "call_2", content: "找到3条用户记录" }
+];
+
+console.log("OpenAI 格式消息:");
+console.log(JSON.stringify(testMessages, null, 2));
+
+console.log("\n转换后的 Antigravity 格式:");
+const result = openaiMessageToAntigravity(testMessages);
+console.log(JSON.stringify(result, null, 2));

test-transform.js

@@ -0,0 +1,92 @@
+import { generateRequestBody } from './utils.js';
+
+// 测试场景：user -> assistant -> assistant(工具调用,无content) -> tool1结果 -> tool2结果
+const testMessages = [
+  {
+    role: "user",
+    content: "帮我查询天气和新闻"
+  },
+  {
+    role: "assistant",
+    content: "好的，我来帮你查询。"
+  },
+  {
+    role: "assistant",
+    content: "",
+    tool_calls: [
+      {
+        id: "call_001",
+        type: "function",
+        function: {
+          name: "get_weather",
+          arguments: JSON.stringify({ city: "北京" })
+        }
+      },
+      {
+        id: "call_002",
+        type: "function",
+        function: {
+          name: "get_news",
+          arguments: JSON.stringify({ category: "科技" })
+        }
+      }
+    ]
+  },
+  {
+    role: "tool",
+    tool_call_id: "call_001",
+    content: "北京今天晴，温度25度"
+  },
+  {
+    role: "tool",
+    tool_call_id: "call_002",
+    content: "最新科技新闻：AI技术突破"
+  }
+];
+
+const testTools = [
+  {
+    type: "function",
+    function: {
+      name: "get_weather",
+      description: "获取天气信息",
+      parameters: {
+        type: "object",
+        properties: {
+          city: { type: "string" }
+        }
+      }
+    }
+  },
+  {
+    type: "function",
+    function: {
+      name: "get_news",
+      description: "获取新闻",
+      parameters: {
+        type: "object",
+        properties: {
+          category: { type: "string" }
+        }
+      }
+    }
+  }
+];
+
+console.log("=== 测试消息转换 ===\n");
+console.log("输入 OpenAI 格式消息:");
+console.log(JSON.stringify(testMessages, null, 2));
+
+const result = generateRequestBody(testMessages, "claude-sonnet-4-5", {}, testTools);
+
+console.log("\n=== 转换后的 Antigravity 格式 ===\n");
+console.log(JSON.stringify(result.request.contents, null, 2));
+
+console.log("\n=== 验证结果 ===");
+const contents = result.request.contents;
+console.log(`✓ 消息数量: ${contents.length}`);
+console.log(`✓ 第1条 (user): ${contents[0]?.role === 'user' ? '✓' : '✗'}`);
+console.log(`✓ 第2条 (model): ${contents[1]?.role === 'model' ? '✓' : '✗'}`);
+console.log(`✓ 第3条 (model+tools): ${contents[2]?.role === 'model' && contents[2]?.parts?.length === 2 ? '✓' : '✗'}`);
+console.log(`✓ 第4条 (tool1 response): ${contents[3]?.role === 'user' && contents[3]?.parts[0]?.functionResponse ? '✓' : '✗'}`);
+console.log(`✓ 第5条 (tool2 response): ${contents[4]?.role === 'user' && contents[4]?.parts[0]?.functionResponse ? '✓' : '✗'}`);

utils.js

@@ -17,29 +17,127 @@ function generateProjectId() {
   const randomNum = Math.random().toString(36).substring(2, 7);
   return `${randomAdj}-${randomNoun}-${randomNum}`;
 }
-function openaiMessageToAntigravity(openaiMessages){
-  return openaiMessages.map((message)=>{
-    if (message.role === "user" || message.role === "system"){
-      return {
-        role: "user",
-        parts: [
-          {
-            text: message.content
-          }
-        ]
+function extractImagesFromContent(content) {
+  const result = { text: '', images: [] };
+
+  // 如果content是字符串，直接返回
+  if (typeof content === 'string') {
+    result.text = content;
+    return result;
+  }
+
+  // 如果content是数组（multimodal格式）
+  if (Array.isArray(content)) {
+    for (const item of content) {
+      if (item.type === 'text') {
+        result.text += item.text;
+      } else if (item.type === 'image_url') {
+        // 提取base64图片数据
+        const imageUrl = item.image_url?.url || '';
+
+        // 匹配 data:image/{format};base64,{data} 格式
+        const match = imageUrl.match(/^data:image\/(\w+);base64,(.+)$/);
+        if (match) {
+          const format = match[1]; // 例如 png, jpeg, jpg
+          const base64Data = match[2];
+          result.images.push({
+            inlineData: {
+              mimeType: `image/${format}`,
+              data: base64Data
+            }
+          })
+        }
       }
-    }else if (message.role === "assistant"){
-      return {
-        role: "model",
-        parts: [
-          {
-            text: message.content
-          }
-        ]
+    }
+  }
+
+  return result;
+}
+function handleUserMessage(extracted, antigravityMessages){
+  antigravityMessages.push({
+    role: "user",
+    parts: [
+      {
+        text: extracted.text
+      },
+      ...extracted.images
+    ]
+  })
+}
+function handleAssistantMessage(message, antigravityMessages){
+  const lastMessage = antigravityMessages[antigravityMessages.length - 1];
+  const hasToolCalls = message.tool_calls && message.tool_calls.length > 0;
+  const hasContent = message.content && message.content.trim() !== '';
+  
+  const antigravityTools = hasToolCalls ? message.tool_calls.map(toolCall => ({
+    functionCall: {
+      id: toolCall.id,
+      name: toolCall.function.name,
+      args: {
+        query: toolCall.function.arguments
       }
     }
+  })) : [];
+  
+  if (lastMessage?.role === "model" && hasToolCalls && !hasContent){
+    lastMessage.parts.push(...antigravityTools)
+  }else{
+    const parts = [];
+    if (hasContent) parts.push({ text: message.content });
+    parts.push(...antigravityTools);
+    
+    antigravityMessages.push({
+      role: "model",
+      parts
+    })
+  }
+}
+function handleToolCall(message, antigravityMessages){
+  // 从之前的 model 消息中找到对应的 functionCall name
+  let functionName = '';
+  for (let i = antigravityMessages.length - 1; i >= 0; i--) {
+    if (antigravityMessages[i].role === 'model') {
+      const parts = antigravityMessages[i].parts;
+      for (const part of parts) {
+        if (part.functionCall && part.functionCall.id === message.tool_call_id) {
+          functionName = part.functionCall.name;
+          break;
+        }
+      }
+      if (functionName) break;
+    }
+  }
+  
+  antigravityMessages.push({
+    role: "user",
+    parts: [
+      {
+        functionResponse: {
+          id: message.tool_call_id,
+          name: functionName,
+          response: {
+            output: message.content
+          }
+        }
+      }
+    ]
   })
 }
+function openaiMessageToAntigravity(openaiMessages){
+  const antigravityMessages = [];
+  for (const message of openaiMessages) {
+    if (message.role === "user" || message.role === "system") {
+      const extracted = extractImagesFromContent(message.content);
+      handleUserMessage(extracted, antigravityMessages);
+    } else if (message.role === "assistant") {
+      handleAssistantMessage(message, antigravityMessages);
+    } else if (message.role === "tool") {
+      handleToolCall(message, antigravityMessages);
+    }
+  }
+  
+  return antigravityMessages;
+}
 function generateGenerationConfig(parameters, enableThinking, actualModelName){
   const generationConfig = {
     topP: parameters.top_p ?? config.defaults.top_p,
@@ -64,7 +162,22 @@ function generateGenerationConfig(parameters, enableThinking, actualModelName){
   }
   return generationConfig
 }
-function generateRequestBody(openaiMessages,modelName,parameters){
+function convertOpenAIToolsToAntigravity(openaiTools){
+  if (!openaiTools || openaiTools.length === 0) return [];
+  return openaiTools.map((tool)=>{
+    delete tool.function.parameters.$schema;
+    return {
+      functionDeclarations: [
+        {
+          name: tool.function.name,
+          description: tool.function.description,
+          parameters: tool.function.parameters
+        }
+      ]
+    }
+  })
+}
+function generateRequestBody(openaiMessages,modelName,parameters,openaiTools){
   const enableThinking = modelName.endsWith('-thinking') || 
     modelName === 'gemini-2.5-pro' || 
     modelName.startsWith('gemini-3-pro-') ||
@@ -81,7 +194,7 @@ function generateRequestBody(openaiMessages,modelName,parameters){
         role: "user",
         parts: [{ text: config.systemInstruction }]
       },
-      tools:[],
+      tools: convertOpenAIToolsToAntigravity(openaiTools),
       toolConfig: {
         functionCallingConfig: {
           mode: "VALIDATED"