Upload 6 files

Dockerfile

@@ -1,27 +1,27 @@
-# 使用 node:20.16.0-alpine 作为基础镜像
-FROM node:20.16.0-alpine
-
-# 设置工作目录
-WORKDIR /app
-
-# 复制 package.json 和 package-lock.json 到工作目录
-COPY package*.json ./
-
-# 设置环境变量
-ENV API_PREFIX=/api \
-  API_KEY=123456 \
-  MAX_RETRY_COUNT=3 \
-  RETRY_DELAY=10000 \
-  PORT=8787
-
-# 安装项目依赖
-RUN npm install
-
-# 复制项目的源代码到工作目录
-COPY . .
-
-# 暴露应用运行的端口（假设应用运行在 3000 端口）
-EXPOSE 8787
-
-# 启动应用
-CMD ["node", "index.js"]
+# 使用 node:20.16.0-alpine 作为基础镜像
+FROM node:20.16.0-alpine
+
+# 设置工作目录
+WORKDIR /app
+
+# 复制 package.json 和 package-lock.json 到工作目录
+COPY package*.json ./
+
+# 设置环境变量
+ENV API_PREFIX=/api \
+  API_KEY=123456 \
+  MAX_RETRY_COUNT=3 \
+  RETRY_DELAY=10000 \
+  PORT=8787
+
+# 安装项目依赖
+RUN npm install
+
+# 复制项目的源代码到工作目录
+COPY . .
+
+# 暴露应用运行的端口（假设应用运行在 3000 端口）
+EXPOSE 8787
+
+# 启动应用
+CMD ["node", "api/index.js"] 

api/index.js

@@ -0,0 +1,303 @@
+import grpc from '@grpc/grpc-js';
+import protoLoader from '@grpc/proto-loader';
+import {AutoRouter, cors, error, json} from 'itty-router';
+import dotenv from 'dotenv';
+import path,{ dirname } from 'path';
+import { fileURLToPath } from 'url';
+import {createServerAdapter} from '@whatwg-node/server';
+import {createServer} from 'http';
+
+// 加载环境变量
+dotenv.config();
+// 获取当前文件的目录路径（ESM 方式）
+const __dirname = dirname(fileURLToPath(import.meta.url));
+// 初始化配置
+class Config {
+        constructor() {
+                this.API_PREFIX = process.env.API_PREFIX || '/';
+                this.API_KEY = process.env.API_KEY || '';
+                this.MAX_RETRY_COUNT = process.env.MAX_RETRY_COUNT || 3;
+                this.RETRY_DELAY = process.env.RETRY_DELAY || 5000;
+                this.COMMON_GRPC = 'runtime-native-io-vertex-inference-grpc-service-lmuw6mcn3q-ul.a.run.app';
+                this.COMMON_PROTO = path.join(__dirname,'..', 'protos', 'VertexInferenceService.proto')
+                this.GPT_GRPC = 'runtime-native-io-gpt-inference-grpc-service-lmuw6mcn3q-ul.a.run.app';
+                this.GPT_PROTO = path.join(__dirname,'..', 'protos', 'GPTInferenceService.proto')
+                this.PORT = process.env.PORT || 8787;
+        }
+}
+class GRPCHandler {
+        constructor(protoFilePath) {
+                // 动态加载传入的 .proto 文件路径
+                this.packageDefinition = protoLoader.loadSync(protoFilePath, {
+                        keepCase: true,
+                        longs: String,
+                        enums: String,
+                        defaults: true,
+                        oneofs: true
+                });
+        }
+}
+const config = new Config();
+// 中间件
+// 添加运行回源
+const { preflight, corsify } = cors({
+	origin: '*',
+	allowMethods: '*',
+	exposeHeaders: '*',
+});
+
+// 添加认证
+const withAuth = (request) => {
+	if (config.API_KEY) {
+		const authHeader = request.headers.get('Authorization');
+		if (!authHeader || !authHeader.startsWith('Bearer ')) {
+			return error(401, 'Unauthorized: Missing or invalid Authorization header');
+		}
+		const token = authHeader.substring(7);
+		if (token !== config.API_KEY) {
+			return error(403, 'Forbidden: Invalid API key');
+		}
+	}
+};
+// 返回运行信息
+const logger = (res, req) => {
+	console.log(req.method, res.status, req.url, Date.now() - req.start, 'ms');
+};
+const router = AutoRouter({
+	before: [preflight, withAuth],
+	missing: () => error(404, '404 not found.'),
+	finally: [corsify, logger],
+});
+// Router路径
+router.get('/', () => json({ message: 'API 服务运行中~' }));
+router.get('/ping', () => json({ message: 'pong' }));
+router.get(config.API_PREFIX + '/v1/models', () =>
+    json({
+            object: 'list',
+            data: [
+                    { id: "gpt-4o-mini", object: "model", owned_by: "pieces-os" },
+                    { id: "gpt-4o", object: "model", owned_by: "pieces-os" },
+                    { id: "gpt-4-turbo", object: "model", owned_by: "pieces-os" },
+                    { id: "gpt-4", object: "model", owned_by: "pieces-os" },
+                    { id: "gpt-3.5-turbo", object: "model", owned_by: "pieces-os" },
+                    { id: "claude-3-sonnet@20240229", object: "model", owned_by: "pieces-os" },
+                    { id: "claude-3-opus@20240229", object: "model", owned_by: "pieces-os" },
+                    { id: "claude-3-haiku@20240307", object: "model", owned_by: "pieces-os" },
+                    { id: "claude-3-5-sonnet@20240620", object: "model", owned_by: "pieces-os" },
+                    { id: "gemini-1.5-flash", object: "model", owned_by: "pieces-os" },
+                    { id: "gemini-1.5-pro", object: "model", owned_by: "pieces-os" },
+                    { id: "chat-bison", object: "model", owned_by: "pieces-os" },
+                    { id: "codechat-bison", object: "model", owned_by: "pieces-os" },
+            ],
+    })
+);
+router.post(config.API_PREFIX + '/v1/chat/completions', (req) => handleCompletion(req));
+
+async function GrpcToPieces(models, message, rules, stream, temperature, top_p) {
+        // 在非GPT类型的模型中，temperature和top_p是无效的
+        // 使用系统的根证书
+        const credentials = grpc.credentials.createSsl();
+        let client,request;
+        if (models.includes('gpt')){
+                // 加载proto文件
+                const packageDefinition = new GRPCHandler(config.GPT_PROTO).packageDefinition;
+                // 构建请求消息
+                request = {
+                        models: models,
+                        messages: [
+                                {role: 0, message: rules}, // system
+                                {role: 1, message: message} // user
+                        ],
+                        temperature:temperature || 0.1,
+                        top_p:top_p ?? 1,
+                }
+                // 获取gRPC对象
+                const GRPCobjects = grpc.loadPackageDefinition(packageDefinition).runtime.aot.machine_learning.parents.gpt;
+                client = new GRPCobjects.GPTInferenceService(config.GPT_GRPC, credentials);
+        } else {
+                // 加载proto文件
+                const packageDefinition = new GRPCHandler(config.COMMON_PROTO).packageDefinition;
+                // 构建请求消息
+                request = {
+                        models: models,
+                        args: {
+                                messages: {
+                                        unknown: 1,
+                                        message: message
+                                },
+                                rules: rules
+                        }
+                };
+                // 获取gRPC对象
+                const GRPCobjects = grpc.loadPackageDefinition(packageDefinition).runtime.aot.machine_learning.parents.vertex;
+                client = new GRPCobjects.VertexInferenceService(config.COMMON_GRPC, credentials);
+        }
+        return await ConvertOpenai(client,request,models,stream);
+}
+
+async function messagesProcess(messages) {
+        let rules = '';
+        let message = '';
+
+        for (const msg of messages) {
+                let role = msg.role;
+                // 格式化为字符串
+                const contentStr = Array.isArray(msg.content)
+                    ? msg.content
+                    .filter((item) => item.text)
+                    .map((item) => item.text)
+                    .join('') || ''
+                    : msg.content;
+                // 判断身份
+                if (role === 'system') {
+                        rules += `system:${contentStr};\r\n`;
+                } else if (['user', 'assistant'].includes(role)) {
+                        message += `${role}:${contentStr};\r\n`;
+                }
+        }
+
+        return { rules, message };
+}
+
+async function ConvertOpenai(client,request,model,stream) {
+        for (let i = 0; i < config.MAX_RETRY_COUNT; i++) {
+                try {
+                        if (stream) {
+                                const call = client.PredictWithStream(request);
+                                const encoder = new TextEncoder();
+                                const ReturnStream = new ReadableStream({
+                                    start(controller) {
+                                            call.on('data', (response) => {
+                                                    let response_code = Number(response.response_code);
+                                                    if (response_code === 204) {
+                                                            // 如果 response_code 是 204，关闭流
+                                                            controller.close()
+                                                            call.destroy()
+                                                    } else if (response_code === 200) {
+                                                            let response_message
+                                                            if (model.includes('gpt')) {
+                                                                    response_message = response.body.message_warpper.message.message;
+                                                            } else {
+                                                                    response_message = response.args.args.args.message;
+                                                            }
+                                                            // 否则，将数据块加入流中
+                                                            controller.enqueue(encoder.encode(`data: ${JSON.stringify(ChatCompletionStreamWithModel(response_message, model))}\n\n`));
+                                                    } else {
+                                                            controller.error(new Error(`Error: stream chunk is not success`));
+                                                            controller.close()
+                                                    }
+                                            })
+                                    }
+                                    });
+                                return new Response(ReturnStream, {
+                                        headers: {
+                                                'Content-Type': 'text/event-stream',
+                                        },
+                                })
+                } else {
+                        const call = await new Promise((resolve, reject) => {
+                                client.Predict(request, (err, response) => {
+                                        if (err) reject(err);
+                                        else resolve(response);
+                                });
+                        });
+                        let response_code = Number(call.response_code);
+                        if (response_code === 200) {
+                                let response_message
+                                if (model.includes('gpt')) {
+                                        response_message = call.body.message_warpper.message.message;
+                                } else {
+                                        response_message = call.args.args.args.message;
+                                }
+                                return new Response(JSON.stringify(ChatCompletionWithModel(response_message, model)), {
+                                                headers: {
+                                                        'Content-Type': 'application/json',
+                                                },
+                                        });
+                                }
+                        }
+                } catch (err) {
+                        console.error(err);
+                        await new Promise((resolve) => setTimeout(resolve, config.RETRY_DELAY));
+                }
+        }
+        return error(500, err.message);
+}
+
+function renameIfNeeded(input) {
+        // 替换的正则表达式
+        const regex = /^(claude-3-(5-sonnet|haiku|sonnet|opus))-(\d{8})$/;
+        const match = input.match(regex);
+        if (match) {
+                return `${match[1]}@${match[3]}`;
+        }
+        return input;
+}
+
+function ChatCompletionWithModel(message, model) {
+        return {
+                id: 'Chat-Nekohy',
+                object: 'chat.completion',
+                created: Date.now(),
+                model,
+                usage: {
+                        prompt_tokens: 0,
+                        completion_tokens: 0,
+                        total_tokens: 0,
+                },
+                choices: [
+                        {
+                                message: {
+                                        content: message,
+                                        role: 'assistant',
+                                },
+                                index: 0,
+                        },
+                ],
+        };
+}
+
+function ChatCompletionStreamWithModel(text, model) {
+        return {
+                id: 'chatcmpl-Nekohy',
+                object: 'chat.completion.chunk',
+                created: 0,
+                model,
+                choices: [
+                        {
+                                index: 0,
+                                delta: {
+                                        content: text,
+                                },
+                                finish_reason: null,
+                        },
+                ],
+        };
+}
+
+async function handleCompletion(request) {
+        try {
+                // todo stream逆向接口
+                // 解析openai格式API请求
+                const { model: inputModel, messages, stream,temperature,top_p} = await request.json();
+                console.log(inputModel,messages,stream)
+                const model = renameIfNeeded(inputModel);
+                // 解析system和user/assistant消息
+                const { rules, message:content } = await messagesProcess(messages);
+                console.log(rules,content)
+                // 响应码，回复的消息
+                return await GrpcToPieces(model, content, rules, stream, temperature, top_p);
+        } catch (err) {
+                return error(500, err.message);
+        }
+}
+
+(async () => {
+	//For Cloudflare Workers
+	if (typeof addEventListener === 'function') return;
+	// For Nodejs
+	const ittyServer = createServerAdapter(router.fetch);
+	console.log(`Listening on http://localhost:${config.PORT}`);
+	const httpServer = createServer(ittyServer);
+	httpServer.listen(config.PORT);
+})();

cloud_model.json

@@ -0,0 +1,214 @@
+{
+    "iterable": [
+        {
+            "version": "t16-v01-i01",
+            "created": {
+                "value": "2023-09-15T16:00:00.000Z",
+                "readable": "about a year ago"
+            },
+            "name": "(PaLM2) Chat Model",
+            "unique": "chat-bison",
+            "provider": "GOOGLE",
+            "maxTokens": {
+                "total": 9000,
+                "input": 8000,
+                "output": 1000
+            }
+        },
+        {
+            "version": "t31-v01-i01",
+            "created": {
+                "value": "2024-07-17T16:00:00.000Z",
+                "readable": "4 months ago"
+            },
+            "name": "GPT-4o Mini Chat Model",
+            "unique": "gpt-4o-mini",
+            "provider": "OPENAI",
+            "maxTokens": {
+                "total": 80384,
+                "input": 64000,
+                "output": 16384
+            }
+        },
+        {
+            "version": "t25-v01-i01",
+            "created": {
+                "value": "2024-05-27T16:00:00.000Z",
+                "readable": "5 months ago"
+            },
+            "name": "Gemini-1.5 Pro Chat Model",
+            "unique": "gemini-1.5-pro",
+            "provider": "GOOGLE",
+            "maxTokens": {
+                "total": 128000,
+                "input": 119808,
+                "output": 8192
+            }
+        },
+        {
+            "version": "t24-v01-i01",
+            "created": {
+                "value": "2024-05-14T16:00:00.000Z",
+                "readable": "6 months ago"
+            },
+            "name": "GPT-4o Chat Model",
+            "unique": "gpt-4o",
+            "provider": "OPENAI",
+            "maxTokens": {
+                "total": 68096,
+                "input": 64000,
+                "output": 4096
+            }
+        },
+        {
+            "version": "t15-v01-i01",
+            "created": {
+                "value": "2023-09-15T16:00:00.000Z",
+                "readable": "about a year ago"
+            },
+            "name": "Codey (PaLM2) Chat Model",
+            "unique": "codechat-bison",
+            "provider": "GOOGLE",
+            "maxTokens": {
+                "total": 7000,
+                "input": 6000,
+                "output": 1000
+            }
+        },
+        {
+            "version": "t28-v01-i01",
+            "created": {
+                "value": "2024-02-28T16:00:00.000Z",
+                "readable": "8 months ago"
+            },
+            "name": "Claude 3 Sonnet Chat Model",
+            "unique": "claude-3-sonnet@20240229",
+            "provider": "ANTHROPIC",
+            "maxTokens": {
+                "total": 40000,
+                "input": 35000,
+                "output": 4096
+            }
+        },
+        {
+            "version": "t22-v01-i01",
+            "created": {
+                "value": "2024-01-07T16:00:00.000Z",
+                "readable": "10 months ago"
+            },
+            "name": "(Gemini) Chat Model",
+            "unique": "gemini-pro",
+            "provider": "GOOGLE",
+            "maxTokens": {
+                "total": 32000,
+                "input": 20000,
+                "output": 8000
+            }
+        },
+        {
+            "version": "t29-v01-i01",
+            "created": {
+                "value": "2024-02-28T16:00:00.000Z",
+                "readable": "8 months ago"
+            },
+            "name": "Claude 3 Opus Chat Model",
+            "unique": "claude-3-opus@20240229",
+            "provider": "ANTHROPIC",
+            "maxTokens": {
+                "total": 40000,
+                "input": 35000,
+                "output": 4096
+            }
+        },
+        {
+            "version": "t23-v01-i01",
+            "created": {
+                "value": "2024-02-08T16:00:00.000Z",
+                "readable": "9 months ago"
+            },
+            "name": "GPT-4 Turbo Chat Model",
+            "unique": "gpt-4-turbo",
+            "provider": "OPENAI",
+            "maxTokens": {
+                "total": 68096,
+                "input": 64000,
+                "output": 4096
+            }
+        },
+        {
+            "version": "t26-v01-i01",
+            "created": {
+                "value": "2024-05-27T16:00:00.000Z",
+                "readable": "5 months ago"
+            },
+            "name": "Gemini-1.5 Flash Chat Model",
+            "unique": "gemini-1.5-flash",
+            "provider": "GOOGLE",
+            "maxTokens": {
+                "total": 128000,
+                "input": 119808,
+                "output": 8192
+            }
+        },
+        {
+            "version": "t27-v01-i01",
+            "created": {
+                "value": "2024-06-19T16:00:00.000Z",
+                "readable": "5 months ago"
+            },
+            "name": "Claude 3.5 Sonnet Chat Model",
+            "unique": "claude-3-5-sonnet@20240620",
+            "provider": "ANTHROPIC",
+            "maxTokens": {
+                "total": 40000,
+                "input": 35000,
+                "output": 4096
+            }
+        },
+        {
+            "version": "t30-v01-i01",
+            "created": {
+                "value": "2024-03-26T16:00:00.000Z",
+                "readable": "7 months ago"
+            },
+            "name": "Claude 3 Haiku Chat Model",
+            "unique": "claude-3-haiku@20240307",
+            "provider": "ANTHROPIC",
+            "maxTokens": {
+                "total": 40000,
+                "input": 35000,
+                "output": 4096
+            }
+        },
+        {
+            "version": "t13-v01-i01",
+            "created": {
+                "value": "2023-09-15T16:00:00.000Z",
+                "readable": "about a year ago"
+            },
+            "name": "GPT-3.5-turbo Chat Model",
+            "unique": "gpt-3.5-turbo",
+            "provider": "OPENAI",
+            "maxTokens": {
+                "total": 16000,
+                "input": 12000,
+                "output": 4000
+            }
+        },
+        {
+            "version": "t14-v01-i01",
+            "created": {
+                "value": "2023-09-15T16:00:00.000Z",
+                "readable": "about a year ago"
+            },
+            "name": "GPT-4 Chat Model",
+            "unique": "gpt-4",
+            "provider": "OPENAI",
+            "maxTokens": {
+                "total": 8000,
+                "input": 4100,
+                "output": 3900
+            }
+        }
+    ]
+}

package.json

@@ -3,10 +3,11 @@
   "version": "1.0.0",
   "type": "module",
   "description": "",
-  "main": "index.js",
+  "main": "api/index.js",
   "scripts": {
-    "dev": "node index.js"
+    "start": "node api/index.js"
   },
+  "author": "Nekohy",
   "private": true,
   "dependencies": {
     "@grpc/grpc-js": "^1.12.2",

protos/GPTInferenceService.proto

@@ -0,0 +1,48 @@
+syntax = "proto3";  // 使用Protobuf版本3
+
+package runtime.aot.machine_learning.parents.gpt;
+
+service GPTInferenceService {
+    rpc Predict (Request) returns (Response);
+    rpc PredictWithStream (Request) returns (stream Response);
+}
+
+// Global
+message Message {
+  uint64 role = 1;  // 值 0 为 system，1 为普通,回复中只有 1
+  string message = 2;
+}
+
+// Requests
+message Request {
+  string models = 1; // 模型名称
+  repeated Message messages = 2; // 消息列表
+  double temperature = 3;  // 采样温度
+  double top_p = 4;  // 核心采样
+}
+
+// Response
+message Response {
+  uint64 response_code = 2; // 返回状态码，200，439 等,204 为终止
+  optional Body body = 4;
+}
+
+message Body{
+  string id = 1; // eg.chatcmpl-ANcM3OsoLf6AXJpO76pDoW7Ry68oc
+  string object = 2; // eg. chat.completion
+  uint64 time = 3; // UNIX时间戳
+  MessageWarpper message_warpper = 4;
+  Unknown unknown = 5;
+}
+
+message MessageWarpper {
+  int64 arg1 = 1; // 也许是常量1
+  Message message = 2; // 回应的消息 wt u need
+}
+
+message Unknown {
+  // 都不知道
+  int64 arg1 = 1;
+  int64 arg2 = 2;
+  int64 arg3 = 3;
+}

protos/VertexInferenceService.proto

@@ -0,0 +1,39 @@
+syntax = "proto3";  // 使用Protobuf版本3
+
+package runtime.aot.machine_learning.parents.vertex;
+
+service VertexInferenceService {
+    rpc Predict (Requests) returns (Response);
+    rpc PredictWithStream (Requests) returns (stream Response);
+}
+//Global
+message Messages{
+    int64 unknown = 1; // 也许只是常量1(请求)/3(回应)
+    string message = 2; // 消息
+}
+
+// Requests
+message Requests {
+    string models = 1; // 模型名称
+    Args args = 2;
+}
+
+message Args {
+  reserved 1;
+  Messages messages = 2; // 消息,可以视作user
+  string rules = 3; // 规则,可以视作system
+}
+
+// Response
+message Response {
+    int64 response_code = 2; // 返回状态码，200，439等
+    Args1 args = 4;
+}
+
+message Args1{
+  Args2 args = 1; // 不知道什么意义的套一层
+}
+
+message Args2{
+    Messages args = 2; // 不知道什么意义的套一层
+}