| | from flask import Flask, request, jsonify |
| | import torch |
| | from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline |
| |
|
| | app = Flask(__name__) |
| |
|
| | |
| | torch.random.manual_seed(0) |
| |
|
| | model = AutoModelForCausalLM.from_pretrained( |
| | "microsoft/Phi-3-mini-4k-instruct", |
| | device_map="cpu", |
| | torch_dtype="auto", |
| | trust_remote_code=True |
| | ) |
| |
|
| | tokenizer = AutoTokenizer.from_pretrained( |
| | "microsoft/Phi-3-mini-4k-instruct" |
| | ) |
| |
|
| | pipe = pipeline( |
| | "text-generation", |
| | model=model, |
| | tokenizer=tokenizer |
| | ) |
| |
|
| | generation_args = { |
| | "max_new_tokens": 500, |
| | "return_full_text": False, |
| | "temperature": 0.0, |
| | "do_sample": False, |
| | } |
| |
|
| | |
| | |
| | |
| | @app.route("/") |
| | def index(): |
| | return """ |
| | <!DOCTYPE html> |
| | <html> |
| | <head> |
| | <meta charset="utf-8"> |
| | <title>Local LLM Chat</title> |
| | |
| | <style> |
| | body{ |
| | font-family: Arial; |
| | background:#111; |
| | color:white; |
| | margin:0; |
| | } |
| | |
| | #chat{ |
| | height:80vh; |
| | overflow-y:auto; |
| | padding:20px; |
| | } |
| | |
| | .message{ |
| | margin-bottom:12px; |
| | } |
| | |
| | .user{ |
| | color:#6cf; |
| | } |
| | |
| | .assistant{ |
| | color:#9f9; |
| | } |
| | |
| | #inputArea{ |
| | position:fixed; |
| | bottom:0; |
| | width:100%; |
| | background:#222; |
| | padding:10px; |
| | } |
| | |
| | #input{ |
| | width:80%; |
| | padding:10px; |
| | font-size:16px; |
| | } |
| | |
| | button{ |
| | padding:10px; |
| | font-size:16px; |
| | } |
| | </style> |
| | </head> |
| | |
| | <body> |
| | |
| | <h2 style="padding:10px;">Local Phi-3 Chat</h2> |
| | |
| | <div id="chat"></div> |
| | |
| | <div id="inputArea"> |
| | <input id="input" placeholder="メッセージを入力..." /> |
| | <button onclick="send()">送信</button> |
| | </div> |
| | |
| | <script> |
| | |
| | let messages = [ |
| | {role:"system",content:"You are a helpful assistant."} |
| | ] |
| | |
| | function add(role,text){ |
| | |
| | const chat=document.getElementById("chat") |
| | |
| | const div=document.createElement("div") |
| | div.className="message "+role |
| | |
| | div.innerText=role+": "+text |
| | |
| | chat.appendChild(div) |
| | chat.scrollTop=chat.scrollHeight |
| | } |
| | |
| | async function send(){ |
| | |
| | const input=document.getElementById("input") |
| | const text=input.value |
| | |
| | if(!text) return |
| | |
| | input.value="" |
| | |
| | add("user",text) |
| | |
| | messages.push({ |
| | role:"user", |
| | content:text |
| | }) |
| | |
| | const res=await fetch("/v1/chat/completions",{ |
| | method:"POST", |
| | headers:{ |
| | "Content-Type":"application/json" |
| | }, |
| | body:JSON.stringify({ |
| | messages:messages |
| | }) |
| | }) |
| | |
| | const data=await res.json() |
| | |
| | const reply=data.choices[0].message.content |
| | |
| | add("assistant",reply) |
| | |
| | messages.push({ |
| | role:"assistant", |
| | content:reply |
| | }) |
| | } |
| | |
| | document.getElementById("input").addEventListener("keypress",function(e){ |
| | if(e.key==="Enter"){ |
| | send() |
| | } |
| | }) |
| | |
| | </script> |
| | |
| | </body> |
| | </html> |
| | """ |
| |
|
| | |
| | |
| | |
| | @app.route("/v1/chat/completions", methods=["POST"]) |
| | def chat_completions(): |
| |
|
| | data = request.json |
| | messages = data.get("messages", []) |
| |
|
| | result = pipe(messages, **generation_args) |
| | text = result[0]["generated_text"] |
| |
|
| | response = { |
| | "id": "chatcmpl-local", |
| | "object": "chat.completion", |
| | "choices": [ |
| | { |
| | "index": 0, |
| | "message": { |
| | "role": "assistant", |
| | "content": text |
| | }, |
| | "finish_reason": "stop" |
| | } |
| | ] |
| | } |
| |
|
| | return jsonify(response) |
| |
|
| |
|
| | if __name__ == "__main__": |
| | app.run(host="0.0.0.0", port=7860) |
