File size: 2,084 Bytes
3c9796e 5b9c8b1 3c9796e 8804f6b 3c9796e 9274314 de70972 3c9796e 5b9c8b1 3c9796e de70972 8804f6b 174c905 de70972 8804f6b 5b9c8b1 8804f6b 5b9c8b1 de70972 5b9c8b1 8804f6b 3c9796e 8804f6b de70972 8804f6b 2ffc227 8804f6b de70972 3c9796e 5b9c8b1 174c905 de70972 5b9c8b1 3c9796e de70972 5b9c8b1 3c9796e 5b9c8b1 3c9796e de70972 5b9c8b1 de70972 5b9c8b1 de70972 3c9796e 5b9c8b1 3c9796e de70972 8804f6b 174c905 de70972 5b9c8b1 de70972 8804f6b de70972 5b9c8b1 3c9796e de70972 5b9c8b1 3c9796e 8804f6b 3c9796e 8804f6b 3c9796e 5b9c8b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | import os
import subprocess
import time
import requests
import gradio as gr
from huggingface_hub import hf_hub_download
MODEL_REPO = "unsloth/Qwen3.5-9B-GGUF"
MODEL_FILE = "Qwen3.5-9B-Q4_K_M.gguf"
LLAMA_DIR = "llama.cpp"
SERVER_PORT = "8000"
def install_dependencies():
subprocess.run(["apt-get", "update"])
subprocess.run([
"apt-get", "install", "-y",
"build-essential",
"cmake",
"git"
])
def download_model():
return hf_hub_download(
repo_id=MODEL_REPO,
filename=MODEL_FILE,
repo_type="model"
)
def setup_llama():
if not os.path.exists(LLAMA_DIR):
subprocess.run([
"git", "clone",
"https://github.com/ggml-org/llama.cpp"
], check=True)
os.makedirs(f"{LLAMA_DIR}/build", exist_ok=True)
subprocess.run([
"cmake",
"-B", "build"
], cwd=LLAMA_DIR, check=True)
subprocess.run([
"cmake",
"--build",
"build",
"-j"
], cwd=LLAMA_DIR, check=True)
def start_server(model_path):
subprocess.Popen([
"./build/bin/llama-server",
"-m", model_path,
"--port", SERVER_PORT,
"-c", "4096",
"-t", "8"
], cwd=LLAMA_DIR)
def wait_for_server():
url = f"http://localhost:{SERVER_PORT}/health"
for _ in range(60):
try:
requests.get(url, timeout=1)
return
except:
time.sleep(2)
def chat(prompt):
r = requests.post(
f"http://localhost:{SERVER_PORT}/v1/chat/completions",
json={
"model": "qwen3.5-9b",
"messages": [
{"role": "user", "content": prompt}
]
},
timeout=300
)
data = r.json()
return data["choices"][0]["message"]["content"]
install_dependencies()
model_path = download_model()
setup_llama()
start_server(model_path)
wait_for_server()
ui = gr.Interface(
fn=chat,
inputs=gr.Textbox(lines=5),
outputs="text",
title="Operon Dev LLM (Qwen3.5-9B)"
)
ui.launch(server_name="0.0.0.0", server_port=7860) |