import os
import re
import time
import traceback
from pathlib import Path
from typing import Any, Dict, List, Tuple
from config import *
import torch
from fastapi import Request
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from gradio import Server
from transformers import AutoModelForCausalLM, AutoTokenizer
# ── ZeroGPU Detection ───────────────────────────────────────────────────
IS_ZEROGPU = False
try:
import spaces
from spaces.config import Config as _ZeroGPUConfig
IS_ZEROGPU = bool(_ZeroGPUConfig.zero_gpu)
except ImportError:
class _DummySpaces:
@staticmethod
def GPU(fn):
return fn
spaces = _DummySpaces()
# ── Configuration ───────────────────────────────────────────────────────
DEFAULT_SYSTEM_PROMPT = (
"You are ChatUI, a helpful local AI assistant. "
"You provide accurate, detailed, and thoughtful responses."
)
STATIC_DIR = Path(__file__).parent / "static"
os.makedirs(STATIC_DIR, exist_ok=True)
# ── Fallback chat templates for tokenizers missing .chat_template ───────
_FALLBACK_TEMPLATES = {
"chatml": (
"{% for message in messages %}"
"{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
"{% endfor %}"
"{% if add_generation_prompt %}"
"{{'<|im_start|>assistant\n'}}"
"{% endif %}"
),
"llama2": (
"{% if messages[0]['role'] == 'system' %}"
"{{'<>\n' + messages[0]['content'] + '\n<>\n\n'}}"
"{% set loop_messages = messages[1:] %}"
"{% else %}"
"{% set loop_messages = messages %}"
"{% endif %}"
"{% for message in loop_messages %}"
"{% if message['role'] == 'user' %}"
"{{'[INST] ' + message['content'] + ' [/INST]'}}"
"{% elif message['role'] == 'assistant' %}"
"{{message['content'] + ''}}"
"{% endif %}"
"{% endfor %}"
),
"phi3": (
"{% for message in messages %}"
"{% if message['role'] == 'system' %}"
"{{'<|system|>\n' + message['content'] + '<|end|>\n'}}"
"{% elif message['role'] == 'user' %}"
"{{'<|user|>\n' + message['content'] + '<|end|>\n'}}"
"{% elif message['role'] == 'assistant' %}"
"{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}"
"{% endif %}"
"{% endfor %}"
"{% if add_generation_prompt %}"
"{{'<|assistant|>\n'}}"
"{% endif %}"
),
"qwen": (
"{% for message in messages %}"
"{% if message['role'] == 'system' %}"
"{{'<|im_start|>system\n' + message['content'] + '<|im_end|>\n'}}"
"{% elif message['role'] == 'user' %}"
"{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}"
"{% elif message['role'] == 'assistant' %}"
"{{'<|im_start|>assistant\n' + message['content'] + '<|im_end|>\n'}}"
"{% endif %}"
"{% endfor %}"
"{% if add_generation_prompt %}"
"{{'<|im_start|>assistant\n'}}"
"{% endif %}"
),
}
def _pick_fallback_template(model_id: str) -> str:
mid = model_id.lower()
if "phi" in mid:
return _FALLBACK_TEMPLATES["phi3"]
if "qwen" in mid:
return _FALLBACK_TEMPLATES["qwen"]
if "tinyllama" in mid or "llama" in mid:
return _FALLBACK_TEMPLATES["llama2"]
return _FALLBACK_TEMPLATES["chatml"]
def _ensure_chat_template(tokenizer, model_id: str):
if getattr(tokenizer, "chat_template", None):
return
tpl = _pick_fallback_template(model_id)
tokenizer.chat_template = tpl
print(f"[chat_template] injected fallback template for {model_id}")
# ── Model cache (lazy) ──────────────────────────────────────────────────
_MODEL_CACHE: Dict[str, Dict[str, Any]] = {}
def _device() -> str:
if IS_ZEROGPU:
return "cuda"
if torch.cuda.is_available():
return "cuda"
if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
return "mps"
return "cpu"
def _dtype():
if IS_ZEROGPU or torch.cuda.is_available():
return torch.float16
return torch.float32
def get_model(model_id: str) -> Dict[str, Any]:
if model_id in _MODEL_CACHE:
return _MODEL_CACHE[model_id]
if model_id not in MODEL_REGISTRY:
raise ValueError(f"Unknown model: {model_id}")
device = _device()
dtype = _dtype()
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
_ensure_chat_template(tokenizer, model_id)
model = AutoModelForCausalLM.from_pretrained(
model_id,
torch_dtype=dtype,
trust_remote_code=True,
low_cpu_mem_usage=True,
).to(device)
model.eval()
_MODEL_CACHE[model_id] = {
"tokenizer": tokenizer,
"model": model,
"device": device,
"loaded_at": time.time(),
}
return _MODEL_CACHE[model_id]
# ── Generation core ─────────────────────────────────────────────────────
def _flatten_content(content) -> str:
if isinstance(content, str):
return content
parts: List[str] = []
for c in content or []:
if isinstance(c, dict) and c.get("type") == "text":
parts.append(c.get("text", ""))
return "\n".join(parts).strip()
def _build_prompt(model_id: str, messages: List[Dict[str, Any]],
system_prompt: str) -> str:
bundle = get_model(model_id)
tokenizer = bundle["tokenizer"]
flat_messages: List[Dict[str, str]] = []
if system_prompt and system_prompt.strip():
flat_messages.append({"role": "system", "content": system_prompt.strip()})
for m in messages:
role = m.get("role", "user")
if role == "system":
continue
text = _flatten_content(m.get("content", ""))
if text:
flat_messages.append({"role": role, "content": text})
if getattr(tokenizer, "chat_template", None):
try:
return tokenizer.apply_chat_template(
flat_messages, tokenize=False, add_generation_prompt=True
)
except Exception as e:
print(f"[chat_template] apply failed for {model_id}: {e}; using manual fallback")
# Manual fallback
lines = [f"{m['role']}: {m['content']}" for m in flat_messages]
lines.append("assistant:")
return "\n".join(lines) + "\n"
@spaces.GPU
def generate(model_id: str, messages: List[Dict[str, Any]],
system_prompt: str, max_tokens: int, temperature: float,
top_p: float = 0.95) -> Tuple[str, str]:
bundle = get_model(model_id)
tokenizer = bundle["tokenizer"]
model = bundle["model"]
device = bundle["device"]
prompt = _build_prompt(model_id, messages, system_prompt)
inputs = tokenizer(prompt, return_tensors="pt").to(device)
do_sample = temperature > 0.01
with torch.inference_mode():
out = model.generate(
**inputs,
max_new_tokens=max_tokens,
do_sample=do_sample,
temperature=max(temperature, 1e-3) if do_sample else 1.0,
top_p=top_p if do_sample else 1.0,
pad_token_id=tokenizer.pad_token_id,
eos_token_id=tokenizer.eos_token_id,
)
new_tokens = out[0][inputs["input_ids"].shape[1]:]
text = tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
reasoning = ""
think_match = re.search(r"]*>(.*?)", text, re.DOTALL | re.IGNORECASE)
if think_match:
reasoning = think_match.group(1).strip()
text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL | re.IGNORECASE).strip()
return text, reasoning
# ── Agent tools ─────────────────────────────────────────────────────────
def _tool_calculator(expression: str) -> str:
try:
if not re.fullmatch(r"[\d+\-*/().%\s]+", expression):
return "Error: only numeric expressions are allowed."
result = eval(expression, {"__builtins__": {}}, {})
return f"= {result}"
except Exception as e:
return f"Error: {e}"
def _tool_web_search(query: str) -> str:
return (
f"[stub search] I cannot reach the web (no API key configured), "
f"but for query '{query}' you might want to check Wikipedia, "
f"Stack Overflow, or arXiv. Use the calculator for any maths."
)
def _tool_file_read(path: str) -> str:
safe_root = Path(__file__).parent.resolve()
try:
target = (safe_root / path).resolve()
target.relative_to(safe_root)
if not target.exists() or not target.is_file():
return f"Error: file not found: {path}"
text = target.read_text(encoding="utf-8", errors="ignore")
if len(text) > 4000:
text = text[:4000] + "\n…[truncated]"
return text
except Exception as e:
return f"Error: {e}"
def _tool_now(_arg: str = "") -> str:
return time.strftime("%Y-%m-%d %H:%M:%S %Z")
AGENT_TOOLS = {
"calculator": _tool_calculator,
"web_search": _tool_web_search,
"file_read": _tool_file_read,
"now": _tool_now,
}
AGENT_TOOL_DESCRIPTION = """You have access to the following tools. To use one,
reply with a single line that starts with `TOOL:` followed by the tool name and
its argument in parentheses, e.g.:
TOOL: calculator(2 * (3 + 4))
TOOL: web_search(latest news about mars)
TOOL: file_read(static/index.html)
TOOL: now()
After receiving the tool output, you may use another tool or give your final
answer. When you are done, start your final message with `FINAL:`.
"""
@spaces.GPU
def run_agent(model_id: str, user_text: str, system_prompt: str,
max_tokens: int, temperature: float,
max_iters: int = 4) -> Dict[str, Any]:
history = [
{"role": "system", "content": system_prompt + "\n\n" + AGENT_TOOL_DESCRIPTION},
{"role": "user", "content": user_text},
]
trace: List[Dict[str, str]] = []
for i in range(max_iters):
answer, _reason = generate(
model_id, history,
system_prompt="",
max_tokens=max_tokens, temperature=temperature,
)
answer_stripped = answer.strip()
if answer_stripped.upper().startswith("FINAL:"):
final = answer_stripped[6:].strip()
trace.append({"step": i + 1, "type": "final", "content": final})
return {"final": final, "trace": trace}
m = re.match(r"\s*TOOL:\s*([A-Za-z_]+)\s*\((.*)\)\s*$",
answer_stripped, re.DOTALL)
if not m:
trace.append({"step": i + 1, "type": "final", "content": answer_stripped})
return {"final": answer_stripped, "trace": trace}
tool_name = m.group(1).strip().lower()
tool_arg = m.group(2).strip()
trace.append({
"step": i + 1,
"type": "tool_call",
"tool": tool_name,
"argument": tool_arg,
})
fn = AGENT_TOOLS.get(tool_name)
if fn is None:
tool_result = (f"Error: unknown tool '{tool_name}'. "
f"Available: {list(AGENT_TOOLS.keys())}")
else:
try:
tool_result = fn(tool_arg)
except Exception as e:
tool_result = f"Error: {e}"
trace.append({
"step": i + 1,
"type": "tool_result",
"tool": tool_name,
"output": tool_result,
})
history.append({"role": "assistant", "content": answer_stripped})
history.append({
"role": "user",
"content": (f"TOOL OUTPUT: {tool_result}\nNow either call another "
f"tool or finish with FINAL: ."),
})
return {
"final": "I ran out of steps before producing a final answer.",
"trace": trace,
}
# ── Gradio Server App ───────────────────────────────────────────────────
app = Server(title="ChatUI Studio", version="2.0.0")
@app.get("/api/models")
async def api_list_models():
return JSONResponse({
"status": "success",
"default": DEFAULT_MODEL,
"models": [
{"id": mid, **meta}
for mid, meta in MODEL_REGISTRY.items()
],
"cached": list(_MODEL_CACHE.keys()),
})
@app.get("/api/tasks")
async def api_list_tasks():
return JSONResponse({
"status": "success",
"tasks": [
{"id": tid, **meta}
for tid, meta in TASK_REGISTRY.items()
],
})
@app.post("/api/load_model")
async def api_load_model(req: Request):
body = await req.json()
model_id = body.get("model_id") or DEFAULT_MODEL
try:
if model_id not in MODEL_REGISTRY:
return JSONResponse({"status": "error", "message": "Unknown model."})
get_model(model_id)
return JSONResponse({
"status": "success",
"model": model_id,
"device": _MODEL_CACHE[model_id]["device"],
})
except Exception as e:
traceback.print_exc()
return JSONResponse({"status": "error", "message": str(e)})
@app.post("/api/run")
async def api_run_inference(req: Request):
try:
body = await req.json()
messages: List[Dict[str, Any]] = body.get("messages", [])
model_id = body.get("model_id") or DEFAULT_MODEL
task = body.get("task") or "chat"
max_tokens_i = int(body.get("max_tokens", 512))
temperature_f = float(body.get("temperature", 0.7))
system_prompt = body.get("system_prompt", "") or ""
if model_id not in MODEL_REGISTRY:
return JSONResponse({"status": "error", "message": f"Unknown model: {model_id}"})
task_meta = TASK_REGISTRY.get(task, TASK_REGISTRY["chat"])
sys_prompt = (system_prompt.strip() or task_meta["system_prompt"])
if task == "agent":
user_text = _flatten_content(messages[-1].get("content", "")) if messages else ""
result = run_agent(
model_id=model_id,
user_text=user_text,
system_prompt=sys_prompt,
max_tokens=max_tokens_i,
temperature=temperature_f,
)
return JSONResponse({
"status": "success",
"task": "agent",
"model": model_id,
"content": result["final"],
"trace": result["trace"],
"reasoning_content": "",
})
content, reasoning = generate(
model_id=model_id,
messages=messages,
system_prompt=sys_prompt,
max_tokens=max_tokens_i,
temperature=temperature_f,
)
return JSONResponse({
"status": "success",
"task": task,
"model": model_id,
"content": content,
"reasoning_content": reasoning or "",
})
except Exception as e:
traceback.print_exc()
return JSONResponse({"status": "error", "message": str(e)})
@app.get("/")
async def homepage():
html_path = STATIC_DIR / "index.html"
if html_path.exists():
with open(html_path, "r", encoding="utf-8") as f:
return HTMLResponse(content=f.read(), status_code=200)
return HTMLResponse(
content="Frontend is building. Please refresh in a few seconds...
",
status_code=200,
)
@app.get("/healthz")
async def healthz():
return JSONResponse({
"status": "ok",
"zerogpu": IS_ZEROGPU,
"device": _device(),
"models_cached": list(_MODEL_CACHE.keys()),
"models_registered": list(MODEL_REGISTRY.keys()),
})
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
if __name__ == "__main__":
import argparse
import inspect
parser = argparse.ArgumentParser(
description="ChatUI Studio — local AI chat server",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--host", "--server-name", dest="server_name", default="0.0.0.0",
help="Host/IP to bind to.",
)
parser.add_argument(
"--port", "--server-port", dest="server_port", type=int, default=7860,
help="Port to listen on.",
)
parser.add_argument(
"--share", dest="share", action="store_true",
help="Create a public Gradio share link.",
)
parser.add_argument(
"--no-share", dest="share", action="store_false",
help="Disable public share link.",
)
parser.set_defaults(share=False)
parser.add_argument(
"--show-error", dest="show_error", action="store_true",
help="Show full error tracebacks in the UI.",
)
parser.add_argument(
"--no-show-error", dest="show_error", action="store_false",
help="Hide error tracebacks in the UI.",
)
parser.set_defaults(show_error=True)
parser.add_argument(
"--quiet", action="store_true",
help="Suppress Gradio startup messages.",
)
parser.add_argument(
"--auth", nargs=2, metavar=("USER", "PASS"), default=None,
help="Enable basic auth with USER and PASS.",
)
parser.add_argument(
"--ssl-keyfile", default=None,
help="Path to SSL key file (enables HTTPS).",
)
parser.add_argument(
"--ssl-certfile", default=None,
help="Path to SSL certificate file.",
)
parser.add_argument(
"--ssl-keyfile-password", default=None,
help="Password for the SSL key file, if any.",
)
parser.add_argument(
"--favicon-path", default=None,
help="Path to a custom favicon.",
)
parser.add_argument(
"--allowed-paths", nargs="*", default=None,
help="Extra paths Gradio is allowed to serve from.",
)
parser.add_argument(
"--blocked-paths", nargs="*", default=None,
help="Paths Gradio must never serve.",
)
parser.add_argument(
"--max-file-size", default=None,
help="Max upload size (e.g. '10mb', '1gb').",
)
args = parser.parse_args()
launch_kwargs = {
"server_name": args.server_name,
"server_port": args.server_port,
"share": args.share,
"show_error": args.show_error,
"quiet": args.quiet,
"auth": tuple(args.auth) if args.auth else None,
}
if args.ssl_keyfile:
launch_kwargs["ssl_keyfile"] = args.ssl_keyfile
if args.ssl_certfile:
launch_kwargs["ssl_certfile"] = args.ssl_certfile
if args.ssl_keyfile_password:
launch_kwargs["ssl_keyfile_password"] = args.ssl_keyfile_password
if args.favicon_path:
launch_kwargs["favicon_path"] = args.favicon_path
if args.allowed_paths:
launch_kwargs["allowed_paths"] = args.allowed_paths
if args.blocked_paths:
launch_kwargs["blocked_paths"] = args.blocked_paths
if args.max_file_size:
launch_kwargs["max_file_size"] = args.max_file_size
# Drop kwargs that aren't supported by this Gradio version.
try:
supported = set(inspect.signature(app.launch).parameters.keys())
if "kwargs" not in supported:
launch_kwargs = {k: v for k, v in launch_kwargs.items() if k in supported}
except (TypeError, ValueError):
pass
app.launch(**launch_kwargs)