"""A server-side Gradio client for an OpenAI-compatible vLLM deployment."""
from __future__ import annotations
import asyncio
import html
import json
import math
import mimetypes
import os
import time
import uuid
from contextlib import aclosing
from functools import partial
from pathlib import Path
os.environ.setdefault("GRADIO_ANALYTICS_ENABLED", "False")
# HF explicitly supports API-proxy Spaces on ZeroGPU using a never-called
# decorated function. Only register it on that runtime; chat stays on CPU.
# See huggingface/skills: huggingface-spaces/references/inference-providers.md.
IS_ZERO_GPU = os.getenv("SPACES_ZERO_GPU", "").strip().lower() not in {
"", "0", "false", "no", "off",
}
if IS_ZERO_GPU:
import spaces
@spaces.GPU(duration=1)
def _zerogpu_compatibility_anchor():
"""Startup compatibility only; never invoked by the application."""
return None
else:
_zerogpu_compatibility_anchor = None
import gradio as gr
from chat_backend import (
ChatAPIError, ChatBackend, Config, build_payload, prepare_images,
redact_image_payload, safe_error,
)
OPTION_KEYS = (
"max_tokens", "inherit_defaults", "temperature", "top_p", "top_k", "min_p",
"repetition_penalty", "presence_penalty", "frequency_penalty", "seed", "stop",
"stream", "include_usage", "history_turns", "thinking", "extra_json", "resize_long_edge",
)
CSS = """
.gradio-container {max-width: 1500px !important; margin: auto !important;}
body {background: #f5f7f6;}
#hero {padding: 25px 4px 22px; border-bottom: 1px solid #dfe7e3; margin-bottom: 18px;}
#hero .eyebrow {font: 600 11px/1.6 ui-monospace,monospace; letter-spacing: .17em; color: #14735d;}
#hero h1 {font-size: clamp(25px,3vw,35px); letter-spacing: -.045em; font-weight: 700; margin: 4px 0 6px; color: #16382d;}
#hero p {font-size: 14px; color: #61776d; margin: 0;}
#hero .badge {float: right; padding: 6px 11px; border: 1px solid #d9e6df; border-radius: 99px;
font: 500 11px/1.5 ui-monospace,monospace; color: #41715e; background: #edf5f0;}
#sidebar {gap: 14px;}
#sidebar .section-title {font-size: 12px; font-weight: 700; color: #46675a; letter-spacing: .08em;}
#mode {background: transparent; border: none;}
#chat {border: 1px solid #e0e9e3; border-radius: 16px; box-shadow: 0 4px 24px #13372d05;}
#composer {border-radius: 14px;}
#send {background: #166b54 !important; border-color: #166b54 !important; color: white !important;}
#send:hover {background: #11553f !important;}
#status {font-size: 12px; min-height: 24px; color: #60746b;}
#mode-note {font-size: 12px; color: #687b72; margin-top: -6px;}
.metrics {display: grid; grid-template-columns: repeat(4,1fr); gap: 8px; margin: 5px 0 3px;}
.metric {background: #edf3ef; border: 1px solid #e3eae5; border-radius: 10px; padding: 11px 13px;}
.metric .label {font-size: 10px; color: #6b7d72; letter-spacing: .02em;}
.metric .value {font: 600 19px/1.65 ui-monospace,monospace; color: #234c3b;}
#footer-note {font-size: 11px; color: #85938b; margin: 16px 4px 0;}
.dark #hero h1 {color: #dcece3;}
.dark #hero p, .dark #status, .dark #mode-note {color: #adc2b6;}
.dark #hero .badge, .dark .metric {background: #1a2d23; border-color: #294334;}
.dark .metric .value {color: #d6eddd;}
.dark .metric .label {color: #a9bfaf;}
@media (max-width: 700px) {
#hero {padding: 16px 2px;} #hero .badge {display: none;}
.metrics {grid-template-columns: repeat(2,1fr);}
#chat {height: 440px !important; min-height: 320px !important;}
}
"""
HEADER = """
IMAGE + TEXT · vLLM
INFERENCE WORKSPACE
vLLM Chat Studio
上传图片,带着上下文继续聊。让每一次生成都可调、可查看。
"""
def new_session():
"""Gradio gives each browser session its own deep copy of this state."""
return {"turns": {"single": [], "multi": []}, "busy": False,
"run_id": None, "active_mode": None}
def image_settings():
settings = {}
for field, variable, default, integer in (
("max_images", "MAX_IMAGES_PER_REQUEST", 8, True),
("max_image_mb", "MAX_IMAGE_MB", 20, False),
("max_image_pixels", "MAX_IMAGE_PIXELS", 40000000, True),
("max_image_payload_mb", "MAX_IMAGE_PAYLOAD_MB", 64, False),
):
try:
raw = os.getenv(variable, str(default))
value = int(raw) if integer else float(raw)
if not math.isfinite(value) or value <= 0:
raise ValueError
except (ValueError, OverflowError):
raise ValueError(f"{variable} 必须是正{'整数' if integer else '数值'}。") from None
settings[field] = value
return settings
def split_composer(value):
"""Only use Gradio-preprocessed upload paths, never a path entered as text."""
if isinstance(value, str) or value is None:
return value or "", []
if not isinstance(value, dict):
raise gr.Error("消息格式无效,请重新输入文字或上传图片。")
text, files = value.get("text") or "", value.get("files") or []
if not isinstance(text, str) or not isinstance(files, list) or any(
not isinstance(path, str) for path in files
):
raise gr.Error("图片上传信息无效,请重新上传。")
return text, files[:]
def composer_value(turn):
files = turn.get("input_files", [])
return {"text": turn["user"], "files": files[:]} if files else turn["user"]
def visible_messages(turns):
"""Presentation messages are never reused as the backend conversation."""
messages = []
for turn in turns:
images = turn.get("images") or [
{"path": path, "name": Path(path).name,
"mime_type": mimetypes.guess_type(path)[0] or "image/png"}
for path in turn.get("input_files", [])
]
if images:
content = ([{"type": "text", "text": turn["user"]}]
if turn["user"] else [])
for picture in images:
content.append({
"type": "file",
"file": {"path": picture["path"], "orig_name": picture["name"],
"mime_type": picture["mime_type"]},
"alt_text": picture["name"],
})
messages.append({"role": "user", "content": content})
else:
messages.append({"role": "user", "content": turn["user"]})
reasoning = turn.get("reasoning", "")
if reasoning:
messages.append({
"role": "assistant", "content": reasoning,
"metadata": {"title": "推理过程", "status": "done"},
})
answer = turn.get("content", "")
status = turn.get("status")
if status == "generating" and not answer:
answer = "*正在生成…*" if not reasoning else "*正在等待最终回答…*"
elif status == "stopped":
answer += "\n\n> 已停止。这条未完成的回答不会加入后续上下文。"
elif status == "error":
answer += "\n\n> 请求未完成。可查看下方状态,修改参数后重新生成。"
elif status == "complete" and not answer:
answer = "*服务端未返回回答正文。*"
if answer:
messages.append({"role": "assistant", "content": answer})
return messages
def metrics_html(turn=None):
turn = turn or {}
usage = turn.get("usage") or {}
elapsed, ttft = turn.get("elapsed"), turn.get("ttft")
items = [
("总耗时", f"{elapsed:.2f}s" if isinstance(elapsed, (int, float)) else "—"),
("首个文本片段", f"{ttft:.2f}s" if isinstance(ttft, (int, float)) else "—"),
("输入 tokens", str(usage.get("prompt_tokens", "—"))),
("输出 tokens", str(usage.get("completion_tokens", "—"))),
]
return '' + "".join(
f'
{html.escape(label)}
'
f'
{html.escape(value)}
' for label, value in items
) + "
"
def mode_note(mode):
if mode == "single":
return "每次只发送 System Prompt 和本次图文;下面的历史记录仅供对照。"
return "自动携带已完成的历史图文;追问同一张图片无需重复上传。"
def snapshot(mode, state, *, input_value=None, model_value=None):
"""One consistent output tuple for generation, cancellation and navigation."""
turns = state["turns"][mode]
last = turns[-1] if turns else {}
busy = state["busy"]
prompt_update = {"interactive": not busy}
if input_value is not None:
prompt_update["value"] = input_value
payload = last.get("payload")
return (
visible_messages(turns), state,
last.get("status_text", "就绪 · 输入问题,开始对话。"),
json.dumps(payload, ensure_ascii=False, indent=2) if payload else "{}",
metrics_html(last), gr.update(**prompt_update),
gr.update(interactive=not busy), gr.update(interactive=busy),
gr.update(interactive=bool(turns) and not busy),
gr.update(interactive=bool(turns) and not busy),
gr.update(interactive=not busy), mode_note(mode),
gr.update(value=model_value) if model_value is not None else gr.skip(),
)
def format_status(turn):
status = turn.get("status")
if status == "generating":
return "生成中 · 可随时停止。"
if status == "complete":
reason = turn.get("finish_reason") or "完成"
note = " · 达到输出长度上限,可提高 max_tokens 后重新生成" if reason == "length" else ""
return f"已完成 · finish_reason={reason}{note}"
return turn.get("status_text", "已停止。")
def make_handlers(backend):
limits = image_settings()
async def refresh_models(current):
try:
models = await backend.list_models()
selected = (current or "").strip() or backend.config.default_model
if not selected and models:
selected = models[0]
# A gateway can expose a valid manually selected model not in /models.
choices = list(dict.fromkeys(models + ([selected] if selected else [])))
return gr.update(choices=choices, value=selected or None), f"已连接 · 发现 {len(models)} 个模型"
except Exception as exc:
selected = (current or "").strip() or backend.config.default_model
return gr.update(value=selected or None), safe_error(exc, backend.config)
async def run(prompt, mode, state, model, system_prompt, *values, retry=False):
if state["busy"]:
raise gr.Error("上一条请求仍在生成,请先停止或等待完成。")
turns = state["turns"][mode]
if retry:
if not turns:
raise gr.Error("还没有可以重新生成的问题。")
question = turns[-1]["user"]
input_files = turns[-1].get("input_files", [])[:]
previous = turns[:-1]
else:
question, input_files = split_composer(prompt)
previous = turns[:]
if not question.strip() and not input_files:
raise gr.Error("请输入问题,或上传至少一张图片。")
options = dict(zip(OPTION_KEYS, values, strict=True))
resize_long_edge = options.pop("resize_long_edge", 0)
options["max_images_per_request"] = limits["max_images"]
options["max_image_payload_mb"] = limits["max_image_payload_mb"]
model_id = (model or "").strip() or backend.config.default_model
run_id = uuid.uuid4().hex
state.update(busy=True, run_id=run_id, active_mode=mode)
turn = {"user": question, "content": "", "reasoning": "", "status": "generating",
"input_files": input_files, "images": [],
"status_text": "正在读取图片…" if input_files else "正在连接模型…",
"usage": {}, "payload": None,
"elapsed": None, "ttft": None, "finish_reason": None}
if retry:
state["turns"][mode] = previous + [turn]
else:
turns.append(turn)
started = time.monotonic()
try:
yield snapshot(mode, state, input_value="")
pictures = await asyncio.to_thread(
prepare_images, input_files,
max_images=limits["max_images"], max_image_mb=limits["max_image_mb"],
max_image_pixels=limits["max_image_pixels"], resize_long_edge=resize_long_edge,
) if input_files else []
if state["run_id"] != run_id:
return
turn["images"] = pictures
payload = build_payload(model_id or "__auto__", system_prompt, question,
previous, mode == "single", options, images=pictures)
if not model_id:
models = await backend.list_models()
if not models:
raise ChatAPIError("/v1/models 没有返回模型,请手动填写模型 ID 或设置 INF_MODEL。")
model_id = models[0]
if state["run_id"] != run_id:
return
payload["model"] = model_id
# Keep the real Base64 in server-side image records and the outgoing
# request only. The preview must not repeat large image data per token.
turn["payload"] = redact_image_payload(payload)
turn["status_text"] = "正在等待模型输出…"
yield snapshot(mode, state, model_value=model_id)
async with aclosing(backend.generate(payload)) as updates:
async for update in updates:
if state["run_id"] != run_id:
return
for key in ("content", "reasoning", "usage", "elapsed", "ttft", "finish_reason"):
if key in update:
turn[key] = update[key]
turn["status"] = "complete" if update.get("complete") else "generating"
turn["status_text"] = format_status(turn)
yield snapshot(mode, state)
if state["run_id"] == run_id and turn["status"] != "complete":
raise ChatAPIError("连接在回答完成前结束,请重新生成。")
except (asyncio.CancelledError, GeneratorExit):
if state["run_id"] == run_id:
turn["status"] = "stopped"
turn["status_text"] = "已停止 · 未完成的回答不会进入后续上下文。"
raise
except Exception as exc:
if state["run_id"] == run_id:
turn["status"] = "error"
turn["status_text"] = "请求失败 · " + safe_error(exc, backend.config)
turn["elapsed"] = time.monotonic() - started
finally:
if state["run_id"] == run_id:
state.update(busy=False, run_id=None, active_mode=None)
# The stop action owns its final UI update; do not overwrite a newer run.
if not state["busy"] and state["turns"][mode] and state["turns"][mode][-1] is turn:
restore = composer_value(turn) if turn["status"] == "error" else None
yield snapshot(mode, state, input_value=restore)
def stop(mode, state):
active = state.get("active_mode") or mode
turns = state["turns"][active]
restore = None
if turns:
turn = turns[-1]
if turn["status"] in {"generating", "stopped"}:
turn["status"] = "stopped"
turn["status_text"] = "已停止 · 未完成的回答不会进入后续上下文。"
restore = composer_value(turn)
state.update(busy=False, run_id=None, active_mode=None)
return snapshot(mode, state, input_value=restore)
def clear(mode, state):
if state["busy"]:
raise gr.Error("请先停止生成。")
state["turns"][mode] = []
return snapshot(mode, state, input_value="")
def switch(mode, state):
if state["busy"]:
raise gr.Error("请先停止生成,再切换模式。")
return snapshot(mode, state, input_value="")
return run, refresh_models, stop, clear, switch
def create_app(config=None, backend=None):
config = config or Config.from_env()
backend = backend or ChatBackend(config)
limits = image_settings()
run, refresh_models, stop, clear, switch = make_handlers(backend)
concurrency = int(os.getenv("MAX_CONCURRENT_REQUESTS", "8"))
if concurrency < 1:
raise ValueError("MAX_CONCURRENT_REQUESTS 必须大于 0。")
with gr.Blocks(title="vLLM Chat Studio", analytics_enabled=False) as demo:
if _zerogpu_compatibility_anchor is not None:
# The startup scan inspects registered Gradio handlers. Merely
# defining a decorated function is insufficient on some versions.
# Do not bind this to load(), send, retry, or any visible control.
gr.Button(visible=False, interactive=False).click(
fn=_zerogpu_compatibility_anchor, inputs=[], outputs=[],
api_visibility="private",
)
session = gr.State(new_session())
gr.HTML(HEADER)
with gr.Row(equal_height=False):
with gr.Column(scale=3, min_width=290, elem_id="sidebar"):
gr.Markdown("模型与指令", elem_classes=["section-title"])
model = gr.Dropdown(
choices=[config.default_model] if config.default_model else [],
value=config.default_model or None, allow_custom_value=True,
label="模型 ID", info="默认 test-model;可手动刷新或填写 served-model-name。",
interactive=True,
)
refresh = gr.Button("刷新模型 / 测试连接", size="sm")
connection = gr.Markdown(
"已读取私有接口配置 · 可直接发送消息。" if config.base_url and config.api_key
else "等待配置 · 请在 Space Settings → Secrets 中添加 INF_BASE_URL 和 INF_API_KEY。",
elem_id="connection",
)
system = gr.Textbox(
label="System Prompt", value="", lines=4,
placeholder="可选:你是一个严谨、简洁的助手。", interactive=True,
)
with gr.Accordion("生成参数", open=True):
max_tokens = gr.Number(value=4096, minimum=1, precision=0,
label="max_tokens · 最大输出长度")
inherit = gr.Checkbox(value=False, label="沿用服务端采样默认值",
info="省略 temperature 等数值采样参数;长度、seed、stop 和高级 JSON 仍生效。")
temperature = gr.Slider(0, 2, value=0.7, step=0.05, label="temperature")
top_p = gr.Slider(0.01, 1, value=0.95, step=0.01, label="top_p")
with gr.Row():
top_k = gr.Number(value=-1, precision=0, label="top_k", info="-1 = 不限制")
min_p = gr.Number(value=0, minimum=0, maximum=1, label="min_p")
repetition = gr.Slider(0.1, 2, value=1, step=0.05, label="repetition_penalty")
with gr.Accordion("惩罚、随机种子与停止词", open=False):
presence = gr.Slider(-2, 2, value=0, step=0.1, label="presence_penalty")
frequency = gr.Slider(-2, 2, value=0, step=0.1, label="frequency_penalty")
seed = gr.Number(value=-1, precision=0, label="seed", info="-1 = 不指定;固定种子不保证跨配置结果一致。")
stops = gr.Textbox(label="stop", lines=3,
placeholder='每行一个停止词,或 JSON 数组,例如 ["\\nEND"]')
with gr.Accordion("会话与输出", open=False):
streaming = gr.Checkbox(value=True, label="流式输出")
include_usage = gr.Checkbox(value=True, label="请求流式 token 统计",
info="若网关不支持 stream_options,可关闭。")
history_turns = gr.Number(value=0, minimum=0, precision=0,
label="携带最近 N 轮完整问答", info="0 = 全部;只影响多轮模式。超出模型上下文时可调小。")
thinking = gr.Radio(
choices=[("服务端默认", "default"), ("开启", "on"), ("关闭", "off")],
value="default", label="Thinking", info="通过 chat_template_kwargs.enable_thinking 控制;需要模型模板支持。",
)
with gr.Accordion("图片设置", open=False):
resize_long_edge = gr.Number(
value=0, minimum=0, precision=0, label="图片最长边 · 像素",
info="0 = 原尺寸;正数只缩小更大的图片,不放大。修改后也适用于重新生成。",
)
gr.Markdown(
f"支持 PNG、JPEG、WebP、BMP、单页 TIFF。每张最多 **{limits['max_image_mb']:g} MiB**,"
f"每次请求最多 **{limits['max_images']} 张**(包含历史图片)。\n\n"
"默认保留原分辨率;模型服务仍可能按自己的视觉处理配置缩放。"
)
with gr.Accordion("高级 JSON 参数", open=False):
gr.Markdown("直接填写请求体字段。可覆盖采样参数;不覆盖模型、消息或流式开关。")
extra = gr.Code(value="{}", language="json", lines=9, interactive=True,
label="额外请求参数")
gr.Markdown('示例:`{"chat_template_kwargs": {"enable_thinking": false}}`')
with gr.Column(scale=7, min_width=360):
mode = gr.Radio(
choices=[("单次问答", "single"), ("多轮聊天", "multi")], value="multi",
label="对话模式", show_label=False, elem_id="mode",
)
note = gr.Markdown(mode_note("multi"), elem_id="mode-note")
chat = gr.Chatbot(
value=[], label="对话", show_label=False, height=510, min_height=320,
elem_id="chat", buttons=["copy"], layout="bubble", sanitize_html=True,
allow_tags=False,
placeholder="发一张图,或者问个问题
文档、图表、截图都可以。请选择支持图片输入的模型。",
latex_delimiters=[{"left": "$$", "right": "$$", "display": True},
{"left": "$", "right": "$", "display": False},
{"left": "\\[", "right": "\\]", "display": True},
{"left": "\\(", "right": "\\)", "display": False}],
)
prompt = gr.MultimodalTextbox(
label="消息与图片", show_label=False,
placeholder="输入问题,或点击 + 上传图片;也可拖入图片…", lines=3,
max_lines=10, elem_id="composer", interactive=True,
sources=["upload"], file_count="multiple",
file_types=[".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"],
submit_btn=False, max_plain_text_length=1000000,
)
with gr.Row():
send = gr.Button("发送", variant="primary", scale=3, elem_id="send")
stop_btn = gr.Button("停止", scale=1, interactive=False)
retry = gr.Button("重新生成", scale=1, interactive=False)
clear_btn = gr.Button("清空当前模式", scale=1, interactive=False)
status = gr.Markdown("就绪 · 输入问题,开始对话。", elem_id="status")
metrics = gr.HTML(metrics_html())
with gr.Accordion("本次请求 JSON", open=False):
request_json = gr.Code(value="{}", language="json", lines=15,
interactive=False, label="请求体结构 · 图片 Base64 已省略")
gr.Markdown("图片编码只在此预览中省略,发送给模型时包含完整图片;此处不包含 Authorization。")
gr.Markdown(
"文字与图片记录保留在当前会话中。刷新或 Space 重启后可能清空。",
elem_id="footer-note",
)
options = [max_tokens, inherit, temperature, top_p, top_k, min_p, repetition,
presence, frequency, seed, stops, streaming, include_usage,
history_turns, thinking, extra, resize_long_edge]
inputs = [prompt, mode, session, model, system, *options]
outputs = [chat, session, status, request_json, metrics, prompt, send, stop_btn,
retry, clear_btn, mode, note, model]
event_options = dict(
inputs=inputs, outputs=outputs, concurrency_limit=concurrency,
concurrency_id="generation", trigger_mode="once", show_progress="hidden",
stream_every=0.08, api_visibility="private",
)
submit_event = gr.on(triggers=[send.click, prompt.submit], fn=run, **event_options)
retry_event = retry.click(fn=partial(run, retry=True), **event_options)
stop_btn.click(
fn=stop, inputs=[mode, session], outputs=outputs,
cancels=[submit_event, retry_event], queue=False, show_progress="hidden",
api_visibility="private",
)
clear_btn.click(fn=clear, inputs=[mode, session], outputs=outputs,
queue=False, api_visibility="private")
def clear_from_chatbox(mode, state):
# Gradio's built-in clear icon must reset backend state as well.
stop(mode, state)
return clear(mode, state)
chat.clear(fn=clear_from_chatbox, inputs=[mode, session], outputs=outputs,
cancels=[submit_event, retry_event], queue=False,
show_progress="hidden", api_visibility="private")
mode.input(fn=switch, inputs=[mode, session], outputs=outputs,
queue=False, api_visibility="private")
refresh.click(fn=refresh_models, inputs=[model], outputs=[model, connection],
concurrency_limit=4, show_progress="hidden", api_visibility="private")
if config.base_url and config.api_key and not config.default_model:
demo.load(fn=refresh_models, inputs=[model], outputs=[model, connection],
concurrency_limit=4, show_progress="hidden", api_visibility="private")
sampling = [temperature, top_p, top_k, min_p, repetition, presence, frequency]
inherit.change(lambda enabled: [gr.update(interactive=not enabled) for _ in sampling],
inputs=[inherit], outputs=sampling, queue=False, api_visibility="private")
demo.queue(default_concurrency_limit=concurrency, max_size=64, api_open=False)
return demo
# Hugging Face's Gradio hot reloader discovers the launched Blocks object by
# its module-level name. Constructing the UI does not call the inference API.
demo = create_app()
def main():
username, password = os.getenv("APP_USERNAME", ""), os.getenv("APP_PASSWORD", "")
if bool(username) != bool(password):
raise ValueError("APP_USERNAME 和 APP_PASSWORD 必须同时设置,或同时留空。")
theme = gr.themes.Soft(
primary_hue="emerald", secondary_hue="slate", neutral_hue="slate",
font=["Inter", "Noto Sans SC", "system-ui", "sans-serif"],
font_mono=["JetBrains Mono", "ui-monospace", "monospace"],
)
demo.launch(
server_name="0.0.0.0", server_port=int(os.getenv("PORT", "7860")),
share=False, show_error=False, theme=theme, css=CSS, footer_links=[],
auth=(username, password) if username else None,
auth_message="登录以使用此 vLLM 工作台。" if username else None,
enable_monitoring=False,
max_file_size=int(image_settings()["max_image_mb"] * 1024 * 1024),
)
if __name__ == "__main__":
main()