Spaces:
Sleeping
Sleeping
Download app.py from GiantPandas/Chat: direct link, hf CLI and curl.
- Browser
- Download file 28.8 kB
-
https://huggingface.co/spaces/GiantPandas/Chat/resolve/main/app.py
- Command line
-
hf download hf://spaces/GiantPandas/Chat/app.py
-
curl -L -o app.py https://huggingface.co/spaces/GiantPandas/Chat/resolve/main/app.py
28.8 kB
| """A server-side Gradio client for an OpenAI-compatible vLLM deployment.""" | |
| from __future__ import annotations | |
| import asyncio | |
| import html | |
| import json | |
| import math | |
| import mimetypes | |
| import os | |
| import time | |
| import uuid | |
| from contextlib import aclosing | |
| from functools import partial | |
| from pathlib import Path | |
| os.environ.setdefault("GRADIO_ANALYTICS_ENABLED", "False") | |
| # HF explicitly supports API-proxy Spaces on ZeroGPU using a never-called | |
| # decorated function. Only register it on that runtime; chat stays on CPU. | |
| # See huggingface/skills: huggingface-spaces/references/inference-providers.md. | |
| IS_ZERO_GPU = os.getenv("SPACES_ZERO_GPU", "").strip().lower() not in { | |
| "", "0", "false", "no", "off", | |
| } | |
| if IS_ZERO_GPU: | |
| import spaces | |
| def _zerogpu_compatibility_anchor(): | |
| """Startup compatibility only; never invoked by the application.""" | |
| return None | |
| else: | |
| _zerogpu_compatibility_anchor = None | |
| import gradio as gr | |
| from chat_backend import ( | |
| ChatAPIError, ChatBackend, Config, build_payload, prepare_images, | |
| redact_image_payload, safe_error, | |
| ) | |
| OPTION_KEYS = ( | |
| "max_tokens", "inherit_defaults", "temperature", "top_p", "top_k", "min_p", | |
| "repetition_penalty", "presence_penalty", "frequency_penalty", "seed", "stop", | |
| "stream", "include_usage", "history_turns", "thinking", "extra_json", "resize_long_edge", | |
| ) | |
| CSS = """ | |
| .gradio-container {max-width: 1500px !important; margin: auto !important;} | |
| body {background: #f5f7f6;} | |
| #hero {padding: 25px 4px 22px; border-bottom: 1px solid #dfe7e3; margin-bottom: 18px;} | |
| #hero .eyebrow {font: 600 11px/1.6 ui-monospace,monospace; letter-spacing: .17em; color: #14735d;} | |
| #hero h1 {font-size: clamp(25px,3vw,35px); letter-spacing: -.045em; font-weight: 700; margin: 4px 0 6px; color: #16382d;} | |
| #hero p {font-size: 14px; color: #61776d; margin: 0;} | |
| #hero .badge {float: right; padding: 6px 11px; border: 1px solid #d9e6df; border-radius: 99px; | |
| font: 500 11px/1.5 ui-monospace,monospace; color: #41715e; background: #edf5f0;} | |
| #sidebar {gap: 14px;} | |
| #sidebar .section-title {font-size: 12px; font-weight: 700; color: #46675a; letter-spacing: .08em;} | |
| #mode {background: transparent; border: none;} | |
| #chat {border: 1px solid #e0e9e3; border-radius: 16px; box-shadow: 0 4px 24px #13372d05;} | |
| #composer {border-radius: 14px;} | |
| #send {background: #166b54 !important; border-color: #166b54 !important; color: white !important;} | |
| #send:hover {background: #11553f !important;} | |
| #status {font-size: 12px; min-height: 24px; color: #60746b;} | |
| #mode-note {font-size: 12px; color: #687b72; margin-top: -6px;} | |
| .metrics {display: grid; grid-template-columns: repeat(4,1fr); gap: 8px; margin: 5px 0 3px;} | |
| .metric {background: #edf3ef; border: 1px solid #e3eae5; border-radius: 10px; padding: 11px 13px;} | |
| .metric .label {font-size: 10px; color: #6b7d72; letter-spacing: .02em;} | |
| .metric .value {font: 600 19px/1.65 ui-monospace,monospace; color: #234c3b;} | |
| #footer-note {font-size: 11px; color: #85938b; margin: 16px 4px 0;} | |
| .dark #hero h1 {color: #dcece3;} | |
| .dark #hero p, .dark #status, .dark #mode-note {color: #adc2b6;} | |
| .dark #hero .badge, .dark .metric {background: #1a2d23; border-color: #294334;} | |
| .dark .metric .value {color: #d6eddd;} | |
| .dark .metric .label {color: #a9bfaf;} | |
| @media (max-width: 700px) { | |
| #hero {padding: 16px 2px;} #hero .badge {display: none;} | |
| .metrics {grid-template-columns: repeat(2,1fr);} | |
| #chat {height: 440px !important; min-height: 320px !important;} | |
| } | |
| """ | |
| HEADER = """ | |
| <div id="hero"> | |
| <span class="badge">IMAGE + TEXT · vLLM</span> | |
| <div class="eyebrow">INFERENCE WORKSPACE</div> | |
| <h1>vLLM Chat Studio</h1> | |
| <p>上传图片,带着上下文继续聊。让每一次生成都可调、可查看。</p> | |
| </div> | |
| """ | |
| def new_session(): | |
| """Gradio gives each browser session its own deep copy of this state.""" | |
| return {"turns": {"single": [], "multi": []}, "busy": False, | |
| "run_id": None, "active_mode": None} | |
| def image_settings(): | |
| settings = {} | |
| for field, variable, default, integer in ( | |
| ("max_images", "MAX_IMAGES_PER_REQUEST", 8, True), | |
| ("max_image_mb", "MAX_IMAGE_MB", 20, False), | |
| ("max_image_pixels", "MAX_IMAGE_PIXELS", 40000000, True), | |
| ("max_image_payload_mb", "MAX_IMAGE_PAYLOAD_MB", 64, False), | |
| ): | |
| try: | |
| raw = os.getenv(variable, str(default)) | |
| value = int(raw) if integer else float(raw) | |
| if not math.isfinite(value) or value <= 0: | |
| raise ValueError | |
| except (ValueError, OverflowError): | |
| raise ValueError(f"{variable} 必须是正{'整数' if integer else '数值'}。") from None | |
| settings[field] = value | |
| return settings | |
| def split_composer(value): | |
| """Only use Gradio-preprocessed upload paths, never a path entered as text.""" | |
| if isinstance(value, str) or value is None: | |
| return value or "", [] | |
| if not isinstance(value, dict): | |
| raise gr.Error("消息格式无效,请重新输入文字或上传图片。") | |
| text, files = value.get("text") or "", value.get("files") or [] | |
| if not isinstance(text, str) or not isinstance(files, list) or any( | |
| not isinstance(path, str) for path in files | |
| ): | |
| raise gr.Error("图片上传信息无效,请重新上传。") | |
| return text, files[:] | |
| def composer_value(turn): | |
| files = turn.get("input_files", []) | |
| return {"text": turn["user"], "files": files[:]} if files else turn["user"] | |
| def visible_messages(turns): | |
| """Presentation messages are never reused as the backend conversation.""" | |
| messages = [] | |
| for turn in turns: | |
| images = turn.get("images") or [ | |
| {"path": path, "name": Path(path).name, | |
| "mime_type": mimetypes.guess_type(path)[0] or "image/png"} | |
| for path in turn.get("input_files", []) | |
| ] | |
| if images: | |
| content = ([{"type": "text", "text": turn["user"]}] | |
| if turn["user"] else []) | |
| for picture in images: | |
| content.append({ | |
| "type": "file", | |
| "file": {"path": picture["path"], "orig_name": picture["name"], | |
| "mime_type": picture["mime_type"]}, | |
| "alt_text": picture["name"], | |
| }) | |
| messages.append({"role": "user", "content": content}) | |
| else: | |
| messages.append({"role": "user", "content": turn["user"]}) | |
| reasoning = turn.get("reasoning", "") | |
| if reasoning: | |
| messages.append({ | |
| "role": "assistant", "content": reasoning, | |
| "metadata": {"title": "推理过程", "status": "done"}, | |
| }) | |
| answer = turn.get("content", "") | |
| status = turn.get("status") | |
| if status == "generating" and not answer: | |
| answer = "*正在生成…*" if not reasoning else "*正在等待最终回答…*" | |
| elif status == "stopped": | |
| answer += "\n\n> 已停止。这条未完成的回答不会加入后续上下文。" | |
| elif status == "error": | |
| answer += "\n\n> 请求未完成。可查看下方状态,修改参数后重新生成。" | |
| elif status == "complete" and not answer: | |
| answer = "*服务端未返回回答正文。*" | |
| if answer: | |
| messages.append({"role": "assistant", "content": answer}) | |
| return messages | |
| def metrics_html(turn=None): | |
| turn = turn or {} | |
| usage = turn.get("usage") or {} | |
| elapsed, ttft = turn.get("elapsed"), turn.get("ttft") | |
| items = [ | |
| ("总耗时", f"{elapsed:.2f}s" if isinstance(elapsed, (int, float)) else "—"), | |
| ("首个文本片段", f"{ttft:.2f}s" if isinstance(ttft, (int, float)) else "—"), | |
| ("输入 tokens", str(usage.get("prompt_tokens", "—"))), | |
| ("输出 tokens", str(usage.get("completion_tokens", "—"))), | |
| ] | |
| return '<div class="metrics">' + "".join( | |
| f'<div class="metric"><div class="label">{html.escape(label)}</div>' | |
| f'<div class="value">{html.escape(value)}</div></div>' for label, value in items | |
| ) + "</div>" | |
| def mode_note(mode): | |
| if mode == "single": | |
| return "每次只发送 System Prompt 和本次图文;下面的历史记录仅供对照。" | |
| return "自动携带已完成的历史图文;追问同一张图片无需重复上传。" | |
| def snapshot(mode, state, *, input_value=None, model_value=None): | |
| """One consistent output tuple for generation, cancellation and navigation.""" | |
| turns = state["turns"][mode] | |
| last = turns[-1] if turns else {} | |
| busy = state["busy"] | |
| prompt_update = {"interactive": not busy} | |
| if input_value is not None: | |
| prompt_update["value"] = input_value | |
| payload = last.get("payload") | |
| return ( | |
| visible_messages(turns), state, | |
| last.get("status_text", "就绪 · 输入问题,开始对话。"), | |
| json.dumps(payload, ensure_ascii=False, indent=2) if payload else "{}", | |
| metrics_html(last), gr.update(**prompt_update), | |
| gr.update(interactive=not busy), gr.update(interactive=busy), | |
| gr.update(interactive=bool(turns) and not busy), | |
| gr.update(interactive=bool(turns) and not busy), | |
| gr.update(interactive=not busy), mode_note(mode), | |
| gr.update(value=model_value) if model_value is not None else gr.skip(), | |
| ) | |
| def format_status(turn): | |
| status = turn.get("status") | |
| if status == "generating": | |
| return "生成中 · 可随时停止。" | |
| if status == "complete": | |
| reason = turn.get("finish_reason") or "完成" | |
| note = " · 达到输出长度上限,可提高 max_tokens 后重新生成" if reason == "length" else "" | |
| return f"已完成 · finish_reason={reason}{note}" | |
| return turn.get("status_text", "已停止。") | |
| def make_handlers(backend): | |
| limits = image_settings() | |
| async def refresh_models(current): | |
| try: | |
| models = await backend.list_models() | |
| selected = (current or "").strip() or backend.config.default_model | |
| if not selected and models: | |
| selected = models[0] | |
| # A gateway can expose a valid manually selected model not in /models. | |
| choices = list(dict.fromkeys(models + ([selected] if selected else []))) | |
| return gr.update(choices=choices, value=selected or None), f"已连接 · 发现 {len(models)} 个模型" | |
| except Exception as exc: | |
| selected = (current or "").strip() or backend.config.default_model | |
| return gr.update(value=selected or None), safe_error(exc, backend.config) | |
| async def run(prompt, mode, state, model, system_prompt, *values, retry=False): | |
| if state["busy"]: | |
| raise gr.Error("上一条请求仍在生成,请先停止或等待完成。") | |
| turns = state["turns"][mode] | |
| if retry: | |
| if not turns: | |
| raise gr.Error("还没有可以重新生成的问题。") | |
| question = turns[-1]["user"] | |
| input_files = turns[-1].get("input_files", [])[:] | |
| previous = turns[:-1] | |
| else: | |
| question, input_files = split_composer(prompt) | |
| previous = turns[:] | |
| if not question.strip() and not input_files: | |
| raise gr.Error("请输入问题,或上传至少一张图片。") | |
| options = dict(zip(OPTION_KEYS, values, strict=True)) | |
| resize_long_edge = options.pop("resize_long_edge", 0) | |
| options["max_images_per_request"] = limits["max_images"] | |
| options["max_image_payload_mb"] = limits["max_image_payload_mb"] | |
| model_id = (model or "").strip() or backend.config.default_model | |
| run_id = uuid.uuid4().hex | |
| state.update(busy=True, run_id=run_id, active_mode=mode) | |
| turn = {"user": question, "content": "", "reasoning": "", "status": "generating", | |
| "input_files": input_files, "images": [], | |
| "status_text": "正在读取图片…" if input_files else "正在连接模型…", | |
| "usage": {}, "payload": None, | |
| "elapsed": None, "ttft": None, "finish_reason": None} | |
| if retry: | |
| state["turns"][mode] = previous + [turn] | |
| else: | |
| turns.append(turn) | |
| started = time.monotonic() | |
| try: | |
| yield snapshot(mode, state, input_value="") | |
| pictures = await asyncio.to_thread( | |
| prepare_images, input_files, | |
| max_images=limits["max_images"], max_image_mb=limits["max_image_mb"], | |
| max_image_pixels=limits["max_image_pixels"], resize_long_edge=resize_long_edge, | |
| ) if input_files else [] | |
| if state["run_id"] != run_id: | |
| return | |
| turn["images"] = pictures | |
| payload = build_payload(model_id or "__auto__", system_prompt, question, | |
| previous, mode == "single", options, images=pictures) | |
| if not model_id: | |
| models = await backend.list_models() | |
| if not models: | |
| raise ChatAPIError("/v1/models 没有返回模型,请手动填写模型 ID 或设置 INF_MODEL。") | |
| model_id = models[0] | |
| if state["run_id"] != run_id: | |
| return | |
| payload["model"] = model_id | |
| # Keep the real Base64 in server-side image records and the outgoing | |
| # request only. The preview must not repeat large image data per token. | |
| turn["payload"] = redact_image_payload(payload) | |
| turn["status_text"] = "正在等待模型输出…" | |
| yield snapshot(mode, state, model_value=model_id) | |
| async with aclosing(backend.generate(payload)) as updates: | |
| async for update in updates: | |
| if state["run_id"] != run_id: | |
| return | |
| for key in ("content", "reasoning", "usage", "elapsed", "ttft", "finish_reason"): | |
| if key in update: | |
| turn[key] = update[key] | |
| turn["status"] = "complete" if update.get("complete") else "generating" | |
| turn["status_text"] = format_status(turn) | |
| yield snapshot(mode, state) | |
| if state["run_id"] == run_id and turn["status"] != "complete": | |
| raise ChatAPIError("连接在回答完成前结束,请重新生成。") | |
| except (asyncio.CancelledError, GeneratorExit): | |
| if state["run_id"] == run_id: | |
| turn["status"] = "stopped" | |
| turn["status_text"] = "已停止 · 未完成的回答不会进入后续上下文。" | |
| raise | |
| except Exception as exc: | |
| if state["run_id"] == run_id: | |
| turn["status"] = "error" | |
| turn["status_text"] = "请求失败 · " + safe_error(exc, backend.config) | |
| turn["elapsed"] = time.monotonic() - started | |
| finally: | |
| if state["run_id"] == run_id: | |
| state.update(busy=False, run_id=None, active_mode=None) | |
| # The stop action owns its final UI update; do not overwrite a newer run. | |
| if not state["busy"] and state["turns"][mode] and state["turns"][mode][-1] is turn: | |
| restore = composer_value(turn) if turn["status"] == "error" else None | |
| yield snapshot(mode, state, input_value=restore) | |
| def stop(mode, state): | |
| active = state.get("active_mode") or mode | |
| turns = state["turns"][active] | |
| restore = None | |
| if turns: | |
| turn = turns[-1] | |
| if turn["status"] in {"generating", "stopped"}: | |
| turn["status"] = "stopped" | |
| turn["status_text"] = "已停止 · 未完成的回答不会进入后续上下文。" | |
| restore = composer_value(turn) | |
| state.update(busy=False, run_id=None, active_mode=None) | |
| return snapshot(mode, state, input_value=restore) | |
| def clear(mode, state): | |
| if state["busy"]: | |
| raise gr.Error("请先停止生成。") | |
| state["turns"][mode] = [] | |
| return snapshot(mode, state, input_value="") | |
| def switch(mode, state): | |
| if state["busy"]: | |
| raise gr.Error("请先停止生成,再切换模式。") | |
| return snapshot(mode, state, input_value="") | |
| return run, refresh_models, stop, clear, switch | |
| def create_app(config=None, backend=None): | |
| config = config or Config.from_env() | |
| backend = backend or ChatBackend(config) | |
| limits = image_settings() | |
| run, refresh_models, stop, clear, switch = make_handlers(backend) | |
| concurrency = int(os.getenv("MAX_CONCURRENT_REQUESTS", "8")) | |
| if concurrency < 1: | |
| raise ValueError("MAX_CONCURRENT_REQUESTS 必须大于 0。") | |
| with gr.Blocks(title="vLLM Chat Studio", analytics_enabled=False) as demo: | |
| if _zerogpu_compatibility_anchor is not None: | |
| # The startup scan inspects registered Gradio handlers. Merely | |
| # defining a decorated function is insufficient on some versions. | |
| # Do not bind this to load(), send, retry, or any visible control. | |
| gr.Button(visible=False, interactive=False).click( | |
| fn=_zerogpu_compatibility_anchor, inputs=[], outputs=[], | |
| api_visibility="private", | |
| ) | |
| session = gr.State(new_session()) | |
| gr.HTML(HEADER) | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=3, min_width=290, elem_id="sidebar"): | |
| gr.Markdown("模型与指令", elem_classes=["section-title"]) | |
| model = gr.Dropdown( | |
| choices=[config.default_model] if config.default_model else [], | |
| value=config.default_model or None, allow_custom_value=True, | |
| label="模型 ID", info="默认 test-model;可手动刷新或填写 served-model-name。", | |
| interactive=True, | |
| ) | |
| refresh = gr.Button("刷新模型 / 测试连接", size="sm") | |
| connection = gr.Markdown( | |
| "已读取私有接口配置 · 可直接发送消息。" if config.base_url and config.api_key | |
| else "等待配置 · 请在 Space Settings → Secrets 中添加 INF_BASE_URL 和 INF_API_KEY。", | |
| elem_id="connection", | |
| ) | |
| system = gr.Textbox( | |
| label="System Prompt", value="", lines=4, | |
| placeholder="可选:你是一个严谨、简洁的助手。", interactive=True, | |
| ) | |
| with gr.Accordion("生成参数", open=True): | |
| max_tokens = gr.Number(value=4096, minimum=1, precision=0, | |
| label="max_tokens · 最大输出长度") | |
| inherit = gr.Checkbox(value=False, label="沿用服务端采样默认值", | |
| info="省略 temperature 等数值采样参数;长度、seed、stop 和高级 JSON 仍生效。") | |
| temperature = gr.Slider(0, 2, value=0.7, step=0.05, label="temperature") | |
| top_p = gr.Slider(0.01, 1, value=0.95, step=0.01, label="top_p") | |
| with gr.Row(): | |
| top_k = gr.Number(value=-1, precision=0, label="top_k", info="-1 = 不限制") | |
| min_p = gr.Number(value=0, minimum=0, maximum=1, label="min_p") | |
| repetition = gr.Slider(0.1, 2, value=1, step=0.05, label="repetition_penalty") | |
| with gr.Accordion("惩罚、随机种子与停止词", open=False): | |
| presence = gr.Slider(-2, 2, value=0, step=0.1, label="presence_penalty") | |
| frequency = gr.Slider(-2, 2, value=0, step=0.1, label="frequency_penalty") | |
| seed = gr.Number(value=-1, precision=0, label="seed", info="-1 = 不指定;固定种子不保证跨配置结果一致。") | |
| stops = gr.Textbox(label="stop", lines=3, | |
| placeholder='每行一个停止词,或 JSON 数组,例如 ["\\nEND"]') | |
| with gr.Accordion("会话与输出", open=False): | |
| streaming = gr.Checkbox(value=True, label="流式输出") | |
| include_usage = gr.Checkbox(value=True, label="请求流式 token 统计", | |
| info="若网关不支持 stream_options,可关闭。") | |
| history_turns = gr.Number(value=0, minimum=0, precision=0, | |
| label="携带最近 N 轮完整问答", info="0 = 全部;只影响多轮模式。超出模型上下文时可调小。") | |
| thinking = gr.Radio( | |
| choices=[("服务端默认", "default"), ("开启", "on"), ("关闭", "off")], | |
| value="default", label="Thinking", info="通过 chat_template_kwargs.enable_thinking 控制;需要模型模板支持。", | |
| ) | |
| with gr.Accordion("图片设置", open=False): | |
| resize_long_edge = gr.Number( | |
| value=0, minimum=0, precision=0, label="图片最长边 · 像素", | |
| info="0 = 原尺寸;正数只缩小更大的图片,不放大。修改后也适用于重新生成。", | |
| ) | |
| gr.Markdown( | |
| f"支持 PNG、JPEG、WebP、BMP、单页 TIFF。每张最多 **{limits['max_image_mb']:g} MiB**," | |
| f"每次请求最多 **{limits['max_images']} 张**(包含历史图片)。\n\n" | |
| "默认保留原分辨率;模型服务仍可能按自己的视觉处理配置缩放。" | |
| ) | |
| with gr.Accordion("高级 JSON 参数", open=False): | |
| gr.Markdown("直接填写请求体字段。可覆盖采样参数;不覆盖模型、消息或流式开关。") | |
| extra = gr.Code(value="{}", language="json", lines=9, interactive=True, | |
| label="额外请求参数") | |
| gr.Markdown('示例:`{"chat_template_kwargs": {"enable_thinking": false}}`') | |
| with gr.Column(scale=7, min_width=360): | |
| mode = gr.Radio( | |
| choices=[("单次问答", "single"), ("多轮聊天", "multi")], value="multi", | |
| label="对话模式", show_label=False, elem_id="mode", | |
| ) | |
| note = gr.Markdown(mode_note("multi"), elem_id="mode-note") | |
| chat = gr.Chatbot( | |
| value=[], label="对话", show_label=False, height=510, min_height=320, | |
| elem_id="chat", buttons=["copy"], layout="bubble", sanitize_html=True, | |
| allow_tags=False, | |
| placeholder="<strong>发一张图,或者问个问题</strong><br>文档、图表、截图都可以。请选择支持图片输入的模型。", | |
| latex_delimiters=[{"left": "$$", "right": "$$", "display": True}, | |
| {"left": "$", "right": "$", "display": False}, | |
| {"left": "\\[", "right": "\\]", "display": True}, | |
| {"left": "\\(", "right": "\\)", "display": False}], | |
| ) | |
| prompt = gr.MultimodalTextbox( | |
| label="消息与图片", show_label=False, | |
| placeholder="输入问题,或点击 + 上传图片;也可拖入图片…", lines=3, | |
| max_lines=10, elem_id="composer", interactive=True, | |
| sources=["upload"], file_count="multiple", | |
| file_types=[".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"], | |
| submit_btn=False, max_plain_text_length=1000000, | |
| ) | |
| with gr.Row(): | |
| send = gr.Button("发送", variant="primary", scale=3, elem_id="send") | |
| stop_btn = gr.Button("停止", scale=1, interactive=False) | |
| retry = gr.Button("重新生成", scale=1, interactive=False) | |
| clear_btn = gr.Button("清空当前模式", scale=1, interactive=False) | |
| status = gr.Markdown("就绪 · 输入问题,开始对话。", elem_id="status") | |
| metrics = gr.HTML(metrics_html()) | |
| with gr.Accordion("本次请求 JSON", open=False): | |
| request_json = gr.Code(value="{}", language="json", lines=15, | |
| interactive=False, label="请求体结构 · 图片 Base64 已省略") | |
| gr.Markdown("图片编码只在此预览中省略,发送给模型时包含完整图片;此处不包含 Authorization。") | |
| gr.Markdown( | |
| "文字与图片记录保留在当前会话中。刷新或 Space 重启后可能清空。", | |
| elem_id="footer-note", | |
| ) | |
| options = [max_tokens, inherit, temperature, top_p, top_k, min_p, repetition, | |
| presence, frequency, seed, stops, streaming, include_usage, | |
| history_turns, thinking, extra, resize_long_edge] | |
| inputs = [prompt, mode, session, model, system, *options] | |
| outputs = [chat, session, status, request_json, metrics, prompt, send, stop_btn, | |
| retry, clear_btn, mode, note, model] | |
| event_options = dict( | |
| inputs=inputs, outputs=outputs, concurrency_limit=concurrency, | |
| concurrency_id="generation", trigger_mode="once", show_progress="hidden", | |
| stream_every=0.08, api_visibility="private", | |
| ) | |
| submit_event = gr.on(triggers=[send.click, prompt.submit], fn=run, **event_options) | |
| retry_event = retry.click(fn=partial(run, retry=True), **event_options) | |
| stop_btn.click( | |
| fn=stop, inputs=[mode, session], outputs=outputs, | |
| cancels=[submit_event, retry_event], queue=False, show_progress="hidden", | |
| api_visibility="private", | |
| ) | |
| clear_btn.click(fn=clear, inputs=[mode, session], outputs=outputs, | |
| queue=False, api_visibility="private") | |
| def clear_from_chatbox(mode, state): | |
| # Gradio's built-in clear icon must reset backend state as well. | |
| stop(mode, state) | |
| return clear(mode, state) | |
| chat.clear(fn=clear_from_chatbox, inputs=[mode, session], outputs=outputs, | |
| cancels=[submit_event, retry_event], queue=False, | |
| show_progress="hidden", api_visibility="private") | |
| mode.input(fn=switch, inputs=[mode, session], outputs=outputs, | |
| queue=False, api_visibility="private") | |
| refresh.click(fn=refresh_models, inputs=[model], outputs=[model, connection], | |
| concurrency_limit=4, show_progress="hidden", api_visibility="private") | |
| if config.base_url and config.api_key and not config.default_model: | |
| demo.load(fn=refresh_models, inputs=[model], outputs=[model, connection], | |
| concurrency_limit=4, show_progress="hidden", api_visibility="private") | |
| sampling = [temperature, top_p, top_k, min_p, repetition, presence, frequency] | |
| inherit.change(lambda enabled: [gr.update(interactive=not enabled) for _ in sampling], | |
| inputs=[inherit], outputs=sampling, queue=False, api_visibility="private") | |
| demo.queue(default_concurrency_limit=concurrency, max_size=64, api_open=False) | |
| return demo | |
| # Hugging Face's Gradio hot reloader discovers the launched Blocks object by | |
| # its module-level name. Constructing the UI does not call the inference API. | |
| demo = create_app() | |
| def main(): | |
| username, password = os.getenv("APP_USERNAME", ""), os.getenv("APP_PASSWORD", "") | |
| if bool(username) != bool(password): | |
| raise ValueError("APP_USERNAME 和 APP_PASSWORD 必须同时设置,或同时留空。") | |
| theme = gr.themes.Soft( | |
| primary_hue="emerald", secondary_hue="slate", neutral_hue="slate", | |
| font=["Inter", "Noto Sans SC", "system-ui", "sans-serif"], | |
| font_mono=["JetBrains Mono", "ui-monospace", "monospace"], | |
| ) | |
| demo.launch( | |
| server_name="0.0.0.0", server_port=int(os.getenv("PORT", "7860")), | |
| share=False, show_error=False, theme=theme, css=CSS, footer_links=[], | |
| auth=(username, password) if username else None, | |
| auth_message="登录以使用此 vLLM 工作台。" if username else None, | |
| enable_monitoring=False, | |
| max_file_size=int(image_settings()["max_image_mb"] * 1024 * 1024), | |
| ) | |
| if __name__ == "__main__": | |
| main() | |