"""Small, framework-independent client for a vLLM Chat Completions server. Only this module sends the server-side credential. Conversation data and request previews contain JSON bodies, never authentication headers. """ from __future__ import annotations import base64 import copy import hashlib import io import html import json import math import os import re import stat import tempfile import time import warnings from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Any from urllib.parse import urlsplit, urlunsplit import httpx from PIL import Image, ImageOps, UnidentifiedImageError DEFAULT_MODEL = "test-model" class ChatAPIError(Exception): """A concise, user-facing configuration, validation, or inference error.""" def normalize_base_url(value: str) -> str: """Accept an origin, an API base, or a complete chat endpoint.""" try: parts = urlsplit(value.strip()) if ( parts.scheme not in {"http", "https"} or not parts.hostname or parts.username is not None or parts.password is not None or parts.query or parts.fragment ): raise ValueError _ = parts.port # Validate a malformed/non-numeric port, if present. except (ValueError, AttributeError): raise ChatAPIError( "INF_BASE_URL 必须是有效的 http/https 地址,且不能包含账号、密码、查询参数或片段。" ) from None path = parts.path.rstrip("/") if path.endswith("/chat/completions"): path = path[: -len("/chat/completions")].rstrip("/") if not path.endswith("/v1"): path += "/v1" return urlunsplit((parts.scheme, parts.netloc, path, "", "")) def _env_bool(name: str, default: bool) -> bool: value = os.environ.get(name, "").strip().lower() if not value: return default if value in {"1", "true", "yes", "on"}: return True if value in {"0", "false", "no", "off"}: return False raise ChatAPIError(f"{name} 必须设置为 true 或 false。") @dataclass(frozen=True, slots=True) class Config: base_url: str = field(default="", repr=False) api_key: str = field(default="", repr=False) verify_ssl: bool = True timeout: float = 600.0 default_model: str = DEFAULT_MODEL def __post_init__(self) -> None: object.__setattr__(self, "base_url", normalize_base_url(self.base_url) if self.base_url.strip() else "") if not isinstance(self.api_key, str) or any(c in self.api_key for c in "\r\n"): raise ChatAPIError("INF_API_KEY 格式无效,请重新检查 Space Secret。") object.__setattr__(self, "api_key", self.api_key.strip()) if not isinstance(self.verify_ssl, bool): raise ChatAPIError("INF_VERIFY_SSL 必须设置为 true 或 false。") if ( isinstance(self.timeout, bool) or not isinstance(self.timeout, (int, float)) or not math.isfinite(self.timeout) or self.timeout <= 0 ): raise ChatAPIError("INF_TIMEOUT 必须是大于 0 的有限秒数。") object.__setattr__(self, "timeout", float(self.timeout)) object.__setattr__(self, "default_model", str(self.default_model or "").strip()) @classmethod def from_env(cls) -> "Config": try: timeout = float(os.environ.get("INF_TIMEOUT", "600")) except ValueError: raise ChatAPIError("INF_TIMEOUT 必须是大于 0 的有限秒数。") from None return cls( base_url=os.environ.get("INF_BASE_URL", "").strip(), api_key=os.environ.get("INF_API_KEY", ""), verify_ssl=_env_bool("INF_VERIFY_SSL", True), timeout=timeout, default_model=os.environ.get("INF_MODEL", "").strip() or DEFAULT_MODEL, ) # Also accept escaped slashes/newlines in a server's JSON error echo. Remove # image data before truncating so an arbitrarily long Base64 prefix cannot leak. _IMAGE_ECHO_RE = re.compile( r"data:image(?:/|\\/)[A-Za-z0-9.+-]+;base64,[A-Za-z0-9+/=_\\\-\s]+", re.IGNORECASE, ) def _redact_image_text(text: str) -> str: return _IMAGE_ECHO_RE.sub("[已隐藏图片 Base64 数据]", text) def _redact_private_text(text: str, config: Config) -> str: if config.api_key: text = text.replace(config.api_key, "[已隐藏密钥]") parts = urlsplit(config.base_url) for value in (config.base_url, parts.netloc, parts.hostname): if value: text = re.sub(re.escape(value), "[已隐藏接口]", text, flags=re.IGNORECASE) return re.sub(r"(?i)\bBearer\s+[^\s\"'<>]+", "Bearer [已隐藏密钥]", text) def _clean_text(value: Any, config: Config, limit: int = 700) -> str: text = _redact_image_text(str(value or "")) # Redact before truncation, so a shortened secret can never survive. text = _redact_private_text(text, config) text = _redact_image_text(html.unescape(text)) text = _redact_private_text(text, config) text = re.sub(r"<[^>]*>", " ", text) text = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", "", text) text = " ".join(text.split()) return text if len(text) <= limit else text[:limit] + "…" def safe_error(exc: BaseException, config: Config) -> str: """Return a bounded error string without credentials or raw HTML.""" if isinstance(exc, ChatAPIError): return _clean_text(exc, config) if isinstance(exc, httpx.TimeoutException): return "请求超时,请检查推理服务,或增大 Space 变量 INF_TIMEOUT。" if isinstance(exc, httpx.ConnectError): if "certificate" in str(exc).lower() or "ssl" in str(exc).lower(): return ( "TLS 证书验证失败。如果此服务需要 curl -k," "可将 Space 变量 INF_VERIFY_SSL 设置为 false。" ) return "无法连接推理服务,请检查 INF_BASE_URL、网络和服务状态。" if isinstance(exc, httpx.HTTPStatusError): return _clean_text(_response_error(exc.response, config), config) if isinstance(exc, httpx.RequestError): return "推理连接异常或已中断,请检查网络后重试。" detail = _clean_text(exc, config) return f"请求失败:{detail}" if detail else "请求失败,请检查配置后重试。" def _error_detail(data: Any) -> str: if isinstance(data, dict): for key in ("error", "message", "detail"): if key in data and data[key] is not None: return _error_detail(data[key]) return "" if isinstance(data, list): return "; ".join(_error_detail(item) for item in data[:3]) return str(data) if data is not None else "" def _response_error(response: httpx.Response, config: Config) -> ChatAPIError: status = response.status_code labels = { 400: "请求参数不被服务接受,请检查模型名、上下文长度和高级参数。", 401: "认证失败,请检查 INF_API_KEY 是否正确。", 403: "服务拒绝访问,请检查 API Key 的权限。", 404: "未找到接口或模型,请检查 INF_BASE_URL 和模型 ID。", 413: "请求过大,请缩短输入或减少携带的历史轮数。", 422: "请求参数格式无效,请检查高级 JSON 和数值范围。", 429: "服务限流或并发已满,请稍后重试。", 502: "推理网关暂时不可用,请检查后端服务。", 503: "推理服务暂时不可用,请稍后重试。", 504: "推理网关超时,请检查后端生成耗时和网关配置。", } message = f"HTTP {status}:" + labels.get(status, "推理请求未成功,请检查服务配置。") try: detail = _error_detail(response.json()) except (ValueError, httpx.ResponseNotRead): try: body = response.text except httpx.ResponseNotRead: body = "" if "html" in response.headers.get("content-type", "").lower() or body.lstrip().startswith("<"): detail = "" else: detail = body detail = _clean_text(detail, config, 450) return ChatAPIError(f"{message} 服务信息:{detail}" if detail else message) def _json_no_constants(value: str) -> None: raise ValueError(f"不允许使用 {value}") def _load_json(text: str, description: str) -> Any: try: return json.loads(text, parse_constant=_json_no_constants) except (ValueError, RecursionError): raise ChatAPIError(f"{description} 不是有效 JSON;请检查括号、引号和数值,不能包含 NaN 或 Infinity。") from None def _finite_tree(value: Any) -> None: pending = [value] while pending: item = pending.pop() if isinstance(item, float) and not math.isfinite(item): raise ChatAPIError("参数必须使用有限数值,不能包含 NaN 或 Infinity。") if isinstance(item, dict): pending.extend(item.values()) elif isinstance(item, list): pending.extend(item) def _number(value: Any, name: str, low: float, high: float | None = None, *, open_low: bool = False) -> float: if isinstance(value, bool) or not isinstance(value, (int, float)): raise ChatAPIError(f"{name} 必须是数值,不能使用布尔值或字符串。") try: valid = math.isfinite(value) and (value > low if open_low else value >= low) except OverflowError: valid = False if not valid or (high is not None and value > high): lower = ">" if open_low else ">=" bounds = f"{lower} {low:g}" + (f" 且 <= {high:g}" if high is not None else "") raise ChatAPIError(f"{name} 必须是有限数值,范围为 {bounds}。") return float(value) def _integer(value: Any, name: str, low: int, high: int = 2**63 - 1) -> int: if isinstance(value, bool) or not isinstance(value, (int, float)): raise ChatAPIError(f"{name} 必须是整数,不能使用布尔值或字符串。") if isinstance(value, float) and (not math.isfinite(value) or not value.is_integer()): raise ChatAPIError(f"{name} 必须是有限整数。") if value < low or value > high: raise ChatAPIError(f"{name} 必须在 {low} 到 {high} 之间。") return int(value) def _bool_option(value: Any, name: str) -> bool: if not isinstance(value, bool): raise ChatAPIError(f"{name} 必须是 true 或 false。") return value def _stop_value(value: Any, *, from_editor: bool = False) -> list[str] | str | None: if value is None or value == "": return None if from_editor: if not isinstance(value, str): raise ChatAPIError("停止序列应填写 JSON 字符串数组,或每行一个停止词。") if not value.strip(): return None if value.lstrip().startswith("["): value = _load_json(value, "停止序列") if not isinstance(value, list): raise ChatAPIError("停止序列 JSON 必须是字符串数组。") else: value = [line.strip() for line in value.splitlines() if line.strip()] if isinstance(value, str): return value if not isinstance(value, list) or any(not isinstance(item, str) or not item for item in value): raise ChatAPIError("stop 必须是非空字符串,或由非空字符串组成的数组。") return value or None _IMAGE_FORMATS = ("PNG", "JPEG", "WEBP", "BMP", "TIFF") _IMAGE_MIME = {"PNG": "image/png", "JPEG": "image/jpeg", "WEBP": "image/webp"} _IMAGE_DATA_RE = re.compile(r"data:(image/(?:png|jpeg|webp));base64,([A-Za-z0-9+/]+={0,2})\Z") def _image_signature(header: bytes) -> bool: return ( header.startswith(b"\x89PNG\r\n\x1a\n") or header.startswith(b"\xff\xd8\xff") or (header.startswith(b"RIFF") and header[8:12] == b"WEBP") or header.startswith(b"BM") or header.startswith((b"II*\x00", b"MM\x00*", b"II+\x00", b"MM\x00+")) ) def _read_uploaded_image(path: Path, max_bytes: int) -> bytes: """Check type and size before reading a bounded uploaded image file.""" try: fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0)) with os.fdopen(fd, "rb") as stream: info = os.fstat(stream.fileno()) if not stat.S_ISREG(info.st_mode): raise ChatAPIError("上传图片必须是普通文件。") if info.st_size <= 0: raise ChatAPIError("上传图片为空。") if info.st_size > max_bytes: raise ChatAPIError("单张图片超过大小限制,请压缩图片或调整 MAX_IMAGE_MB。") header = stream.read(min(32, max_bytes + 1)) if len(header) > max_bytes: raise ChatAPIError("单张图片超过大小限制,请压缩图片或调整 MAX_IMAGE_MB。") if not _image_signature(header): raise ChatAPIError("仅支持 PNG、JPEG、WebP、BMP 或 TIFF 静态图片;文件内容与图片格式不符。") data = header + stream.read(max_bytes - len(header) + 1) if len(data) > max_bytes: raise ChatAPIError("单张图片超过大小限制,请压缩图片或调整 MAX_IMAGE_MB。") return data except ChatAPIError: raise except (OSError, ValueError): raise ChatAPIError("无法读取上传图片,请重新上传后重试。") from None def _save_prepared_png(source: Path, data: bytes) -> str: """Keep the browser preview and the API image byte-for-byte identical.""" target = source.parent / f"prepared-{hashlib.sha256(data).hexdigest()}.png" temporary = None try: with tempfile.NamedTemporaryFile(dir=source.parent, prefix=".prepared-", suffix=".png", delete=False) as stream: temporary = stream.name stream.write(data) os.replace(temporary, target) temporary = None return str(target) except OSError: raise ChatAPIError("无法保存处理后的图片,请重新上传后重试。") from None finally: if temporary: try: os.unlink(temporary) except OSError: pass def _png_image(image: Image.Image, *, resizing: bool) -> Image.Image: if image.mode == "P" and resizing: return image.convert("RGBA" if "transparency" in image.info else "RGB") if image.mode == "1" and resizing: return image.convert("L") if image.mode in {"CMYK", "YCbCr", "LAB", "HSV"}: return image.convert("RGB") if image.mode.startswith("I;16"): return image.convert("I" if resizing else "I;16") if image.mode not in {"1", "L", "LA", "P", "RGB", "RGBA", "I"}: raise ChatAPIError("此图片的像素格式暂不支持,请先转换为普通 PNG 或 JPEG。") return image def prepare_images( paths: Sequence[str | os.PathLike[str]] | str | os.PathLike[str] | None, *, max_images: int = 8, max_image_mb: float = 20, max_image_pixels: int = 40_000_000, resize_long_edge: int = 0, ) -> list[dict[str, Any]]: """Validate local uploads and prepare static OpenAI-compatible data URLs. PNG/JPEG/WebP bytes are preserved unless EXIF orientation or an explicitly requested resize changes the image. Transformed images and BMP/TIFF become lossless PNG, with a matching preview next to the original uploaded file. ``size_bytes`` is the final decoded image size; MB settings use 1024**2. """ count_limit = _integer(max_images, "max_images", 1) pixel_limit = _integer(max_image_pixels, "max_image_pixels", 1) long_edge = _integer(resize_long_edge, "resize_long_edge", 0) byte_limit = int(_number(max_image_mb, "max_image_mb", 0, open_low=True) * 1024**2) if byte_limit < 1: raise ChatAPIError("max_image_mb 至少需要允许 1 字节。") if paths is None: return [] if isinstance(paths, (str, os.PathLike)): paths = [paths] if not isinstance(paths, Sequence): raise ChatAPIError("上传图片列表格式无效,请重新上传。") if len(paths) > count_limit: raise ChatAPIError(f"本次最多上传 {count_limit} 张图片,请减少图片数量。") records = [] for supplied in paths: if not isinstance(supplied, (str, os.PathLike)): raise ChatAPIError("上传图片路径格式无效,请重新上传。") path = Path(supplied).absolute() raw = _read_uploaded_image(path, byte_limit) try: with warnings.catch_warnings(): warnings.simplefilter("error", Image.DecompressionBombWarning) with Image.open(io.BytesIO(raw), formats=list(_IMAGE_FORMATS)) as image: if getattr(image, "is_animated", False) or getattr(image, "n_frames", 1) != 1: raise ChatAPIError("暂不支持动图或多页图片,请将需要的帧或页面另存为静态图片。") if image.width <= 0 or image.height <= 0 or image.width * image.height > pixel_limit: raise ChatAPIError(f"图片像素数超过限制(最多 {pixel_limit:,} 像素),请先缩小图片。") image.verify() # Loading after verify catches incomplete compressed pixel data. with Image.open(io.BytesIO(raw), formats=list(_IMAGE_FORMATS)) as image: image.load() orientation = image.getexif().get(274, 1) rotated = orientation in {2, 3, 4, 5, 6, 7, 8} output = ImageOps.exif_transpose(image) if rotated else image resizing = bool(long_edge and max(output.size) > long_edge) as_png = rotated or resizing or image.format not in _IMAGE_MIME if as_png: output = _png_image(output, resizing=resizing) if resizing: ratio = long_edge / max(output.size) size = tuple(max(1, round(dimension * ratio)) for dimension in output.size) output = output.resize(size, Image.Resampling.LANCZOS) if output.mode == "I": low, high = output.getextrema() if low < 0 or high > 65535: raise ChatAPIError("此 TIFF 的像素范围暂不支持,请先转换为普通 PNG 或 JPEG。") output = output.convert("I;16") buffer = io.BytesIO() output.save(buffer, format="PNG") encoded, mime = buffer.getvalue(), "image/png" else: encoded, mime = raw, _IMAGE_MIME[image.format] width, height = output.size except ChatAPIError: raise except (Image.DecompressionBombError, Image.DecompressionBombWarning): raise ChatAPIError("图片像素数过大,请先缩小图片再上传。") from None except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError): raise ChatAPIError("图片已损坏或格式无效,无法完整解码;请重新导出图片后上传。") from None if len(encoded) > byte_limit: raise ChatAPIError("图片转换为 PNG 后超过大小限制,请设置较小的图片最长边,或调整 MAX_IMAGE_MB。") preview = _save_prepared_png(path, encoded) if as_png else str(path) records.append({ "path": preview, "name": path.name, "mime_type": mime, "data_url": f"data:{mime};base64," + base64.b64encode(encoded).decode("ascii"), "width": width, "height": height, "size_bytes": len(encoded), }) return records def _image_records(images: Sequence[Mapping[str, Any]] | None) -> list[Mapping[str, Any]]: if images is None: return [] if not isinstance(images, (list, tuple)) or any(not isinstance(item, Mapping) for item in images): raise ChatAPIError("图片记录格式无效,请重新上传图片。") return list(images) def _image_message(text: str, images: Sequence[Mapping[str, Any]]) -> str | list[dict[str, Any]]: if not images: return text blocks = [{"type": "text", "text": text}] if text.strip() else [] blocks.extend({"type": "image_url", "image_url": {"url": record["data_url"]}} for record in images) return blocks def _validate_request_images(images: Sequence[Mapping[str, Any]], options: Mapping[str, Any]) -> None: count_limit = _integer(options.get("max_images_per_request", 8), "max_images_per_request", 1) budget = _number(options.get("max_image_payload_mb", 64), "max_image_payload_mb", 0, open_low=True) if len(images) > count_limit: raise ChatAPIError( f"当前请求共包含 {len(images)} 张图片,超过每次请求 {count_limit} 张的限制。" "历史图片也计入总数,请减少携带的历史轮数、清空会话,或移除当前图片。" ) total = 0 for record in images: data_url = record.get("data_url") if not isinstance(data_url, str): raise ChatAPIError("图片数据缺失,请重新上传图片。") total += len(data_url) if total > budget * 1024**2: raise ChatAPIError( f"当前请求的图片 Base64 数据超过 {budget:g} MiB 限制(包含历史图片和编码膨胀)。" "请减少历史轮数、图片数量,或设置较小的图片最长边。" ) match = _IMAGE_DATA_RE.fullmatch(data_url) if not match or (match.end(2) - match.start(2)) % 4 or record.get("mime_type", match.group(1)) != match.group(1): raise ChatAPIError("图片数据格式无效,请重新上传图片。") def redact_image_payload(payload: Any) -> Any: """Copy a JSON preview without embedding uploaded image bytes in the UI.""" if isinstance(payload, dict): return {key: redact_image_payload(value) for key, value in payload.items()} if isinstance(payload, list): return [redact_image_payload(value) for value in payload] if isinstance(payload, str) and "data:image" in payload.lower(): match = _IMAGE_DATA_RE.fullmatch(payload) if match: return f"[图片 {match.group(1)};Base64 已隐藏,编码长度 {match.end(2) - match.start(2)} 字符]" return _redact_image_text(payload) return copy.deepcopy(payload) _SAMPLING_DEFAULTS = { "temperature": 0.7, "top_p": 0.9, "top_k": -1, "min_p": 0.0, "repetition_penalty": 1.0, "presence_penalty": 0.0, "frequency_penalty": 0.0, } _PROTECTED = {"messages", "model", "stream", "n"} _CLIENT_ONLY = { "api_key", "base_url", "url", "headers", "extra_headers", "extra_query", "query", "timeout", "request_timeout", "http_client", "transport", "verify", "verify_ssl", "inherit_defaults", "include_usage", "history_turns", "single_turn", "system_prompt", "prompt", "thinking", "extra_json", "images", "image_paths", "resize_long_edge", "max_images_per_request", "max_image_payload_mb", "max_images", "max_image_mb", "max_image_pixels", } _TOOL_FIELDS = {"tools", "tool_choice", "parallel_tool_calls", "functions", "function_call"} def _validate_payload_options(payload: dict[str, Any]) -> None: _finite_tree(payload) for name, low, high, open_low in ( ("temperature", 0, 2, False), ("top_p", 0, 1, True), ("min_p", 0, 1, False), ("repetition_penalty", 0, None, True), ("presence_penalty", -2, 2, False), ("frequency_penalty", -2, 2, False), ): if name in payload: payload[name] = _number(payload[name], name, low, high, open_low=open_low) for name, low in (("max_tokens", 1), ("max_completion_tokens", 1), ("min_tokens", 0), ("top_k", -1), ("seed", -(2**63))): if name in payload: payload[name] = _integer(payload[name], name, low) if payload.get("seed") == -1: payload.pop("seed") if "min_tokens" in payload: maximum = payload.get("max_completion_tokens", payload.get("max_tokens")) if maximum is not None and payload["min_tokens"] > maximum: raise ChatAPIError("min_tokens 不能大于最大生成 token 数。") if "stop" in payload: value = _stop_value(payload["stop"]) if value is None: payload.pop("stop") else: payload["stop"] = value if "stop_token_ids" in payload: if not isinstance(payload["stop_token_ids"], list): raise ChatAPIError("stop_token_ids 必须是整数数组。") payload["stop_token_ids"] = [_integer(v, "stop_token_ids", 0) for v in payload["stop_token_ids"]] if "chat_template_kwargs" in payload and not isinstance(payload["chat_template_kwargs"], dict): raise ChatAPIError("chat_template_kwargs 必须是 JSON 对象。") if "stream_options" in payload: if not payload["stream"]: raise ChatAPIError("stream_options 仅可用于流式输出。") if not isinstance(payload["stream_options"], dict): raise ChatAPIError("stream_options 必须是 JSON 对象。") for name in ("include_usage", "continuous_usage_stats"): if name in payload["stream_options"]: _bool_option(payload["stream_options"][name], f"stream_options.{name}") for name in ("ignore_eos", "skip_special_tokens", "include_stop_str_in_output", "logprobs", "return_token_ids"): if name in payload: _bool_option(payload[name], name) def build_payload( model: str, system_prompt: str, prompt: str, turns: Sequence[Mapping[str, Any]] | None, single_turn: bool, options: Mapping[str, Any] | None, *, images: Sequence[Mapping[str, Any]] | None = None, ) -> dict[str, Any]: """Build one request; advanced JSON overrides controls at the top level. ``inherit_defaults`` omits the seven numeric sampling controls. Explicit token limit, seed, stop, reasoning preference and advanced JSON still apply. Only completed turns enter history; UI-only reasoning is never replayed. Image limits cover all retained historical images and the current uploads. """ options = dict(options or {}) current_images = _image_records(images) if not isinstance(model, str) or not model.strip(): raise ChatAPIError("请选择或输入模型 ID,也可设置 Space 变量 INF_MODEL。") if prompt is None: prompt = "" if not isinstance(prompt, str): raise ChatAPIError("问题必须是文本。") if not prompt.strip() and not current_images: raise ChatAPIError("请先输入问题或上传图片。") if not isinstance(system_prompt, str): raise ChatAPIError("系统提示词必须是文本。") _bool_option(single_turn, "single_turn") inherit = _bool_option(options.get("inherit_defaults", False), "inherit_defaults") stream = _bool_option(options.get("stream", True), "stream") include_usage = _bool_option(options.get("include_usage", True), "include_usage") history_turns = _integer(options.get("history_turns", 0), "history_turns", 0) history = [] all_images = [] if not single_turn: completed = [turn for turn in (turns or []) if turn.get("status") == "complete"] if history_turns: completed = completed[-history_turns:] for turn in completed: user, content = turn.get("user"), turn.get("content") if not isinstance(user, str) or not isinstance(content, str): raise ChatAPIError("历史消息格式无效,请清空会话后重试。") historical_images = _image_records(turn.get("images")) all_images.extend(historical_images) history.append((user, content, historical_images)) all_images.extend(current_images) _validate_request_images(all_images, options) messages: list[dict[str, Any]] = [] if system_prompt.strip(): messages.append({"role": "system", "content": system_prompt}) for user, content, historical_images in history: messages.extend(( {"role": "user", "content": _image_message(user, historical_images)}, {"role": "assistant", "content": content}, )) messages.append({"role": "user", "content": _image_message(prompt, current_images)}) payload: dict[str, Any] = { "model": model.strip(), "messages": messages, "stream": stream, "n": 1, "max_tokens": options.get("max_tokens", 4096), } if not inherit: payload.update({name: options.get(name, default) for name, default in _SAMPLING_DEFAULTS.items()}) seed = options.get("seed", -1) if seed is not None: payload["seed"] = seed stop = _stop_value(options.get("stop", ""), from_editor=True) if stop is not None: payload["stop"] = stop if stream and include_usage: payload["stream_options"] = {"include_usage": True} thinking = options.get("thinking", "default") if thinking not in {"default", "on", "off"}: raise ChatAPIError("thinking 必须是 default、on 或 off。") if thinking != "default": payload["chat_template_kwargs"] = {"enable_thinking": thinking == "on"} extra_text = options.get("extra_json", "") if not isinstance(extra_text, str): raise ChatAPIError("高级参数必须填写为 JSON 对象。") extras = _load_json(extra_text, "高级参数") if extra_text.strip() else {} if not isinstance(extras, dict): raise ChatAPIError("高级参数必须是 JSON 对象,例如 {\"top_k\": 20}。") if "extra_body" in extras: raise ChatAPIError("请直接填写 top_k 等字段,不要嵌套 extra_body;该包装仅用于 OpenAI Python SDK。") blocked = sorted(set(extras) & (_PROTECTED | _CLIENT_ONLY | _TOOL_FIELDS)) if blocked: raise ChatAPIError("高级参数不能设置以下字段:" + "、".join(blocked) + "。模型、消息、输出模式由界面管理;此界面不执行工具。") payload.update(extras) if "max_completion_tokens" in extras: payload.pop("max_tokens", None) _validate_payload_options(payload) return payload def _text_part(value: Any) -> str: if value is None: return "" if isinstance(value, str): return value if isinstance(value, list): return "".join(item.get("text", "") for item in value if isinstance(item, dict) and isinstance(item.get("text"), str)) raise ChatAPIError("服务返回了无法识别的文本内容格式。") def _check_error(data: Any, config: Config) -> dict[str, Any]: if not isinstance(data, dict): raise ChatAPIError("服务返回格式不符合 Chat Completions 协议。") if data.get("error") is not None or data.get("object") == "error": detail = _clean_text(_error_detail(data), config, 500) raise ChatAPIError("推理服务返回错误" + (f":{detail}" if detail else ",请检查请求参数。")) return data def _first_choice(data: dict[str, Any], *, required: bool) -> dict[str, Any] | None: choices = data.get("choices", []) if not isinstance(choices, list): raise ChatAPIError("服务返回的 choices 格式无效。") choice = next((item for item in choices if isinstance(item, dict) and item.get("index", 0) == 0), None) if required and choice is None: raise ChatAPIError("服务没有返回模型回答,请检查接口地址和模型配置。") return choice async def _sse_events(response: httpx.Response) -> AsyncIterator[str]: data_lines: list[str] = [] async for line in response.aiter_lines(): if not line: if data_lines: yield "\n".join(data_lines) data_lines.clear() elif line.startswith("data:"): value = line[5:] data_lines.append(value[1:] if value.startswith(" ") else value) # SSE comments, IDs and retry/event metadata need no UI update. if data_lines: yield "\n".join(data_lines) class ChatBackend: def __init__(self, config: Config, transport: httpx.AsyncBaseTransport | None = None) -> None: self.config = config self.transport = transport def _client(self) -> httpx.AsyncClient: if not self.config.base_url: raise ChatAPIError("尚未配置 INF_BASE_URL,请在 Space Settings → Secrets 中添加接口地址。") if not self.config.api_key: raise ChatAPIError("尚未配置 INF_API_KEY,请在 Space Settings → Secrets 中添加 API Key。") return httpx.AsyncClient( base_url=self.config.base_url + "/", headers={"Authorization": f"Bearer {self.config.api_key}", "Content-Type": "application/json"}, verify=self.config.verify_ssl, timeout=httpx.Timeout(self.config.timeout, connect=min(20.0, self.config.timeout)), transport=self.transport, follow_redirects=False, trust_env=False, ) async def list_models(self) -> list[str]: try: async with self._client() as client: response = await client.get("models") if not 200 <= response.status_code < 300: raise _response_error(response, self.config) data = _check_error(_load_json(response.text, "模型列表返回内容"), self.config) if not isinstance(data.get("data"), list): raise ChatAPIError("模型列表格式无效,可在界面手动输入模型 ID,或设置 INF_MODEL。") ids = [item["id"] for item in data["data"] if isinstance(item, dict) and isinstance(item.get("id"), str) and item["id"].strip()] return list(dict.fromkeys(ids)) except httpx.HTTPError as exc: raise ChatAPIError(safe_error(exc, self.config)) from None async def generate(self, payload: dict[str, Any]) -> AsyncIterator[dict[str, Any]]: """Yield immutable snapshots and close all HTTP resources on cancellation. ``elapsed`` and ``ttft`` are client-observed seconds. For non-streaming requests, ttft is the response arrival time, not server token latency. A stream becomes complete only after its [DONE] marker is received. """ stream = _bool_option(payload.get("stream", False), "stream") started = time.perf_counter() state: dict[str, Any] = { "content": "", "reasoning": "", "usage": {}, "finish_reason": None, "elapsed": 0.0, "ttft": None, "complete": False, } def snapshot() -> dict[str, Any]: state["elapsed"] = time.perf_counter() - started return copy.deepcopy(state) def consume(data: dict[str, Any], *, streaming: bool) -> bool: changed = False usage = data.get("usage") if isinstance(usage, dict): state["usage"].update(usage) changed = True choice = _first_choice(data, required=not streaming) if choice is not None: message = choice.get("delta" if streaming else "message") or {} if not isinstance(message, dict): raise ChatAPIError("服务返回的消息格式无效。") content = _text_part(message.get("content")) reasoning = _text_part(message.get("reasoning") or message.get("reasoning_content")) if content or reasoning: if state["ttft"] is None: state["ttft"] = time.perf_counter() - started state["content"] += content state["reasoning"] += reasoning changed = True if choice.get("finish_reason") is not None: state["finish_reason"] = choice["finish_reason"] changed = True return changed try: async with self._client() as client: if not stream: response = await client.post("chat/completions", json=payload) if not 200 <= response.status_code < 300: raise _response_error(response, self.config) data = _check_error(_load_json(response.text, "模型返回内容"), self.config) consume(data, streaming=False) state["complete"] = True yield snapshot() return async with client.stream( "POST", "chat/completions", json=payload, headers={"Accept": "text/event-stream"}, ) as response: if not 200 <= response.status_code < 300: await response.aread() raise _response_error(response, self.config) if "application/json" in response.headers.get("content-type", "").lower(): # Some gateways ignore stream=true and return a full response. await response.aread() data = _check_error(_load_json(response.text, "模型返回内容"), self.config) consume(data, streaming=False) state["complete"] = True yield snapshot() return done = False async for event in _sse_events(response): if event.strip() == "[DONE]": done = True break if not event.strip(): continue data = _check_error(_load_json(event, "流式返回内容"), self.config) if consume(data, streaming=True): yield snapshot() if not done: raise ChatAPIError("流式连接提前中断,未收到 [DONE]。已显示的内容可能不完整,请重试。") state["complete"] = True yield snapshot() except httpx.HTTPError as exc: raise ChatAPIError(safe_error(exc, self.config)) from None