waveshaker's picture
Update app.py
82d2d53 verified
Raw History Blame Contribute Delete
123 kB
"""Universal Lora Studio: Comfy Render Backend + FastAPI Data Backend + Native Static Frontend + ZeroGPU Token Passthrough + Config Driven""" """Comfy Version: c1eece62cbf1805d4f64286d7f5592e6a215e6ba""" import base64 import asyncio import hashlib import glob import json import os import pathlib import random import re import shutil import socket import subprocess import sys import traceback import uuid import time import contextvars import logging from typing import Any import requests from huggingface_hub import hf_hub_download, list_repo_files from tqdm.auto import tqdm import tqdm as tqdm_lib import gradio as gr from gradio.context import LocalContext from fastapi import FastAPI, BackgroundTasks, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from pydantic import BaseModel import uvicorn import threading from model_config import MODEL_CONFIG from fastapi import HTTPException, Depends, Header, Query # ⚠️ 必须在 load_char_loras_from_repo() 之前定义 —— 它内部要读这个常量。 # 放在文件后面的"静态挂载"段里会导致启动首次扫描 NameError,触发词全空且被 300s 缓存锁住。 LORA_META_URL = "https://huggingface.co/spaces/waveshaker/loras_gallery/resolve/main/models.json" # ========================================================================== # 🛠️ Environment Patches & Global Paths # ========================================================================== import builtins _raw_space_id = os.environ.get("SPACE_ID", "local_space") SPACE_NAME = _raw_space_id.split("/")[-1] _original_print = builtins.print def _prefixed_print(*args, **kwargs): if args: args = (f"[{SPACE_NAME}] {args[0]}", *args[1:]) _original_print(*args, **kwargs) builtins.print = _prefixed_print logging.getLogger("root").setLevel(logging.ERROR) try: import spaces except ImportError: class _SpacesFallback: @staticmethod def GPU(**_kwargs): def decorate(function): return function return decorate spaces = _SpacesFallback() import torch if hasattr(torch.cuda, 'get_device_properties'): _orig_get_device_properties = torch.cuda.get_device_properties def _patched_get_device_properties(device): props = _orig_get_device_properties(device) if not hasattr(props, 'is_integrated'): props.is_integrated = False return props torch.cuda.get_device_properties = _patched_get_device_properties ROOT = pathlib.Path(__file__).resolve().parent COMFY = ROOT / "ComfyUI" MODELS = COMFY / "models" INPUT = COMFY / "input" OUTPUT = COMFY / "output" COMFY_COMMIT = "c1eece62cbf1805d4f64286d7f5592e6a215e6ba" ZG_ENFORCE = os.environ.get("ZG_ENFORCE", "1").strip().lower() in ("1", "true", "yes", "on") # ========================================================================== # 🔥 Warmup Constants # ========================================================================== WARMUP_BOOT_FULL = True WARMUP_BOOT_DELAY = 1.0 # 让第一个 health 先过 WARMUP_BOOT_CAP_MB = 24000 # 全量 sweep 的硬上限 WARMUP_PROBE_BYTES = 8 * 1024 * 1024 WARMUP_MIN_MB = 200 # 小于此体积不预热(冷读也就 1s) WARMUP_TTL = 900 # warm 状态有效期(秒) WARMUP_CHUNK = 32 * 1024 * 1024 WARMUP_VERIFY_BYTES = 32 * 1024 * 1024 WARMUP_WARM_MBPS = 600 # 校验读速阈值:低于此值说明没真进 cache WARMUP_LITE_MB = 200 # 进入界面时的"阶梯一级"读取量 WARMUP_STATE_FILE = "/tmp/warmup_state.json" WARMUP_EPOCH_FILE = "/tmp/warmup_epoch.txt" WARMUP_LAST_FILE = str(ROOT / ".warmup_last.json") WARMUP_PRE_GPU_BUDGET = 8.0 # 申请 GPU 前最多等多久(覆盖"已经很接近完成") WARMUP_RESIDUAL_WAIT = 5.0 # GPU 内残余等待上限(烧 GPU 预算,务必小) SYNC_ON_BOOT = True # 启动时是否 os.sync(默认关:只刷脏页,不预热) # ========================================================================== # 🧠 Global Runtime State # ========================================================================== _comfy_ready = False _nodes_ready = False tasks_status = {} _last_used_architecture = None _last_used_unet = None # _previous_arch_for_gc = None # ========================================================================== # ⚙️ Comfy Config # ========================================================================== SAMPLERS = ["euler", "euler_ancestral", "euler_a", "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_sde", "heun", "lms","lcm"] SCHEDULERS = ["simple", "beta", "normal", "karras", "exponential", "sgm_uniform"] UNET_CHOICES = {} BASE_LORAS_CHOICES = {} CHAR_LORA_CHOICES = {} # ========================================================================== # 📊 Load Config & LoRAs List # ========================================================================== def load_config_from_json(): global UNET_CHOICES, BASE_LORAS_CHOICES try: print(f"正在获取远端 JSON 配置: {MODEL_CONFIG['config_url']}") res = requests.get(MODEL_CONFIG['config_url'], timeout=5) if res.ok: full_data = res.json() root_mapping = MODEL_CONFIG['json_keys']['root_mapping'] for remote_key, local_engine in root_mapping.items(): engine_data = full_data.get(remote_key, {}) display_engine = MODEL_CONFIG["models"].get(local_engine, {}).get("name", local_engine) for item in engine_data.get(MODEL_CONFIG['json_keys']['unets'], []): display_name = f"{display_engine} | {item['name']}" UNET_CHOICES[display_name] = {"repo": item["repo"], "subfolder": item["subfolder"], "filename": item["filename"], "engine": local_engine} for item in engine_data.get(MODEL_CONFIG['json_keys']['base_loras'], []): display_name = f"{item['name']}" BASE_LORAS_CHOICES[display_name] = {"repo": item["repo"], "subfolder": item["subfolder"], "filename": item["filename"], "defaultEnable": str(item.get("defaultEnable", "false")).lower() == "true", "engine": local_engine} print("✅ 成功加载 JSON 配置") return except Exception as e: print(f"⚠️ JSON 配置加载失败: {e}") for engine, cfg in MODEL_CONFIG.get("models", {}).items(): display_engine = cfg.get("name", engine) for name, info in cfg.get("fallback_unets", {}).items(): UNET_CHOICES[f"{display_engine} | {name}"] = {**info, "engine": engine} for name, info in cfg.get("fallback_base_loras", {}).items(): BASE_LORAS_CHOICES[f"{name}"] = {**info, "engine": engine} # ========================================================================== # 📝 预设 Prompt(与 models.json 同源:gallery space 的静态文件) # - 由 App Manager 的【Presets】Tab 维护并上传 # - 只服务 studio:按 path 取单条,前端做"三条护栏"后决定要不要写进 textarea # ========================================================================== GALLERY_SPACE_ID = "waveshaker/loras_gallery" _GALLERY_SUBDOMAIN = GALLERY_SPACE_ID.replace("/", "-").replace("_", "-").lower() PROMPTS_URL = f"https://{_GALLERY_SUBDOMAIN}.static.hf.space/prompts.json" PROMPTS_TTL = 300 # 与 /api/loras 的扫描节奏保持一致 PROMPTS_RETRY_GAP = 30 # 拉取失败后 30s 内不再重试(保护冷启动) _PROMPTS_CACHE = {} # path -> prompt 文本 _PROMPTS_META = {} # {"count": n, "$updated": ...},仅用于日志 _PROMPTS_LAST_TRY = 0.0 _PROMPTS_LAST_OK = 0.0 _PROMPTS_LOCK = threading.Lock() # 若文件顶部没 import threading,请补上 LAST_SCAN_TIME = 0 def _meta_flag(v) -> bool: """models.json 里 showcase/dataset 是目录名字符串; 但元数据拉取失败时会回退到 old_flat(上一次扫描的结果), 那时这个值已经是 bool 了 —— 必须区分,因为 str(False) == "False" 是 truthy。""" if isinstance(v, bool): return v return bool(str(v or "").strip()) def _parse_prompts_payload(data): """统一解析:{"$version":1, "presets": {path: {"prompt": ...}}}""" presets = data.get("presets", data) if isinstance(data, dict) else {} clean = {} if isinstance(presets, dict): for p, v in presets.items(): if not isinstance(v, dict): continue txt = str(v.get("prompt") or "").strip() if txt: clean[p] = txt return clean def _load_prompts_index(force: bool = False) -> dict: """拉取并缓存 prompts.json。TTL 300s;失败保留上次结果;**绝不抛异常**。""" global _PROMPTS_CACHE, _PROMPTS_META, _PROMPTS_LAST_TRY, _PROMPTS_LAST_OK def _fresh(): return _PROMPTS_CACHE and (time.time() - _PROMPTS_LAST_OK) < PROMPTS_TTL def _cooling(): return _PROMPTS_CACHE and (time.time() - _PROMPTS_LAST_TRY) < PROMPTS_RETRY_GAP if not force and (_fresh() or _cooling()): return _PROMPTS_CACHE with _PROMPTS_LOCK: if not force and (_fresh() or _cooling()): return _PROMPTS_CACHE _PROMPTS_LAST_TRY = time.time() data = None # 主路径:static CDN(带 ?t= 破缓存) try: import urllib.request url = f"{PROMPTS_URL}?t={int(_PROMPTS_LAST_TRY)}" req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout=15) as resp: data = json.loads(resp.read().decode("utf-8")) except Exception as e: print(f" └ ℹ️ prompts.json static 拉取失败({type(e).__name__}),尝试走 Hub API…") # 兜底:Hub API 直读(绕开 static CDN) if data is None: try: from huggingface_hub import hf_hub_download fp = hf_hub_download(repo_id=GALLERY_SPACE_ID, filename="prompts.json", repo_type="space") with open(fp, "r", encoding="utf-8") as f: data = json.load(f) except Exception as e: if _PROMPTS_CACHE: print(f" └ ⚠️ prompts.json 拉取失败,沿用上次缓存 {len(_PROMPTS_CACHE)} 条: {type(e).__name__}") else: print(f" └ ⚠️ prompts.json 拉取失败(视为无预设,不影响图库/生成): {type(e).__name__}") return _PROMPTS_CACHE try: _PROMPTS_CACHE = _parse_prompts_payload(data) _PROMPTS_META = { "count": len(_PROMPTS_CACHE), "updated": (data.get("$updated") if isinstance(data, dict) else None), "version": (data.get("$version") if isinstance(data, dict) else None), } _PROMPTS_LAST_OK = time.time() print(f" └ prompts.json 就绪: {_PROMPTS_META['count']} 条" f"($version={_PROMPTS_META['version']}, $updated={_PROMPTS_META['updated']})") except Exception as e: print(f" └ ⚠️ prompts.json 解析异常: {type(e).__name__}: {e}") return _PROMPTS_CACHE def _has_prompt(path: str) -> bool: """字典查询 + 一个时间戳判断,成本可以忽略,所以可以直接在 1034 次循环里调用""" try: return bool(path) and path in _load_prompts_index() except Exception: return False def load_char_loras_from_repo(): global CHAR_LORA_CHOICES, LAST_SCAN_TIME current_time = time.time() if current_time - LAST_SCAN_TIME < 300: print("⚡ 使用缓存,跳过重新扫描 LoRA 列表") return print(f"🔄 触发动态刷新,正在重新扫描 {MODEL_CONFIG['char_lora_repo']}...") exclude_list = [x.lower() for x in MODEL_CONFIG.get("exclude_folders", [])] new_char_choices = {} # 🌟 拉取 trigger word 元数据(key = 完整仓库路径,如 moss/CN/xxx.safetensors) lora_meta = {} meta_ok = False try: r = requests.get(LORA_META_URL, timeout=10) if r.ok: lora_meta = r.json().get("items", {}) or {} meta_ok = True print(f"✅ 已加载 trigger 元数据: {len(lora_meta)} 条") else: print(f"⚠️ trigger 元数据拉取失败: HTTP {r.status_code}") except Exception as e: print(f"⚠️ trigger 元数据拉取失败: {e}") # 多形态索引:仓库全路径 / 去前缀路径 / 纯文件名,任一命中即可 meta_index = {} for k, v in (lora_meta or {}).items(): meta_index[k] = v meta_index[k.lstrip("./")] = v meta_index[os.path.basename(k)] = v # 上一次成功扫描的结果(元数据拉取失败时兜底,避免全站触发词瞬间消失) old_flat = {} for _eng, _m in CHAR_LORA_CHOICES.items(): for _fn, _v in (_m or {}).items(): old_flat[_fn] = _v try: files = list_repo_files(repo_id=MODEL_CONFIG['char_lora_repo'], token=os.environ.get("HF_TOKEN")) for engine, cfg in MODEL_CONFIG.get("models", {}).items(): prefix = cfg.get("char_lora_prefix", "") new_char_choices[engine] = {} for f in files: if f.startswith(prefix) and f.endswith(".safetensors") and "loras/" not in f: relative_path = f[len(prefix):] parts = relative_path.split("/") if len(parts) > 1 and parts[0].lower() in exclude_list: continue filename = parts[-1] webp_path = f.replace(".safetensors", ".webp") webp_url = f"https://huggingface.co/{MODEL_CONFIG['char_lora_repo']}/resolve/main/{webp_path}" # 🌟 合并 title / trigger info = ( meta_index.get(f) or meta_index.get(relative_path) or meta_index.get(filename) or {} ) if not info and not meta_ok: info = old_flat.get(filename, {}) # 元数据失败 → 复用上次缓存 # trigger:数组原样透传(只 trim / 去空),字符串原样透传 # ⚠️ 绝不 join 成 ", ",也绝不在后端按逗号拆分 —— 拆分只由前端 normalizeTriggers 负责 raw_trigger = info.get("trigger", "") if isinstance(raw_trigger, (list, tuple)): triggers = [str(t).strip() for t in raw_trigger if str(t).strip()] else: triggers = str(raw_trigger or "").strip() title = (info.get("title") or "").strip() or filename.replace(".safetensors", "") # 🌟 只传布尔:前端用自己的 path 拼深链,不需要完整 URL # 命名用 has_xxx,避免和 models.json 里的目录名字符串混淆 new_char_choices[engine][filename] = { "path": f, "webp": webp_url, "title": title, "trigger": triggers, # list 或 str,前端都能吃 "has_showcase": _meta_flag(info.get("showcase")), "has_dataset": _meta_flag(info.get("dataset")), "has_prompt": _has_prompt(f), # 🌟 供前端决定是否显示「套用推荐 prompt」 } CHAR_LORA_CHOICES.clear() CHAR_LORA_CHOICES.update(new_char_choices) LAST_SCAN_TIME = current_time total = sum(len(eng) for eng in CHAR_LORA_CHOICES.values()) matched = sum(1 for eng in CHAR_LORA_CHOICES.values() for v in eng.values() if v.get("trigger")) print(f"✅ 成功刷新角色 LoRA: {[f'{k}: {len(v)}个' for k, v in CHAR_LORA_CHOICES.items()]} | trigger 覆盖 {matched}/{total}") # ✅ 新增:分架构覆盖率,数据缺口一眼可见 per = " | ".join( f"{eng}:{sum(1 for v in m.values() if v.get('trigger'))}/{len(m)}" for eng, m in CHAR_LORA_CHOICES.items() ) print(f" └ trigger 分架构: {per}") per_show = " | ".join( f"{eng}:{sum(1 for v in m.values() if v.get('has_showcase'))}/{len(m)}" for eng, m in CHAR_LORA_CHOICES.items() ) print(f" └ showcase 分架构: {per_show}") per_prompt = " | ".join( f"{eng}:{sum(1 for v in m.values() if v.get('has_prompt'))}/{len(m)}" for eng, m in CHAR_LORA_CHOICES.items() ) print(f" └ prompt 覆盖: {per_prompt}") # 🌟 孤儿检测:presets 里的 path 在 models.json 里不存在 = 深链进来静默不生效,最难查 try: pidx = _load_prompts_index() known = {v.get("path") for m in CHAR_LORA_CHOICES.values() for v in m.values() if v.get("path")} orphans = [k for k in pidx if k not in known] if orphans: print(f" └ ⚠️ prompts.json 孤儿 {len(orphans)} 条(path 不在 models.json 里): {orphans[:5]}") elif pidx: print(f" └ ✅ prompts.json 全部命中({len(pidx)}/{len(pidx)})") except Exception as e: print(f" └ ⚠️ prompts 孤儿检测跳过: {type(e).__name__}") except Exception as e: print(f"⚠️ 动态刷新失败: {e}") load_config_from_json() load_char_loras_from_repo() # ========================================================================== # ⬇️ Download Comfy Models & Init # ========================================================================== def _run(command: list[str], cwd: pathlib.Path | None = None, check: bool = True) -> None: subprocess.run(command, cwd=str(cwd) if cwd else None, check=check) def _install_filtered_requirements(path: pathlib.Path) -> None: if not path.exists(): return blocked = {"torch", "torchvision", "torchaudio", "transformers", "huggingface-hub", "accelerate"} requirements = [] for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines(): item = raw.strip() if not item or item.startswith("#"): continue package = re.split(r"[<>=!~;\[\s]", item.lower().replace("_", "-"), maxsplit=1)[0] if package not in blocked: requirements.append(item) if requirements: _run([sys.executable, "-m", "pip", "install", "--no-cache-dir", *requirements], check=False) def _download_if_needed(repo: str, subfolder: str, filename: str, dest_root: pathlib.Path) -> None: dest_path = dest_root / filename if dest_path.exists(): return print(f"⏳ Downloading: {filename} ...") dest_root.mkdir(parents=True, exist_ok=True) hf_hub_download(repo_id=repo, subfolder=subfolder if subfolder else None, filename=filename, local_dir=str(dest_root)) src = dest_root / subfolder / filename if subfolder else dest_root / filename if src.exists() and src != dest_path: shutil.move(str(src), str(dest_path)) def _ensure_comfy() -> None: global _comfy_ready if _comfy_ready: return if not COMFY.exists(): print(f"📦 Cloning ComfyUI repo...") _run(["git", "clone", "https://github.com/comfyanonymous/ComfyUI.git", str(COMFY)]) # 已经停在目标 commit → 跳过 fetch/checkout(每次重启省一次网络往返 + 磁盘 churn) def _head() -> str: try: return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=str(COMFY), text=True).strip() except Exception: return "" current_commit = _head() if current_commit != COMFY_COMMIT: print(f"🔄 Fetching and checking out ComfyUI commit: {COMFY_COMMIT[:8]}...") _run(["git", "fetch", "origin"], cwd=COMFY) _run(["git", "checkout", COMFY_COMMIT], cwd=COMFY) current_commit = _head() else: print(f"✅ ComfyUI 已在目标 commit,跳过 fetch/checkout") print(f"✅ ComfyUI 当前版本 Commit: {current_commit or 'unknown'}") _install_filtered_requirements(COMFY / "requirements.txt") for folder in ("diffusion_models", "text_encoders", "vae", "loras"): (MODELS / folder).mkdir(parents=True, exist_ok=True) INPUT.mkdir(parents=True, exist_ok=True) OUTPUT.mkdir(parents=True, exist_ok=True) _comfy_ready = True def _ensure_models() -> None: print("开始全量下载基础模型...") for engine_cfg in MODEL_CONFIG["models"].values(): for repo, file_path in engine_cfg.get("companion_downloads", []): path = pathlib.Path(file_path) if "text_encoders" in str(path): dest_root = MODELS / "text_encoders" elif "vae" in str(path): dest_root = MODELS / "vae" else: dest_root = MODELS / path.parent _download_if_needed(repo, str(path.parent), path.name, dest_root) for name, cfg in UNET_CHOICES.items(): _download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], MODELS / "diffusion_models") lora_root = MODELS / "loras" for name, cfg in BASE_LORAS_CHOICES.items(): _download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], lora_root) # ❌ 不再在这里 os.sync + 全量预热: # ① 同步跑在 import 阶段 → uvicorn 还没 listen,/api/health 打不进来 # ② 会阻塞启动几十秒,且和后面的 sweep 重复 # ✅ 预热统一交给 WarmupManager,在 @app.on_event("startup") 的后台线程里做 if SYNC_ON_BOOT: try: _t0 = time.time() os.sync() print(f"✅ 脏页已刷盘 (os.sync 耗时: {round(time.time() - _t0, 2)}s)", flush=True) except Exception as e: print(f"⚠️ os.sync 失败: {e}", flush=True) def _init_comfy_nodes() -> None: global _nodes_ready if _nodes_ready: return # 1. 获取绝对路径,防止相对路径在某些环境下解析失败 comfy_path = str(COMFY.resolve()) # 2. 确保绝对路径在 sys.path 的最前面 if comfy_path not in sys.path: sys.path.insert(0, comfy_path) # 3. 切换工作目录,确保 ComfyUI 内部的相对路径依赖(如 custom_nodes 等)正常 os.chdir(comfy_path) # 4. 增加一道硬校验:如果没克隆成功,直接报错提示,而不是抛出难懂的 import error if not os.path.exists(os.path.join(comfy_path, "execution.py")): raise RuntimeError(f"❌ ComfyUI execution.py not found in {comfy_path}. Did git clone fail?") # 5. 开始导入 import execution, nodes, server from app.assets.manager import default_asset_manager loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) server_instance = server.PromptServer(loop, default_asset_manager()) execution.PromptQueue(server_instance) loop.run_until_complete(nodes.init_extra_nodes()) _nodes_ready = True print("✅ ComfyUI 节点初始化完成!", flush=True) # ========================================================================== # 💻 Prepare Runtime # ========================================================================== def _warmup_node_imports() -> None: """预热 Comfy 节点类导入,消除首次 execute 的 import 开销""" try: import nodes as comfy_nodes for class_name in ["UNETLoader", "CLIPLoader", "DualCLIPLoader", "VAELoader", "LoraLoaderModelOnly", "KSampler", "VAEDecode", "SaveImage", "CLIPTextEncode", "ConditioningZeroOut", "EmptyLatentImage"]: if hasattr(comfy_nodes, class_name): getattr(comfy_nodes, class_name) print("[预热] ✅ 节点类导入完成", flush=True) except Exception as e: print(f"[预热] ⚠️ {e}", flush=True) def _prepare_runtime() -> None: _ensure_comfy() _ensure_models() _init_comfy_nodes() _warmup_node_imports() # 🌟 P1: 预热节点类 try: print("⬇️ Starting up...") _prepare_runtime() except Exception as e: print(f"⚠️ Startup failed: {e}") # ========================================================================== # ♻️ Reusable Executor Singleton (preserves Comfy model cache across tasks) # ========================================================================== _exec_lock = threading.Lock() # 串行化执行,GPU 一次一个任务 # ⚠️ 必须是 Event(有 set/clear/is_set)。Semaphore 没有这些方法。 # 用「计数器 + 锁」而不是裸 Event,避免并发请求互相 clear。 _gen_inflight = threading.Event() _gen_inflight_count = 0 _gen_inflight_lock = threading.Lock() def _gen_enter(): global _gen_inflight_count with _gen_inflight_lock: _gen_inflight_count += 1 _gen_inflight.set() def _gen_leave(): global _gen_inflight_count with _gen_inflight_lock: _gen_inflight_count = max(0, _gen_inflight_count - 1) if _gen_inflight_count == 0: _gen_inflight.clear() _exec_state = { "loop": None, "server": None, "executor": None, } _init_lock = threading.Lock() def _get_executor(): if _exec_state["executor"] is not None: return _exec_state["loop"], _exec_state["executor"] with _init_lock: if _exec_state["executor"] is not None: return _exec_state["loop"], _exec_state["executor"] # 👇 从这里开始全部缩进 4 格 if str(COMFY) not in sys.path: sys.path.insert(0, str(COMFY)) import execution, server from app.assets.manager import default_asset_manager loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) server_instance = server.PromptServer(loop, default_asset_manager()) executor = execution.PromptExecutor( server_instance, cache_type=execution.CacheType.RAM_PRESSURE, cache_args={"lru": 8, "ram": 16.0, "ram_inactive": 0}, ) _exec_state.update(loop=loop, server=server_instance, executor=executor) print("[执行器] 🆕 子进程内新建执行器(父进程状态不可继承)", flush=True) # 👇 新增这一行,方便你在日志里观察当前 VRAM 状态 try: import torch allocated = torch.cuda.memory_allocated() / (1024**3) reserved = torch.cuda.memory_reserved() / (1024**3) print(f"[显存监控] 已分配: {allocated:.2f} GB | 已保留: {reserved:.2f} GB", flush=True) except Exception: pass return loop, executor # ========================================================================== # 🧩 Loader Chain Cache (stable node ids => Comfy node-cache hits, no re-patch) # ========================================================================== _loader_chain_cache: dict[str, tuple[dict, str]] = {} def _combo_key(engine: str, unet_name: str, base_loras: list[tuple[str, float]], char_lora: str | None, char_strength: float) -> str: base = tuple((f, round(float(s), 3)) for f, s in base_loras) return f"{engine}|{unet_name}|{base}|{char_lora or ''}|{round(float(char_strength), 3)}" def _build_clip_inputs(engine_cfg: dict, clip_loader_node: str) -> dict: clip_type = engine_cfg.get("clip_type", "moss_image") clip_inputs = {"type": clip_type} te_filenames = engine_cfg.get("local_te_filename", []) if clip_loader_node == "DualCLIPLoader": if len(te_filenames) >= 2: if "clip_l" in te_filenames[0].lower(): clip_inputs["clip_name1"] = te_filenames[0] clip_inputs["clip_name2"] = te_filenames[1] else: clip_inputs["clip_name1"] = te_filenames[1] clip_inputs["clip_name2"] = te_filenames[0] elif len(te_filenames) == 1: clip_inputs["clip_name1"] = te_filenames[0] else: if te_filenames: clip_inputs["clip_name"] = te_filenames[0] if "clip_device" in MODEL_CONFIG.get("loader_args", {}): clip_inputs["device"] = MODEL_CONFIG["loader_args"]["clip_device"] return clip_inputs def _build_loader_chain(engine, unet_name, base_loras, char_lora, char_strength): """ 只构建 1..N 的 loader 链(unet/clip/vae + lora 叠加), 返回 (chain_nodes, final_model_node_id)。 chain_nodes 的节点 id 从 "1" 开始连续,保证相同组合 -> 相同 id/inputs。 """ engine_cfg = MODEL_CONFIG["models"].get(engine, {}) nodes = MODEL_CONFIG["node_types"] clip_loader_node = engine_cfg.get("clip_loader_override", nodes["clip_loader"]) vae_filename = engine_cfg.get("local_vae_filename", "") chain: dict[str, Any] = { "1": {"class_type": nodes["unet_loader"], "inputs": {"unet_name": unet_name, "weight_dtype": "default"}}, "2": {"class_type": clip_loader_node, "inputs": _build_clip_inputs(engine_cfg, clip_loader_node)}, "3": {"class_type": nodes["vae_loader"], "inputs": {"vae_name": vae_filename}}, } cur = "1" idx = 4 for lora_file, strength in base_loras: chain[str(idx)] = {"class_type": nodes["lora_loader"], "inputs": {"model": _ref(cur), "lora_name": lora_file, "strength_model": strength}} cur = str(idx); idx += 1 if char_lora: chain[str(idx)] = {"class_type": nodes["lora_loader"], "inputs": {"model": _ref(cur), "lora_name": char_lora, "strength_model": char_strength}} cur = str(idx); idx += 1 return chain, cur # ========================================================================== # 🔄 Progress Interceptor # ========================================================================== _current_generation_task_id = contextvars.ContextVar("current_generation_task_id", default=None) _current_download_task_id = contextvars.ContextVar("current_download_task_id", default=None) # 进度节流:task_id -> (上次落盘的步数, 时间戳) _last_metric_step: dict[str, tuple[int, float]] = {} class UnifiedProgressTqdm(tqdm): def update(self, n=1): super().update(n) download_task_id = _current_download_task_id.get() gen_task_id = _current_generation_task_id.get() # 终态守卫:已结束的任务不再写入(防 ContextVar 残留污染) if gen_task_id and tasks_status.get(gen_task_id, {}).get("status") in ("success", "error"): return if download_task_id and tasks_status.get(download_task_id, {}).get("status") in ("success", "error"): return if download_task_id and download_task_id in tasks_status: if self.total and self.total > 0: percent = int((self.n / self.total) * 100) tasks_status[download_task_id]["message"] = f"📥 Downloading LoRA: {percent}%..." else: tasks_status[download_task_id]["message"] = f"📥 Downloading LoRA: {self.n} units..." return if not gen_task_id or gen_task_id not in tasks_status: return # 节流:步数没变且距上次落盘 <100ms 就跳过;首次必写(保证 load_time 精度) now = time.time() last = _last_metric_step.get(gen_task_id) if last is not None and last[0] == self.n and (now - last[1]) < 0.1: return _last_metric_step[gen_task_id] = (self.n, now) has_steps = bool(self.total and self.total > 1) step_msg = None if has_steps: step_msg = f"🎨 Rendering: {self.n}/{self.total} steps ({int(self.n / self.total * 100)}%)" def _m(data): dirty = False if "load_time" not in data or data.get("load_time", -1) == -1: load_start = data.get("load_start") if load_start is not None and data.get("load_time_source") != "load_models_gpu": data["load_time"] = round(now - load_start, 2) data["load_time_source"] = "tqdm_first_update" print(f"[探针] 🎯 tqdm 首次 update load_time: {data['load_time']}s", flush=True) dirty = True if step_msg and data.get("phase") != step_msg: data["phase"] = step_msg dirty = True return dirty _update_metric(gen_task_id, _m) if has_steps: tasks_status[gen_task_id]["message"] = step_msg print(f"[进度拦截] Task {gen_task_id[:8]} | {self.n}/{self.total} steps ({int(self.n / self.total * 100)}%)") # ========================================================================== # 🔁 tqdm Patch: Re-entrant Ref-counting (no save/restore leak) # ========================================================================== _tqdm_patch_lock = threading.RLock() _tqdm_patch_depth = 0 _tqdm_patch_originals: dict[str, Any] = {} def _tqdm_patch_enter() -> None: global _tqdm_patch_depth import tqdm as tqdm_lib import tqdm.auto as tqdm_auto with _tqdm_patch_lock: if _tqdm_patch_depth == 0: _tqdm_patch_originals["lib"] = tqdm_lib.tqdm _tqdm_patch_originals["auto"] = tqdm_auto.tqdm tqdm_lib.tqdm = UnifiedProgressTqdm tqdm_auto.tqdm = UnifiedProgressTqdm _tqdm_patch_depth += 1 def _tqdm_patch_exit() -> None: global _tqdm_patch_depth import tqdm as tqdm_lib import tqdm.auto as tqdm_auto with _tqdm_patch_lock: if _tqdm_patch_depth > 0: _tqdm_patch_depth -= 1 if _tqdm_patch_depth == 0 and "lib" in _tqdm_patch_originals: tqdm_lib.tqdm = _tqdm_patch_originals.pop("lib") tqdm_auto.tqdm = _tqdm_patch_originals.pop("auto") # ========================================================================== # ⚛️ Comfy Backend & Workflow # ========================================================================== import psutil def print_ram_status(stage: str): mem = psutil.virtual_memory() process = psutil.Process(os.getpid()) proc_mem_gb = process.memory_info().rss / (1024 ** 3) total_gb = mem.total / (1024 ** 3) avail_gb = mem.available / (1024 ** 3) # 🌟 新增:读取真实的 Cgroup 限制 cgroup_limit_gb = -1 try: # 尝试读取 Cgroup v2 with open("/sys/fs/cgroup/memory.max", "r") as f: val = f.read().strip() if val != "max": cgroup_limit_gb = int(val) / (1024 ** 3) except FileNotFoundError: try: # 尝试读取 Cgroup v1 with open("/sys/fs/cgroup/memory/memory.limit_in_bytes", "r") as f: val = f.read().strip() # v1 经常用一个巨大的数字表示无限制 if int(val) < (1 << 60): cgroup_limit_gb = int(val) / (1024 ** 3) except Exception: pass cgroup_str = f" | 容器真实限额: {cgroup_limit_gb:.2f} GB" if cgroup_limit_gb > 0 else " | 容器真实限额: 未知/无限制" print(f"[RAM监控 - {stage}] 宿主机总内存: {total_gb:.2f} GB | 宿主机可用: {avail_gb:.2f} GB | 本进程占用: {proc_mem_gb:.2f} GB | 系统已用: {mem.percent}%{cgroup_str}") def _io_probe(): try: c = psutil.Process().io_counters() return {"read_chars": c.read_chars, "read_bytes": c.read_bytes} except Exception: return {"read_chars": 0, "read_bytes": 0} def _mem_probe(): """返回 (host_Cached_MB, host_Avail_MB, cg_current_GB, cg_file_GB, cg_inactive_file_GB)""" cached = avail = -1 try: with open("/proc/meminfo") as f: m = {l.split(":")[0]: int(l.split()[1]) // 1024 for l in f if len(l.split()) >= 2} cached, avail = m.get("Cached", -1), m.get("MemAvailable", -1) except Exception: pass cur = file_gb = inact_gb = -1.0 try: with open("/sys/fs/cgroup/memory.current") as f: cur = int(f.read().strip()) / 1024**3 with open("/sys/fs/cgroup/memory.stat") as f: for line in f: p = line.split() if len(p) != 2: continue if p[0] == "file": file_gb = int(p[1]) / 1024**3 elif p[0] == "inactive_file": inact_gb = int(p[1]) / 1024**3 except Exception: pass return cached, avail, cur, file_gb, inact_gb def _gpu_id(): try: dev = torch.cuda.current_device() p = torch.cuda.get_device_properties(dev) return f"dev{dev}/uuid={getattr(p, 'uuid', 'n/a')}" except Exception as e: return f"n/a({e})" # -------------------------------------------------------------------------- # 📊 Metric Files: Atomic Write + Read-Modify-Write (/tmp/metrics_<task_id>.json) # -------------------------------------------------------------------------- _metric_lock = threading.RLock() def _metric_path(task_id: str) -> str: return f"/tmp/metrics_{task_id}.json" def _read_metric(task_id: str) -> dict: try: with open(_metric_path(task_id), "r") as f: return json.load(f) except Exception: return {} def _atomic_dump(task_id: str, data: dict) -> None: """必须在 _metric_lock 内调用:临时文件 + os.replace,读者永远读不到半截 JSON。""" path = _metric_path(task_id) tmp = f"{path}.{os.getpid()}.{threading.get_ident()}.tmp" try: with open(tmp, "w") as f: json.dump(data, f) os.replace(tmp, path) except Exception: try: os.remove(tmp) except Exception: pass raise def _update_metric(task_id: str, mutator) -> dict: """锁内读-改-写。mutator(data) 就地修改并返回 True 表示需要落盘。返回最新 data。""" if not task_id: return {} try: with _metric_lock: data = _read_metric(task_id) if mutator(data): _atomic_dump(task_id, data) return data except Exception as e: print(f"[探针] metric 更新失败: {e}", flush=True) return {} def _write_metric(task_id: str, **updates) -> None: def _m(d): d.update(updates) return True _update_metric(task_id, _m) # -------------------------------------------------------------------------- def _ref(node: str, output: int = 0) -> list[Any]: return [node, output] def select_base_loras(engine: str, lora_states: tuple) -> list[tuple[str, float]]: selected = [] # 找出当前引擎专属的 Base LoRA 列表,保持顺序 valid_loras = [] for name, cfg in BASE_LORAS_CHOICES.items(): if cfg["engine"] == engine: valid_loras.append((name, cfg)) # 解析前端传来的勾选状态 for i in range(0, len(lora_states), 2): enable = lora_states[i] try: strength = float(lora_states[i + 1]) except (ValueError, TypeError): strength = 1.0 if enable and strength > 0: # 如果前端的数组长度和当前引擎实际可用的 LoRA 列表长度一致 idx = i // 2 if idx < len(valid_loras): lora_name, lora_cfg = valid_loras[idx] selected.append((lora_cfg["filename"], strength)) # 你的探针(保留观察) print(f"[LoRA选择探针] ✅ 成功匹配并加载: {lora_name} ({strength})") return selected def build_workflow( prompt: str, unet_name: str, engine: str, width: int, height: int, steps: int, cfg: float, seed: int, sampler: str, scheduler: str, base_loras: list[tuple[str, float]], char_lora: str | None, char_strength: float, ) -> dict[str, Any]: # --- loader 链:命中缓存则复用(节点 id / inputs 完全一致) --- key = _combo_key(engine, unet_name, base_loras, char_lora, char_strength) cached = _loader_chain_cache.get(key) if cached is not None: chain, current_model_node = cached workflow = dict(chain) # 浅拷贝,下游不改 loader 链 else: chain, current_model_node = _build_loader_chain( engine, unet_name, base_loras, char_lora, char_strength) _loader_chain_cache[key] = (chain, current_model_node) # 只保留最近 8 个组合,避免内存无限增长 if len(_loader_chain_cache) > 8: _loader_chain_cache.pop(next(iter(_loader_chain_cache))) workflow = dict(chain) # --- 下游节点:id 从链末端续接,与原逻辑完全一致 --- nodes = MODEL_CONFIG["node_types"] node_idx = max(int(k) for k in chain.keys()) + 1 final_prompt = prompt.strip() workflow[str(node_idx)] = {"class_type": "CLIPTextEncode", "inputs": {"clip": _ref("2"), "text": final_prompt}} workflow[str(node_idx+1)] = {"class_type": "ConditioningZeroOut", "inputs": {"conditioning": _ref(str(node_idx))}} workflow[str(node_idx+2)] = {"class_type": "EmptyLatentImage", "inputs": {"width": width, "height": height, "batch_size": 1}} workflow[str(node_idx+3)] = {"class_type": "KSampler", "inputs": {"model": _ref(current_model_node), "positive": _ref(str(node_idx)), "negative": _ref(str(node_idx+1)), "latent_image": _ref(str(node_idx+2)), "seed": seed, "steps": steps, "cfg": cfg, "sampler_name": sampler, "scheduler": scheduler, "denoise": 1.0}} workflow[str(node_idx+4)] = {"class_type": "VAEDecode", "inputs": {"samples": _ref(str(node_idx+3)), "vae": _ref("3")}} workflow[str(node_idx+5)] = {"class_type": "SaveImage", "inputs": {"images": _ref(str(node_idx+4)), "filename_prefix": MODEL_CONFIG["name"].lower()}} return workflow def _log_workflow_summary(workflow: dict[str, Any], task_id: str, load_time_val: float, wf_time: float, image: str = None): """独立提取并打印单行 JSON 摘要日志(不影响主流程逻辑)""" try: # 获取当前日期时间 current_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) # 1. 提取 UNet unet_node = next((v for v in workflow.values() if v.get("class_type") in ["UNETLoader", "CheckpointLoaderSimple"]), None) unet_name = unet_node["inputs"].get("unet_name", unet_node["inputs"].get("ckpt_name", "Unknown")) if unet_node else "Unknown" # 2. 提取 VAE vae_node = next((v for v in workflow.values() if v.get("class_type") == "VAELoader"), None) vae_name = vae_node["inputs"].get("vae_name", "Unknown") if vae_node else "Unknown" # 3. 提取 CLIP / TE te_name = "Unknown" clip_node = next((v for v in workflow.values() if v.get("class_type") in ["CLIPLoader", "DualCLIPLoader"]), None) if clip_node: if clip_node["class_type"] == "DualCLIPLoader": te_name = f"{clip_node['inputs'].get('clip_name1')} + {clip_node['inputs'].get('clip_name2')}" else: te_name = clip_node["inputs"].get("clip_name", "Unknown") # 4. 提取 LoRA loras = [] for v in workflow.values(): if v.get("class_type") in ["LoraLoader", "LoraLoaderModelOnly"]: l_name = v["inputs"].get("lora_name", "Unknown").replace(".safetensors", "") l_str = v["inputs"].get("strength_model", 1.0) loras.append(f"{l_name}({l_str})") lora_str = ",".join(loras) if loras else "None" # 5. 提取 Sampler 参数和尺寸 sampler_node = next((v for v in workflow.values() if v.get("class_type") == "KSampler"), None) latent_node = next((v for v in workflow.values() if v.get("class_type") == "EmptyLatentImage"), None) sampler_str, steps, cfg, seed, w, h = "", 0, 0, 0, 0, 0 if sampler_node: s = sampler_node["inputs"] sampler_str = f"{s.get('sampler_name')}/{s.get('scheduler')}" steps, cfg, seed = s.get('steps'), s.get('cfg'), s.get('seed') if latent_node: w, h = latent_node["inputs"].get("width"), latent_node["inputs"].get("height") # 6. 提取 Prompt (不截断,完全显示) prompt_text = "None" for v in workflow.values(): if v.get("class_type") == "CLIPTextEncode": p = v["inputs"].get("text", "") prompt_text = p break # 去除换行符,防止破坏单行展示;如果有单引号,转义一下 prompt_text = prompt_text.replace("\n", " ").replace("'", "\\'") task_short = task_id[:8] if task_id else 'None' # 打印单行 JSON print(f"✅ [{task_short}] {{" f"'date': '{current_time_str}', " f"'unet': '{unet_name}', " f"'te': '{te_name}', " f"'vae': '{vae_name}', " f"'lora': '{lora_str}', " f"'size': '{w}x{h}', " f"'seed': {seed}, " f"'sampler': '{sampler_str} {steps}s cfg{cfg}', " f"'prompt': '{prompt_text}', " f"'load': {load_time_val}s, " f"'total': {wf_time}s, " f"'image': '{image or 'None'}'" f"}}", flush=True) except Exception as e: print(f"[Log Format Error]: {e}", flush=True) def _execute_workflow(workflow: dict[str, Any], task_id: str = None) -> list[str]: print_ram_status("任务开始前") # 新增:任务画像 try: import sys as _sys _cuda_init = torch.cuda.is_initialized() _alloc = torch.cuda.memory_allocated() / (1024**3) _resv = torch.cuda.memory_reserved() / (1024**3) print( f"[任务画像] task={task_id[:8] if task_id else 'None'} " f"cuda_init={_cuda_init} alloc={_alloc:.2f}GB reserved={_resv:.2f}GB " f"py_modules={len(_sys.modules)} " f"executor_ready={'yes' if _exec_state['executor'] else 'no'}", flush=True, ) except Exception as _e: print(f"[任务画像] 失败: {_e}", flush=True) _tqdm_patch_enter() try: loop, executor = _get_executor() prompt_id = str(uuid.uuid4()) save_id = next(k for k, v in workflow.items() if v.get("class_type") == "SaveImage") if task_id and task_id in tasks_status: tasks_status[task_id]["message"] = "🔄 Executing Workflow..." _write_metric(task_id, phase="🔄 Executing Workflow...", load_start=time.time()) with _exec_lock: # ← 串行,复用同一 executor # 清理上一次的 history/status,避免状态串味 executor.history_result = {} executor.status_messages = [] executor.success = True workflow_start = time.time() io0 = _io_probe() m0 = _mem_probe() # (host_cached, host_avail, cg_cur, cg_file, cg_inact) gpu = _gpu_id() v0 = torch.cuda.memory_allocated() / 1024**3 if torch.cuda.is_available() else 0.0 executor.execute(workflow, prompt_id, extra_data={}, execute_outputs=[save_id]) wf_time = round(time.time() - workflow_start, 2) io1 = _io_probe() m1 = _mem_probe() v1 = torch.cuda.memory_allocated() / 1024**3 if torch.cuda.is_available() else 0.0 print( f"[IO探针] {task_id[:8] if task_id else 'None'} " f"disk_read={(io1['read_bytes']-io0['read_bytes'])/1024**3:.2f}GB " f"syscall_read={(io1['read_chars']-io0['read_chars'])/1024**3:.2f}GB " f"cg_cur={m0[2]:.2f}->{m1[2]:.2f}GB " f"cg_file={m0[3]:.2f}->{m1[3]:.2f}GB " f"cg_inact_file={m0[4]:.2f}->{m1[4]:.2f}GB " f"cg_anon={m0[2]-m0[3]:.2f}->{m1[2]-m1[3]:.2f}GB " f"hostAvail={m0[1]}->{m1[1]}MB " f"VRAM={v0:.2f}->{v1:.2f}GB " f"gpu={gpu} wf={wf_time}s", flush=True, ) _write_metric(task_id, workflow=wf_time) # 兜底补录 load_time 逻辑保持不变 load_time_val = 0 if task_id: def _m(data): if "load_time" not in data or data.get("load_time", -1) == -1: data["load_time"] = -1 data["load_time_source"] = "cache_hit" print("[探针] ⚡ 缓存命中(或无需重新加载显存),耗时 < 1s", flush=True) return True return False load_time_val = _update_metric(task_id, _m).get("load_time", 0) # --- 先收路径(失败时为空),再打摘要,保证成功/失败都有日志 --- paths = [] if executor.success: for output in executor.history_result.get("outputs", {}).values(): for items in output.values(): if not isinstance(items, list): continue for item in items: if not isinstance(item, dict) or not item.get("filename"): continue candidate = OUTPUT / item.get("subfolder", "") / item["filename"] if candidate.exists(): paths.append(str(candidate)) if not paths: # ⚠️ 兜底 glob 只在成功时执行,否则会捞到上一次的旧图 paths = sorted([p for p in glob.glob(str(OUTPUT / "**" / "*.png"), recursive=True)], key=lambda p: os.path.getmtime(p), reverse=True) image_name = os.path.basename(paths[0]) if paths else None _log_workflow_summary(workflow, task_id, load_time_val, wf_time, image=image_name) print_ram_status("工作流执行完毕后") if not executor.success: message = executor.status_messages[-1] if executor.status_messages else "ComfyUI execution failed" raise RuntimeError(str(message)) return paths[:1] finally: _tqdm_patch_exit() def prepare_env(char_enable: bool, char_lora: str, unet_choice: str, task_id: str = None): _current_download_task_id.set(task_id) _tqdm_patch_enter() try: engine = UNET_CHOICES.get(unet_choice, {}).get("engine", "MOSS") if char_enable and char_lora: lora_root = MODELS / "loras" dest = lora_root / char_lora if not dest.exists(): file_path = CHAR_LORA_CHOICES.get(engine, {}).get(char_lora, {}).get("path") if file_path: if task_id and task_id in tasks_status: tasks_status[task_id]["message"] = f"📥 Downloading LoRA ({char_lora})..." hf_hub_download(repo_id=MODEL_CONFIG['char_lora_repo'], filename=file_path, local_dir=str(lora_root)) src = lora_root / file_path if src.exists(): shutil.move(str(src), str(dest)) if task_id and task_id in tasks_status: tasks_status[task_id]["message"] = "📦 LoRA downloaded, loading..." finally: _tqdm_patch_exit() _current_download_task_id.set(None) # ========================================================================== # 🔥 GPU-side Residual Warmup Wait (same /tmp channel as _read_metric) # ========================================================================== def _read_warm_state(): try: with open(WARMUP_STATE_FILE) as f: return json.load(f) except Exception: return None def _warm_wait_in_gpu(unet_choice: str, budget: float = WARMUP_RESIDUAL_WAIT) -> bool: """只在"已经非常接近完成"时等,最多 budget 秒。 注意:这段是在 GPU 切片内执行,等待即烧配额,所以必须能提前放弃。""" t0 = time.time() while time.time() - t0 < budget: st = _read_warm_state() if st and st.get("target") == unet_choice: if st.get("state") == "ready": print(f"[预热] ✅ GPU 内等到 ready({time.time() - t0:.2f}s)", flush=True) return True # ⚠️ 进度太低 = 还在冷读,等满 budget 也等不到。立刻放行,省下配额。 p = float(st.get("progress") or 0.0) if p < 0.9: saved = budget - (time.time() - t0) print(f"[预热] ⏭️ 进度仅 {p:.2f},放弃残余等待(省 {saved:.1f}s GPU)", flush=True) return False elif time.time() - t0 > 0.5: # 0.5 秒内 target 还没轮到本模型(比如 sweep 占着),也别等 print( f"[预热] ⏭️ target={st.get('target') if st else None} ≠ 本模型,放弃残余等待", flush=True, ) return False time.sleep(0.25) print(f"[预热] ⏳ GPU 内残余等待超时({budget}s),转冷加载", flush=True) return False # ========================================================================== # 🎬 Generation # ========================================================================== def _generate_impl( prompt: str, unet_choice: str, custom_w: int, custom_h: int, steps: int, cfg: float, seed: int, sampler: str, scheduler: str, char_enable: bool, char_lora: str, char_strength: float, *lora_states, task_id: str = None, ): # ↓↓↓ 原 generate() 的完整函数体从这里开始,一行都不用改 ↓↓↓ _current_generation_task_id.set(task_id) # 疑似bug 1:拿到 ZeroGPU 显卡后,还要先看一眼状态文件再决定要不要干活,结果因为状态文件被后台串台覆写,硬生生白等满 5 秒。 # _warm_wait_in_gpu(unet_choice) # global _previous_arch_for_gc print(f"[Token探针] task={task_id} | token_present={bool(LocalContext.request.get(None) and LocalContext.request.get(None).headers.get('x-ip-token'))}", flush=True) # 获取当前架构 # current_arch = UNET_CHOICES.get(unet_choice, {}).get("engine", "unknown") # 🚫 【已注释】原因:经过极限测试,跨架构排队时会导致 ZeroGPU 重新申请显存,拖慢加载。 # 交给 ComfyUI 原生的 RAM_PRESSURE 机制(lru=8, ram=16.0)平滑过渡效果更好。 # if _previous_arch_for_gc is not None and _previous_arch_for_gc != current_arch: # import gc # if torch.cuda.is_available(): # torch.cuda.empty_cache() # gc.collect() # print(f"[显存清理] 架构切换 {_previous_arch_for_gc} -> {current_arch},已清理碎片", flush=True) # # _previous_arch_for_gc = current_arch if task_id: if task_id not in tasks_status: tasks_status[task_id] = {} now_ts = time.time() start_ts = tasks_status[task_id].get("queue_start_time", now_ts) raw_diff = now_ts - start_ts print(f"[排队探针] 拿到卡时间: {now_ts:.2f}, 排队起点时间: {start_ts:.2f}, 原始差值: {raw_diff:.2f}s", flush=True) q_time = max(0.0, round(raw_diff, 2)) tasks_status[task_id]["metrics_queue"] = q_time tasks_status[task_id]["message"] = "⚙️ Allocating GPU..." _write_metric(task_id, queue=q_time, phase="⚙️ Allocating GPU...") effective_seed = int(seed) unet_info = UNET_CHOICES[unet_choice] unet_name = unet_info["filename"] engine = unet_info["engine"] try: workflow = build_workflow( prompt=prompt, unet_name=unet_name, engine=engine, width=int(custom_w), height=int(custom_h), steps=int(steps), cfg=float(cfg), seed=effective_seed, sampler=sampler, scheduler=scheduler, base_loras=select_base_loras(engine, lora_states), char_lora=char_lora if (char_enable and char_lora) else None, char_strength=float(char_strength), ) print(f"[GPU 探针] 📦 模型节点构建完毕,准备调用 ComfyUI 执行引擎...", flush=True) img_path = _execute_workflow(workflow, task_id)[0] return img_path, workflow except Exception as exc: print(traceback.format_exc(), flush=True) raise gr.Error(f"generation failed: {str(exc)[:500]}") from exc # duration 是上限不是计费;30 → 90 让冷路径不被 30s 直接杀掉 @spaces.GPU(duration=40) def generate(*args, **kwargs): _ctx = LocalContext.request.get(None) if not (_ctx and _ctx.headers.get("x-ip-token")): print("[合规告警] GPU 调用缺少 Gradio 请求上下文 —— 本次不计费!", flush=True) return _generate_impl(*args, **kwargs) # # ========================================================================== # # 🖥️ Gradio UI (Dummy for ZeroGPU) # # ========================================================================== # with gr.Blocks() as demo: # pass # demo.queue() # ========================================================================== # 🧹 Cleanup # ========================================================================== # 专门用于保护 tasks_status 字典的状态锁 _status_lock = threading.RLock() _model_switch_lock = threading.Lock() # ---- Cleanup Policy ---- _STATUS_TTL = 3600 # 终态任务内存保留 60 分钟 _STATUS_MAX_ENTRIES = 200 # 兜底:最多保留 200 条终态 _OUTPUT_TTL = 24 * 3600 # 输出图片保留 24 小时 _cleanup_last = 0.0 def _cleanup_stale(force: bool = False) -> None: """定期清理 tasks_status / 残留 metric / 过期输出图片。每 5 分钟最多跑一次。""" global _cleanup_last now = time.time() if not force and (now - _cleanup_last) < 300: return _cleanup_last = now # 1) tasks_status + 残留 metric with _status_lock: dead = [ tid for tid, st in tasks_status.items() if st.get("status") in ("success", "error") and (now - st.get("finished_at", now)) > _STATUS_TTL ] for tid in dead: tasks_status.pop(tid, None) _last_metric_step.pop(tid, None) try: os.remove(_metric_path(tid)) except Exception: pass if len(tasks_status) > _STATUS_MAX_ENTRIES: finished = sorted( ((tid, st.get("finished_at", 0)) for tid, st in tasks_status.items() if st.get("status") in ("success", "error")), key=lambda x: x[1], ) for tid, _ in finished[: len(tasks_status) - _STATUS_MAX_ENTRIES]: tasks_status.pop(tid, None) _last_metric_step.pop(tid, None) live = {st["filename"] for st in tasks_status.values() if st.get("filename")} # 2) 输出图片(只删顶层 .png/.webp,跳过仍被任务引用的) deleted = freed = 0 for p in OUTPUT.glob("*"): if not p.is_file() or p.suffix.lower() not in (".png", ".webp"): continue if p.name in live: continue try: st = p.stat() if (now - st.st_mtime) > _OUTPUT_TTL: freed += st.st_size p.unlink() deleted += 1 except Exception: continue if deleted: print(f"[清理] 🧹 删除 {deleted} 个过期输出,释放 {freed / 1024 / 1024:.1f} MB", flush=True) def _read_last_unet(): try: with open(WARMUP_LAST_FILE) as f: return json.load(f).get("unet_choice") except Exception: return None # ========================================================================== # 🔥 WarmupManager: File-level Page-cache Warmup State Machine # ========================================================================== class WarmupManager: def __init__(self): self._lock = threading.RLock() self._plan = None self._thread = None self._seq = 0 self._files = {} # target -> {file_key: record} self._resident_at = 0.0 self._resident_unet = None self.boot_id = self._read_boot_id() self.epoch = self._bump_epoch() self._load_state() # 检测换机 → 全部 cold self._resume = None # 被 P0 打断的 P1 计划 self._active = None # 正在执行中的计划(worker 取走后 _plan 为 None) # ---------- 基础设施 ---------- @staticmethod def _cache_budget_mb(): """file cache 可用预算:cgroup 限额的 60% 减去当前匿名页,再受硬上限约束。""" limit = 0 try: with open("/sys/fs/cgroup/memory.max") as f: raw = f.read().strip() if raw != "max": limit = int(raw) / 1024 / 1024 except Exception: pass anon = 0 try: with open("/sys/fs/cgroup/memory.stat") as f: for line in f: p = line.split() if len(p) == 2 and p[0] == "anon": anon = int(p[1]) / 1024 / 1024 except Exception: pass if not limit: return WARMUP_BOOT_CAP_MB return max(0, min(WARMUP_BOOT_CAP_MB, limit * 0.6 - anon)) def probe(self, rec): """廉价探针:随机读 8MB,判断这个文件现在是否已在 cache 里(~5ms)。""" try: size = os.path.getsize(rec["path"]) if size <= WARMUP_PROBE_BYTES: return None off = random.randint(0, size - WARMUP_PROBE_BYTES) t0 = time.time() with open(rec["path"], "rb") as f: f.seek(off) f.read(WARMUP_PROBE_BYTES) mbps = (WARMUP_PROBE_BYTES / 1024 / 1024) / max(1e-6, time.time() - t0) rec["verify_mbps"] = round(mbps, 1) return mbps except Exception: return None @staticmethod def _read_boot_id(): try: with open("/proc/sys/kernel/random/boot_id") as f: return f.read().strip() except Exception: return "unknown" @staticmethod def _bump_epoch(): try: with open(WARMUP_EPOCH_FILE) as f: cur = int(f.read().strip()) except Exception: cur = 0 cur += 1 try: with open(WARMUP_EPOCH_FILE, "w") as f: f.write(str(cur)) except Exception: pass return cur def _load_state(self): """读上次的状态文件:boot_id 不同 → 换机了,全部作废。 ⚠️ 必须整体 try:它在 __init__ 里跑,抛异常会让整个 app 起不来。""" try: st = _read_warm_state() if not st or not isinstance(st, dict): return if st.get("boot_id") != self.boot_id: print("[预热] 🧊 boot_id 变化 → 容器换机,全部置 cold", flush=True) return files_all = st.get("files_all") or {} if not isinstance(files_all, dict): return now = time.time() for target, recs in files_all.items(): if not isinstance(recs, dict): continue for r in recs.values(): if not isinstance(r, dict): continue if r.get("state") == "warm" and (now - r.get("verified_at", 0)) > WARMUP_TTL: r["state"] = "stale" self._files[target] = recs print(f"[预热] ♻️ 继承上次状态(同机),targets={list(self._files)}", flush=True) except Exception as e: print(f"[预热] ⚠️ 状态继承失败,按全冷处理: {e}", flush=True) self._files = {} def _write_state(self, snapshot): try: with self._lock: snapshot["files_all"] = { t: {k: dict(v) for k, v in recs.items()} for t, recs in self._files.items() } tmp = f"{WARMUP_STATE_FILE}.{os.getpid()}.tmp" with open(tmp, "w") as f: json.dump(snapshot, f) os.replace(tmp, WARMUP_STATE_FILE) except Exception as e: print(f"[预热] ⚠️ 状态落盘失败: {e}", flush=True) # ---------- 文件清单 ---------- def _rec(self, path, file_key, role): try: size = os.path.getsize(path) / 1024 / 1024 except Exception: size = 0.0 return { "file_key": file_key, "role": role, "path": str(path), "name": os.path.basename(str(path)), "size_mb": round(size, 1), "state": "cold", "progress": 0.0, "verified_at": 0.0, "warm_ms": 0, "mbps": 0.0, "verify_mbps": 0.0, "gen": 0, "error": None, } def _target_files(self, unet_choice, char_lora=None, char_enable=True): info = UNET_CHOICES.get(unet_choice) if not info: return [] engine = info["engine"] cfg = MODEL_CONFIG["models"].get(engine, {}) out = [] for i, te in enumerate(cfg.get("local_te_filename", [])): out.append(self._rec(MODELS / "text_encoders" / te, f"te{i}", "te")) vae = cfg.get("local_vae_filename", "") if vae: out.append(self._rec(MODELS / "vae" / vae, "vae", "vae")) # ⚠️ 不预热 character LoRA: # · 大 LoRA 已在 UNet merge 阶段吃掉,不单独读文件 # · character LoRA 全部 < 220MB,冷读 < 1s,且 prepare_env 会按需下载 # 少这两类记录 → /api/warmup/status 里不会再出现一堆 skipped 噪音 i = 0 for name, bc in BASE_LORAS_CHOICES.items(): if bc["engine"] == engine: p = MODELS / "loras" / bc["filename"] if p.exists(): # ✅ 存在才预热 out.append(self._rec(p, f"blora{i}", "lora")) i += 1 # if char_enable and char_lora: # out.append(self._rec(MODELS / "loras" / char_lora, "char_lora", "char_lora")) # ✅ UNET 最后追加 → 最后被读 → LRU 上最新鲜,最不容易被回收 out.append(self._rec(MODELS / "diffusion_models" / info["filename"], "unet", "unet")) head = sorted([r for r in out if r["role"] != "unet"], key=lambda r: r["size_mb"]) tail = [r for r in out if r["role"] == "unet"] for r in head + tail: # ✅ 全部 append 完之后再打 owner r["owner"] = unet_choice return head + tail # ---------- 热度判断 ---------- def _is_resident(self, unet_choice): """权重仍在内存/显存里(刚生成过)→ 文件预热无意义。""" return (self._resident_unet == unet_choice and (time.time() - self._resident_at) < 600) def mark_resident(self, unet_choice): self._resident_unet = unet_choice self._resident_at = time.time() def mark_stale(self, unet_choice): """生成完成 → 文件 cache 大概率被挤掉,标记失效。 注意:不动 _resident_at —— 权重还在内存里,这是两件事。""" with self._lock: for r in self._files.get(unet_choice, {}).values(): if r["state"] == "warm": r["state"] = "stale" _REQUIRED_ROLES = ("unet", "te", "vae") @classmethod def _is_required(cls, r): """只有 unet/te/vae 参与"就绪"判定;LoRA 失败不影响 ready。""" return r.get("role") in cls._REQUIRED_ROLES and r.get("size_mb", 0) >= WARMUP_MIN_MB def ready(self, unet_choice): with self._lock: need = [r for r in self._files.get(unet_choice, {}).values() if self._is_required(r)] return bool(need) and all(r["state"] == "warm" for r in need) def progress(self, unet_choice): with self._lock: need = [r for r in self._files.get(unet_choice, {}).values() if self._is_required(r)] total = sum(r["size_mb"] for r in need) or 1 done = sum(r["size_mb"] for r in need if r["state"] in ("warm", "skipped")) return round(done / total, 3) # ---------- 单文件预热 ---------- def _warm_file(self, rec, limit_bytes=None, plan_seq=None, records=None, plan=None): path = pathlib.Path(rec["path"]) if not path.exists(): rec.update(state="failed", error="file not found") return False # ① 内核异步预读:不占用户态带宽,网络卷上也能触发 readahead try: fd = os.open(str(path), os.O_RDONLY) try: os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_WILLNEED) finally: os.close(fd) except Exception: pass # ② 分块顺序读:拿进度 + 确保真的进 cache size = os.path.getsize(str(path)) cap = min(size, limit_bytes) if limit_bytes else size t0, read = time.time(), 0 try: with open(str(path), "rb") as f: while read < cap: if self._superseded(plan_seq): rec.update(state="stale", error="superseded") return False if _gen_inflight.is_set(): # ← 新增:让路给真正的模型加载 time.sleep(0.25) continue chunk = f.read(min(WARMUP_CHUNK, cap - read)) if not chunk: break read += len(chunk) rec["progress"] = round(read / max(1, size), 3) if records is not None and plan is not None: self._flush(plan, records) except Exception as e: rec.update(state="failed", error=str(e)[:120]) return False elapsed = max(0.001, time.time() - t0) rec["warm_ms"] = int(elapsed * 1000) rec["mbps"] = round((read / 1024 / 1024) / elapsed, 1) # ③ 阶梯一级:只标 partial,不算 ready if limit_bytes and cap < size: rec.update(state="partial", error=None) return True # ④ 校验:随机 32MB 测真实热读速度,避免"读了但被挤掉"的假 warm if not self._verify(rec): rec.update(state="stale", error="verify failed (cache evicted?)") return False rec.update(state="warm", progress=1.0, verified_at=time.time(), gen=self.epoch, error=None) return True def _verify(self, rec): try: size = os.path.getsize(rec["path"]) if size <= WARMUP_VERIFY_BYTES: return True off = random.randint(0, size - WARMUP_VERIFY_BYTES) t0 = time.time() with open(rec["path"], "rb") as f: f.seek(off) f.read(WARMUP_VERIFY_BYTES) mbps = (WARMUP_VERIFY_BYTES / 1024 / 1024) / max(1e-6, time.time() - t0) rec["verify_mbps"] = round(mbps, 1) return mbps >= WARMUP_WARM_MBPS except Exception as e: rec["error"] = f"verify: {e}" return False # ---------- 计划调度 ---------- def _superseded(self, plan_seq): # request()/worker 每次都会 self._seq += 1,所以"我的 seq 不再是当前 seq"就是被抢占了 with self._lock: return plan_seq is not None and self._seq != plan_seq def _flush(self, plan, records): # ① 先把 live 进度合并进 self._files —— 否则 ready()/progress()/wait_ready() 全程失效 with self._lock: for rec in records.values(): owner = rec.get("owner") or plan["target"] self._files.setdefault(owner, {})[rec["file_key"]] = rec big = [r for r in records.values() if self._is_required(r)] total = sum(r["size_mb"] for r in big) or 1 done = sum(r["size_mb"] for r in big if r["state"] in ("warm", "skipped")) if big and all(r["state"] == "warm" for r in big): overall = "ready" elif any(r["state"] == "failed" for r in big) and not any(r["state"] == "warm" for r in big): overall = "failed" else: overall = "warming" cur = next((r for r in big if r["state"] in ("warming", "cold")), None) label = f"{cur['role'].upper()} {int(cur['progress'] * 100)}%" if cur else "finalizing" snapshot = { "boot_id": self.boot_id, "epoch": self.epoch, "seq": plan["seq"], "target": plan["target"], "reason": plan.get("reason"), "state": overall, "progress": round(done / total, 3), "done_mb": round(done, 1), "total_mb": round(total, 1), "label": label, "updated_at": time.time(), "files": {k: {kk: v[kk] for kk in ("role", "name", "size_mb", "state", "progress", "mbps", "verify_mbps", "error")} for k, v in records.items()}, } # ② 锁外落盘 + 写 tasks_status —— 避免 self._lock 与 _status_lock 顺序反转 self._write_state(snapshot) tid = plan.get("task_id") if tid: with _status_lock: st = tasks_status.get(tid) if st is not None and st.get("status") == "processing": st["warmup"] = {"state": overall, "progress": snapshot["progress"], "label": label} def _worker(self): while True: with self._lock: plan = self._plan self._plan = None # 疑似bug 2:选了 Krea2,后台暂停了 MOSS;一旦 Krea2 读完,后台立刻又把 MOSS 捞出来继续读,直接把刚进内存的 Krea2 又挤出去了。 # if plan is None and self._resume: # plan, self._resume = self._resume, None # self._seq += 1 # plan = {**plan, "seq": self._seq} self._active = plan if plan is None: time.sleep(0.2) continue try: self._run_plan(plan) except Exception as e: print(f"[预热] ⚠️ plan 异常: {e}", flush=True) finally: with self._lock: if self._active is plan: self._active = None def request_sweep(self, reason="boot_sweep"): """启动全量 sweep:预算内、给"上次用过的模型"预留额度、并让它排最后读。""" budget = self._cache_budget_mb() last = _read_last_unet() # ① 先给 last 预留,保证它一定进预算 last_files = self._target_files(last) if last in UNET_CHOICES else [] last_big = [r for r in last_files if r["size_mb"] >= WARMUP_MIN_MB] reserved = sum(r["size_mb"] for r in last_big) budget_others = max(0.0, budget - reserved) # ② 其余模型按体积升序,先读小的 others = [u for u in UNET_CHOICES if u != last] files, used = [], 0.0 for u in others: for rec in self._target_files(u): if rec["size_mb"] < WARMUP_MIN_MB: continue if used + rec["size_mb"] > budget_others: rec.update(state="skipped", error="over_budget") files.append(rec) continue used += rec["size_mb"] files.append(rec) # ③ last 的 UNET 放最后 → LRU 最新鲜 files.extend(last_big) used += reserved print(f"[预热] 📋 sweep 计划: {len(files)} 文件 / {used:.0f}MB " f"/ 预算 {budget:.0f}MB (为 last 预留 {reserved:.0f}MB)", flush=True) with self._lock: self._seq += 1 self._plan = {"seq": self._seq, "target": "__sweep__", "priority": 1, "files": files, "lite": False, "reason": reason, "task_id": None} if self._thread is None or not self._thread.is_alive(): self._thread = threading.Thread(target=self._worker, daemon=True, name="warmup-worker") self._thread.start() def _run_plan(self, plan): seq, target = plan["seq"], plan["target"] files, lite = plan["files"], plan.get("lite") records = {} t0 = time.time() big_total = sum(r["size_mb"] for r in files if r["size_mb"] >= WARMUP_MIN_MB) print(f"[预热] 🔥 start target={target} reason={plan.get('reason')} " f"files={len(files)} big={big_total:.0f}MB lite={lite}", flush=True) # for rec in files: # if self._superseded(seq): # done_mb = sum( # r.get("size_mb", 0) for r in records.values() # if r.get("state") in ("warm", "skipped") # ) # pct = (done_mb / big_total * 100) if big_total else 0.0 # print( # f"[预热] ⏹️ 被新目标抢占 (seq={seq})," # f"已完成 {done_mb:.0f}MB/{big_total:.0f}MB ({pct:.0f}%)," # f"用时 {time.time() - t0:.1f}s", # flush=True, # ) # break # records[f"{rec['owner']}::{rec['file_key']}"] = rec # # ✅ 预算裁剪:request_sweep() 已判定超预算的记录,直接跳过,别再读 # if rec.get("state") == "skipped": # self._flush(plan, records) # continue # if rec["size_mb"] < WARMUP_MIN_MB: # rec.update(state="skipped", progress=1.0) # self._flush(plan, records) # continue for rec in files: if self._superseded(seq): print(f"[预热] ⏹️ 被新目标抢占 (seq={seq})", flush=True) break # ✅ 预算裁剪:request_sweep() 已判定超预算的记录 # ⚠️ 修复:不写入 records,也不写入 self._files,让它保持 cold 状态 # 否则一旦被标为 skipped,ready() 永远等不到它,后续也不会自动预热 if rec.get("state") == "skipped": continue records[f"{rec['owner']}::{rec['file_key']}"] = rec if rec["size_mb"] < WARMUP_MIN_MB: rec.update(state="skipped", progress=1.0) self._flush(plan, records) continue old = self._files.get(rec["owner"], {}).get(rec["file_key"]) if (old and old["state"] == "warm" and old["size_mb"] == rec["size_mb"] and (time.time() - old.get("verified_at", 0)) < WARMUP_TTL): rec.update({**old, "state": "warm"}) # 直接复用,不重读 self._flush(plan, records) continue old = self._files.get(rec["owner"], {}).get(rec["file_key"]) if (old and old["state"] == "warm" and old["size_mb"] == rec["size_mb"] and (time.time() - old.get("verified_at", 0)) < WARMUP_TTL): rec.update({**old, "state": "warm"}) # 直接复用,不重读 self._flush(plan, records) continue rec["state"] = "warming" limit = WARMUP_LITE_MB * 1024 * 1024 if (lite and rec["role"] == "unet") else None ok = self._warm_file(rec, limit_bytes=limit, plan_seq=seq, records=records, plan=plan) self._flush(plan, records) if not ok and rec["state"] == "failed": print(f"[预热] ⚠️ {rec['name']} 失败: {rec['error']}", flush=True) self._flush(plan, records) big = [r for r in records.values() if r["size_mb"] >= WARMUP_MIN_MB] warm_n = sum(1 for r in big if r["state"] == "warm") skip_n = sum(1 for r in big if r["state"] == "skipped") fail_n = sum(1 for r in big if r["state"] == "failed") read_mb = sum(r["size_mb"] for r in big if r["state"] == "warm") dt = max(0.1, time.time() - t0) print(f"[预热] 📊 {target}: warm={warm_n} skip={skip_n} fail={fail_n} " f"read={read_mb:.0f}MB 用时={dt:.1f}s 均速={read_mb/dt:.0f}MB/s", flush=True) def _publish_ready(self, unet_choice): """短路路径也要刷新状态文件,否则 _warm_wait_in_gpu 认不出"已就绪", 会白等满 5 秒 GPU 预算。""" try: self._write_state({ "boot_id": self.boot_id, "epoch": self.epoch, "target": unet_choice, "state": "ready", "progress": 1.0, "files": {}, "updated_at": time.time(), }) except Exception: pass # ---------- 对外 API ---------- def request(self, unet_choice, char_lora=None, char_enable=True, lite=False, reason="manual", task_id=None, priority=0): if not unet_choice or unet_choice not in UNET_CHOICES: return {"target": None, "error": "unknown unet_choice"} # ① 权重还在内存里 → 文件预热无意义 if reason in ("submit", "select") and self._is_resident(unet_choice): self._publish_ready(unet_choice) return {"target": unet_choice, "skipped": "resident"} # ② 全热且未过期 → 秒回 if self.ready(unet_choice) and not lite: self._publish_ready(unet_choice) return {"target": unet_choice, "skipped": "already_warm"} with self._lock: # P0 抢占正在跑/正在排队的 P1(启动 sweep / touch),进度不丢,稍后续跑 cur = self._plan or self._active # 疑似bug 已经关掉了全库盲扫(boot_sweep),唯一可能的低优先级任务只剩下开机时的 lite 预热(仅 200MB)或心跳 touch。如果还留着这个“死灰复燃”机制,反而会导致你切换新模型后,旧模型的低优先级任务在后台重新复活,白白抢占磁盘 I/O 和容器内存。 # if priority == 0 and cur and cur.get("priority", 0) > 0: # self._resume = cur # 🌟 如果正在预热或排队的就是同一个完整模型,直接复用,不自己打断自己 if cur and cur.get("target") == unet_choice and not cur.get("lite") and not lite: if task_id and not cur.get("task_id"): cur["task_id"] = task_id # 顺便把新任务的 task_id 挂上去更新前端进度条 return {"target": unet_choice, "coalesced": "already_running"} self._seq += 1 self._plan = { "seq": self._seq, "target": unet_choice, "priority": priority, "files": self._target_files(unet_choice, char_lora, char_enable), "lite": lite, "reason": reason, "task_id": task_id, } if self._thread is None or not self._thread.is_alive(): self._thread = threading.Thread(target=self._worker, daemon=True, name="warmup-worker") self._thread.start() return {"target": unet_choice, "queued": True, "seq": self._seq} def wait_ready(self, unet_choice, timeout=WARMUP_PRE_GPU_BUDGET): """申请 GPU 前的短暂等待:只等"已经很接近完成"的情况。""" t0 = time.time() while time.time() - t0 < timeout: if self.ready(unet_choice): return True # 冷启动(进度 <50%)→ 别等,直接放行,让预热与 GPU 排队并行 if time.time() - t0 > 0.5 and self.progress(unet_choice) < 0.5: return False time.sleep(0.2) return self.ready(unet_choice) def snapshot(self): with self._lock: target = self._plan["target"] if self._plan else (self._resident_unet or None) st = _read_warm_state() or {} st.setdefault("epoch", self.epoch) st.setdefault("boot_id", self.boot_id) return st warmup = WarmupManager() def _boot_warmup(): """两段式: ① 阶梯一级(<1s):上次模型的 UNET 头部 → 用户第一眼就有"部分热" ② 全量 sweep(P1,可被抢占):等 ① 跑完再启动,否则会把 ① 顶掉 """ last = _read_last_unet() target = last if last in UNET_CHOICES else next(iter(UNET_CHOICES), None) lite_requested = False if target: rec = warmup._rec(MODELS / "diffusion_models" / UNET_CHOICES[target]["filename"], "unet", "unet") mbps = warmup.probe(rec) if mbps is not None and mbps >= WARMUP_WARM_MBPS: print(f"[预热] ⚡ boot: UNET 已在 cache 中 ({mbps:.0f}MB/s),跳过 lite", flush=True) else: print(f"[预热] 🥾 boot: UNET 探测 {mbps} MB/s → 做阶梯一级", flush=True) warmup.request(target, lite=True, reason="boot", priority=1) lite_requested = True # ← 新增 if not WARMUP_BOOT_FULL: return if lite_requested: # ← 只在发起了才等 t0 = time.time() while time.time() - t0 < 20: with warmup._lock: r = warmup._files.get(target, {}).get("unet") if r and r.get("state") in ("partial", "warm", "failed", "stale"): break time.sleep(0.2) # 疑似bug:服务器一启动,不管你用不用,后台就会把 MOSS、Flux、Krea2 等所有模型按顺序全读一遍。 # 注释掉后,启动时只预热你上次用过的那 1 个模型(_boot_warmup 的第一段阶梯预热依然保留)。 # warmup.request_sweep(reason="boot_sweep") # ========================================================================== # 🌐 FastAPI & API Routes # ========================================================================== class GenerateRequest(BaseModel): prompt: str unet_choice: str custom_w: int = 1024 custom_h: int = 1024 steps: int = 8 cfg: float = 1.0 seed: int = -1 sampler: str = "euler" scheduler: str = "simple" char_enable: bool = True char_lora: str = "" char_strength: float = 1.0 base_lora_states: list = [] app = FastAPI() app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str = None, request: Request = None): global _last_used_architecture, _last_used_unet _current_generation_task_id.set(task_id) _gen_entered = False # ← 新增 # 初始状态 with _status_lock: tasks_status[task_id] = {"status": "processing", "message": "⏳ Queue for GPU..."} print(f"[Token探针] task={task_id[:8]} | route_token_present={bool(ip_token)}", flush=True) # --- 阶段 0:预热(与 GPU 排队并行,不阻塞 HTTP) --- try: _wres = warmup.request( req.unet_choice, req.char_lora if req.char_enable else None, char_enable=req.char_enable, reason="submit", task_id=task_id, ) if not (_wres or {}).get("skipped"): # ← 只在真的排了任务时才显示预热阶段 with _status_lock: tasks_status[task_id]["message"] = "🔥 Preparing model files..." # 只在"接近完成"时短暂等待;冷启动立即放行,避免把 GPU 排队串到预热后面 _warm_ok = warmup.wait_ready(req.unet_choice, timeout=WARMUP_PRE_GPU_BUDGET) print(f"[预热] gate ready={_warm_ok} progress={warmup.progress(req.unet_choice)}", flush=True) except Exception as _e: print(f"[预热] ⚠️ gate 异常: {_e}", flush=True) try: # --- 阶段 1:环境准备与模型下载 --- with _status_lock: tasks_status[task_id]["message"] = "📥 Downloading LoRA / Preparing environment..." download_start = time.time() prepare_env(req.char_enable, req.char_lora, req.unet_choice, task_id=task_id) real_download_time = round(time.time() - download_start, 2) current_arch = UNET_CHOICES.get(req.unet_choice, {}).get("engine", "unknown") need_heavy_load = (_last_used_architecture != current_arch) or (_last_used_unet != req.unet_choice) if need_heavy_load: with _model_switch_lock: if _last_used_architecture != current_arch: # 状态提示保留,卸载动作已移至 generate() 内部 with _status_lock: tasks_status[task_id]["message"] = f"🔍 Resolving Base Model: {current_arch.upper()}..." _last_used_architecture = current_arch _last_used_unet = req.unet_choice elif _last_used_unet != req.unet_choice: with _status_lock: tasks_status[task_id]["message"] = "🔄 Swapping Model Checkpoint..." _last_used_unet = req.unet_choice else: with _status_lock: tasks_status[task_id]["message"] = "🎨 GPU processing..." final_seed = req.seed if req.seed >= 0 else random.randint(0, 2**32 - 1) with _status_lock: tasks_status[task_id]["seed"] = final_seed tasks_status[task_id]["queue_start_time"] = time.time() tasks_status[task_id]["message"] = "⏳ Requesting GPU resources..." print(f"[GPU 探针] 🔴 发起 Request,申请 GPU 资源...", flush=True) ctx = contextvars.copy_context() def _run_generate(): if request is not None: LocalContext.request.set(request) return generate( req.prompt, req.unet_choice, req.custom_w, req.custom_h, req.steps, req.cfg, final_seed, req.sampler, req.scheduler, req.char_enable, req.char_lora, req.char_strength, *req.base_lora_states, task_id=task_id, ) # --- 阶段 2:核心执行(阻塞式,前端此时只看到进度条走动) --- _gen_enter() # ← 新增 _gen_entered = True # ← 新增 image_path, real_workflow = ctx.run(_run_generate) # --- 阶段 3:死磕后处理(前端此时处于静默等待,避免出现“快好了”的错觉) --- from PIL import Image original_filename = os.path.basename(image_path) webp_filename = original_filename.replace(".png", ".webp") output_webp_path = os.path.join(OUTPUT, webp_filename) with Image.open(image_path) as img: metadata = img.info PROMPT_TAG = 0x0110 EXTRA_METADATA_TAG = 0x010f output_metadata = img.getexif() for key, value in metadata.items(): if key == 'prompt': output_metadata[PROMPT_TAG] = "Prompt:" + value elif key == 'workflow': output_metadata[EXTRA_METADATA_TAG] = "Workflow:" + value EXTRA_METADATA_TAG -= 1 if img.mode in ("RGBA", "P"): img = img.convert("RGB") # 🌟 性能妥协:method=4, quality=90,速度极快,肉眼无感 img.save( output_webp_path, "WEBP", lossless=False, quality=90, method=4, exif=output_metadata ) try: os.remove(image_path) except Exception: pass image_url = f"/output/{webp_filename}" try: _webp_kb = os.path.getsize(output_webp_path) / 1024 except Exception: _webp_kb = -1 print(f"[产出] task={task_id[:8]} url={image_url} file={webp_filename} " f"size={_webp_kb:.0f}KB seed={final_seed}", flush=True) # --- 阶段 4:提取指标 --- final_queue = 0 final_workflow = 0 final_load_time = 0 if os.path.exists(_metric_path(task_id)): saved_metrics = _read_metric(task_id) final_queue = saved_metrics.get("queue", 0) final_workflow = saved_metrics.get("workflow", 0) final_load_time = saved_metrics.get("load_time", 0) try: os.remove(_metric_path(task_id)) except Exception: pass # --- 阶段 5:高光时刻!直接抛出最终结果,抛弃一切中间态日志 --- with _status_lock: tasks_status[task_id] = { "status": "success", "image": image_url, "filename": webp_filename, "seed": final_seed, "finished_at": time.time(), "metrics": { "download": real_download_time, "queue": final_queue, "workflow": final_workflow, "load_time": final_load_time, "steps": req.steps, "resolved_seed": final_seed }, "anonymous": ip_token is None, } # --- 阶段 6:回温(在锁外执行) --- # ① 记住这次用的模型 → 下次启动 boot sweep 把它排最后(LRU 最新鲜) try: with open(WARMUP_LAST_FILE, "w") as f: json.dump({"unet_choice": req.unet_choice, "at": time.time()}, f) except Exception: pass # ② 模型加载必然把 file cache 挤掉,所以"刚生成完"这一刻文件往往已经不热了。 # 后台补一次 WILLNEED,保证同一会话的下一次点击是热的。 try: warmup.mark_resident(req.unet_choice) # 权重在内存,下一次 submit 可跳过预热 warmup.mark_stale(req.unet_choice) # 但文件 cache 大概率已被挤掉 warmup.request(req.unet_choice, req.char_lora if req.char_enable else None, char_enable=req.char_enable, reason="post_gen") except Exception as _e: print(f"[预热] ⚠️ 回温异常: {_e}", flush=True) except Exception as e: with _status_lock: tasks_status[task_id] = { "status": "error", "message": str(e), "finished_at": time.time(), # ← 新增 } finally: if _gen_entered: # ← 新增 _gen_leave() _current_generation_task_id.set(None) # ========================================================================== # 🖥️ Gradio GPU Endpoint —— ZeroGPU 归因必须发生在 Gradio 的请求栈里 # ========================================================================== GRADIO_CALL_URL = os.environ.get( "GRADIO_CALL_URL", "http://127.0.0.1:7861/gradio_api/call/generate" ) def _gpu_generate(task_id: str, payload_json: str) -> str: """Gradio 侧入口:同步跑完整个生成。进度仍由前端轮询 /api/status 读取。""" # ⚠️ 合规闸门:没有 ZeroGPU 凭据时默认拒绝; # ZG_ENFORCE=0 时只告警不拦截(排查/应急用) _ctx = LocalContext.request.get(None) _tok = _ctx.headers.get("x-ip-token") if _ctx else None if not _tok: print(f"[合规] task={task_id[:8]} 无 ZeroGPU 凭据 (enforce={ZG_ENFORCE})", flush=True) if ZG_ENFORCE: with _status_lock: tasks_status[task_id] = { "status": "error", "message": "GPU 仅在 huggingface.co 页面内可用。请勿直接访问 .hf.space —— " "该入口无法关联到账号额度,因此已被禁用。", "finished_at": time.time(), } return task_id # ZG_ENFORCE=0 → 只打日志,继续往下跑 try: req = GenerateRequest(**json.loads(payload_json)) except Exception as e: with _status_lock: tasks_status[task_id] = { "status": "error", "message": f"bad payload: {e}", "finished_at": time.time(), } return task_id # ⚠️ ip_token 和 request 必须传 None。 # ZeroGPU 的计费凭据来自 Gradio 队列工作线程自带的请求上下文, # 手动把 FastAPI 的 Request 塞进 LocalContext 不会被 @spaces.GPU 认。 # 一旦改成传值,会静默退化成不计费(日志上仍显示 token_present=True)。 background_generate_task(task_id, req, None, None) return task_id with gr.Blocks() as demo: _g_tid = gr.Textbox(visible=False, value="") _g_payload = gr.Textbox(visible=False, value="") _g_out = gr.Textbox(visible=False) _g_btn = gr.Button(visible=False) _g_btn.click(_gpu_generate, [_g_tid, _g_payload], _g_out, api_name="generate") demo.queue(default_concurrency_limit=1) @app.on_event("startup") async def _on_startup(): def _delayed(): time.sleep(WARMUP_BOOT_DELAY) # 让第一个 /api/health 秒回 try: _boot_warmup() except Exception as e: print(f"[预热] ⚠️ boot 异常: {e}", flush=True) threading.Thread(target=_delayed, daemon=True).start() @app.get("/api/models") async def get_models(): models_data = {} global_default_prompt = MODEL_CONFIG.get("default_prompt", "") for unet_name, unet_info in UNET_CHOICES.items(): engine = unet_info["engine"] engine_name = MODEL_CONFIG["models"].get(engine, {}).get("name", engine) if " | " in unet_name: model_specific_name = unet_name.split(" | ", 1)[1] else: model_specific_name = unet_name display_unet_name = f"{engine_name} | {model_specific_name}" base_loras_for_engine = [] for name, cfg in BASE_LORAS_CHOICES.items(): if cfg["engine"] == engine: base_loras_for_engine.append({ "name": name, "filename": cfg["filename"], "defaultEnable": cfg.get("defaultEnable", False) }) engine_cfg = MODEL_CONFIG["models"].get(engine, {}) final_prompt = engine_cfg.get("default_prompt_override", global_default_prompt) # 🌟 核心新增:提取 TE 和 VAE 信息 # TE 可能是列表,用 " + " 连接起来展示 te_list = engine_cfg.get("local_te_filename", []) te_display = " + ".join([f.replace(".safetensors", "") for f in te_list]) if te_list else "Unknown" # VAE 通常是字符串 vae_display = engine_cfg.get("local_vae_filename", "Unknown").replace(".safetensors", "") w_state = ("resident" if (warmup._resident_unet == unet_name and (time.time() - warmup._resident_at) < 600) else "warm" if warmup.ready(unet_name) else "cold") models_data[display_unet_name] = { "engine": engine, "defaults": engine_cfg.get("defaults", {}), "base_loras": base_loras_for_engine, "default_prompt": final_prompt, "te_name": te_display, # 👈 传给前端 "vae_name": vae_display, # 👈 传给前端 "warm_state": w_state, # resident / warm / cold } return {"models": models_data} @app.get("/api/models/warm") async def api_models_warm(): now = time.time() out = {} for name in UNET_CHOICES: out[name] = ("resident" if (warmup._resident_unet == name and (now - warmup._resident_at) < 600) else "warm" if warmup.ready(name) else "cold") return {"warm": out} @app.get("/api/loras") async def get_loras(): # ⚠️ load_char_loras_from_repo 里有同步 requests.get + list_repo_files(网络 IO)。 # 直接在 async 路由里调用会阻塞 event loop,期间所有请求(含 /api/config) # 都无法处理,HF 代理超时后会 reset 连接 → 前端表现为 "Failed to fetch"。 # 扔进线程池执行,event loop 保持畅通。 await asyncio.to_thread(load_char_loras_from_repo) return {"loras": CHAR_LORA_CHOICES} from fastapi.responses import JSONResponse # 若文件顶部已导入可删这行 @app.get("/api/prompt") def api_prompt(path: str = "", file: str = ""): """ 取单条预设。 ?path=<models.json 的 key> 推荐,唯一键 ?file=<filename> 兜底;同文件名多条 → 409 ambiguous,绝不猜 """ idx = _load_prompts_index() key = (path or "").strip().strip("/") if key: txt = idx.get(key) if txt: return {"ok": True, "path": key, "prompt": txt, "total": len(idx)} # 大小写不敏感兜底(分享链接被人手改过的情况) low = key.lower() for k, v in idx.items(): if k.lower() == low: return {"ok": True, "path": k, "prompt": v, "total": len(idx), "normalized": True} return JSONResponse(status_code=404, content={"ok": False, "reason": "not_found", "path": key, "total": len(idx)}) name = (file or "").strip() if name: low = name.lower() hits = [(k, v) for k, v in idx.items() if k.split("/")[-1].lower() == low] if len(hits) == 1: return {"ok": True, "path": hits[0][0], "prompt": hits[0][1], "total": len(idx)} if len(hits) > 1: return JSONResponse(status_code=409, content={"ok": False, "reason": "ambiguous", "candidates": [k for k, _ in hits[:5]]}) return JSONResponse(status_code=404, content={"ok": False, "reason": "not_found", "file": name, "total": len(idx)}) return JSONResponse(status_code=400, content={"ok": False, "reason": "missing_param"}) @app.get("/api/config") async def get_sampler_config(): return { "samplers": SAMPLERS, "schedulers": SCHEDULERS, "root_mapping": MODEL_CONFIG.get("json_keys", {}).get("root_mapping", {}), "default_prompt": MODEL_CONFIG.get("default_prompt", ""), } @app.get("/api/health") async def health_check(): return { "status": "ok", "frontend": { "sync_ok": FRONTEND_SYNC_OK, "error": FRONTEND_SYNC_ERR, # 有缓存但本次没同步成功 = 页面能开,但你刚 push 的改动不在里面 "stale": (not FRONTEND_SYNC_OK) and bool(FRONTEND_MANIFEST), "files": FRONTEND_MANIFEST, }, } @app.post("/api/generate") async def handle_generate(req: GenerateRequest, background_tasks: BackgroundTasks, request: Request): # ---- 服务端硬校验(对齐 index.html 的 min/max)---- import math problems = [] if not (16 <= req.custom_w <= 1536) or req.custom_w % 8: problems.append(f"width 需在 16–1536(当前 {req.custom_w})") if not (16 <= req.custom_h <= 1536) or req.custom_h % 8: problems.append(f"height 需在 16–1536(当前 {req.custom_h})") if not (4 <= req.steps <= 30): problems.append(f"steps 需在 4–30(当前 {req.steps})") if not (1.0 <= req.cfg <= 10.0): problems.append(f"cfg 需在 1–10(当前 {req.cfg})") if not req.prompt or not req.prompt.strip(): problems.append("prompt 不能为空") elif len(req.prompt) > 4000: problems.append(f"prompt 超长({len(req.prompt)} > 4000 字符)") if not math.isfinite(req.char_strength): problems.append(f"char_strength 必须是有限数值(当前 {req.char_strength})") if req.seed != -1 and not (0 <= req.seed <= 2**32 - 1): problems.append(f"seed 需为 -1 或 0–4294967295(当前 {req.seed})") if len(req.base_lora_states) % 2 != 0: problems.append("base_lora_states 必须是 [enabled, strength, ...] 的偶数长度数组") if problems: raise HTTPException(status_code=400, detail=";".join(problems)) if req.char_enable and req.char_lora: if not req.char_lora.endswith(".safetensors"): req.char_lora = f"{req.char_lora}.safetensors" _cleanup_stale() # 顺手清理,每 5 分钟最多一次 task_id = str(uuid.uuid4()) ip_token = request.headers.get("x-ip-token") tasks_status[task_id] = {"status": "queued", "message": "⏳ 任务已进入队列..."} # 把 ZeroGPU 相关的头原样转交给本地 Gradio,让 GPU 调用落在 Gradio 的请求栈里 _fwd = { k: v for k, v in request.headers.items() if k.lower() in ("x-ip-token", "x-zerogpu-token", "x-zerogpu-uuid") } def _fire(): try: r = requests.post( GRADIO_CALL_URL, json={"data": [task_id, req.model_dump_json()]}, headers=_fwd, timeout=10, ) if r.status_code != 200: print(f"[gradio] 转交失败 HTTP {r.status_code}: {r.text[:200]}", flush=True) with _status_lock: if tasks_status.get(task_id, {}).get("status") == "queued": tasks_status[task_id] = { "status": "error", "message": f"GPU 调度入口异常 (HTTP {r.status_code})", "finished_at": time.time(), } except Exception as e: print(f"[gradio] 转交异常: {type(e).__name__}: {e}", flush=True) with _status_lock: if tasks_status.get(task_id, {}).get("status") == "queued": tasks_status[task_id] = { "status": "error", "message": f"GPU 调度入口不可达: {e}", "finished_at": time.time(), } threading.Thread(target=_fire, daemon=True).start() # 仍然瞬间返回,前端照常轮询 /api/status return {"task_id": task_id, "status": "queued"} @app.get("/api/status/{task_id}") async def check_status(task_id: str): status_data = dict(tasks_status.get(task_id, {"status": "not_found"})) if status_data.get("status") == "processing": msg = status_data.get("message", "") w = status_data.get("warmup") # 预热阶段优先显示,但不覆盖已经进入采样/加载的文案 if (w and w.get("state") == "warming" and not status_data.get("queue_start_time") # ← 还没开始申请 GPU and not msg.startswith(("🎨", "🔄 Executing", "📦"))): status_data["message"] = ( f"🔥 Preparing model files: {w.get('label', '...')} " f"({int(w.get('progress', 0) * 100)}%)" ) else: file_data = _read_metric(task_id) if "phase" in file_data: status_data["message"] = file_data["phase"] return status_data # ========================================================================== # 🔥 Warmup Endpoints # ========================================================================== class WarmupRequest(BaseModel): unet_choice: str char_lora: str = "" char_enable: bool = True mode: str = "full" # lite(进入界面) | full(选中模型) | touch(心跳) reason: str = "manual" # 公开接口的防滥用冷却:同一目标 3s 内只接受一次,全局 1s 内只接受一次计划变更 _warmup_req_log: dict[str, float] = {} _warmup_req_lock = threading.Lock() WARMUP_REQ_COOLDOWN = 3.0 WARMUP_REQ_GLOBAL = 0.3 # 原 1.0 太激进,会吞掉 init→选模型 的合法连发 _warmup_last_accept = 0.0 @app.post("/api/warmup") async def api_warmup(req: WarmupRequest): global _warmup_last_accept if req.unet_choice not in UNET_CHOICES: raise HTTPException(status_code=400, detail="unknown unet_choice") now = time.time() key = f"{req.unet_choice}|{req.mode}|{req.char_lora}" with _warmup_req_lock: if now - _warmup_req_log.get(key, 0.0) < WARMUP_REQ_COOLDOWN: return {"status": "coalesced", "target": req.unet_choice} if now - _warmup_last_accept < WARMUP_REQ_GLOBAL: return {"status": "coalesced", "target": req.unet_choice} _warmup_req_log[key] = now _warmup_last_accept = now # 顺带清理,防止字典无限增长 if len(_warmup_req_log) > 200: for k in [k for k, t in _warmup_req_log.items() if now - t > 60]: _warmup_req_log.pop(k, None) res = warmup.request( req.unet_choice, req.char_lora or None, char_enable=req.char_enable, lite=(req.mode == "lite"), reason=req.reason, priority=0 if req.reason in ("select", "enter", "submit") else 1, ) return {"status": "accepted", **res} @app.get("/api/warmup/status") async def api_warmup_status(): return warmup.snapshot() # ========================================================================== # 🔧 Warmup Debug Endpoint (disabled by default) # Set WARMUP_DEBUG_TOKEN in Space Secrets; send X-Debug-Token header or ?token= # ========================================================================== import hmac WARMUP_DEBUG_TOKEN = os.environ.get("WARMUP_DEBUG_TOKEN", "").strip() def _require_debug_token( x_debug_token: str = Header(None, alias="X-Debug-Token"), token: str = Query(None), ): """三层守卫: ① 没配 WARMUP_DEBUG_TOKEN → 直接 404,接口等同不存在(外部无法探测) ② 配了但请求没带 token → 403 ③ token 不匹配 → 403(用 compare_digest 防时序攻击) """ if not WARMUP_DEBUG_TOKEN: raise HTTPException(status_code=404, detail="Not Found") provided = (x_debug_token or token or "").strip() if not provided or not hmac.compare_digest(provided, WARMUP_DEBUG_TOKEN): raise HTTPException(status_code=403, detail="forbidden") @app.post("/api/debug/warmup", dependencies=[Depends(_require_debug_token)]) async def debug_warmup( action: str = Query("status"), unet_choice: str = Query(""), mode: str = Query("full"), ): """仅用于测试预热状态机。生产环境不配 WARMUP_DEBUG_TOKEN 即自动消失。""" def _all_files(u: str): return warmup._target_files(u) if u in UNET_CHOICES else [] # ---------- status:完整快照(含内存里的 _files) ---------- if action == "status": snap = warmup.snapshot() with warmup._lock: snap["in_memory_targets"] = list(warmup._files.keys()) snap["plan"] = { "seq": warmup._plan["seq"], "target": warmup._plan["target"], "reason": warmup._plan.get("reason"), "priority": warmup._plan.get("priority"), } if warmup._plan else None snap["resume"] = bool(warmup._resume) snap["resident"] = {"unet": warmup._resident_unet, "age_s": round(time.time() - warmup._resident_at, 1)} return snap # ---------- info:环境/预算/文件清单 ---------- if action == "info": models = {} for u, cfg in UNET_CHOICES.items(): files = [] for r in _all_files(u): files.append({ "name": r["name"], "role": r["role"], "size_mb": r["size_mb"], "exists": os.path.exists(r["path"]), }) models[u] = { "engine": cfg.get("engine"), "total_mb": round(sum(f["size_mb"] for f in files), 1), "files": files, } return { "boot_id": warmup.boot_id, "epoch": warmup.epoch, "seq": warmup._seq, "cgroup_budget_mb": round(warmup._cache_budget_mb(), 1), "warmup_min_mb": WARMUP_MIN_MB, "warmup_ttl_s": WARMUP_TTL, "warm_mbps_threshold": WARMUP_WARM_MBPS, "state_file": WARMUP_STATE_FILE, "last_file": WARMUP_LAST_FILE, "last_unet": _read_last_unet(), "models": models, } # ---------- probe:这个模型现在热不热(返回 MB/s,>=600 视为热) ---------- if action == "probe": if not unet_choice: raise HTTPException(400, "unet_choice required") info = UNET_CHOICES.get(unet_choice) if not info: raise HTTPException(400, "unknown unet_choice") out = {"unet": unet_choice, "files": {}} for r in _all_files(unet_choice): if r["size_mb"] < WARMUP_MIN_MB: continue out["files"][r["name"]] = warmup.probe(r) # MB/s 或 None return out # ---------- evict:把指定模型的文件踢出 page cache(造"冷") ---------- if action in ("evict", "evict_all"): targets = list(UNET_CHOICES) if action == "evict_all" else [unet_choice] evicted = [] for u in targets: for r in _all_files(u): if r["size_mb"] < WARMUP_MIN_MB: continue try: fd = os.open(r["path"], os.O_RDONLY) try: os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) finally: os.close(fd) evicted.append(r["name"]) except Exception as e: print(f"[debug] evict {r['name']} 失败: {e}", flush=True) with warmup._lock: warmup._files.pop(u, None) # 同步把状态文件改成 cold,避免 GPU 内残余等待误判为 ready warmup._write_state({ "boot_id": warmup.boot_id, "epoch": warmup.epoch, "target": unet_choice or "__all__", "state": "cold", "progress": 0.0, "files": {}, "updated_at": time.time(), }) return {"evicted_count": len(evicted), "evicted": evicted} # ---------- age:把所有 verified_at 往前拨 1 小时 → 触发 TTL 过期 ---------- if action == "age": n = 0 with warmup._lock: for recs in warmup._files.values(): for r in recs.values(): if r.get("verified_at"): r["verified_at"] -= 3600 n += 1 warmup._write_state(_read_warm_state() or {}) # 用已老化的内存覆盖 files_all return {"aged_records": n} # ---------- fake_boot_id:伪造"换机",测试全部置 cold ---------- if action == "fake_boot_id": st = _read_warm_state() or {} st["boot_id"] = "deadbeef" * 4 warmup._write_state(st) return {"boot_id": st["boot_id"], "note": "重启进程后生效"} # ---------- reset_state:清空 /tmp 状态文件(完全冷启动) ---------- if action == "reset_state": removed = [] for p in (WARMUP_STATE_FILE, WARMUP_EPOCH_FILE): try: os.remove(p) removed.append(p) except Exception: pass with warmup._lock: warmup._files.clear() warmup._plan = None warmup._resume = None return {"removed": removed} # ---------- warm:手动触发一次预热(等同前端"选中模型") ---------- if action == "warm": if unet_choice not in UNET_CHOICES: raise HTTPException(400, "unknown unet_choice") res = warmup.request( unet_choice, None, char_enable=False, lite=(mode == "lite"), reason="debug", priority=0, ) return {"status": "accepted", **res} raise HTTPException(400, f"unknown action: {action}") # ========================================================================== # 📁 Static Files Sync & Mount # ========================================================================== from model_config import MODEL_CONFIG, SYNC_FROM_GUI_SPACE UI_SPACE_URL = MODEL_CONFIG.get("ui_space_url", "").rstrip("/") FRONTEND_CACHE_DIR = ROOT / "frontend_cache" # 🌟 指向新文件夹 # ---- 前端指纹:让"这一版到底拉到新前端没有"变成机器可回答 ---- # 动机:同步失败时旧缓存仍可服务(页面正常打开), # 肉眼无法区分"没生效"和"没拉到",这是排查黑洞。 FRONTEND_MANIFEST = {} # {"index.html": {"md5": "a1b2c3d4", "size": n, "mtime": t}, ...} FRONTEND_SYNC_OK = False # 本次启动三个文件是否全部拉取成功 FRONTEND_SYNC_ERR = "" # 失败摘要(供 /api/health 展示) def sync_ui_assets(): global FRONTEND_MANIFEST, FRONTEND_SYNC_OK, FRONTEND_SYNC_ERR if not SYNC_FROM_GUI_SPACE: print("📦 [Frontend Cache] SYNC_FROM_GUI_SPACE = False,跳过同步,使用本地缓存。") return if not UI_SPACE_URL: print("⚠️ 未配置 ui_space_url,跳过前端同步。") return FRONTEND_CACHE_DIR.mkdir(parents=True, exist_ok=True) files_to_sync = ["index.html", "style.css", "app.js"] print(f"🔄 正在从 {UI_SPACE_URL} 同步前端文件到 /frontend_cache...") ok = True errs = [] for f in files_to_sync: try: res = requests.get(f"{UI_SPACE_URL}/{f.lstrip('/')}?t={int(time.time())}", timeout=10) if res.status_code == 200: (FRONTEND_CACHE_DIR / f).write_text(res.text, encoding="utf-8") print(f"✅ 已更新: {f}") else: ok = False; errs.append(f"{f}=HTTP{res.status_code}") print(f"⚠️ 同步 {f} 失败,状态码: {res.status_code}") except Exception as e: ok = False; errs.append(f"{f}={type(e).__name__}") print(f"⚠️ 同步 {f} 网络异常: {e}") FRONTEND_SYNC_OK = ok FRONTEND_SYNC_ERR = ";".join(errs) # 指纹:内容 md5(前8位) + size + mtime —— 三者任一变化都能被看出来 man = {} for f in files_to_sync: p = FRONTEND_CACHE_DIR / f if not p.exists(): continue data = p.read_bytes() st = p.stat() man[f] = { "md5": hashlib.md5(data).hexdigest()[:8], "size": st.st_size, "mtime": int(st.st_mtime), } FRONTEND_MANIFEST = man fingerprint = " | ".join(f"{k}={v['md5']}" for k, v in man.items()) if ok: print(f"🧩 [Frontend] 指纹: {fingerprint}") else: # 🚨 大字告警:这次没拉到,正在用旧缓存 → 你改的东西不会生效 print(f"🚨 [Frontend] 同步不完整,正在使用旧缓存!({FRONTEND_SYNC_ERR})") print(f"🚨 [Frontend] 实际生效的指纹: {fingerprint}") # 1. 启动时执行同步 sync_ui_assets() # 🌟 核心修改:必须在挂载根路径 "/" 之前,先挂载 /output 目录 app.mount("/output", StaticFiles(directory=str(OUTPUT)), name="output") # 2. 挂载静态文件夹 (前端) if (FRONTEND_CACHE_DIR / "index.html").exists(): # html=True 会让根路径自动返回 index.html app.mount("/", StaticFiles(directory=str(FRONTEND_CACHE_DIR), html=True), name="frontend_cache") else: @app.get("/") async def no_ui(): return {"error": "UI files not found in /frontend_cache."} # ========================================================================== # 🚀 Server Startup # ========================================================================== if __name__ == "__main__": os.environ["GRADIO_SSR_MODE"] = "false" # 2. 启动 ZeroGPU 探针 def trigger_gpu_scan(): demo.launch( server_name="127.0.0.1", server_port=7861, share=False, prevent_thread_lock=True, ) print("🚀 Triggering ZeroGPU scan on 7861...") threading.Thread(target=trigger_gpu_scan, daemon=True).start() # 3. 等 ZeroGPU 探针(7861)真正 listen —— 端口探测比固定 sleep 更快也更稳 def _wait_port(port: int, timeout: float = 12.0) -> bool: t0 = time.time() while time.time() - t0 < timeout: try: with socket.create_connection(("127.0.0.1", port), timeout=0.3): print(f"✅ 端口 {port} 已就绪 ({round(time.time() - t0, 2)}s)") return True except OSError: time.sleep(0.15) print(f"⚠️ 端口 {port} 等待超时,继续启动主服务") return False # 不是"必要步骤":探测失败必须降级继续,绝不能杀掉启动 try: _wait_port(7861) except Exception as _e: print(f"⚠️ 端口探测异常,按固定 sleep 兜底: {type(_e).__name__}: {_e}") time.sleep(4) # 4. 最后启动主 API 服务 print("🌐 Starting main API Gateway on 7860...") uvicorn.run(app, host="0.0.0.0", port=7860, log_level="info")
"""Universal Lora Studio: Comfy Render Backend + FastAPI Data Backend + Native Static Frontend + ZeroGPU Token Passthrough + Config Driven"""
"""Comfy Version: c1eece62cbf1805d4f64286d7f5592e6a215e6ba"""
import base64
import asyncio
import hashlib
import glob
import json
import os
import pathlib
import random
import re
import shutil
import socket
import subprocess
import sys
import traceback
import uuid
import time
import contextvars
import logging
from typing import Any
import requests
from huggingface_hub import hf_hub_download, list_repo_files
from tqdm.auto import tqdm
import tqdm as tqdm_lib
import gradio as gr
from gradio.context import LocalContext
from fastapi import FastAPI, BackgroundTasks, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
import uvicorn
import threading
from model_config import MODEL_CONFIG
from fastapi import HTTPException, Depends, Header, Query
# ⚠️ 必须在 load_char_loras_from_repo() 之前定义 —— 它内部要读这个常量。
# 放在文件后面的"静态挂载"段里会导致启动首次扫描 NameError,触发词全空且被 300s 缓存锁住。
LORA_META_URL = "https://huggingface.co/spaces/waveshaker/loras_gallery/resolve/main/models.json"
# ==========================================================================
# 🛠️ Environment Patches & Global Paths
# ==========================================================================
import builtins
_raw_space_id = os.environ.get("SPACE_ID", "local_space")
SPACE_NAME = _raw_space_id.split("/")[-1]
_original_print = builtins.print
def _prefixed_print(*args, **kwargs):
if args:
args = (f"[{SPACE_NAME}] {args[0]}", *args[1:])
_original_print(*args, **kwargs)
builtins.print = _prefixed_print
logging.getLogger("root").setLevel(logging.ERROR)
try:
import spaces
except ImportError:
class _SpacesFallback:
@staticmethod
def GPU(**_kwargs):
def decorate(function):
return function
return decorate
spaces = _SpacesFallback()
import torch
if hasattr(torch.cuda, 'get_device_properties'):
_orig_get_device_properties = torch.cuda.get_device_properties
def _patched_get_device_properties(device):
props = _orig_get_device_properties(device)
if not hasattr(props, 'is_integrated'):
props.is_integrated = False
return props
torch.cuda.get_device_properties = _patched_get_device_properties
ROOT = pathlib.Path(__file__).resolve().parent
COMFY = ROOT / "ComfyUI"
MODELS = COMFY / "models"
INPUT = COMFY / "input"
OUTPUT = COMFY / "output"
COMFY_COMMIT = "c1eece62cbf1805d4f64286d7f5592e6a215e6ba"
ZG_ENFORCE = os.environ.get("ZG_ENFORCE", "1").strip().lower() in ("1", "true", "yes", "on")
# ==========================================================================
# 🔥 Warmup Constants
# ==========================================================================
WARMUP_BOOT_FULL = True
WARMUP_BOOT_DELAY = 1.0 # 让第一个 health 先过
WARMUP_BOOT_CAP_MB = 24000 # 全量 sweep 的硬上限
WARMUP_PROBE_BYTES = 8 * 1024 * 1024
WARMUP_MIN_MB = 200 # 小于此体积不预热(冷读也就 1s)
WARMUP_TTL = 900 # warm 状态有效期(秒)
WARMUP_CHUNK = 32 * 1024 * 1024