Spaces:
Running on Zero
Running on Zero
Update app.py
Browse files
app.py
CHANGED
|
@@ -31,14 +31,15 @@ import uvicorn
|
|
| 31 |
import threading
|
| 32 |
|
| 33 |
from model_config import MODEL_CONFIG
|
|
|
|
|
|
|
| 34 |
# ⚠️ 必须在 load_char_loras_from_repo() 之前定义 —— 它内部要读这个常量。
|
| 35 |
# 放在文件后面的"静态挂载"段里会导致启动首次扫描 NameError,触发词全空且被 300s 缓存锁住。
|
| 36 |
LORA_META_URL = "https://huggingface.co/spaces/waveshaker/loras_gallery/resolve/main/models.json"
|
| 37 |
|
| 38 |
|
| 39 |
-
|
| 40 |
# ==========================================================================
|
| 41 |
-
# 🛠️ Environment Patches & Global Paths
|
| 42 |
# ==========================================================================
|
| 43 |
|
| 44 |
import builtins
|
|
@@ -84,7 +85,7 @@ OUTPUT = COMFY / "output"
|
|
| 84 |
COMFY_COMMIT = "c1eece62cbf1805d4f64286d7f5592e6a215e6ba"
|
| 85 |
|
| 86 |
# ==========================================================================
|
| 87 |
-
# 🔥 Warmup
|
| 88 |
# ==========================================================================
|
| 89 |
WARMUP_BOOT_FULL = True
|
| 90 |
WARMUP_BOOT_DELAY = 1.0 # 让第一个 health 先过
|
|
@@ -102,6 +103,9 @@ WARMUP_LAST_FILE = str(ROOT / ".warmup_last.json")
|
|
| 102 |
WARMUP_PRE_GPU_BUDGET = 8.0 # 申请 GPU 前最多等多久(覆盖"已经很接近完成")
|
| 103 |
WARMUP_RESIDUAL_WAIT = 5.0 # GPU 内残余等待上限(烧 GPU 预算,务必小)
|
| 104 |
SYNC_ON_BOOT = False # 启动时是否 os.sync(默认关:只刷脏页,不预热)
|
|
|
|
|
|
|
|
|
|
| 105 |
# ==========================================================================
|
| 106 |
|
| 107 |
_comfy_ready = False
|
|
@@ -246,6 +250,13 @@ def load_char_loras_from_repo():
|
|
| 246 |
total = sum(len(eng) for eng in CHAR_LORA_CHOICES.values())
|
| 247 |
matched = sum(1 for eng in CHAR_LORA_CHOICES.values() for v in eng.values() if v.get("trigger"))
|
| 248 |
print(f"✅ 成功刷新角色 LoRA: {[f'{k}: {len(v)}个' for k, v in CHAR_LORA_CHOICES.items()]} | trigger 覆盖 {matched}/{total}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
except Exception as e:
|
| 250 |
print(f"⚠️ 动态刷新失败: {e}")
|
| 251 |
|
|
@@ -270,6 +281,17 @@ def _install_filtered_requirements(path: pathlib.Path) -> None:
|
|
| 270 |
if requirements:
|
| 271 |
_run([sys.executable, "-m", "pip", "install", "--no-cache-dir", *requirements], check=False)
|
| 272 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
def _ensure_comfy() -> None:
|
| 274 |
global _comfy_ready
|
| 275 |
if _comfy_ready: return
|
|
@@ -301,6 +323,35 @@ def _ensure_comfy() -> None:
|
|
| 301 |
|
| 302 |
_comfy_ready = True
|
| 303 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 304 |
def _init_comfy_nodes() -> None:
|
| 305 |
global _nodes_ready
|
| 306 |
if _nodes_ready: return
|
|
@@ -333,78 +384,38 @@ def _init_comfy_nodes() -> None:
|
|
| 333 |
print("✅ ComfyUI 节点初始化完成!", flush=True)
|
| 334 |
|
| 335 |
# ==========================================================================
|
| 336 |
-
#
|
| 337 |
# ==========================================================================
|
| 338 |
-
|
| 339 |
-
|
| 340 |
|
| 341 |
-
def
|
| 342 |
-
|
| 343 |
-
if dest_path.exists(): return
|
| 344 |
-
print(f"⏳ Downloading: {filename} ...")
|
| 345 |
-
dest_root.mkdir(parents=True, exist_ok=True)
|
| 346 |
-
hf_hub_download(repo_id=repo, subfolder=subfolder if subfolder else None, filename=filename, local_dir=str(dest_root))
|
| 347 |
-
src = dest_root / subfolder / filename if subfolder else dest_root / filename
|
| 348 |
-
if src.exists() and src != dest_path:
|
| 349 |
-
shutil.move(str(src), str(dest_path))
|
| 350 |
-
|
| 351 |
-
# 主动把大文件拉到本地 Page Cache,避免 ZeroGPU worker 跨网络 mmap 缺页
|
| 352 |
-
def _warmup_page_cache(file_path: pathlib.Path):
|
| 353 |
-
if not file_path.exists():
|
| 354 |
-
return
|
| 355 |
-
# 只预热大于 500MB 的大文件(量化大模型、大 TE),小文件(如 VAE)没必要
|
| 356 |
-
if file_path.stat().st_size < 500 * 1024 * 1024:
|
| 357 |
-
return
|
| 358 |
-
|
| 359 |
-
print(f"[预热] 🔥 正在拉取文件到本地缓存: {file_path.name} ({file_path.stat().st_size / 1024 / 1024:.1f} MB)...", flush=True)
|
| 360 |
-
_t0 = time.time()
|
| 361 |
-
|
| 362 |
try:
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
break
|
| 371 |
-
_elapsed = round(time.time() - _t0, 2)
|
| 372 |
-
print(f"[预热] ✅ {file_path.name} 已载入本地缓存,耗时: {_elapsed}s", flush=True)
|
| 373 |
except Exception as e:
|
| 374 |
-
print(f"[预热] ⚠️ {
|
| 375 |
|
| 376 |
|
| 377 |
-
def
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
if "text_encoders" in str(path): dest_root = MODELS / "text_encoders"
|
| 383 |
-
elif "vae" in str(path): dest_root = MODELS / "vae"
|
| 384 |
-
else: dest_root = MODELS / path.parent
|
| 385 |
-
_download_if_needed(repo, str(path.parent), path.name, dest_root)
|
| 386 |
-
for name, cfg in UNET_CHOICES.items():
|
| 387 |
-
_download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], MODELS / "diffusion_models")
|
| 388 |
-
lora_root = MODELS / "loras"
|
| 389 |
-
for name, cfg in BASE_LORAS_CHOICES.items():
|
| 390 |
-
_download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], lora_root)
|
| 391 |
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
try:
|
| 398 |
-
_t0 = time.time()
|
| 399 |
-
os.sync()
|
| 400 |
-
print(f"✅ 脏页已刷盘 (os.sync 耗时: {round(time.time() - _t0, 2)}s)", flush=True)
|
| 401 |
-
except Exception as e:
|
| 402 |
-
print(f"⚠️ os.sync 失败: {e}", flush=True)
|
| 403 |
|
| 404 |
-
#
|
| 405 |
-
# ♻️
|
| 406 |
-
#
|
| 407 |
-
import threading
|
| 408 |
|
| 409 |
_exec_lock = threading.Lock() # 串行化执行,GPU 一次一个任务
|
| 410 |
|
|
@@ -470,9 +481,9 @@ def _get_executor():
|
|
| 470 |
except Exception:
|
| 471 |
pass
|
| 472 |
return loop, executor
|
| 473 |
-
#
|
| 474 |
-
# 🧩
|
| 475 |
-
#
|
| 476 |
|
| 477 |
_loader_chain_cache: dict[str, tuple[dict, str]] = {}
|
| 478 |
|
|
@@ -481,6 +492,26 @@ def _combo_key(engine: str, unet_name: str,
|
|
| 481 |
base = tuple((f, round(float(s), 3)) for f, s in base_loras)
|
| 482 |
return f"{engine}|{unet_name}|{base}|{char_lora or ''}|{round(float(char_strength), 3)}"
|
| 483 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 484 |
def _build_loader_chain(engine, unet_name, base_loras, char_lora, char_strength):
|
| 485 |
"""
|
| 486 |
只构建 1..N 的 loader 链(unet/clip/vae + lora 叠加),
|
|
@@ -509,36 +540,6 @@ def _build_loader_chain(engine, unet_name, base_loras, char_lora, char_strength)
|
|
| 509 |
cur = str(idx); idx += 1
|
| 510 |
return chain, cur
|
| 511 |
|
| 512 |
-
# ==========================================================================
|
| 513 |
-
# 💻 Prepare runtime
|
| 514 |
-
# ==========================================================================
|
| 515 |
-
|
| 516 |
-
def _warmup_node_imports() -> None:
|
| 517 |
-
"""预热 Comfy 节点类导入,消除首次 execute 的 import 开销"""
|
| 518 |
-
try:
|
| 519 |
-
import nodes as comfy_nodes
|
| 520 |
-
for class_name in ["UNETLoader", "CLIPLoader", "DualCLIPLoader", "VAELoader",
|
| 521 |
-
"LoraLoaderModelOnly", "KSampler", "VAEDecode", "SaveImage",
|
| 522 |
-
"CLIPTextEncode", "ConditioningZeroOut", "EmptyLatentImage"]:
|
| 523 |
-
if hasattr(comfy_nodes, class_name):
|
| 524 |
-
getattr(comfy_nodes, class_name)
|
| 525 |
-
print("[预热] ✅ 节点类导入完成", flush=True)
|
| 526 |
-
except Exception as e:
|
| 527 |
-
print(f"[预热] ⚠️ {e}", flush=True)
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
def _prepare_runtime() -> None:
|
| 531 |
-
_ensure_comfy()
|
| 532 |
-
_ensure_models()
|
| 533 |
-
_init_comfy_nodes()
|
| 534 |
-
_warmup_node_imports() # 🌟 P1: 预热节点类
|
| 535 |
-
|
| 536 |
-
try:
|
| 537 |
-
print("⬇️ Starting up...")
|
| 538 |
-
_prepare_runtime()
|
| 539 |
-
except Exception as e:
|
| 540 |
-
print(f"⚠️ Startup failed: {e}")
|
| 541 |
-
|
| 542 |
|
| 543 |
# ==========================================================================
|
| 544 |
# 🔄 Progress Interceptor
|
|
@@ -604,6 +605,36 @@ class UnifiedProgressTqdm(tqdm):
|
|
| 604 |
if has_steps:
|
| 605 |
tasks_status[gen_task_id]["message"] = step_msg
|
| 606 |
print(f"[进度拦截] Task {gen_task_id[:8]} | {self.n}/{self.total} steps ({int(self.n / self.total * 100)}%)")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 607 |
|
| 608 |
# ==========================================================================
|
| 609 |
# ⚛️ Comfy Backend & Workflow
|
|
@@ -641,8 +672,51 @@ def print_ram_status(stage: str):
|
|
| 641 |
print(f"[RAM监控 - {stage}] 宿主机总内存: {total_gb:.2f} GB | 宿主机可用: {avail_gb:.2f} GB | 本进程占用: {proc_mem_gb:.2f} GB | ���统已用: {mem.percent}%{cgroup_str}")
|
| 642 |
|
| 643 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 644 |
# --------------------------------------------------------------------------
|
| 645 |
-
# 📊 Metric
|
| 646 |
# --------------------------------------------------------------------------
|
| 647 |
_metric_lock = threading.RLock()
|
| 648 |
|
|
@@ -690,40 +764,83 @@ def _write_metric(task_id: str, **updates) -> None:
|
|
| 690 |
d.update(updates)
|
| 691 |
return True
|
| 692 |
_update_metric(task_id, _m)
|
| 693 |
-
|
| 694 |
-
# --------------------------------------------------------------------------
|
| 695 |
-
# 🔁 tqdm Patch:可重入引用计数,杜绝并发 save/restore 泄漏
|
| 696 |
-
# --------------------------------------------------------------------------
|
| 697 |
-
_tqdm_patch_lock = threading.RLock()
|
| 698 |
-
_tqdm_patch_depth = 0
|
| 699 |
-
_tqdm_patch_originals: dict[str, Any] = {}
|
| 700 |
-
|
| 701 |
-
def _tqdm_patch_enter() -> None:
|
| 702 |
-
global _tqdm_patch_depth
|
| 703 |
-
import tqdm as tqdm_lib
|
| 704 |
-
import tqdm.auto as tqdm_auto
|
| 705 |
-
with _tqdm_patch_lock:
|
| 706 |
-
if _tqdm_patch_depth == 0:
|
| 707 |
-
_tqdm_patch_originals["lib"] = tqdm_lib.tqdm
|
| 708 |
-
_tqdm_patch_originals["auto"] = tqdm_auto.tqdm
|
| 709 |
-
tqdm_lib.tqdm = UnifiedProgressTqdm
|
| 710 |
-
tqdm_auto.tqdm = UnifiedProgressTqdm
|
| 711 |
-
_tqdm_patch_depth += 1
|
| 712 |
-
|
| 713 |
-
def _tqdm_patch_exit() -> None:
|
| 714 |
-
global _tqdm_patch_depth
|
| 715 |
-
import tqdm as tqdm_lib
|
| 716 |
-
import tqdm.auto as tqdm_auto
|
| 717 |
-
with _tqdm_patch_lock:
|
| 718 |
-
if _tqdm_patch_depth > 0:
|
| 719 |
-
_tqdm_patch_depth -= 1
|
| 720 |
-
if _tqdm_patch_depth == 0 and "lib" in _tqdm_patch_originals:
|
| 721 |
-
tqdm_lib.tqdm = _tqdm_patch_originals.pop("lib")
|
| 722 |
-
tqdm_auto.tqdm = _tqdm_patch_originals.pop("auto")
|
| 723 |
# --------------------------------------------------------------------------
|
| 724 |
def _ref(node: str, output: int = 0) -> list[Any]:
|
| 725 |
return [node, output]
|
| 726 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 727 |
def _log_workflow_summary(workflow: dict[str, Any], task_id: str, load_time_val: float, wf_time: float, image: str = None):
|
| 728 |
"""独立提取并打印单行 JSON 摘要日志(不影响主流程逻辑)"""
|
| 729 |
try:
|
|
@@ -798,49 +915,6 @@ def _log_workflow_summary(workflow: dict[str, Any], task_id: str, load_time_val:
|
|
| 798 |
|
| 799 |
except Exception as e:
|
| 800 |
print(f"[Log Format Error]: {e}", flush=True)
|
| 801 |
-
|
| 802 |
-
|
| 803 |
-
def _io_probe():
|
| 804 |
-
try:
|
| 805 |
-
c = psutil.Process().io_counters()
|
| 806 |
-
return {"read_chars": c.read_chars, "read_bytes": c.read_bytes}
|
| 807 |
-
except Exception:
|
| 808 |
-
return {"read_chars": 0, "read_bytes": 0}
|
| 809 |
-
|
| 810 |
-
def _mem_probe():
|
| 811 |
-
"""返回 (host_Cached_MB, host_Avail_MB, cg_current_GB, cg_file_GB, cg_inactive_file_GB)"""
|
| 812 |
-
cached = avail = -1
|
| 813 |
-
try:
|
| 814 |
-
with open("/proc/meminfo") as f:
|
| 815 |
-
m = {l.split(":")[0]: int(l.split()[1]) // 1024 for l in f if len(l.split()) >= 2}
|
| 816 |
-
cached, avail = m.get("Cached", -1), m.get("MemAvailable", -1)
|
| 817 |
-
except Exception:
|
| 818 |
-
pass
|
| 819 |
-
|
| 820 |
-
cur = file_gb = inact_gb = -1.0
|
| 821 |
-
try:
|
| 822 |
-
with open("/sys/fs/cgroup/memory.current") as f:
|
| 823 |
-
cur = int(f.read().strip()) / 1024**3
|
| 824 |
-
with open("/sys/fs/cgroup/memory.stat") as f:
|
| 825 |
-
for line in f:
|
| 826 |
-
p = line.split()
|
| 827 |
-
if len(p) != 2:
|
| 828 |
-
continue
|
| 829 |
-
if p[0] == "file":
|
| 830 |
-
file_gb = int(p[1]) / 1024**3
|
| 831 |
-
elif p[0] == "inactive_file":
|
| 832 |
-
inact_gb = int(p[1]) / 1024**3
|
| 833 |
-
except Exception:
|
| 834 |
-
pass
|
| 835 |
-
return cached, avail, cur, file_gb, inact_gb
|
| 836 |
-
|
| 837 |
-
def _gpu_id():
|
| 838 |
-
try:
|
| 839 |
-
dev = torch.cuda.current_device()
|
| 840 |
-
p = torch.cuda.get_device_properties(dev)
|
| 841 |
-
return f"dev{dev}/uuid={getattr(p, 'uuid', 'n/a')}"
|
| 842 |
-
except Exception as e:
|
| 843 |
-
return f"n/a({e})"
|
| 844 |
|
| 845 |
def _execute_workflow(workflow: dict[str, Any], task_id: str = None) -> list[str]:
|
| 846 |
print_ram_status("任务开始前")
|
|
@@ -973,105 +1047,11 @@ def prepare_env(char_enable: bool, char_lora: str, unet_choice: str, task_id: st
|
|
| 973 |
tasks_status[task_id]["message"] = "📦 LoRA downloaded, loading..."
|
| 974 |
finally:
|
| 975 |
_tqdm_patch_exit()
|
| 976 |
-
_current_download_task_id.set(None)
|
| 977 |
-
|
| 978 |
-
def select_base_loras(engine: str, lora_states: tuple) -> list[tuple[str, float]]:
|
| 979 |
-
selected = []
|
| 980 |
-
|
| 981 |
-
# 找出当前引擎专属的 Base LoRA 列表,保持顺序
|
| 982 |
-
valid_loras = []
|
| 983 |
-
for name, cfg in BASE_LORAS_CHOICES.items():
|
| 984 |
-
if cfg["engine"] == engine:
|
| 985 |
-
valid_loras.append((name, cfg))
|
| 986 |
-
|
| 987 |
-
# 解析前端传来的勾选状态
|
| 988 |
-
for i in range(0, len(lora_states), 2):
|
| 989 |
-
enable = lora_states[i]
|
| 990 |
-
try:
|
| 991 |
-
strength = float(lora_states[i + 1])
|
| 992 |
-
except (ValueError, TypeError):
|
| 993 |
-
strength = 1.0
|
| 994 |
-
|
| 995 |
-
if enable and strength > 0:
|
| 996 |
-
# 如果前端的数组长度和当前引擎实际可用的 LoRA 列表长度一致
|
| 997 |
-
idx = i // 2
|
| 998 |
-
if idx < len(valid_loras):
|
| 999 |
-
lora_name, lora_cfg = valid_loras[idx]
|
| 1000 |
-
selected.append((lora_cfg["filename"], strength))
|
| 1001 |
-
|
| 1002 |
-
# 你的探针(保留观察)
|
| 1003 |
-
print(f"[LoRA选择探针] ✅ 成功匹配并加载: {lora_name} ({strength})")
|
| 1004 |
-
|
| 1005 |
-
return selected
|
| 1006 |
-
|
| 1007 |
-
def _build_clip_inputs(engine_cfg: dict, clip_loader_node: str) -> dict:
|
| 1008 |
-
clip_type = engine_cfg.get("clip_type", "moss_image")
|
| 1009 |
-
clip_inputs = {"type": clip_type}
|
| 1010 |
-
te_filenames = engine_cfg.get("local_te_filename", [])
|
| 1011 |
-
if clip_loader_node == "DualCLIPLoader":
|
| 1012 |
-
if len(te_filenames) >= 2:
|
| 1013 |
-
if "clip_l" in te_filenames[0].lower():
|
| 1014 |
-
clip_inputs["clip_name1"] = te_filenames[0]
|
| 1015 |
-
clip_inputs["clip_name2"] = te_filenames[1]
|
| 1016 |
-
else:
|
| 1017 |
-
clip_inputs["clip_name1"] = te_filenames[1]
|
| 1018 |
-
clip_inputs["clip_name2"] = te_filenames[0]
|
| 1019 |
-
elif len(te_filenames) == 1:
|
| 1020 |
-
clip_inputs["clip_name1"] = te_filenames[0]
|
| 1021 |
-
else:
|
| 1022 |
-
if te_filenames: clip_inputs["clip_name"] = te_filenames[0]
|
| 1023 |
-
if "clip_device" in MODEL_CONFIG.get("loader_args", {}):
|
| 1024 |
-
clip_inputs["device"] = MODEL_CONFIG["loader_args"]["clip_device"]
|
| 1025 |
-
return clip_inputs
|
| 1026 |
-
|
| 1027 |
-
def build_workflow(
|
| 1028 |
-
prompt: str,
|
| 1029 |
-
unet_name: str,
|
| 1030 |
-
engine: str,
|
| 1031 |
-
width: int,
|
| 1032 |
-
height: int,
|
| 1033 |
-
steps: int,
|
| 1034 |
-
cfg: float,
|
| 1035 |
-
seed: int,
|
| 1036 |
-
sampler: str,
|
| 1037 |
-
scheduler: str,
|
| 1038 |
-
base_loras: list[tuple[str, float]],
|
| 1039 |
-
char_lora: str | None,
|
| 1040 |
-
char_strength: float,
|
| 1041 |
-
) -> dict[str, Any]:
|
| 1042 |
-
# --- loader 链:命中缓存则复用(节点 id / inputs 完全一致) ---
|
| 1043 |
-
key = _combo_key(engine, unet_name, base_loras, char_lora, char_strength)
|
| 1044 |
-
cached = _loader_chain_cache.get(key)
|
| 1045 |
-
if cached is not None:
|
| 1046 |
-
chain, current_model_node = cached
|
| 1047 |
-
workflow = dict(chain) # 浅拷贝,下游不改 loader 链
|
| 1048 |
-
else:
|
| 1049 |
-
chain, current_model_node = _build_loader_chain(
|
| 1050 |
-
engine, unet_name, base_loras, char_lora, char_strength)
|
| 1051 |
-
_loader_chain_cache[key] = (chain, current_model_node)
|
| 1052 |
-
# 只保留最近 8 个组合,避免内存无限增长
|
| 1053 |
-
if len(_loader_chain_cache) > 8:
|
| 1054 |
-
_loader_chain_cache.pop(next(iter(_loader_chain_cache)))
|
| 1055 |
-
workflow = dict(chain)
|
| 1056 |
-
|
| 1057 |
-
# --- 下游节点:id 从链末端续接,与原逻辑完全一致 ---
|
| 1058 |
-
nodes = MODEL_CONFIG["node_types"]
|
| 1059 |
-
node_idx = max(int(k) for k in chain.keys()) + 1
|
| 1060 |
-
|
| 1061 |
-
final_prompt = prompt.strip()
|
| 1062 |
-
workflow[str(node_idx)] = {"class_type": "CLIPTextEncode", "inputs": {"clip": _ref("2"), "text": final_prompt}}
|
| 1063 |
-
workflow[str(node_idx+1)] = {"class_type": "ConditioningZeroOut", "inputs": {"conditioning": _ref(str(node_idx))}}
|
| 1064 |
-
workflow[str(node_idx+2)] = {"class_type": "EmptyLatentImage", "inputs": {"width": width, "height": height, "batch_size": 1}}
|
| 1065 |
-
|
| 1066 |
-
workflow[str(node_idx+3)] = {"class_type": "KSampler", "inputs": {"model": _ref(current_model_node), "positive": _ref(str(node_idx)), "negative": _ref(str(node_idx+1)), "latent_image": _ref(str(node_idx+2)), "seed": seed, "steps": steps, "cfg": cfg, "sampler_name": sampler, "scheduler": scheduler, "denoise": 1.0}}
|
| 1067 |
-
workflow[str(node_idx+4)] = {"class_type": "VAEDecode", "inputs": {"samples": _ref(str(node_idx+3)), "vae": _ref("3")}}
|
| 1068 |
-
workflow[str(node_idx+5)] = {"class_type": "SaveImage", "inputs": {"images": _ref(str(node_idx+4)), "filename_prefix": MODEL_CONFIG["name"].lower()}}
|
| 1069 |
-
return workflow
|
| 1070 |
|
| 1071 |
-
# 🌟 增加全局变量用于追踪 GPU 域内的架构切换
|
| 1072 |
|
| 1073 |
# ==========================================================================
|
| 1074 |
-
# 🔥 GPU
|
| 1075 |
# ==========================================================================
|
| 1076 |
def _read_warm_state():
|
| 1077 |
try:
|
|
@@ -1083,17 +1063,40 @@ def _read_warm_state():
|
|
| 1083 |
|
| 1084 |
def _warm_wait_in_gpu(unet_choice: str, budget: float = WARMUP_RESIDUAL_WAIT) -> bool:
|
| 1085 |
"""只在"已经非常接近完成"时等,最多 budget 秒。
|
| 1086 |
-
注意:这段是在 GPU 切片内执行,等待即烧配额,所以
|
| 1087 |
t0 = time.time()
|
| 1088 |
while time.time() - t0 < budget:
|
| 1089 |
st = _read_warm_state()
|
| 1090 |
-
|
| 1091 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1092 |
time.sleep(0.25)
|
|
|
|
| 1093 |
print(f"[预热] ⏳ GPU 内残余等待超时({budget}s),转冷加载", flush=True)
|
| 1094 |
return False
|
| 1095 |
|
| 1096 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1097 |
def _generate_impl(
|
| 1098 |
prompt: str, unet_choice: str, custom_w: int, custom_h: int,
|
| 1099 |
steps: int, cfg: float, seed: int, sampler: str, scheduler: str,
|
|
@@ -1174,16 +1177,15 @@ demo.queue()
|
|
| 1174 |
|
| 1175 |
|
| 1176 |
# ==========================================================================
|
| 1177 |
-
# 🧹
|
| 1178 |
# ==========================================================================
|
| 1179 |
|
| 1180 |
|
| 1181 |
-
import threading
|
| 1182 |
# 专门用于保护 tasks_status 字典的状态锁
|
| 1183 |
_status_lock = threading.RLock()
|
| 1184 |
_model_switch_lock = threading.Lock()
|
| 1185 |
|
| 1186 |
-
# ----
|
| 1187 |
_STATUS_TTL = 3600 # 终态任务内存保留 60 分钟
|
| 1188 |
_STATUS_MAX_ENTRIES = 200 # 兜底:最多保留 200 条终态
|
| 1189 |
_OUTPUT_TTL = 24 * 3600 # 输出图片保留 24 小时
|
|
@@ -1242,9 +1244,16 @@ def _cleanup_stale(force: bool = False) -> None:
|
|
| 1242 |
if deleted:
|
| 1243 |
print(f"[清理] 🧹 删除 {deleted} 个过期输出,释放 {freed / 1024 / 1024:.1f} MB", flush=True)
|
| 1244 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1245 |
|
| 1246 |
# ==========================================================================
|
| 1247 |
-
# 🔥 WarmupManager
|
| 1248 |
# ==========================================================================
|
| 1249 |
class WarmupManager:
|
| 1250 |
def __init__(self):
|
|
@@ -1662,7 +1671,17 @@ class WarmupManager:
|
|
| 1662 |
|
| 1663 |
for rec in files:
|
| 1664 |
if self._superseded(seq):
|
| 1665 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1666 |
break
|
| 1667 |
records[f"{rec['owner']}::{rec['file_key']}"] = rec
|
| 1668 |
|
|
@@ -1769,6 +1788,40 @@ class WarmupManager:
|
|
| 1769 |
|
| 1770 |
|
| 1771 |
warmup = WarmupManager()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1772 |
|
| 1773 |
# ==========================================================================
|
| 1774 |
# 🌐 FastAPI & API Routes
|
|
@@ -1788,6 +1841,17 @@ class GenerateRequest(BaseModel):
|
|
| 1788 |
char_strength: float = 1.0
|
| 1789 |
base_lora_states: list = []
|
| 1790 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1791 |
|
| 1792 |
def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str = None, request: Request = None):
|
| 1793 |
global _last_used_architecture, _last_used_unet
|
|
@@ -1803,14 +1867,15 @@ def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str =
|
|
| 1803 |
|
| 1804 |
# --- 阶段 0:预热(与 GPU 排队并行,不阻塞 HTTP) ---
|
| 1805 |
try:
|
| 1806 |
-
warmup.request(
|
| 1807 |
req.unet_choice,
|
| 1808 |
req.char_lora if req.char_enable else None,
|
| 1809 |
char_enable=req.char_enable,
|
| 1810 |
reason="submit", task_id=task_id,
|
| 1811 |
)
|
| 1812 |
-
|
| 1813 |
-
|
|
|
|
| 1814 |
# 只在"接近完成"时短暂等待;冷启动立即放行,避免把 GPU 排队串到预热后面
|
| 1815 |
_warm_ok = warmup.wait_ready(req.unet_choice, timeout=WARMUP_PRE_GPU_BUDGET)
|
| 1816 |
print(f"[预热] gate ready={_warm_ok} progress={warmup.progress(req.unet_choice)}", flush=True)
|
|
@@ -1983,49 +2048,6 @@ def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str =
|
|
| 1983 |
_gen_leave()
|
| 1984 |
_current_generation_task_id.set(None)
|
| 1985 |
|
| 1986 |
-
app = FastAPI()
|
| 1987 |
-
|
| 1988 |
-
def _read_last_unet():
|
| 1989 |
-
try:
|
| 1990 |
-
with open(WARMUP_LAST_FILE) as f:
|
| 1991 |
-
return json.load(f).get("unet_choice")
|
| 1992 |
-
except Exception:
|
| 1993 |
-
return None
|
| 1994 |
-
|
| 1995 |
-
def _boot_warmup():
|
| 1996 |
-
"""两段式:
|
| 1997 |
-
① 阶梯一级(<1s):上次模型的 UNET 头部 → 用户第一眼就有"部分热"
|
| 1998 |
-
② 全量 sweep(P1,可被抢占):等 ① 跑完再启动,否则会把 ① 顶掉
|
| 1999 |
-
"""
|
| 2000 |
-
last = _read_last_unet()
|
| 2001 |
-
target = last if last in UNET_CHOICES else next(iter(UNET_CHOICES), None)
|
| 2002 |
-
|
| 2003 |
-
lite_requested = False
|
| 2004 |
-
if target:
|
| 2005 |
-
rec = warmup._rec(MODELS / "diffusion_models" / UNET_CHOICES[target]["filename"],
|
| 2006 |
-
"unet", "unet")
|
| 2007 |
-
mbps = warmup.probe(rec)
|
| 2008 |
-
if mbps is not None and mbps >= WARMUP_WARM_MBPS:
|
| 2009 |
-
print(f"[预热] ⚡ boot: UNET 已在 cache 中 ({mbps:.0f}MB/s),跳过 lite", flush=True)
|
| 2010 |
-
else:
|
| 2011 |
-
print(f"[预热] 🥾 boot: UNET 探测 {mbps} MB/s → 做阶梯一级", flush=True)
|
| 2012 |
-
warmup.request(target, lite=True, reason="boot", priority=1)
|
| 2013 |
-
lite_requested = True # ← 新增
|
| 2014 |
-
|
| 2015 |
-
if not WARMUP_BOOT_FULL:
|
| 2016 |
-
return
|
| 2017 |
-
|
| 2018 |
-
if lite_requested: # ← 只在发起了才等
|
| 2019 |
-
t0 = time.time()
|
| 2020 |
-
while time.time() - t0 < 20:
|
| 2021 |
-
with warmup._lock:
|
| 2022 |
-
r = warmup._files.get(target, {}).get("unet")
|
| 2023 |
-
if r and r.get("state") in ("partial", "warm", "failed", "stale"):
|
| 2024 |
-
break
|
| 2025 |
-
time.sleep(0.2)
|
| 2026 |
-
|
| 2027 |
-
warmup.request_sweep(reason="boot_sweep")
|
| 2028 |
-
|
| 2029 |
|
| 2030 |
@app.on_event("startup")
|
| 2031 |
async def _on_startup():
|
|
@@ -2036,17 +2058,75 @@ async def _on_startup():
|
|
| 2036 |
except Exception as e:
|
| 2037 |
print(f"[预热] ⚠️ boot 异常: {e}", flush=True)
|
| 2038 |
threading.Thread(target=_delayed, daemon=True).start()
|
| 2039 |
-
|
| 2040 |
|
| 2041 |
-
app.
|
| 2042 |
-
|
| 2043 |
-
|
| 2044 |
-
|
| 2045 |
-
allow_methods=["*"],
|
| 2046 |
-
allow_headers=["*"],
|
| 2047 |
-
)
|
| 2048 |
|
| 2049 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2050 |
|
| 2051 |
@app.post("/api/generate")
|
| 2052 |
async def handle_generate(req: GenerateRequest, background_tasks: BackgroundTasks, request: Request):
|
|
@@ -2085,7 +2165,6 @@ async def handle_generate(req: GenerateRequest, background_tasks: BackgroundTask
|
|
| 2085 |
tasks_status[task_id] = {"status": "queued", "message": "⏳ 任务已进入队列..."}
|
| 2086 |
|
| 2087 |
# 👇 抛弃 BackgroundTasks,使用原生守护线程,直接与 HTTP 返回脱钩
|
| 2088 |
-
import threading
|
| 2089 |
t = threading.Thread(
|
| 2090 |
target=background_generate_task,
|
| 2091 |
args=(task_id, req, ip_token, request),
|
|
@@ -2116,70 +2195,10 @@ async def check_status(task_id: str):
|
|
| 2116 |
status_data["message"] = file_data["phase"]
|
| 2117 |
return status_data
|
| 2118 |
|
| 2119 |
-
@app.get("/api/models")
|
| 2120 |
-
async def get_models():
|
| 2121 |
-
models_data = {}
|
| 2122 |
-
global_default_prompt = MODEL_CONFIG.get("default_prompt", "")
|
| 2123 |
-
|
| 2124 |
-
for unet_name, unet_info in UNET_CHOICES.items():
|
| 2125 |
-
engine = unet_info["engine"]
|
| 2126 |
-
engine_name = MODEL_CONFIG["models"].get(engine, {}).get("name", engine)
|
| 2127 |
-
|
| 2128 |
-
if " | " in unet_name:
|
| 2129 |
-
model_specific_name = unet_name.split(" | ", 1)[1]
|
| 2130 |
-
else:
|
| 2131 |
-
model_specific_name = unet_name
|
| 2132 |
-
|
| 2133 |
-
display_unet_name = f"{engine_name} | {model_specific_name}"
|
| 2134 |
-
|
| 2135 |
-
base_loras_for_engine = []
|
| 2136 |
-
for name, cfg in BASE_LORAS_CHOICES.items():
|
| 2137 |
-
if cfg["engine"] == engine:
|
| 2138 |
-
base_loras_for_engine.append({
|
| 2139 |
-
"name": name,
|
| 2140 |
-
"filename": cfg["filename"],
|
| 2141 |
-
"defaultEnable": cfg.get("defaultEnable", False)
|
| 2142 |
-
})
|
| 2143 |
-
|
| 2144 |
-
engine_cfg = MODEL_CONFIG["models"].get(engine, {})
|
| 2145 |
-
final_prompt = engine_cfg.get("default_prompt_override", global_default_prompt)
|
| 2146 |
-
|
| 2147 |
-
# 🌟 核心新增:提取 TE 和 VAE 信息
|
| 2148 |
-
# TE 可能是列表,用 " + " 连接起来展示
|
| 2149 |
-
te_list = engine_cfg.get("local_te_filename", [])
|
| 2150 |
-
te_display = " + ".join([f.replace(".safetensors", "") for f in te_list]) if te_list else "Unknown"
|
| 2151 |
-
|
| 2152 |
-
# VAE 通常是字符串
|
| 2153 |
-
vae_display = engine_cfg.get("local_vae_filename", "Unknown").replace(".safetensors", "")
|
| 2154 |
-
|
| 2155 |
-
models_data[display_unet_name] = {
|
| 2156 |
-
"engine": engine,
|
| 2157 |
-
"defaults": engine_cfg.get("defaults", {}),
|
| 2158 |
-
"base_loras": base_loras_for_engine,
|
| 2159 |
-
"default_prompt": final_prompt,
|
| 2160 |
-
"te_name": te_display, # 👈 传给前端
|
| 2161 |
-
"vae_name": vae_display # 👈 传给前端
|
| 2162 |
-
}
|
| 2163 |
-
|
| 2164 |
-
return {"models": models_data}
|
| 2165 |
-
|
| 2166 |
-
@app.get("/api/loras")
|
| 2167 |
-
async def get_loras():
|
| 2168 |
-
# ⚠️ load_char_loras_from_repo 里有同步 requests.get + list_repo_files(网络 IO)。
|
| 2169 |
-
# 直接在 async 路由里调用会阻塞 event loop,期间所有请求(含 /api/config)
|
| 2170 |
-
# 都无法处理,HF 代理超时后会 reset 连接 → 前端表现为 "Failed to fetch"。
|
| 2171 |
-
# 扔进线程池执行,event loop 保持畅通。
|
| 2172 |
-
await asyncio.to_thread(load_char_loras_from_repo)
|
| 2173 |
-
return {"loras": CHAR_LORA_CHOICES}
|
| 2174 |
|
| 2175 |
-
|
| 2176 |
-
|
| 2177 |
-
|
| 2178 |
-
"samplers": SAMPLERS,
|
| 2179 |
-
"schedulers": SCHEDULERS,
|
| 2180 |
-
"root_mapping": MODEL_CONFIG["json_keys"]["root_mapping"],
|
| 2181 |
-
"default_prompt": MODEL_CONFIG.get("default_prompt", ""),
|
| 2182 |
-
}
|
| 2183 |
|
| 2184 |
class WarmupRequest(BaseModel):
|
| 2185 |
unet_choice: str
|
|
@@ -2233,8 +2252,8 @@ async def api_warmup_status():
|
|
| 2233 |
return warmup.snapshot()
|
| 2234 |
|
| 2235 |
# ==========================================================================
|
| 2236 |
-
# 🔧
|
| 2237 |
-
# WARMUP_DEBUG_TOKEN
|
| 2238 |
# ==========================================================================
|
| 2239 |
import hmac
|
| 2240 |
|
|
@@ -2401,10 +2420,6 @@ async def debug_warmup(
|
|
| 2401 |
|
| 2402 |
raise HTTPException(400, f"unknown action: {action}")
|
| 2403 |
|
| 2404 |
-
@app.get("/api/health")
|
| 2405 |
-
async def health_check():
|
| 2406 |
-
return {"status": "ok"}
|
| 2407 |
-
|
| 2408 |
# ==========================================================================
|
| 2409 |
# 📁 Static Files Sync & Mount
|
| 2410 |
# ==========================================================================
|
|
@@ -2440,9 +2455,7 @@ def sync_ui_assets():
|
|
| 2440 |
# 1. 启动时执行同步
|
| 2441 |
sync_ui_assets()
|
| 2442 |
|
| 2443 |
-
# ==========================================================
|
| 2444 |
# 🌟 核心修改:必须在挂载根路径 "/" 之前,先挂载 /output 目录
|
| 2445 |
-
# ==========================================================
|
| 2446 |
app.mount("/output", StaticFiles(directory=str(OUTPUT)), name="output")
|
| 2447 |
|
| 2448 |
# 2. 挂载静态文件夹 (前端)
|
|
|
|
| 31 |
import threading
|
| 32 |
|
| 33 |
from model_config import MODEL_CONFIG
|
| 34 |
+
|
| 35 |
+
from fastapi import HTTPException, Depends, Header, Query
|
| 36 |
# ⚠️ 必须在 load_char_loras_from_repo() 之前定义 —— 它内部要读这个常量。
|
| 37 |
# 放在文件后面的"静态挂载"段里会导致启动首次扫描 NameError,触发词全空且被 300s 缓存锁住。
|
| 38 |
LORA_META_URL = "https://huggingface.co/spaces/waveshaker/loras_gallery/resolve/main/models.json"
|
| 39 |
|
| 40 |
|
|
|
|
| 41 |
# ==========================================================================
|
| 42 |
+
# 🛠️ Environment Patches & Global Paths
|
| 43 |
# ==========================================================================
|
| 44 |
|
| 45 |
import builtins
|
|
|
|
| 85 |
COMFY_COMMIT = "c1eece62cbf1805d4f64286d7f5592e6a215e6ba"
|
| 86 |
|
| 87 |
# ==========================================================================
|
| 88 |
+
# 🔥 Warmup Constants
|
| 89 |
# ==========================================================================
|
| 90 |
WARMUP_BOOT_FULL = True
|
| 91 |
WARMUP_BOOT_DELAY = 1.0 # 让第一个 health 先过
|
|
|
|
| 103 |
WARMUP_PRE_GPU_BUDGET = 8.0 # 申请 GPU 前最多等多久(覆盖"已经很接近完成")
|
| 104 |
WARMUP_RESIDUAL_WAIT = 5.0 # GPU 内残余等待上限(烧 GPU 预算,务必小)
|
| 105 |
SYNC_ON_BOOT = False # 启动时是否 os.sync(默认关:只刷脏页,不预热)
|
| 106 |
+
|
| 107 |
+
# ==========================================================================
|
| 108 |
+
# 🧠 Global Runtime State
|
| 109 |
# ==========================================================================
|
| 110 |
|
| 111 |
_comfy_ready = False
|
|
|
|
| 250 |
total = sum(len(eng) for eng in CHAR_LORA_CHOICES.values())
|
| 251 |
matched = sum(1 for eng in CHAR_LORA_CHOICES.values() for v in eng.values() if v.get("trigger"))
|
| 252 |
print(f"✅ 成功刷新角色 LoRA: {[f'{k}: {len(v)}个' for k, v in CHAR_LORA_CHOICES.items()]} | trigger 覆盖 {matched}/{total}")
|
| 253 |
+
|
| 254 |
+
# ✅ 新增:分架构覆盖率,数据缺口一眼可见
|
| 255 |
+
per = " | ".join(
|
| 256 |
+
f"{eng}:{sum(1 for v in m.values() if v.get('trigger'))}/{len(m)}"
|
| 257 |
+
for eng, m in CHAR_LORA_CHOICES.items()
|
| 258 |
+
)
|
| 259 |
+
print(f" └ trigger 分架构: {per}")
|
| 260 |
except Exception as e:
|
| 261 |
print(f"⚠️ 动态刷新失败: {e}")
|
| 262 |
|
|
|
|
| 281 |
if requirements:
|
| 282 |
_run([sys.executable, "-m", "pip", "install", "--no-cache-dir", *requirements], check=False)
|
| 283 |
|
| 284 |
+
|
| 285 |
+
def _download_if_needed(repo: str, subfolder: str, filename: str, dest_root: pathlib.Path) -> None:
|
| 286 |
+
dest_path = dest_root / filename
|
| 287 |
+
if dest_path.exists(): return
|
| 288 |
+
print(f"⏳ Downloading: {filename} ...")
|
| 289 |
+
dest_root.mkdir(parents=True, exist_ok=True)
|
| 290 |
+
hf_hub_download(repo_id=repo, subfolder=subfolder if subfolder else None, filename=filename, local_dir=str(dest_root))
|
| 291 |
+
src = dest_root / subfolder / filename if subfolder else dest_root / filename
|
| 292 |
+
if src.exists() and src != dest_path:
|
| 293 |
+
shutil.move(str(src), str(dest_path))
|
| 294 |
+
|
| 295 |
def _ensure_comfy() -> None:
|
| 296 |
global _comfy_ready
|
| 297 |
if _comfy_ready: return
|
|
|
|
| 323 |
|
| 324 |
_comfy_ready = True
|
| 325 |
|
| 326 |
+
|
| 327 |
+
def _ensure_models() -> None:
|
| 328 |
+
print("开始全量下载基础模型...")
|
| 329 |
+
for engine_cfg in MODEL_CONFIG["models"].values():
|
| 330 |
+
for repo, file_path in engine_cfg.get("companion_downloads", []):
|
| 331 |
+
path = pathlib.Path(file_path)
|
| 332 |
+
if "text_encoders" in str(path): dest_root = MODELS / "text_encoders"
|
| 333 |
+
elif "vae" in str(path): dest_root = MODELS / "vae"
|
| 334 |
+
else: dest_root = MODELS / path.parent
|
| 335 |
+
_download_if_needed(repo, str(path.parent), path.name, dest_root)
|
| 336 |
+
for name, cfg in UNET_CHOICES.items():
|
| 337 |
+
_download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], MODELS / "diffusion_models")
|
| 338 |
+
lora_root = MODELS / "loras"
|
| 339 |
+
for name, cfg in BASE_LORAS_CHOICES.items():
|
| 340 |
+
_download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], lora_root)
|
| 341 |
+
|
| 342 |
+
# ❌ 不再在这里 os.sync + 全量预热:
|
| 343 |
+
# ① 同步跑在 import 阶段 → uvicorn 还没 listen,/api/health 打不进来
|
| 344 |
+
# ② 会阻塞启动几十秒,且和后面的 sweep 重复
|
| 345 |
+
# ✅ 预热统一交给 WarmupManager,在 @app.on_event("startup") 的后台线程里做
|
| 346 |
+
if SYNC_ON_BOOT:
|
| 347 |
+
try:
|
| 348 |
+
_t0 = time.time()
|
| 349 |
+
os.sync()
|
| 350 |
+
print(f"✅ 脏页已刷盘 (os.sync 耗时: {round(time.time() - _t0, 2)}s)", flush=True)
|
| 351 |
+
except Exception as e:
|
| 352 |
+
print(f"⚠️ os.sync 失败: {e}", flush=True)
|
| 353 |
+
|
| 354 |
+
|
| 355 |
def _init_comfy_nodes() -> None:
|
| 356 |
global _nodes_ready
|
| 357 |
if _nodes_ready: return
|
|
|
|
| 384 |
print("✅ ComfyUI 节点初始化完成!", flush=True)
|
| 385 |
|
| 386 |
# ==========================================================================
|
| 387 |
+
# 💻 Prepare Runtime
|
| 388 |
# ==========================================================================
|
|
|
|
|
|
|
| 389 |
|
| 390 |
+
def _warmup_node_imports() -> None:
|
| 391 |
+
"""预热 Comfy 节点类导入,消除首次 execute 的 import 开销"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 392 |
try:
|
| 393 |
+
import nodes as comfy_nodes
|
| 394 |
+
for class_name in ["UNETLoader", "CLIPLoader", "DualCLIPLoader", "VAELoader",
|
| 395 |
+
"LoraLoaderModelOnly", "KSampler", "VAEDecode", "SaveImage",
|
| 396 |
+
"CLIPTextEncode", "ConditioningZeroOut", "EmptyLatentImage"]:
|
| 397 |
+
if hasattr(comfy_nodes, class_name):
|
| 398 |
+
getattr(comfy_nodes, class_name)
|
| 399 |
+
print("[预热] ✅ 节点类导入完成", flush=True)
|
|
|
|
|
|
|
|
|
|
| 400 |
except Exception as e:
|
| 401 |
+
print(f"[预热] ⚠️ {e}", flush=True)
|
| 402 |
|
| 403 |
|
| 404 |
+
def _prepare_runtime() -> None:
|
| 405 |
+
_ensure_comfy()
|
| 406 |
+
_ensure_models()
|
| 407 |
+
_init_comfy_nodes()
|
| 408 |
+
_warmup_node_imports() # 🌟 P1: 预热节点类
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
|
| 410 |
+
try:
|
| 411 |
+
print("⬇️ Starting up...")
|
| 412 |
+
_prepare_runtime()
|
| 413 |
+
except Exception as e:
|
| 414 |
+
print(f"⚠️ Startup failed: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 415 |
|
| 416 |
+
# ==========================================================================
|
| 417 |
+
# ♻️ Reusable Executor Singleton (preserves Comfy model cache across tasks)
|
| 418 |
+
# ==========================================================================
|
|
|
|
| 419 |
|
| 420 |
_exec_lock = threading.Lock() # 串行化执行,GPU 一次一个任务
|
| 421 |
|
|
|
|
| 481 |
except Exception:
|
| 482 |
pass
|
| 483 |
return loop, executor
|
| 484 |
+
# ==========================================================================
|
| 485 |
+
# 🧩 Loader Chain Cache (stable node ids => Comfy node-cache hits, no re-patch)
|
| 486 |
+
# ==========================================================================
|
| 487 |
|
| 488 |
_loader_chain_cache: dict[str, tuple[dict, str]] = {}
|
| 489 |
|
|
|
|
| 492 |
base = tuple((f, round(float(s), 3)) for f, s in base_loras)
|
| 493 |
return f"{engine}|{unet_name}|{base}|{char_lora or ''}|{round(float(char_strength), 3)}"
|
| 494 |
|
| 495 |
+
def _build_clip_inputs(engine_cfg: dict, clip_loader_node: str) -> dict:
|
| 496 |
+
clip_type = engine_cfg.get("clip_type", "moss_image")
|
| 497 |
+
clip_inputs = {"type": clip_type}
|
| 498 |
+
te_filenames = engine_cfg.get("local_te_filename", [])
|
| 499 |
+
if clip_loader_node == "DualCLIPLoader":
|
| 500 |
+
if len(te_filenames) >= 2:
|
| 501 |
+
if "clip_l" in te_filenames[0].lower():
|
| 502 |
+
clip_inputs["clip_name1"] = te_filenames[0]
|
| 503 |
+
clip_inputs["clip_name2"] = te_filenames[1]
|
| 504 |
+
else:
|
| 505 |
+
clip_inputs["clip_name1"] = te_filenames[1]
|
| 506 |
+
clip_inputs["clip_name2"] = te_filenames[0]
|
| 507 |
+
elif len(te_filenames) == 1:
|
| 508 |
+
clip_inputs["clip_name1"] = te_filenames[0]
|
| 509 |
+
else:
|
| 510 |
+
if te_filenames: clip_inputs["clip_name"] = te_filenames[0]
|
| 511 |
+
if "clip_device" in MODEL_CONFIG.get("loader_args", {}):
|
| 512 |
+
clip_inputs["device"] = MODEL_CONFIG["loader_args"]["clip_device"]
|
| 513 |
+
return clip_inputs
|
| 514 |
+
|
| 515 |
def _build_loader_chain(engine, unet_name, base_loras, char_lora, char_strength):
|
| 516 |
"""
|
| 517 |
只构建 1..N 的 loader 链(unet/clip/vae + lora 叠加),
|
|
|
|
| 540 |
cur = str(idx); idx += 1
|
| 541 |
return chain, cur
|
| 542 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 543 |
|
| 544 |
# ==========================================================================
|
| 545 |
# 🔄 Progress Interceptor
|
|
|
|
| 605 |
if has_steps:
|
| 606 |
tasks_status[gen_task_id]["message"] = step_msg
|
| 607 |
print(f"[进度拦截] Task {gen_task_id[:8]} | {self.n}/{self.total} steps ({int(self.n / self.total * 100)}%)")
|
| 608 |
+
|
| 609 |
+
# ==========================================================================
|
| 610 |
+
# 🔁 tqdm Patch: Re-entrant Ref-counting (no save/restore leak)
|
| 611 |
+
# ==========================================================================
|
| 612 |
+
_tqdm_patch_lock = threading.RLock()
|
| 613 |
+
_tqdm_patch_depth = 0
|
| 614 |
+
_tqdm_patch_originals: dict[str, Any] = {}
|
| 615 |
+
|
| 616 |
+
def _tqdm_patch_enter() -> None:
|
| 617 |
+
global _tqdm_patch_depth
|
| 618 |
+
import tqdm as tqdm_lib
|
| 619 |
+
import tqdm.auto as tqdm_auto
|
| 620 |
+
with _tqdm_patch_lock:
|
| 621 |
+
if _tqdm_patch_depth == 0:
|
| 622 |
+
_tqdm_patch_originals["lib"] = tqdm_lib.tqdm
|
| 623 |
+
_tqdm_patch_originals["auto"] = tqdm_auto.tqdm
|
| 624 |
+
tqdm_lib.tqdm = UnifiedProgressTqdm
|
| 625 |
+
tqdm_auto.tqdm = UnifiedProgressTqdm
|
| 626 |
+
_tqdm_patch_depth += 1
|
| 627 |
+
|
| 628 |
+
def _tqdm_patch_exit() -> None:
|
| 629 |
+
global _tqdm_patch_depth
|
| 630 |
+
import tqdm as tqdm_lib
|
| 631 |
+
import tqdm.auto as tqdm_auto
|
| 632 |
+
with _tqdm_patch_lock:
|
| 633 |
+
if _tqdm_patch_depth > 0:
|
| 634 |
+
_tqdm_patch_depth -= 1
|
| 635 |
+
if _tqdm_patch_depth == 0 and "lib" in _tqdm_patch_originals:
|
| 636 |
+
tqdm_lib.tqdm = _tqdm_patch_originals.pop("lib")
|
| 637 |
+
tqdm_auto.tqdm = _tqdm_patch_originals.pop("auto")
|
| 638 |
|
| 639 |
# ==========================================================================
|
| 640 |
# ⚛️ Comfy Backend & Workflow
|
|
|
|
| 672 |
print(f"[RAM监控 - {stage}] 宿主机总内存: {total_gb:.2f} GB | 宿主机可用: {avail_gb:.2f} GB | 本进程占用: {proc_mem_gb:.2f} GB | ���统已用: {mem.percent}%{cgroup_str}")
|
| 673 |
|
| 674 |
|
| 675 |
+
def _io_probe():
|
| 676 |
+
try:
|
| 677 |
+
c = psutil.Process().io_counters()
|
| 678 |
+
return {"read_chars": c.read_chars, "read_bytes": c.read_bytes}
|
| 679 |
+
except Exception:
|
| 680 |
+
return {"read_chars": 0, "read_bytes": 0}
|
| 681 |
+
|
| 682 |
+
def _mem_probe():
|
| 683 |
+
"""返回 (host_Cached_MB, host_Avail_MB, cg_current_GB, cg_file_GB, cg_inactive_file_GB)"""
|
| 684 |
+
cached = avail = -1
|
| 685 |
+
try:
|
| 686 |
+
with open("/proc/meminfo") as f:
|
| 687 |
+
m = {l.split(":")[0]: int(l.split()[1]) // 1024 for l in f if len(l.split()) >= 2}
|
| 688 |
+
cached, avail = m.get("Cached", -1), m.get("MemAvailable", -1)
|
| 689 |
+
except Exception:
|
| 690 |
+
pass
|
| 691 |
+
|
| 692 |
+
cur = file_gb = inact_gb = -1.0
|
| 693 |
+
try:
|
| 694 |
+
with open("/sys/fs/cgroup/memory.current") as f:
|
| 695 |
+
cur = int(f.read().strip()) / 1024**3
|
| 696 |
+
with open("/sys/fs/cgroup/memory.stat") as f:
|
| 697 |
+
for line in f:
|
| 698 |
+
p = line.split()
|
| 699 |
+
if len(p) != 2:
|
| 700 |
+
continue
|
| 701 |
+
if p[0] == "file":
|
| 702 |
+
file_gb = int(p[1]) / 1024**3
|
| 703 |
+
elif p[0] == "inactive_file":
|
| 704 |
+
inact_gb = int(p[1]) / 1024**3
|
| 705 |
+
except Exception:
|
| 706 |
+
pass
|
| 707 |
+
return cached, avail, cur, file_gb, inact_gb
|
| 708 |
+
|
| 709 |
+
def _gpu_id():
|
| 710 |
+
try:
|
| 711 |
+
dev = torch.cuda.current_device()
|
| 712 |
+
p = torch.cuda.get_device_properties(dev)
|
| 713 |
+
return f"dev{dev}/uuid={getattr(p, 'uuid', 'n/a')}"
|
| 714 |
+
except Exception as e:
|
| 715 |
+
return f"n/a({e})"
|
| 716 |
+
|
| 717 |
+
|
| 718 |
# --------------------------------------------------------------------------
|
| 719 |
+
# 📊 Metric Files: Atomic Write + Read-Modify-Write (/tmp/metrics_<task_id>.json)
|
| 720 |
# --------------------------------------------------------------------------
|
| 721 |
_metric_lock = threading.RLock()
|
| 722 |
|
|
|
|
| 764 |
d.update(updates)
|
| 765 |
return True
|
| 766 |
_update_metric(task_id, _m)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 767 |
# --------------------------------------------------------------------------
|
| 768 |
def _ref(node: str, output: int = 0) -> list[Any]:
|
| 769 |
return [node, output]
|
| 770 |
|
| 771 |
+
def select_base_loras(engine: str, lora_states: tuple) -> list[tuple[str, float]]:
|
| 772 |
+
selected = []
|
| 773 |
+
|
| 774 |
+
# 找出当前引擎专属的 Base LoRA 列表,保持顺序
|
| 775 |
+
valid_loras = []
|
| 776 |
+
for name, cfg in BASE_LORAS_CHOICES.items():
|
| 777 |
+
if cfg["engine"] == engine:
|
| 778 |
+
valid_loras.append((name, cfg))
|
| 779 |
+
|
| 780 |
+
# 解析前端传来的勾选状态
|
| 781 |
+
for i in range(0, len(lora_states), 2):
|
| 782 |
+
enable = lora_states[i]
|
| 783 |
+
try:
|
| 784 |
+
strength = float(lora_states[i + 1])
|
| 785 |
+
except (ValueError, TypeError):
|
| 786 |
+
strength = 1.0
|
| 787 |
+
|
| 788 |
+
if enable and strength > 0:
|
| 789 |
+
# 如果前端的数组长度和当前引擎实际可用的 LoRA 列表长度一致
|
| 790 |
+
idx = i // 2
|
| 791 |
+
if idx < len(valid_loras):
|
| 792 |
+
lora_name, lora_cfg = valid_loras[idx]
|
| 793 |
+
selected.append((lora_cfg["filename"], strength))
|
| 794 |
+
|
| 795 |
+
# 你的探针(保留观察)
|
| 796 |
+
print(f"[LoRA选择探针] ✅ 成功匹配并加载: {lora_name} ({strength})")
|
| 797 |
+
|
| 798 |
+
return selected
|
| 799 |
+
|
| 800 |
+
def build_workflow(
|
| 801 |
+
prompt: str,
|
| 802 |
+
unet_name: str,
|
| 803 |
+
engine: str,
|
| 804 |
+
width: int,
|
| 805 |
+
height: int,
|
| 806 |
+
steps: int,
|
| 807 |
+
cfg: float,
|
| 808 |
+
seed: int,
|
| 809 |
+
sampler: str,
|
| 810 |
+
scheduler: str,
|
| 811 |
+
base_loras: list[tuple[str, float]],
|
| 812 |
+
char_lora: str | None,
|
| 813 |
+
char_strength: float,
|
| 814 |
+
) -> dict[str, Any]:
|
| 815 |
+
# --- loader 链:命中缓存则复用(节点 id / inputs 完全一致) ---
|
| 816 |
+
key = _combo_key(engine, unet_name, base_loras, char_lora, char_strength)
|
| 817 |
+
cached = _loader_chain_cache.get(key)
|
| 818 |
+
if cached is not None:
|
| 819 |
+
chain, current_model_node = cached
|
| 820 |
+
workflow = dict(chain) # 浅拷贝,下游不改 loader 链
|
| 821 |
+
else:
|
| 822 |
+
chain, current_model_node = _build_loader_chain(
|
| 823 |
+
engine, unet_name, base_loras, char_lora, char_strength)
|
| 824 |
+
_loader_chain_cache[key] = (chain, current_model_node)
|
| 825 |
+
# 只保留最近 8 个组合,避免内存无限增长
|
| 826 |
+
if len(_loader_chain_cache) > 8:
|
| 827 |
+
_loader_chain_cache.pop(next(iter(_loader_chain_cache)))
|
| 828 |
+
workflow = dict(chain)
|
| 829 |
+
|
| 830 |
+
# --- 下游节点:id 从链末端续接,与原逻辑完全一致 ---
|
| 831 |
+
nodes = MODEL_CONFIG["node_types"]
|
| 832 |
+
node_idx = max(int(k) for k in chain.keys()) + 1
|
| 833 |
+
|
| 834 |
+
final_prompt = prompt.strip()
|
| 835 |
+
workflow[str(node_idx)] = {"class_type": "CLIPTextEncode", "inputs": {"clip": _ref("2"), "text": final_prompt}}
|
| 836 |
+
workflow[str(node_idx+1)] = {"class_type": "ConditioningZeroOut", "inputs": {"conditioning": _ref(str(node_idx))}}
|
| 837 |
+
workflow[str(node_idx+2)] = {"class_type": "EmptyLatentImage", "inputs": {"width": width, "height": height, "batch_size": 1}}
|
| 838 |
+
|
| 839 |
+
workflow[str(node_idx+3)] = {"class_type": "KSampler", "inputs": {"model": _ref(current_model_node), "positive": _ref(str(node_idx)), "negative": _ref(str(node_idx+1)), "latent_image": _ref(str(node_idx+2)), "seed": seed, "steps": steps, "cfg": cfg, "sampler_name": sampler, "scheduler": scheduler, "denoise": 1.0}}
|
| 840 |
+
workflow[str(node_idx+4)] = {"class_type": "VAEDecode", "inputs": {"samples": _ref(str(node_idx+3)), "vae": _ref("3")}}
|
| 841 |
+
workflow[str(node_idx+5)] = {"class_type": "SaveImage", "inputs": {"images": _ref(str(node_idx+4)), "filename_prefix": MODEL_CONFIG["name"].lower()}}
|
| 842 |
+
return workflow
|
| 843 |
+
|
| 844 |
def _log_workflow_summary(workflow: dict[str, Any], task_id: str, load_time_val: float, wf_time: float, image: str = None):
|
| 845 |
"""独立提取并打印单行 JSON 摘要日志(不影响主流程逻辑)"""
|
| 846 |
try:
|
|
|
|
| 915 |
|
| 916 |
except Exception as e:
|
| 917 |
print(f"[Log Format Error]: {e}", flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 918 |
|
| 919 |
def _execute_workflow(workflow: dict[str, Any], task_id: str = None) -> list[str]:
|
| 920 |
print_ram_status("任务开始前")
|
|
|
|
| 1047 |
tasks_status[task_id]["message"] = "📦 LoRA downloaded, loading..."
|
| 1048 |
finally:
|
| 1049 |
_tqdm_patch_exit()
|
| 1050 |
+
_current_download_task_id.set(None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1051 |
|
|
|
|
| 1052 |
|
| 1053 |
# ==========================================================================
|
| 1054 |
+
# 🔥 GPU-side Residual Warmup Wait (same /tmp channel as _read_metric)
|
| 1055 |
# ==========================================================================
|
| 1056 |
def _read_warm_state():
|
| 1057 |
try:
|
|
|
|
| 1063 |
|
| 1064 |
def _warm_wait_in_gpu(unet_choice: str, budget: float = WARMUP_RESIDUAL_WAIT) -> bool:
|
| 1065 |
"""只在"已经非常接近完成"时等,最多 budget 秒。
|
| 1066 |
+
注意:这段是在 GPU 切片内执行,等待即烧配额,所以必须能提前放弃。"""
|
| 1067 |
t0 = time.time()
|
| 1068 |
while time.time() - t0 < budget:
|
| 1069 |
st = _read_warm_state()
|
| 1070 |
+
|
| 1071 |
+
if st and st.get("target") == unet_choice:
|
| 1072 |
+
if st.get("state") == "ready":
|
| 1073 |
+
print(f"[预热] ✅ GPU 内等到 ready({time.time() - t0:.2f}s)", flush=True)
|
| 1074 |
+
return True
|
| 1075 |
+
# ⚠️ 进度太低 = 还在冷读,等满 budget 也等不到。立刻放行,省下配额。
|
| 1076 |
+
p = float(st.get("progress") or 0.0)
|
| 1077 |
+
if p < 0.9:
|
| 1078 |
+
saved = budget - (time.time() - t0)
|
| 1079 |
+
print(f"[预热] ⏭️ 进度仅 {p:.2f},放弃残余等待(省 {saved:.1f}s GPU)", flush=True)
|
| 1080 |
+
return False
|
| 1081 |
+
|
| 1082 |
+
elif time.time() - t0 > 0.5:
|
| 1083 |
+
# 0.5 秒内 target 还没轮���本模型(比如 sweep 占着),也别等
|
| 1084 |
+
print(
|
| 1085 |
+
f"[预热] ⏭️ target={st.get('target') if st else None} ≠ 本模型,放弃残余等待",
|
| 1086 |
+
flush=True,
|
| 1087 |
+
)
|
| 1088 |
+
return False
|
| 1089 |
+
|
| 1090 |
time.sleep(0.25)
|
| 1091 |
+
|
| 1092 |
print(f"[预热] ⏳ GPU 内残余等待超时({budget}s),转冷加载", flush=True)
|
| 1093 |
return False
|
| 1094 |
|
| 1095 |
|
| 1096 |
+
# ==========================================================================
|
| 1097 |
+
# 🎬 Generation
|
| 1098 |
+
# ==========================================================================
|
| 1099 |
+
|
| 1100 |
def _generate_impl(
|
| 1101 |
prompt: str, unet_choice: str, custom_w: int, custom_h: int,
|
| 1102 |
steps: int, cfg: float, seed: int, sampler: str, scheduler: str,
|
|
|
|
| 1177 |
|
| 1178 |
|
| 1179 |
# ==========================================================================
|
| 1180 |
+
# 🧹 Cleanup
|
| 1181 |
# ==========================================================================
|
| 1182 |
|
| 1183 |
|
|
|
|
| 1184 |
# 专门用于保护 tasks_status 字典的状态锁
|
| 1185 |
_status_lock = threading.RLock()
|
| 1186 |
_model_switch_lock = threading.Lock()
|
| 1187 |
|
| 1188 |
+
# ---- Cleanup Policy ----
|
| 1189 |
_STATUS_TTL = 3600 # 终态任务内存保留 60 分钟
|
| 1190 |
_STATUS_MAX_ENTRIES = 200 # 兜底:最多保留 200 条终态
|
| 1191 |
_OUTPUT_TTL = 24 * 3600 # 输出图片保留 24 小时
|
|
|
|
| 1244 |
if deleted:
|
| 1245 |
print(f"[清理] 🧹 删除 {deleted} 个过期输出,释放 {freed / 1024 / 1024:.1f} MB", flush=True)
|
| 1246 |
|
| 1247 |
+
def _read_last_unet():
|
| 1248 |
+
try:
|
| 1249 |
+
with open(WARMUP_LAST_FILE) as f:
|
| 1250 |
+
return json.load(f).get("unet_choice")
|
| 1251 |
+
except Exception:
|
| 1252 |
+
return None
|
| 1253 |
+
|
| 1254 |
|
| 1255 |
# ==========================================================================
|
| 1256 |
+
# 🔥 WarmupManager: File-level Page-cache Warmup State Machine
|
| 1257 |
# ==========================================================================
|
| 1258 |
class WarmupManager:
|
| 1259 |
def __init__(self):
|
|
|
|
| 1671 |
|
| 1672 |
for rec in files:
|
| 1673 |
if self._superseded(seq):
|
| 1674 |
+
done_mb = sum(
|
| 1675 |
+
r.get("size_mb", 0) for r in records.values()
|
| 1676 |
+
if r.get("state") in ("warm", "skipped")
|
| 1677 |
+
)
|
| 1678 |
+
pct = (done_mb / big_total * 100) if big_total else 0.0
|
| 1679 |
+
print(
|
| 1680 |
+
f"[预热] ⏹️ 被新目标抢占 (seq={seq}),"
|
| 1681 |
+
f"已完成 {done_mb:.0f}MB/{big_total:.0f}MB ({pct:.0f}%),"
|
| 1682 |
+
f"用时 {time.time() - t0:.1f}s",
|
| 1683 |
+
flush=True,
|
| 1684 |
+
)
|
| 1685 |
break
|
| 1686 |
records[f"{rec['owner']}::{rec['file_key']}"] = rec
|
| 1687 |
|
|
|
|
| 1788 |
|
| 1789 |
|
| 1790 |
warmup = WarmupManager()
|
| 1791 |
+
|
| 1792 |
+
def _boot_warmup():
|
| 1793 |
+
"""两段式:
|
| 1794 |
+
① 阶梯一级(<1s):上次模型的 UNET 头部 → 用户第一眼就有"部分热"
|
| 1795 |
+
② 全量 sweep(P1,可被抢占):等 ① 跑完再启动,否则会把 ① 顶掉
|
| 1796 |
+
"""
|
| 1797 |
+
last = _read_last_unet()
|
| 1798 |
+
target = last if last in UNET_CHOICES else next(iter(UNET_CHOICES), None)
|
| 1799 |
+
|
| 1800 |
+
lite_requested = False
|
| 1801 |
+
if target:
|
| 1802 |
+
rec = warmup._rec(MODELS / "diffusion_models" / UNET_CHOICES[target]["filename"],
|
| 1803 |
+
"unet", "unet")
|
| 1804 |
+
mbps = warmup.probe(rec)
|
| 1805 |
+
if mbps is not None and mbps >= WARMUP_WARM_MBPS:
|
| 1806 |
+
print(f"[预热] ⚡ boot: UNET 已在 cache 中 ({mbps:.0f}MB/s),跳过 lite", flush=True)
|
| 1807 |
+
else:
|
| 1808 |
+
print(f"[预热] 🥾 boot: UNET 探测 {mbps} MB/s → 做阶梯一级", flush=True)
|
| 1809 |
+
warmup.request(target, lite=True, reason="boot", priority=1)
|
| 1810 |
+
lite_requested = True # ← 新增
|
| 1811 |
+
|
| 1812 |
+
if not WARMUP_BOOT_FULL:
|
| 1813 |
+
return
|
| 1814 |
+
|
| 1815 |
+
if lite_requested: # ← 只在发起了才等
|
| 1816 |
+
t0 = time.time()
|
| 1817 |
+
while time.time() - t0 < 20:
|
| 1818 |
+
with warmup._lock:
|
| 1819 |
+
r = warmup._files.get(target, {}).get("unet")
|
| 1820 |
+
if r and r.get("state") in ("partial", "warm", "failed", "stale"):
|
| 1821 |
+
break
|
| 1822 |
+
time.sleep(0.2)
|
| 1823 |
+
|
| 1824 |
+
warmup.request_sweep(reason="boot_sweep")
|
| 1825 |
|
| 1826 |
# ==========================================================================
|
| 1827 |
# 🌐 FastAPI & API Routes
|
|
|
|
| 1841 |
char_strength: float = 1.0
|
| 1842 |
base_lora_states: list = []
|
| 1843 |
|
| 1844 |
+
app = FastAPI()
|
| 1845 |
+
|
| 1846 |
+
|
| 1847 |
+
app.add_middleware(
|
| 1848 |
+
CORSMiddleware,
|
| 1849 |
+
allow_origins=["*"],
|
| 1850 |
+
allow_credentials=True,
|
| 1851 |
+
allow_methods=["*"],
|
| 1852 |
+
allow_headers=["*"],
|
| 1853 |
+
)
|
| 1854 |
+
|
| 1855 |
|
| 1856 |
def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str = None, request: Request = None):
|
| 1857 |
global _last_used_architecture, _last_used_unet
|
|
|
|
| 1867 |
|
| 1868 |
# --- 阶段 0:预热(与 GPU 排队并行,不阻塞 HTTP) ---
|
| 1869 |
try:
|
| 1870 |
+
_wres = warmup.request(
|
| 1871 |
req.unet_choice,
|
| 1872 |
req.char_lora if req.char_enable else None,
|
| 1873 |
char_enable=req.char_enable,
|
| 1874 |
reason="submit", task_id=task_id,
|
| 1875 |
)
|
| 1876 |
+
if not (_wres or {}).get("skipped"): # ← 只在真的排了任务时才显示预热阶段
|
| 1877 |
+
with _status_lock:
|
| 1878 |
+
tasks_status[task_id]["message"] = "🔥 Preparing model files..."
|
| 1879 |
# 只在"接近完成"时短暂等待;冷启动立即放行,避免把 GPU 排队串到预热后面
|
| 1880 |
_warm_ok = warmup.wait_ready(req.unet_choice, timeout=WARMUP_PRE_GPU_BUDGET)
|
| 1881 |
print(f"[预热] gate ready={_warm_ok} progress={warmup.progress(req.unet_choice)}", flush=True)
|
|
|
|
| 2048 |
_gen_leave()
|
| 2049 |
_current_generation_task_id.set(None)
|
| 2050 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2051 |
|
| 2052 |
@app.on_event("startup")
|
| 2053 |
async def _on_startup():
|
|
|
|
| 2058 |
except Exception as e:
|
| 2059 |
print(f"[预热] ⚠️ boot 异常: {e}", flush=True)
|
| 2060 |
threading.Thread(target=_delayed, daemon=True).start()
|
|
|
|
| 2061 |
|
| 2062 |
+
@app.get("/api/models")
|
| 2063 |
+
async def get_models():
|
| 2064 |
+
models_data = {}
|
| 2065 |
+
global_default_prompt = MODEL_CONFIG.get("default_prompt", "")
|
|
|
|
|
|
|
|
|
|
| 2066 |
|
| 2067 |
+
for unet_name, unet_info in UNET_CHOICES.items():
|
| 2068 |
+
engine = unet_info["engine"]
|
| 2069 |
+
engine_name = MODEL_CONFIG["models"].get(engine, {}).get("name", engine)
|
| 2070 |
+
|
| 2071 |
+
if " | " in unet_name:
|
| 2072 |
+
model_specific_name = unet_name.split(" | ", 1)[1]
|
| 2073 |
+
else:
|
| 2074 |
+
model_specific_name = unet_name
|
| 2075 |
+
|
| 2076 |
+
display_unet_name = f"{engine_name} | {model_specific_name}"
|
| 2077 |
+
|
| 2078 |
+
base_loras_for_engine = []
|
| 2079 |
+
for name, cfg in BASE_LORAS_CHOICES.items():
|
| 2080 |
+
if cfg["engine"] == engine:
|
| 2081 |
+
base_loras_for_engine.append({
|
| 2082 |
+
"name": name,
|
| 2083 |
+
"filename": cfg["filename"],
|
| 2084 |
+
"defaultEnable": cfg.get("defaultEnable", False)
|
| 2085 |
+
})
|
| 2086 |
+
|
| 2087 |
+
engine_cfg = MODEL_CONFIG["models"].get(engine, {})
|
| 2088 |
+
final_prompt = engine_cfg.get("default_prompt_override", global_default_prompt)
|
| 2089 |
+
|
| 2090 |
+
# 🌟 核心新增:提取 TE 和 VAE 信息
|
| 2091 |
+
# TE 可能是列表,用 " + " 连接起来展示
|
| 2092 |
+
te_list = engine_cfg.get("local_te_filename", [])
|
| 2093 |
+
te_display = " + ".join([f.replace(".safetensors", "") for f in te_list]) if te_list else "Unknown"
|
| 2094 |
+
|
| 2095 |
+
# VAE 通常是字符串
|
| 2096 |
+
vae_display = engine_cfg.get("local_vae_filename", "Unknown").replace(".safetensors", "")
|
| 2097 |
+
|
| 2098 |
+
models_data[display_unet_name] = {
|
| 2099 |
+
"engine": engine,
|
| 2100 |
+
"defaults": engine_cfg.get("defaults", {}),
|
| 2101 |
+
"base_loras": base_loras_for_engine,
|
| 2102 |
+
"default_prompt": final_prompt,
|
| 2103 |
+
"te_name": te_display, # 👈 传给前端
|
| 2104 |
+
"vae_name": vae_display # 👈 传给前端
|
| 2105 |
+
}
|
| 2106 |
+
|
| 2107 |
+
return {"models": models_data}
|
| 2108 |
+
|
| 2109 |
+
@app.get("/api/loras")
|
| 2110 |
+
async def get_loras():
|
| 2111 |
+
# ⚠️ load_char_loras_from_repo 里有同步 requests.get + list_repo_files(网络 IO)。
|
| 2112 |
+
# 直接在 async 路由里调用会阻塞 event loop,期间所有请求(含 /api/config)
|
| 2113 |
+
# 都无法处理,HF 代理超时后会 reset 连接 → 前端表现为 "Failed to fetch"。
|
| 2114 |
+
# 扔进线程池执行,event loop 保持畅通。
|
| 2115 |
+
await asyncio.to_thread(load_char_loras_from_repo)
|
| 2116 |
+
return {"loras": CHAR_LORA_CHOICES}
|
| 2117 |
+
|
| 2118 |
+
@app.get("/api/config")
|
| 2119 |
+
async def get_sampler_config():
|
| 2120 |
+
return {
|
| 2121 |
+
"samplers": SAMPLERS,
|
| 2122 |
+
"schedulers": SCHEDULERS,
|
| 2123 |
+
"root_mapping": MODEL_CONFIG.get("json_keys", {}).get("root_mapping", {}),
|
| 2124 |
+
"default_prompt": MODEL_CONFIG.get("default_prompt", ""),
|
| 2125 |
+
}
|
| 2126 |
+
|
| 2127 |
+
@app.get("/api/health")
|
| 2128 |
+
async def health_check():
|
| 2129 |
+
return {"status": "ok"}
|
| 2130 |
|
| 2131 |
@app.post("/api/generate")
|
| 2132 |
async def handle_generate(req: GenerateRequest, background_tasks: BackgroundTasks, request: Request):
|
|
|
|
| 2165 |
tasks_status[task_id] = {"status": "queued", "message": "⏳ 任务已进入队列..."}
|
| 2166 |
|
| 2167 |
# 👇 抛弃 BackgroundTasks,使用原生守护线程,直接与 HTTP 返回脱钩
|
|
|
|
| 2168 |
t = threading.Thread(
|
| 2169 |
target=background_generate_task,
|
| 2170 |
args=(task_id, req, ip_token, request),
|
|
|
|
| 2195 |
status_data["message"] = file_data["phase"]
|
| 2196 |
return status_data
|
| 2197 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2198 |
|
| 2199 |
+
# ==========================================================================
|
| 2200 |
+
# 🔥 Warmup Endpoints
|
| 2201 |
+
# ==========================================================================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2202 |
|
| 2203 |
class WarmupRequest(BaseModel):
|
| 2204 |
unet_choice: str
|
|
|
|
| 2252 |
return warmup.snapshot()
|
| 2253 |
|
| 2254 |
# ==========================================================================
|
| 2255 |
+
# 🔧 Warmup Debug Endpoint (disabled by default)
|
| 2256 |
+
# Set WARMUP_DEBUG_TOKEN in Space Secrets; send X-Debug-Token header or ?token=
|
| 2257 |
# ==========================================================================
|
| 2258 |
import hmac
|
| 2259 |
|
|
|
|
| 2420 |
|
| 2421 |
raise HTTPException(400, f"unknown action: {action}")
|
| 2422 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2423 |
# ==========================================================================
|
| 2424 |
# 📁 Static Files Sync & Mount
|
| 2425 |
# ==========================================================================
|
|
|
|
| 2455 |
# 1. 启动时执行同步
|
| 2456 |
sync_ui_assets()
|
| 2457 |
|
|
|
|
| 2458 |
# 🌟 核心修改:必须在挂载根路径 "/" 之前,先挂载 /output 目录
|
|
|
|
| 2459 |
app.mount("/output", StaticFiles(directory=str(OUTPUT)), name="output")
|
| 2460 |
|
| 2461 |
# 2. 挂载静态文件夹 (前端)
|