waveshaker commited on
Commit
02fefcd
·
verified ·
1 Parent(s): dce3bdf

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +418 -405
app.py CHANGED
@@ -31,14 +31,15 @@ import uvicorn
31
  import threading
32
 
33
  from model_config import MODEL_CONFIG
 
 
34
  # ⚠️ 必须在 load_char_loras_from_repo() 之前定义 —— 它内部要读这个常量。
35
  # 放在文件后面的"静态挂载"段里会导致启动首次扫描 NameError,触发词全空且被 300s 缓存锁住。
36
  LORA_META_URL = "https://huggingface.co/spaces/waveshaker/loras_gallery/resolve/main/models.json"
37
 
38
 
39
-
40
  # ==========================================================================
41
- # 🛠️ Environment Patches & Global Paths
42
  # ==========================================================================
43
 
44
  import builtins
@@ -84,7 +85,7 @@ OUTPUT = COMFY / "output"
84
  COMFY_COMMIT = "c1eece62cbf1805d4f64286d7f5592e6a215e6ba"
85
 
86
  # ==========================================================================
87
- # 🔥 Warmup 常量
88
  # ==========================================================================
89
  WARMUP_BOOT_FULL = True
90
  WARMUP_BOOT_DELAY = 1.0 # 让第一个 health 先过
@@ -102,6 +103,9 @@ WARMUP_LAST_FILE = str(ROOT / ".warmup_last.json")
102
  WARMUP_PRE_GPU_BUDGET = 8.0 # 申请 GPU 前最多等多久(覆盖"已经很接近完成")
103
  WARMUP_RESIDUAL_WAIT = 5.0 # GPU 内残余等待上限(烧 GPU 预算,务必小)
104
  SYNC_ON_BOOT = False # 启动时是否 os.sync(默认关:只刷脏页,不预热)
 
 
 
105
  # ==========================================================================
106
 
107
  _comfy_ready = False
@@ -246,6 +250,13 @@ def load_char_loras_from_repo():
246
  total = sum(len(eng) for eng in CHAR_LORA_CHOICES.values())
247
  matched = sum(1 for eng in CHAR_LORA_CHOICES.values() for v in eng.values() if v.get("trigger"))
248
  print(f"✅ 成功刷新角色 LoRA: {[f'{k}: {len(v)}个' for k, v in CHAR_LORA_CHOICES.items()]} | trigger 覆盖 {matched}/{total}")
 
 
 
 
 
 
 
249
  except Exception as e:
250
  print(f"⚠️ 动态刷新失败: {e}")
251
 
@@ -270,6 +281,17 @@ def _install_filtered_requirements(path: pathlib.Path) -> None:
270
  if requirements:
271
  _run([sys.executable, "-m", "pip", "install", "--no-cache-dir", *requirements], check=False)
272
 
 
 
 
 
 
 
 
 
 
 
 
273
  def _ensure_comfy() -> None:
274
  global _comfy_ready
275
  if _comfy_ready: return
@@ -301,6 +323,35 @@ def _ensure_comfy() -> None:
301
 
302
  _comfy_ready = True
303
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
304
  def _init_comfy_nodes() -> None:
305
  global _nodes_ready
306
  if _nodes_ready: return
@@ -333,78 +384,38 @@ def _init_comfy_nodes() -> None:
333
  print("✅ ComfyUI 节点初始化完成!", flush=True)
334
 
335
  # ==========================================================================
336
- # ⚡ Comfy optimization
337
  # ==========================================================================
338
-
339
-
340
 
341
- def _download_if_needed(repo: str, subfolder: str, filename: str, dest_root: pathlib.Path) -> None:
342
- dest_path = dest_root / filename
343
- if dest_path.exists(): return
344
- print(f"⏳ Downloading: {filename} ...")
345
- dest_root.mkdir(parents=True, exist_ok=True)
346
- hf_hub_download(repo_id=repo, subfolder=subfolder if subfolder else None, filename=filename, local_dir=str(dest_root))
347
- src = dest_root / subfolder / filename if subfolder else dest_root / filename
348
- if src.exists() and src != dest_path:
349
- shutil.move(str(src), str(dest_path))
350
-
351
- # 主动把大文件拉到本地 Page Cache,避免 ZeroGPU worker 跨网络 mmap 缺页
352
- def _warmup_page_cache(file_path: pathlib.Path):
353
- if not file_path.exists():
354
- return
355
- # 只预热大于 500MB 的大文件(量化大模型、大 TE),小文件(如 VAE)没必要
356
- if file_path.stat().st_size < 500 * 1024 * 1024:
357
- return
358
-
359
- print(f"[预热] 🔥 正在拉取文件到本地缓存: {file_path.name} ({file_path.stat().st_size / 1024 / 1024:.1f} MB)...", flush=True)
360
- _t0 = time.time()
361
-
362
  try:
363
- # 顺序读取整个文件。这里不要用 mmap,直接 f.read() 才能最快触发底层网络拉取和本地缓存
364
- with open(file_path, "rb") as f:
365
- # 每次读 32MB,避免瞬间吃掉几十 GB 内存导致 OOM
366
- chunk_size = 32 * 1024 * 1024
367
- while True:
368
- chunk = f.read(chunk_size)
369
- if not chunk:
370
- break
371
- _elapsed = round(time.time() - _t0, 2)
372
- print(f"[预热] ✅ {file_path.name} 已载入本地缓存,耗时: {_elapsed}s", flush=True)
373
  except Exception as e:
374
- print(f"[预热] ⚠️ {file_path.name} 预热失败: {e}", flush=True)
375
 
376
 
377
- def _ensure_models() -> None:
378
- print("开始全量下载基础模型...")
379
- for engine_cfg in MODEL_CONFIG["models"].values():
380
- for repo, file_path in engine_cfg.get("companion_downloads", []):
381
- path = pathlib.Path(file_path)
382
- if "text_encoders" in str(path): dest_root = MODELS / "text_encoders"
383
- elif "vae" in str(path): dest_root = MODELS / "vae"
384
- else: dest_root = MODELS / path.parent
385
- _download_if_needed(repo, str(path.parent), path.name, dest_root)
386
- for name, cfg in UNET_CHOICES.items():
387
- _download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], MODELS / "diffusion_models")
388
- lora_root = MODELS / "loras"
389
- for name, cfg in BASE_LORAS_CHOICES.items():
390
- _download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], lora_root)
391
 
392
- # ❌ 不再在这里 os.sync + 全量预热:
393
- # ① 同步跑在 import 阶段 → uvicorn 还没 listen,/api/health 打不进来
394
- # ② 会阻塞启动几十秒,且和后面的 sweep 重复
395
- # ✅ 预热统一交给 WarmupManager,在 @app.on_event("startup") 的后台线程里做
396
- if SYNC_ON_BOOT:
397
- try:
398
- _t0 = time.time()
399
- os.sync()
400
- print(f"✅ 脏页已刷盘 (os.sync 耗时: {round(time.time() - _t0, 2)}s)", flush=True)
401
- except Exception as e:
402
- print(f"⚠️ os.sync 失败: {e}", flush=True)
403
 
404
- # --------------------------------------------------------------------------
405
- # ♻️ Reusable Executor Singleton (preserves Comfy model cache across tasks)
406
- # --------------------------------------------------------------------------
407
- import threading
408
 
409
  _exec_lock = threading.Lock() # 串行化执行,GPU 一次一个任务
410
 
@@ -470,9 +481,9 @@ def _get_executor():
470
  except Exception:
471
  pass
472
  return loop, executor
473
- # --------------------------------------------------------------------------
474
- # 🧩 Loader Chain Cache (stable node ids => Comfy node-cache hits, no re-patch
475
- # --------------------------------------------------------------------------
476
 
477
  _loader_chain_cache: dict[str, tuple[dict, str]] = {}
478
 
@@ -481,6 +492,26 @@ def _combo_key(engine: str, unet_name: str,
481
  base = tuple((f, round(float(s), 3)) for f, s in base_loras)
482
  return f"{engine}|{unet_name}|{base}|{char_lora or ''}|{round(float(char_strength), 3)}"
483
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
484
  def _build_loader_chain(engine, unet_name, base_loras, char_lora, char_strength):
485
  """
486
  只构建 1..N 的 loader 链(unet/clip/vae + lora 叠加),
@@ -509,36 +540,6 @@ def _build_loader_chain(engine, unet_name, base_loras, char_lora, char_strength)
509
  cur = str(idx); idx += 1
510
  return chain, cur
511
 
512
- # ==========================================================================
513
- # 💻 Prepare runtime
514
- # ==========================================================================
515
-
516
- def _warmup_node_imports() -> None:
517
- """预热 Comfy 节点类导入,消除首次 execute 的 import 开销"""
518
- try:
519
- import nodes as comfy_nodes
520
- for class_name in ["UNETLoader", "CLIPLoader", "DualCLIPLoader", "VAELoader",
521
- "LoraLoaderModelOnly", "KSampler", "VAEDecode", "SaveImage",
522
- "CLIPTextEncode", "ConditioningZeroOut", "EmptyLatentImage"]:
523
- if hasattr(comfy_nodes, class_name):
524
- getattr(comfy_nodes, class_name)
525
- print("[预热] ✅ 节点类导入完成", flush=True)
526
- except Exception as e:
527
- print(f"[预热] ⚠️ {e}", flush=True)
528
-
529
-
530
- def _prepare_runtime() -> None:
531
- _ensure_comfy()
532
- _ensure_models()
533
- _init_comfy_nodes()
534
- _warmup_node_imports() # 🌟 P1: 预热节点类
535
-
536
- try:
537
- print("⬇️ Starting up...")
538
- _prepare_runtime()
539
- except Exception as e:
540
- print(f"⚠️ Startup failed: {e}")
541
-
542
 
543
  # ==========================================================================
544
  # 🔄 Progress Interceptor
@@ -604,6 +605,36 @@ class UnifiedProgressTqdm(tqdm):
604
  if has_steps:
605
  tasks_status[gen_task_id]["message"] = step_msg
606
  print(f"[进度拦截] Task {gen_task_id[:8]} | {self.n}/{self.total} steps ({int(self.n / self.total * 100)}%)")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
607
 
608
  # ==========================================================================
609
  # ⚛️ Comfy Backend & Workflow
@@ -641,8 +672,51 @@ def print_ram_status(stage: str):
641
  print(f"[RAM监控 - {stage}] 宿主机总内存: {total_gb:.2f} GB | 宿主机可用: {avail_gb:.2f} GB | 本进程占用: {proc_mem_gb:.2f} GB | ���统已用: {mem.percent}%{cgroup_str}")
642
 
643
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
644
  # --------------------------------------------------------------------------
645
- # 📊 Metric 文件:原子写入 + 统一读改写(/tmp/metrics_<task_id>.json)
646
  # --------------------------------------------------------------------------
647
  _metric_lock = threading.RLock()
648
 
@@ -690,40 +764,83 @@ def _write_metric(task_id: str, **updates) -> None:
690
  d.update(updates)
691
  return True
692
  _update_metric(task_id, _m)
693
-
694
- # --------------------------------------------------------------------------
695
- # 🔁 tqdm Patch:可重入引用计数,杜绝并发 save/restore 泄漏
696
- # --------------------------------------------------------------------------
697
- _tqdm_patch_lock = threading.RLock()
698
- _tqdm_patch_depth = 0
699
- _tqdm_patch_originals: dict[str, Any] = {}
700
-
701
- def _tqdm_patch_enter() -> None:
702
- global _tqdm_patch_depth
703
- import tqdm as tqdm_lib
704
- import tqdm.auto as tqdm_auto
705
- with _tqdm_patch_lock:
706
- if _tqdm_patch_depth == 0:
707
- _tqdm_patch_originals["lib"] = tqdm_lib.tqdm
708
- _tqdm_patch_originals["auto"] = tqdm_auto.tqdm
709
- tqdm_lib.tqdm = UnifiedProgressTqdm
710
- tqdm_auto.tqdm = UnifiedProgressTqdm
711
- _tqdm_patch_depth += 1
712
-
713
- def _tqdm_patch_exit() -> None:
714
- global _tqdm_patch_depth
715
- import tqdm as tqdm_lib
716
- import tqdm.auto as tqdm_auto
717
- with _tqdm_patch_lock:
718
- if _tqdm_patch_depth > 0:
719
- _tqdm_patch_depth -= 1
720
- if _tqdm_patch_depth == 0 and "lib" in _tqdm_patch_originals:
721
- tqdm_lib.tqdm = _tqdm_patch_originals.pop("lib")
722
- tqdm_auto.tqdm = _tqdm_patch_originals.pop("auto")
723
  # --------------------------------------------------------------------------
724
  def _ref(node: str, output: int = 0) -> list[Any]:
725
  return [node, output]
726
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
727
  def _log_workflow_summary(workflow: dict[str, Any], task_id: str, load_time_val: float, wf_time: float, image: str = None):
728
  """独立提取并打印单行 JSON 摘要日志(不影响主流程逻辑)"""
729
  try:
@@ -798,49 +915,6 @@ def _log_workflow_summary(workflow: dict[str, Any], task_id: str, load_time_val:
798
 
799
  except Exception as e:
800
  print(f"[Log Format Error]: {e}", flush=True)
801
-
802
-
803
- def _io_probe():
804
- try:
805
- c = psutil.Process().io_counters()
806
- return {"read_chars": c.read_chars, "read_bytes": c.read_bytes}
807
- except Exception:
808
- return {"read_chars": 0, "read_bytes": 0}
809
-
810
- def _mem_probe():
811
- """返回 (host_Cached_MB, host_Avail_MB, cg_current_GB, cg_file_GB, cg_inactive_file_GB)"""
812
- cached = avail = -1
813
- try:
814
- with open("/proc/meminfo") as f:
815
- m = {l.split(":")[0]: int(l.split()[1]) // 1024 for l in f if len(l.split()) >= 2}
816
- cached, avail = m.get("Cached", -1), m.get("MemAvailable", -1)
817
- except Exception:
818
- pass
819
-
820
- cur = file_gb = inact_gb = -1.0
821
- try:
822
- with open("/sys/fs/cgroup/memory.current") as f:
823
- cur = int(f.read().strip()) / 1024**3
824
- with open("/sys/fs/cgroup/memory.stat") as f:
825
- for line in f:
826
- p = line.split()
827
- if len(p) != 2:
828
- continue
829
- if p[0] == "file":
830
- file_gb = int(p[1]) / 1024**3
831
- elif p[0] == "inactive_file":
832
- inact_gb = int(p[1]) / 1024**3
833
- except Exception:
834
- pass
835
- return cached, avail, cur, file_gb, inact_gb
836
-
837
- def _gpu_id():
838
- try:
839
- dev = torch.cuda.current_device()
840
- p = torch.cuda.get_device_properties(dev)
841
- return f"dev{dev}/uuid={getattr(p, 'uuid', 'n/a')}"
842
- except Exception as e:
843
- return f"n/a({e})"
844
 
845
  def _execute_workflow(workflow: dict[str, Any], task_id: str = None) -> list[str]:
846
  print_ram_status("任务开始前")
@@ -973,105 +1047,11 @@ def prepare_env(char_enable: bool, char_lora: str, unet_choice: str, task_id: st
973
  tasks_status[task_id]["message"] = "📦 LoRA downloaded, loading..."
974
  finally:
975
  _tqdm_patch_exit()
976
- _current_download_task_id.set(None)
977
-
978
- def select_base_loras(engine: str, lora_states: tuple) -> list[tuple[str, float]]:
979
- selected = []
980
-
981
- # 找出当前引擎专属的 Base LoRA 列表,保持顺序
982
- valid_loras = []
983
- for name, cfg in BASE_LORAS_CHOICES.items():
984
- if cfg["engine"] == engine:
985
- valid_loras.append((name, cfg))
986
-
987
- # 解析前端传来的勾选状态
988
- for i in range(0, len(lora_states), 2):
989
- enable = lora_states[i]
990
- try:
991
- strength = float(lora_states[i + 1])
992
- except (ValueError, TypeError):
993
- strength = 1.0
994
-
995
- if enable and strength > 0:
996
- # 如果前端的数组长度和当前引擎实际可用的 LoRA 列表长度一致
997
- idx = i // 2
998
- if idx < len(valid_loras):
999
- lora_name, lora_cfg = valid_loras[idx]
1000
- selected.append((lora_cfg["filename"], strength))
1001
-
1002
- # 你的探针(保留观察)
1003
- print(f"[LoRA选择探针] ✅ 成功匹配并加载: {lora_name} ({strength})")
1004
-
1005
- return selected
1006
-
1007
- def _build_clip_inputs(engine_cfg: dict, clip_loader_node: str) -> dict:
1008
- clip_type = engine_cfg.get("clip_type", "moss_image")
1009
- clip_inputs = {"type": clip_type}
1010
- te_filenames = engine_cfg.get("local_te_filename", [])
1011
- if clip_loader_node == "DualCLIPLoader":
1012
- if len(te_filenames) >= 2:
1013
- if "clip_l" in te_filenames[0].lower():
1014
- clip_inputs["clip_name1"] = te_filenames[0]
1015
- clip_inputs["clip_name2"] = te_filenames[1]
1016
- else:
1017
- clip_inputs["clip_name1"] = te_filenames[1]
1018
- clip_inputs["clip_name2"] = te_filenames[0]
1019
- elif len(te_filenames) == 1:
1020
- clip_inputs["clip_name1"] = te_filenames[0]
1021
- else:
1022
- if te_filenames: clip_inputs["clip_name"] = te_filenames[0]
1023
- if "clip_device" in MODEL_CONFIG.get("loader_args", {}):
1024
- clip_inputs["device"] = MODEL_CONFIG["loader_args"]["clip_device"]
1025
- return clip_inputs
1026
-
1027
- def build_workflow(
1028
- prompt: str,
1029
- unet_name: str,
1030
- engine: str,
1031
- width: int,
1032
- height: int,
1033
- steps: int,
1034
- cfg: float,
1035
- seed: int,
1036
- sampler: str,
1037
- scheduler: str,
1038
- base_loras: list[tuple[str, float]],
1039
- char_lora: str | None,
1040
- char_strength: float,
1041
- ) -> dict[str, Any]:
1042
- # --- loader 链:命中缓存则复用(节点 id / inputs 完全一致) ---
1043
- key = _combo_key(engine, unet_name, base_loras, char_lora, char_strength)
1044
- cached = _loader_chain_cache.get(key)
1045
- if cached is not None:
1046
- chain, current_model_node = cached
1047
- workflow = dict(chain) # 浅拷贝,下游不改 loader 链
1048
- else:
1049
- chain, current_model_node = _build_loader_chain(
1050
- engine, unet_name, base_loras, char_lora, char_strength)
1051
- _loader_chain_cache[key] = (chain, current_model_node)
1052
- # 只保留最近 8 个组合,避免内存无限增长
1053
- if len(_loader_chain_cache) > 8:
1054
- _loader_chain_cache.pop(next(iter(_loader_chain_cache)))
1055
- workflow = dict(chain)
1056
-
1057
- # --- 下游节点:id 从链末端续接,与原逻辑完全一致 ---
1058
- nodes = MODEL_CONFIG["node_types"]
1059
- node_idx = max(int(k) for k in chain.keys()) + 1
1060
-
1061
- final_prompt = prompt.strip()
1062
- workflow[str(node_idx)] = {"class_type": "CLIPTextEncode", "inputs": {"clip": _ref("2"), "text": final_prompt}}
1063
- workflow[str(node_idx+1)] = {"class_type": "ConditioningZeroOut", "inputs": {"conditioning": _ref(str(node_idx))}}
1064
- workflow[str(node_idx+2)] = {"class_type": "EmptyLatentImage", "inputs": {"width": width, "height": height, "batch_size": 1}}
1065
-
1066
- workflow[str(node_idx+3)] = {"class_type": "KSampler", "inputs": {"model": _ref(current_model_node), "positive": _ref(str(node_idx)), "negative": _ref(str(node_idx+1)), "latent_image": _ref(str(node_idx+2)), "seed": seed, "steps": steps, "cfg": cfg, "sampler_name": sampler, "scheduler": scheduler, "denoise": 1.0}}
1067
- workflow[str(node_idx+4)] = {"class_type": "VAEDecode", "inputs": {"samples": _ref(str(node_idx+3)), "vae": _ref("3")}}
1068
- workflow[str(node_idx+5)] = {"class_type": "SaveImage", "inputs": {"images": _ref(str(node_idx+4)), "filename_prefix": MODEL_CONFIG["name"].lower()}}
1069
- return workflow
1070
 
1071
- # 🌟 增加全局变量用于追踪 GPU 域内的架构切换
1072
 
1073
  # ==========================================================================
1074
- # 🔥 GPU 内残余预热等待(跨进程读 /tmp 状态文件,与 _read_metric 同一机制)
1075
  # ==========================================================================
1076
  def _read_warm_state():
1077
  try:
@@ -1083,17 +1063,40 @@ def _read_warm_state():
1083
 
1084
  def _warm_wait_in_gpu(unet_choice: str, budget: float = WARMUP_RESIDUAL_WAIT) -> bool:
1085
  """只在"已经非常接近完成"时等,最多 budget 秒。
1086
- 注意:这段是在 GPU 切片内执行,等待即烧配额,所以上限必须小。"""
1087
  t0 = time.time()
1088
  while time.time() - t0 < budget:
1089
  st = _read_warm_state()
1090
- if st and st.get("target") == unet_choice and st.get("state") == "ready":
1091
- return True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1092
  time.sleep(0.25)
 
1093
  print(f"[预热] ⏳ GPU 内残余等待超时({budget}s),转冷加载", flush=True)
1094
  return False
1095
 
1096
 
 
 
 
 
1097
  def _generate_impl(
1098
  prompt: str, unet_choice: str, custom_w: int, custom_h: int,
1099
  steps: int, cfg: float, seed: int, sampler: str, scheduler: str,
@@ -1174,16 +1177,15 @@ demo.queue()
1174
 
1175
 
1176
  # ==========================================================================
1177
- # 🧹 Clean
1178
  # ==========================================================================
1179
 
1180
 
1181
- import threading
1182
  # 专门用于保护 tasks_status 字典的状态锁
1183
  _status_lock = threading.RLock()
1184
  _model_switch_lock = threading.Lock()
1185
 
1186
- # ---- 清理策略 ----
1187
  _STATUS_TTL = 3600 # 终态任务内存保留 60 分钟
1188
  _STATUS_MAX_ENTRIES = 200 # 兜底:最多保留 200 条终态
1189
  _OUTPUT_TTL = 24 * 3600 # 输出图片保留 24 小时
@@ -1242,9 +1244,16 @@ def _cleanup_stale(force: bool = False) -> None:
1242
  if deleted:
1243
  print(f"[清理] 🧹 删除 {deleted} 个过期输出,释放 {freed / 1024 / 1024:.1f} MB", flush=True)
1244
 
 
 
 
 
 
 
 
1245
 
1246
  # ==========================================================================
1247
- # 🔥 WarmupManager:文件级 page-cache 预热状态机
1248
  # ==========================================================================
1249
  class WarmupManager:
1250
  def __init__(self):
@@ -1662,7 +1671,17 @@ class WarmupManager:
1662
 
1663
  for rec in files:
1664
  if self._superseded(seq):
1665
- print(f"[预热] ⏹️ 被新目标抢占 (seq={seq})", flush=True)
 
 
 
 
 
 
 
 
 
 
1666
  break
1667
  records[f"{rec['owner']}::{rec['file_key']}"] = rec
1668
 
@@ -1769,6 +1788,40 @@ class WarmupManager:
1769
 
1770
 
1771
  warmup = WarmupManager()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1772
 
1773
  # ==========================================================================
1774
  # 🌐 FastAPI & API Routes
@@ -1788,6 +1841,17 @@ class GenerateRequest(BaseModel):
1788
  char_strength: float = 1.0
1789
  base_lora_states: list = []
1790
 
 
 
 
 
 
 
 
 
 
 
 
1791
 
1792
  def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str = None, request: Request = None):
1793
  global _last_used_architecture, _last_used_unet
@@ -1803,14 +1867,15 @@ def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str =
1803
 
1804
  # --- 阶段 0:预热(与 GPU 排队并行,不阻塞 HTTP) ---
1805
  try:
1806
- warmup.request(
1807
  req.unet_choice,
1808
  req.char_lora if req.char_enable else None,
1809
  char_enable=req.char_enable,
1810
  reason="submit", task_id=task_id,
1811
  )
1812
- with _status_lock:
1813
- tasks_status[task_id]["message"] = "🔥 Preparing model files..."
 
1814
  # 只在"接近完成"时短暂等待;冷启动立即放行,避免把 GPU 排队串到预热后面
1815
  _warm_ok = warmup.wait_ready(req.unet_choice, timeout=WARMUP_PRE_GPU_BUDGET)
1816
  print(f"[预热] gate ready={_warm_ok} progress={warmup.progress(req.unet_choice)}", flush=True)
@@ -1983,49 +2048,6 @@ def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str =
1983
  _gen_leave()
1984
  _current_generation_task_id.set(None)
1985
 
1986
- app = FastAPI()
1987
-
1988
- def _read_last_unet():
1989
- try:
1990
- with open(WARMUP_LAST_FILE) as f:
1991
- return json.load(f).get("unet_choice")
1992
- except Exception:
1993
- return None
1994
-
1995
- def _boot_warmup():
1996
- """两段式:
1997
- ① 阶梯一级(<1s):上次模型的 UNET 头部 → 用户第一眼就有"部分热"
1998
- ② 全量 sweep(P1,可被抢占):等 ① 跑完再启动,否则会把 ① 顶掉
1999
- """
2000
- last = _read_last_unet()
2001
- target = last if last in UNET_CHOICES else next(iter(UNET_CHOICES), None)
2002
-
2003
- lite_requested = False
2004
- if target:
2005
- rec = warmup._rec(MODELS / "diffusion_models" / UNET_CHOICES[target]["filename"],
2006
- "unet", "unet")
2007
- mbps = warmup.probe(rec)
2008
- if mbps is not None and mbps >= WARMUP_WARM_MBPS:
2009
- print(f"[预热] ⚡ boot: UNET 已在 cache 中 ({mbps:.0f}MB/s),跳过 lite", flush=True)
2010
- else:
2011
- print(f"[预热] 🥾 boot: UNET 探测 {mbps} MB/s → 做阶梯一级", flush=True)
2012
- warmup.request(target, lite=True, reason="boot", priority=1)
2013
- lite_requested = True # ← 新增
2014
-
2015
- if not WARMUP_BOOT_FULL:
2016
- return
2017
-
2018
- if lite_requested: # ← 只在发起了才等
2019
- t0 = time.time()
2020
- while time.time() - t0 < 20:
2021
- with warmup._lock:
2022
- r = warmup._files.get(target, {}).get("unet")
2023
- if r and r.get("state") in ("partial", "warm", "failed", "stale"):
2024
- break
2025
- time.sleep(0.2)
2026
-
2027
- warmup.request_sweep(reason="boot_sweep")
2028
-
2029
 
2030
  @app.on_event("startup")
2031
  async def _on_startup():
@@ -2036,17 +2058,75 @@ async def _on_startup():
2036
  except Exception as e:
2037
  print(f"[预热] ⚠️ boot 异常: {e}", flush=True)
2038
  threading.Thread(target=_delayed, daemon=True).start()
2039
-
2040
 
2041
- app.add_middleware(
2042
- CORSMiddleware,
2043
- allow_origins=["*"],
2044
- allow_credentials=True,
2045
- allow_methods=["*"],
2046
- allow_headers=["*"],
2047
- )
2048
 
2049
- from fastapi import HTTPException, Depends, Header, Query
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2050
 
2051
  @app.post("/api/generate")
2052
  async def handle_generate(req: GenerateRequest, background_tasks: BackgroundTasks, request: Request):
@@ -2085,7 +2165,6 @@ async def handle_generate(req: GenerateRequest, background_tasks: BackgroundTask
2085
  tasks_status[task_id] = {"status": "queued", "message": "⏳ 任务已进入队列..."}
2086
 
2087
  # 👇 抛弃 BackgroundTasks,使用原生守护线程,直接与 HTTP 返回脱钩
2088
- import threading
2089
  t = threading.Thread(
2090
  target=background_generate_task,
2091
  args=(task_id, req, ip_token, request),
@@ -2116,70 +2195,10 @@ async def check_status(task_id: str):
2116
  status_data["message"] = file_data["phase"]
2117
  return status_data
2118
 
2119
- @app.get("/api/models")
2120
- async def get_models():
2121
- models_data = {}
2122
- global_default_prompt = MODEL_CONFIG.get("default_prompt", "")
2123
-
2124
- for unet_name, unet_info in UNET_CHOICES.items():
2125
- engine = unet_info["engine"]
2126
- engine_name = MODEL_CONFIG["models"].get(engine, {}).get("name", engine)
2127
-
2128
- if " | " in unet_name:
2129
- model_specific_name = unet_name.split(" | ", 1)[1]
2130
- else:
2131
- model_specific_name = unet_name
2132
-
2133
- display_unet_name = f"{engine_name} | {model_specific_name}"
2134
-
2135
- base_loras_for_engine = []
2136
- for name, cfg in BASE_LORAS_CHOICES.items():
2137
- if cfg["engine"] == engine:
2138
- base_loras_for_engine.append({
2139
- "name": name,
2140
- "filename": cfg["filename"],
2141
- "defaultEnable": cfg.get("defaultEnable", False)
2142
- })
2143
-
2144
- engine_cfg = MODEL_CONFIG["models"].get(engine, {})
2145
- final_prompt = engine_cfg.get("default_prompt_override", global_default_prompt)
2146
-
2147
- # 🌟 核心新增:提取 TE 和 VAE 信息
2148
- # TE 可能是列表,用 " + " 连接起来展示
2149
- te_list = engine_cfg.get("local_te_filename", [])
2150
- te_display = " + ".join([f.replace(".safetensors", "") for f in te_list]) if te_list else "Unknown"
2151
-
2152
- # VAE 通常是字符串
2153
- vae_display = engine_cfg.get("local_vae_filename", "Unknown").replace(".safetensors", "")
2154
-
2155
- models_data[display_unet_name] = {
2156
- "engine": engine,
2157
- "defaults": engine_cfg.get("defaults", {}),
2158
- "base_loras": base_loras_for_engine,
2159
- "default_prompt": final_prompt,
2160
- "te_name": te_display, # 👈 传给前端
2161
- "vae_name": vae_display # 👈 传给前端
2162
- }
2163
-
2164
- return {"models": models_data}
2165
-
2166
- @app.get("/api/loras")
2167
- async def get_loras():
2168
- # ⚠️ load_char_loras_from_repo 里有同步 requests.get + list_repo_files(网络 IO)。
2169
- # 直接在 async 路由里调用会阻塞 event loop,期间所有请求(含 /api/config)
2170
- # 都无法处理,HF 代理超时后会 reset 连接 → 前端表现为 "Failed to fetch"。
2171
- # 扔进线程池执行,event loop 保持畅通。
2172
- await asyncio.to_thread(load_char_loras_from_repo)
2173
- return {"loras": CHAR_LORA_CHOICES}
2174
 
2175
- @app.get("/api/config")
2176
- async def get_sampler_config():
2177
- return {
2178
- "samplers": SAMPLERS,
2179
- "schedulers": SCHEDULERS,
2180
- "root_mapping": MODEL_CONFIG["json_keys"]["root_mapping"],
2181
- "default_prompt": MODEL_CONFIG.get("default_prompt", ""),
2182
- }
2183
 
2184
  class WarmupRequest(BaseModel):
2185
  unet_choice: str
@@ -2233,8 +2252,8 @@ async def api_warmup_status():
2233
  return warmup.snapshot()
2234
 
2235
  # ==========================================================================
2236
- # 🔧 预热调试端点(默认完全关闭;开启需在 Space Settings → Secrets 配置
2237
- # WARMUP_DEBUG_TOKEN,之后请求必须带 X-Debug-Token 头或 ?token= 参数)
2238
  # ==========================================================================
2239
  import hmac
2240
 
@@ -2401,10 +2420,6 @@ async def debug_warmup(
2401
 
2402
  raise HTTPException(400, f"unknown action: {action}")
2403
 
2404
- @app.get("/api/health")
2405
- async def health_check():
2406
- return {"status": "ok"}
2407
-
2408
  # ==========================================================================
2409
  # 📁 Static Files Sync & Mount
2410
  # ==========================================================================
@@ -2440,9 +2455,7 @@ def sync_ui_assets():
2440
  # 1. 启动时执行同步
2441
  sync_ui_assets()
2442
 
2443
- # ==========================================================
2444
  # 🌟 核心修改:必须在挂载根路径 "/" 之前,先挂载 /output 目录
2445
- # ==========================================================
2446
  app.mount("/output", StaticFiles(directory=str(OUTPUT)), name="output")
2447
 
2448
  # 2. 挂载静态文件夹 (前端)
 
31
  import threading
32
 
33
  from model_config import MODEL_CONFIG
34
+
35
+ from fastapi import HTTPException, Depends, Header, Query
36
  # ⚠️ 必须在 load_char_loras_from_repo() 之前定义 —— 它内部要读这个常量。
37
  # 放在文件后面的"静态挂载"段里会导致启动首次扫描 NameError,触发词全空且被 300s 缓存锁住。
38
  LORA_META_URL = "https://huggingface.co/spaces/waveshaker/loras_gallery/resolve/main/models.json"
39
 
40
 
 
41
  # ==========================================================================
42
+ # 🛠️ Environment Patches & Global Paths
43
  # ==========================================================================
44
 
45
  import builtins
 
85
  COMFY_COMMIT = "c1eece62cbf1805d4f64286d7f5592e6a215e6ba"
86
 
87
  # ==========================================================================
88
+ # 🔥 Warmup Constants
89
  # ==========================================================================
90
  WARMUP_BOOT_FULL = True
91
  WARMUP_BOOT_DELAY = 1.0 # 让第一个 health 先过
 
103
  WARMUP_PRE_GPU_BUDGET = 8.0 # 申请 GPU 前最多等多久(覆盖"已经很接近完成")
104
  WARMUP_RESIDUAL_WAIT = 5.0 # GPU 内残余等待上限(烧 GPU 预算,务必小)
105
  SYNC_ON_BOOT = False # 启动时是否 os.sync(默认关:只刷脏页,不预热)
106
+
107
+ # ==========================================================================
108
+ # 🧠 Global Runtime State
109
  # ==========================================================================
110
 
111
  _comfy_ready = False
 
250
  total = sum(len(eng) for eng in CHAR_LORA_CHOICES.values())
251
  matched = sum(1 for eng in CHAR_LORA_CHOICES.values() for v in eng.values() if v.get("trigger"))
252
  print(f"✅ 成功刷新角色 LoRA: {[f'{k}: {len(v)}个' for k, v in CHAR_LORA_CHOICES.items()]} | trigger 覆盖 {matched}/{total}")
253
+
254
+ # ✅ 新增:分架构覆盖率,数据缺口一眼可见
255
+ per = " | ".join(
256
+ f"{eng}:{sum(1 for v in m.values() if v.get('trigger'))}/{len(m)}"
257
+ for eng, m in CHAR_LORA_CHOICES.items()
258
+ )
259
+ print(f" └ trigger 分架构: {per}")
260
  except Exception as e:
261
  print(f"⚠️ 动态刷新失败: {e}")
262
 
 
281
  if requirements:
282
  _run([sys.executable, "-m", "pip", "install", "--no-cache-dir", *requirements], check=False)
283
 
284
+
285
+ def _download_if_needed(repo: str, subfolder: str, filename: str, dest_root: pathlib.Path) -> None:
286
+ dest_path = dest_root / filename
287
+ if dest_path.exists(): return
288
+ print(f"⏳ Downloading: {filename} ...")
289
+ dest_root.mkdir(parents=True, exist_ok=True)
290
+ hf_hub_download(repo_id=repo, subfolder=subfolder if subfolder else None, filename=filename, local_dir=str(dest_root))
291
+ src = dest_root / subfolder / filename if subfolder else dest_root / filename
292
+ if src.exists() and src != dest_path:
293
+ shutil.move(str(src), str(dest_path))
294
+
295
  def _ensure_comfy() -> None:
296
  global _comfy_ready
297
  if _comfy_ready: return
 
323
 
324
  _comfy_ready = True
325
 
326
+
327
+ def _ensure_models() -> None:
328
+ print("开始全量下载基础模型...")
329
+ for engine_cfg in MODEL_CONFIG["models"].values():
330
+ for repo, file_path in engine_cfg.get("companion_downloads", []):
331
+ path = pathlib.Path(file_path)
332
+ if "text_encoders" in str(path): dest_root = MODELS / "text_encoders"
333
+ elif "vae" in str(path): dest_root = MODELS / "vae"
334
+ else: dest_root = MODELS / path.parent
335
+ _download_if_needed(repo, str(path.parent), path.name, dest_root)
336
+ for name, cfg in UNET_CHOICES.items():
337
+ _download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], MODELS / "diffusion_models")
338
+ lora_root = MODELS / "loras"
339
+ for name, cfg in BASE_LORAS_CHOICES.items():
340
+ _download_if_needed(cfg["repo"], cfg["subfolder"], cfg["filename"], lora_root)
341
+
342
+ # ❌ 不再在这里 os.sync + 全量预热:
343
+ # ① 同步跑在 import 阶段 → uvicorn 还没 listen,/api/health 打不进来
344
+ # ② 会阻塞启动几十秒,且和后面的 sweep 重复
345
+ # ✅ 预热统一交给 WarmupManager,在 @app.on_event("startup") 的后台线程里做
346
+ if SYNC_ON_BOOT:
347
+ try:
348
+ _t0 = time.time()
349
+ os.sync()
350
+ print(f"✅ 脏页已刷盘 (os.sync 耗时: {round(time.time() - _t0, 2)}s)", flush=True)
351
+ except Exception as e:
352
+ print(f"⚠️ os.sync 失败: {e}", flush=True)
353
+
354
+
355
  def _init_comfy_nodes() -> None:
356
  global _nodes_ready
357
  if _nodes_ready: return
 
384
  print("✅ ComfyUI 节点初始化完成!", flush=True)
385
 
386
  # ==========================================================================
387
+ # 💻 Prepare Runtime
388
  # ==========================================================================
 
 
389
 
390
+ def _warmup_node_imports() -> None:
391
+ """预热 Comfy 节点类导入,消除首次 execute 的 import 开销"""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
392
  try:
393
+ import nodes as comfy_nodes
394
+ for class_name in ["UNETLoader", "CLIPLoader", "DualCLIPLoader", "VAELoader",
395
+ "LoraLoaderModelOnly", "KSampler", "VAEDecode", "SaveImage",
396
+ "CLIPTextEncode", "ConditioningZeroOut", "EmptyLatentImage"]:
397
+ if hasattr(comfy_nodes, class_name):
398
+ getattr(comfy_nodes, class_name)
399
+ print("[预热] ✅ 节点类导入完成", flush=True)
 
 
 
400
  except Exception as e:
401
+ print(f"[预热] ⚠️ {e}", flush=True)
402
 
403
 
404
+ def _prepare_runtime() -> None:
405
+ _ensure_comfy()
406
+ _ensure_models()
407
+ _init_comfy_nodes()
408
+ _warmup_node_imports() # 🌟 P1: 预热节点类
 
 
 
 
 
 
 
 
 
409
 
410
+ try:
411
+ print("⬇️ Starting up...")
412
+ _prepare_runtime()
413
+ except Exception as e:
414
+ print(f"⚠️ Startup failed: {e}")
 
 
 
 
 
 
415
 
416
+ # ==========================================================================
417
+ # ♻️ Reusable Executor Singleton (preserves Comfy model cache across tasks)
418
+ # ==========================================================================
 
419
 
420
  _exec_lock = threading.Lock() # 串行化执行,GPU 一次一个任务
421
 
 
481
  except Exception:
482
  pass
483
  return loop, executor
484
+ # ==========================================================================
485
+ # 🧩 Loader Chain Cache (stable node ids => Comfy node-cache hits, no re-patch)
486
+ # ==========================================================================
487
 
488
  _loader_chain_cache: dict[str, tuple[dict, str]] = {}
489
 
 
492
  base = tuple((f, round(float(s), 3)) for f, s in base_loras)
493
  return f"{engine}|{unet_name}|{base}|{char_lora or ''}|{round(float(char_strength), 3)}"
494
 
495
+ def _build_clip_inputs(engine_cfg: dict, clip_loader_node: str) -> dict:
496
+ clip_type = engine_cfg.get("clip_type", "moss_image")
497
+ clip_inputs = {"type": clip_type}
498
+ te_filenames = engine_cfg.get("local_te_filename", [])
499
+ if clip_loader_node == "DualCLIPLoader":
500
+ if len(te_filenames) >= 2:
501
+ if "clip_l" in te_filenames[0].lower():
502
+ clip_inputs["clip_name1"] = te_filenames[0]
503
+ clip_inputs["clip_name2"] = te_filenames[1]
504
+ else:
505
+ clip_inputs["clip_name1"] = te_filenames[1]
506
+ clip_inputs["clip_name2"] = te_filenames[0]
507
+ elif len(te_filenames) == 1:
508
+ clip_inputs["clip_name1"] = te_filenames[0]
509
+ else:
510
+ if te_filenames: clip_inputs["clip_name"] = te_filenames[0]
511
+ if "clip_device" in MODEL_CONFIG.get("loader_args", {}):
512
+ clip_inputs["device"] = MODEL_CONFIG["loader_args"]["clip_device"]
513
+ return clip_inputs
514
+
515
  def _build_loader_chain(engine, unet_name, base_loras, char_lora, char_strength):
516
  """
517
  只构建 1..N 的 loader 链(unet/clip/vae + lora 叠加),
 
540
  cur = str(idx); idx += 1
541
  return chain, cur
542
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
543
 
544
  # ==========================================================================
545
  # 🔄 Progress Interceptor
 
605
  if has_steps:
606
  tasks_status[gen_task_id]["message"] = step_msg
607
  print(f"[进度拦截] Task {gen_task_id[:8]} | {self.n}/{self.total} steps ({int(self.n / self.total * 100)}%)")
608
+
609
+ # ==========================================================================
610
+ # 🔁 tqdm Patch: Re-entrant Ref-counting (no save/restore leak)
611
+ # ==========================================================================
612
+ _tqdm_patch_lock = threading.RLock()
613
+ _tqdm_patch_depth = 0
614
+ _tqdm_patch_originals: dict[str, Any] = {}
615
+
616
+ def _tqdm_patch_enter() -> None:
617
+ global _tqdm_patch_depth
618
+ import tqdm as tqdm_lib
619
+ import tqdm.auto as tqdm_auto
620
+ with _tqdm_patch_lock:
621
+ if _tqdm_patch_depth == 0:
622
+ _tqdm_patch_originals["lib"] = tqdm_lib.tqdm
623
+ _tqdm_patch_originals["auto"] = tqdm_auto.tqdm
624
+ tqdm_lib.tqdm = UnifiedProgressTqdm
625
+ tqdm_auto.tqdm = UnifiedProgressTqdm
626
+ _tqdm_patch_depth += 1
627
+
628
+ def _tqdm_patch_exit() -> None:
629
+ global _tqdm_patch_depth
630
+ import tqdm as tqdm_lib
631
+ import tqdm.auto as tqdm_auto
632
+ with _tqdm_patch_lock:
633
+ if _tqdm_patch_depth > 0:
634
+ _tqdm_patch_depth -= 1
635
+ if _tqdm_patch_depth == 0 and "lib" in _tqdm_patch_originals:
636
+ tqdm_lib.tqdm = _tqdm_patch_originals.pop("lib")
637
+ tqdm_auto.tqdm = _tqdm_patch_originals.pop("auto")
638
 
639
  # ==========================================================================
640
  # ⚛️ Comfy Backend & Workflow
 
672
  print(f"[RAM监控 - {stage}] 宿主机总内存: {total_gb:.2f} GB | 宿主机可用: {avail_gb:.2f} GB | 本进程占用: {proc_mem_gb:.2f} GB | ���统已用: {mem.percent}%{cgroup_str}")
673
 
674
 
675
+ def _io_probe():
676
+ try:
677
+ c = psutil.Process().io_counters()
678
+ return {"read_chars": c.read_chars, "read_bytes": c.read_bytes}
679
+ except Exception:
680
+ return {"read_chars": 0, "read_bytes": 0}
681
+
682
+ def _mem_probe():
683
+ """返回 (host_Cached_MB, host_Avail_MB, cg_current_GB, cg_file_GB, cg_inactive_file_GB)"""
684
+ cached = avail = -1
685
+ try:
686
+ with open("/proc/meminfo") as f:
687
+ m = {l.split(":")[0]: int(l.split()[1]) // 1024 for l in f if len(l.split()) >= 2}
688
+ cached, avail = m.get("Cached", -1), m.get("MemAvailable", -1)
689
+ except Exception:
690
+ pass
691
+
692
+ cur = file_gb = inact_gb = -1.0
693
+ try:
694
+ with open("/sys/fs/cgroup/memory.current") as f:
695
+ cur = int(f.read().strip()) / 1024**3
696
+ with open("/sys/fs/cgroup/memory.stat") as f:
697
+ for line in f:
698
+ p = line.split()
699
+ if len(p) != 2:
700
+ continue
701
+ if p[0] == "file":
702
+ file_gb = int(p[1]) / 1024**3
703
+ elif p[0] == "inactive_file":
704
+ inact_gb = int(p[1]) / 1024**3
705
+ except Exception:
706
+ pass
707
+ return cached, avail, cur, file_gb, inact_gb
708
+
709
+ def _gpu_id():
710
+ try:
711
+ dev = torch.cuda.current_device()
712
+ p = torch.cuda.get_device_properties(dev)
713
+ return f"dev{dev}/uuid={getattr(p, 'uuid', 'n/a')}"
714
+ except Exception as e:
715
+ return f"n/a({e})"
716
+
717
+
718
  # --------------------------------------------------------------------------
719
+ # 📊 Metric Files: Atomic Write + Read-Modify-Write (/tmp/metrics_<task_id>.json)
720
  # --------------------------------------------------------------------------
721
  _metric_lock = threading.RLock()
722
 
 
764
  d.update(updates)
765
  return True
766
  _update_metric(task_id, _m)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
767
  # --------------------------------------------------------------------------
768
  def _ref(node: str, output: int = 0) -> list[Any]:
769
  return [node, output]
770
 
771
+ def select_base_loras(engine: str, lora_states: tuple) -> list[tuple[str, float]]:
772
+ selected = []
773
+
774
+ # 找出当前引擎专属的 Base LoRA 列表,保持顺序
775
+ valid_loras = []
776
+ for name, cfg in BASE_LORAS_CHOICES.items():
777
+ if cfg["engine"] == engine:
778
+ valid_loras.append((name, cfg))
779
+
780
+ # 解析前端传来的勾选状态
781
+ for i in range(0, len(lora_states), 2):
782
+ enable = lora_states[i]
783
+ try:
784
+ strength = float(lora_states[i + 1])
785
+ except (ValueError, TypeError):
786
+ strength = 1.0
787
+
788
+ if enable and strength > 0:
789
+ # 如果前端的数组长度和当前引擎实际可用的 LoRA 列表长度一致
790
+ idx = i // 2
791
+ if idx < len(valid_loras):
792
+ lora_name, lora_cfg = valid_loras[idx]
793
+ selected.append((lora_cfg["filename"], strength))
794
+
795
+ # 你的探针(保留观察)
796
+ print(f"[LoRA选择探针] ✅ 成功匹配并加载: {lora_name} ({strength})")
797
+
798
+ return selected
799
+
800
+ def build_workflow(
801
+ prompt: str,
802
+ unet_name: str,
803
+ engine: str,
804
+ width: int,
805
+ height: int,
806
+ steps: int,
807
+ cfg: float,
808
+ seed: int,
809
+ sampler: str,
810
+ scheduler: str,
811
+ base_loras: list[tuple[str, float]],
812
+ char_lora: str | None,
813
+ char_strength: float,
814
+ ) -> dict[str, Any]:
815
+ # --- loader 链:命中缓存则复用(节点 id / inputs 完全一致) ---
816
+ key = _combo_key(engine, unet_name, base_loras, char_lora, char_strength)
817
+ cached = _loader_chain_cache.get(key)
818
+ if cached is not None:
819
+ chain, current_model_node = cached
820
+ workflow = dict(chain) # 浅拷贝,下游不改 loader 链
821
+ else:
822
+ chain, current_model_node = _build_loader_chain(
823
+ engine, unet_name, base_loras, char_lora, char_strength)
824
+ _loader_chain_cache[key] = (chain, current_model_node)
825
+ # 只保留最近 8 个组合,避免内存无限增长
826
+ if len(_loader_chain_cache) > 8:
827
+ _loader_chain_cache.pop(next(iter(_loader_chain_cache)))
828
+ workflow = dict(chain)
829
+
830
+ # --- 下游节点:id 从链末端续接,与原逻辑完全一致 ---
831
+ nodes = MODEL_CONFIG["node_types"]
832
+ node_idx = max(int(k) for k in chain.keys()) + 1
833
+
834
+ final_prompt = prompt.strip()
835
+ workflow[str(node_idx)] = {"class_type": "CLIPTextEncode", "inputs": {"clip": _ref("2"), "text": final_prompt}}
836
+ workflow[str(node_idx+1)] = {"class_type": "ConditioningZeroOut", "inputs": {"conditioning": _ref(str(node_idx))}}
837
+ workflow[str(node_idx+2)] = {"class_type": "EmptyLatentImage", "inputs": {"width": width, "height": height, "batch_size": 1}}
838
+
839
+ workflow[str(node_idx+3)] = {"class_type": "KSampler", "inputs": {"model": _ref(current_model_node), "positive": _ref(str(node_idx)), "negative": _ref(str(node_idx+1)), "latent_image": _ref(str(node_idx+2)), "seed": seed, "steps": steps, "cfg": cfg, "sampler_name": sampler, "scheduler": scheduler, "denoise": 1.0}}
840
+ workflow[str(node_idx+4)] = {"class_type": "VAEDecode", "inputs": {"samples": _ref(str(node_idx+3)), "vae": _ref("3")}}
841
+ workflow[str(node_idx+5)] = {"class_type": "SaveImage", "inputs": {"images": _ref(str(node_idx+4)), "filename_prefix": MODEL_CONFIG["name"].lower()}}
842
+ return workflow
843
+
844
  def _log_workflow_summary(workflow: dict[str, Any], task_id: str, load_time_val: float, wf_time: float, image: str = None):
845
  """独立提取并打印单行 JSON 摘要日志(不影响主流程逻辑)"""
846
  try:
 
915
 
916
  except Exception as e:
917
  print(f"[Log Format Error]: {e}", flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
918
 
919
  def _execute_workflow(workflow: dict[str, Any], task_id: str = None) -> list[str]:
920
  print_ram_status("任务开始前")
 
1047
  tasks_status[task_id]["message"] = "📦 LoRA downloaded, loading..."
1048
  finally:
1049
  _tqdm_patch_exit()
1050
+ _current_download_task_id.set(None)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1051
 
 
1052
 
1053
  # ==========================================================================
1054
+ # 🔥 GPU-side Residual Warmup Wait (same /tmp channel as _read_metric)
1055
  # ==========================================================================
1056
  def _read_warm_state():
1057
  try:
 
1063
 
1064
  def _warm_wait_in_gpu(unet_choice: str, budget: float = WARMUP_RESIDUAL_WAIT) -> bool:
1065
  """只在"已经非常接近完成"时等,最多 budget 秒。
1066
+ 注意:这段是在 GPU 切片内执行,等待即烧配额,所以必须能提前放弃。"""
1067
  t0 = time.time()
1068
  while time.time() - t0 < budget:
1069
  st = _read_warm_state()
1070
+
1071
+ if st and st.get("target") == unet_choice:
1072
+ if st.get("state") == "ready":
1073
+ print(f"[预热] ✅ GPU 内等到 ready({time.time() - t0:.2f}s)", flush=True)
1074
+ return True
1075
+ # ⚠️ 进度太低 = 还在冷读,等满 budget 也等不到。立刻放行,省下配额。
1076
+ p = float(st.get("progress") or 0.0)
1077
+ if p < 0.9:
1078
+ saved = budget - (time.time() - t0)
1079
+ print(f"[预热] ⏭️ 进度仅 {p:.2f},放弃残余等待(省 {saved:.1f}s GPU)", flush=True)
1080
+ return False
1081
+
1082
+ elif time.time() - t0 > 0.5:
1083
+ # 0.5 秒内 target 还没轮���本模型(比如 sweep 占着),也别等
1084
+ print(
1085
+ f"[预热] ⏭️ target={st.get('target') if st else None} ≠ 本模型,放弃残余等待",
1086
+ flush=True,
1087
+ )
1088
+ return False
1089
+
1090
  time.sleep(0.25)
1091
+
1092
  print(f"[预热] ⏳ GPU 内残余等待超时({budget}s),转冷加载", flush=True)
1093
  return False
1094
 
1095
 
1096
+ # ==========================================================================
1097
+ # 🎬 Generation
1098
+ # ==========================================================================
1099
+
1100
  def _generate_impl(
1101
  prompt: str, unet_choice: str, custom_w: int, custom_h: int,
1102
  steps: int, cfg: float, seed: int, sampler: str, scheduler: str,
 
1177
 
1178
 
1179
  # ==========================================================================
1180
+ # 🧹 Cleanup
1181
  # ==========================================================================
1182
 
1183
 
 
1184
  # 专门用于保护 tasks_status 字典的状态锁
1185
  _status_lock = threading.RLock()
1186
  _model_switch_lock = threading.Lock()
1187
 
1188
+ # ---- Cleanup Policy ----
1189
  _STATUS_TTL = 3600 # 终态任务内存保留 60 分钟
1190
  _STATUS_MAX_ENTRIES = 200 # 兜底:最多保留 200 条终态
1191
  _OUTPUT_TTL = 24 * 3600 # 输出图片保留 24 小时
 
1244
  if deleted:
1245
  print(f"[清理] 🧹 删除 {deleted} 个过期输出,释放 {freed / 1024 / 1024:.1f} MB", flush=True)
1246
 
1247
+ def _read_last_unet():
1248
+ try:
1249
+ with open(WARMUP_LAST_FILE) as f:
1250
+ return json.load(f).get("unet_choice")
1251
+ except Exception:
1252
+ return None
1253
+
1254
 
1255
  # ==========================================================================
1256
+ # 🔥 WarmupManager: File-level Page-cache Warmup State Machine
1257
  # ==========================================================================
1258
  class WarmupManager:
1259
  def __init__(self):
 
1671
 
1672
  for rec in files:
1673
  if self._superseded(seq):
1674
+ done_mb = sum(
1675
+ r.get("size_mb", 0) for r in records.values()
1676
+ if r.get("state") in ("warm", "skipped")
1677
+ )
1678
+ pct = (done_mb / big_total * 100) if big_total else 0.0
1679
+ print(
1680
+ f"[预热] ⏹️ 被新目标抢占 (seq={seq}),"
1681
+ f"已完成 {done_mb:.0f}MB/{big_total:.0f}MB ({pct:.0f}%),"
1682
+ f"用时 {time.time() - t0:.1f}s",
1683
+ flush=True,
1684
+ )
1685
  break
1686
  records[f"{rec['owner']}::{rec['file_key']}"] = rec
1687
 
 
1788
 
1789
 
1790
  warmup = WarmupManager()
1791
+
1792
+ def _boot_warmup():
1793
+ """两段式:
1794
+ ① 阶梯一级(<1s):上次模型的 UNET 头部 → 用户第一眼就有"部分热"
1795
+ ② 全量 sweep(P1,可被抢占):等 ① 跑完再启动,否则会把 ① 顶掉
1796
+ """
1797
+ last = _read_last_unet()
1798
+ target = last if last in UNET_CHOICES else next(iter(UNET_CHOICES), None)
1799
+
1800
+ lite_requested = False
1801
+ if target:
1802
+ rec = warmup._rec(MODELS / "diffusion_models" / UNET_CHOICES[target]["filename"],
1803
+ "unet", "unet")
1804
+ mbps = warmup.probe(rec)
1805
+ if mbps is not None and mbps >= WARMUP_WARM_MBPS:
1806
+ print(f"[预热] ⚡ boot: UNET 已在 cache 中 ({mbps:.0f}MB/s),跳过 lite", flush=True)
1807
+ else:
1808
+ print(f"[预热] 🥾 boot: UNET 探测 {mbps} MB/s → 做阶梯一级", flush=True)
1809
+ warmup.request(target, lite=True, reason="boot", priority=1)
1810
+ lite_requested = True # ← 新增
1811
+
1812
+ if not WARMUP_BOOT_FULL:
1813
+ return
1814
+
1815
+ if lite_requested: # ← 只在发起了才等
1816
+ t0 = time.time()
1817
+ while time.time() - t0 < 20:
1818
+ with warmup._lock:
1819
+ r = warmup._files.get(target, {}).get("unet")
1820
+ if r and r.get("state") in ("partial", "warm", "failed", "stale"):
1821
+ break
1822
+ time.sleep(0.2)
1823
+
1824
+ warmup.request_sweep(reason="boot_sweep")
1825
 
1826
  # ==========================================================================
1827
  # 🌐 FastAPI & API Routes
 
1841
  char_strength: float = 1.0
1842
  base_lora_states: list = []
1843
 
1844
+ app = FastAPI()
1845
+
1846
+
1847
+ app.add_middleware(
1848
+ CORSMiddleware,
1849
+ allow_origins=["*"],
1850
+ allow_credentials=True,
1851
+ allow_methods=["*"],
1852
+ allow_headers=["*"],
1853
+ )
1854
+
1855
 
1856
  def background_generate_task(task_id: str, req: GenerateRequest, ip_token: str = None, request: Request = None):
1857
  global _last_used_architecture, _last_used_unet
 
1867
 
1868
  # --- 阶段 0:预热(与 GPU 排队并行,不阻塞 HTTP) ---
1869
  try:
1870
+ _wres = warmup.request(
1871
  req.unet_choice,
1872
  req.char_lora if req.char_enable else None,
1873
  char_enable=req.char_enable,
1874
  reason="submit", task_id=task_id,
1875
  )
1876
+ if not (_wres or {}).get("skipped"): # ← 只在真的排了任务时才显示预热阶段
1877
+ with _status_lock:
1878
+ tasks_status[task_id]["message"] = "🔥 Preparing model files..."
1879
  # 只在"接近完成"时短暂等待;冷启动立即放行,避免把 GPU 排队串到预热后面
1880
  _warm_ok = warmup.wait_ready(req.unet_choice, timeout=WARMUP_PRE_GPU_BUDGET)
1881
  print(f"[预热] gate ready={_warm_ok} progress={warmup.progress(req.unet_choice)}", flush=True)
 
2048
  _gen_leave()
2049
  _current_generation_task_id.set(None)
2050
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2051
 
2052
  @app.on_event("startup")
2053
  async def _on_startup():
 
2058
  except Exception as e:
2059
  print(f"[预热] ⚠️ boot 异常: {e}", flush=True)
2060
  threading.Thread(target=_delayed, daemon=True).start()
 
2061
 
2062
+ @app.get("/api/models")
2063
+ async def get_models():
2064
+ models_data = {}
2065
+ global_default_prompt = MODEL_CONFIG.get("default_prompt", "")
 
 
 
2066
 
2067
+ for unet_name, unet_info in UNET_CHOICES.items():
2068
+ engine = unet_info["engine"]
2069
+ engine_name = MODEL_CONFIG["models"].get(engine, {}).get("name", engine)
2070
+
2071
+ if " | " in unet_name:
2072
+ model_specific_name = unet_name.split(" | ", 1)[1]
2073
+ else:
2074
+ model_specific_name = unet_name
2075
+
2076
+ display_unet_name = f"{engine_name} | {model_specific_name}"
2077
+
2078
+ base_loras_for_engine = []
2079
+ for name, cfg in BASE_LORAS_CHOICES.items():
2080
+ if cfg["engine"] == engine:
2081
+ base_loras_for_engine.append({
2082
+ "name": name,
2083
+ "filename": cfg["filename"],
2084
+ "defaultEnable": cfg.get("defaultEnable", False)
2085
+ })
2086
+
2087
+ engine_cfg = MODEL_CONFIG["models"].get(engine, {})
2088
+ final_prompt = engine_cfg.get("default_prompt_override", global_default_prompt)
2089
+
2090
+ # 🌟 核心新增:提取 TE 和 VAE 信息
2091
+ # TE 可能是列表,用 " + " 连接起来展示
2092
+ te_list = engine_cfg.get("local_te_filename", [])
2093
+ te_display = " + ".join([f.replace(".safetensors", "") for f in te_list]) if te_list else "Unknown"
2094
+
2095
+ # VAE 通常是字符串
2096
+ vae_display = engine_cfg.get("local_vae_filename", "Unknown").replace(".safetensors", "")
2097
+
2098
+ models_data[display_unet_name] = {
2099
+ "engine": engine,
2100
+ "defaults": engine_cfg.get("defaults", {}),
2101
+ "base_loras": base_loras_for_engine,
2102
+ "default_prompt": final_prompt,
2103
+ "te_name": te_display, # 👈 传给前端
2104
+ "vae_name": vae_display # 👈 传给前端
2105
+ }
2106
+
2107
+ return {"models": models_data}
2108
+
2109
+ @app.get("/api/loras")
2110
+ async def get_loras():
2111
+ # ⚠️ load_char_loras_from_repo 里有同步 requests.get + list_repo_files(网络 IO)。
2112
+ # 直接在 async 路由里调用会阻塞 event loop,期间所有请求(含 /api/config)
2113
+ # 都无法处理,HF 代理超时后会 reset 连接 → 前端表现为 "Failed to fetch"。
2114
+ # 扔进线程池执行,event loop 保持畅通。
2115
+ await asyncio.to_thread(load_char_loras_from_repo)
2116
+ return {"loras": CHAR_LORA_CHOICES}
2117
+
2118
+ @app.get("/api/config")
2119
+ async def get_sampler_config():
2120
+ return {
2121
+ "samplers": SAMPLERS,
2122
+ "schedulers": SCHEDULERS,
2123
+ "root_mapping": MODEL_CONFIG.get("json_keys", {}).get("root_mapping", {}),
2124
+ "default_prompt": MODEL_CONFIG.get("default_prompt", ""),
2125
+ }
2126
+
2127
+ @app.get("/api/health")
2128
+ async def health_check():
2129
+ return {"status": "ok"}
2130
 
2131
  @app.post("/api/generate")
2132
  async def handle_generate(req: GenerateRequest, background_tasks: BackgroundTasks, request: Request):
 
2165
  tasks_status[task_id] = {"status": "queued", "message": "⏳ 任务已进入队列..."}
2166
 
2167
  # 👇 抛弃 BackgroundTasks,使用原生守护线程,直接与 HTTP 返回脱钩
 
2168
  t = threading.Thread(
2169
  target=background_generate_task,
2170
  args=(task_id, req, ip_token, request),
 
2195
  status_data["message"] = file_data["phase"]
2196
  return status_data
2197
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2198
 
2199
+ # ==========================================================================
2200
+ # 🔥 Warmup Endpoints
2201
+ # ==========================================================================
 
 
 
 
 
2202
 
2203
  class WarmupRequest(BaseModel):
2204
  unet_choice: str
 
2252
  return warmup.snapshot()
2253
 
2254
  # ==========================================================================
2255
+ # 🔧 Warmup Debug Endpoint (disabled by default)
2256
+ # Set WARMUP_DEBUG_TOKEN in Space Secrets; send X-Debug-Token header or ?token=
2257
  # ==========================================================================
2258
  import hmac
2259
 
 
2420
 
2421
  raise HTTPException(400, f"unknown action: {action}")
2422
 
 
 
 
 
2423
  # ==========================================================================
2424
  # 📁 Static Files Sync & Mount
2425
  # ==========================================================================
 
2455
  # 1. 启动时执行同步
2456
  sync_ui_assets()
2457
 
 
2458
  # 🌟 核心修改:必须在挂载根路径 "/" 之前,先挂载 /output 目录
 
2459
  app.mount("/output", StaticFiles(directory=str(OUTPUT)), name="output")
2460
 
2461
  # 2. 挂载静态文件夹 (前端)