waveshaker commited on
Commit
664de0f
·
verified ·
1 Parent(s): 29aa76d

Upload 2 files

Browse files
Files changed (2) hide show
  1. models_manager_v5.bat +15 -0
  2. models_manager_v5.py +1217 -0
models_manager_v5.bat ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ @echo off
2
+ chcp 65001 >nul
3
+ title LoRA 仓库管理工具
4
+ cd /d "%~dp0"
5
+
6
+ python models_manager_v5.py
7
+
8
+ :: 检查上一句命令的返回值,如果非 0(代表报错崩溃了),就暂停
9
+ if %errorlevel% neq 0 (
10
+ echo.
11
+ echo ==========================================
12
+ echo ❌ 程序异常退出!错误码: %errorlevel%
13
+ echo ==========================================
14
+ pause
15
+ )
models_manager_v5.py ADDED
@@ -0,0 +1,1217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # models_manager.py
2
+ import json
3
+ import os
4
+ import sys
5
+ import time
6
+ import threading
7
+ import traceback
8
+ import tkinter as tk
9
+ from tkinter import ttk, messagebox, filedialog
10
+ from tkinter.scrolledtext import ScrolledText
11
+ import urllib.request
12
+ import urllib.error
13
+ from pathlib import Path
14
+ import subprocess
15
+ import tempfile
16
+ import pandas as pd
17
+ from datasets import Dataset, Features, Image as HFImage, Value
18
+ from huggingface_hub import HfApi, DatasetCard
19
+ from PIL import Image
20
+
21
+ from huggingface_hub import HfApi
22
+ from PIL import Image # 需 pip install Pillow
23
+ import os
24
+ for k in ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy", "ALL_PROXY", "all_proxy"]:
25
+ os.environ.pop(k, None)
26
+
27
+ # ================= 配置区 =================
28
+ SITE_CONFIG_FILE = "site_config.json"
29
+ SPACE_ID = "waveshaker/character_loras_gallery"
30
+ MODEL_REPO_ID = "waveshaker/character_loras"
31
+ DATASET_REPO_ID = "waveshaker/dataset_collections"
32
+ DATASET_BASE = "https://huggingface.co/datasets/waveshaker/dataset_collections/tree/main"
33
+
34
+ OUTPUT_FILE = "models.json"
35
+ TOKEN_FILE = "hf_token.txt"
36
+ LOG_FILE = "models_manager.log"
37
+ HTTP_TIMEOUT = 20
38
+ # ==========================================
39
+
40
+ TOP_ORDER = {"krea2": 1, "anima": 2, "flux1_dev": 3, "pony": 4, "sd15": 5}
41
+ SUB_ORDER = {"CN": 1, "KR": 2, "JP": 3, "Fictional": 4, "WE": 5}
42
+ IMG_EXTS = {".jpg", ".jpeg", ".png", ".webp"}
43
+
44
+ def custom_sort_key(file_path):
45
+ parts = file_path.split('/')
46
+ top_dir = parts[0] if len(parts) > 0 else ""
47
+ sub_dir = parts[1] if len(parts) > 1 else ""
48
+ return (TOP_ORDER.get(top_dir, 999), SUB_ORDER.get(sub_dir, 999), file_path)
49
+
50
+ # ---------- 可选拖拽 ----------
51
+ try:
52
+ from tkinterdnd2 import DND_FILES, TkinterDnD
53
+ _HAS_DND = True
54
+ except Exception:
55
+ DND_FILES = None
56
+ TkinterDnD = None
57
+ _HAS_DND = False
58
+
59
+ class ModelsManager(tk.Tk):
60
+ def __init__(self):
61
+ super().__init__()
62
+ self.title("LoRA 仓库管理工具")
63
+ self.geometry("1250x850")
64
+
65
+ # 全局 Token
66
+ self.token_var = tk.StringVar(value=self._load_token())
67
+
68
+ # 🌟 新增:保存 site_config 数据
69
+ self.site_config_data = {"models": [], "categories": []}
70
+
71
+ # JSON 相关状态
72
+ self.items = {}
73
+ self.remote_files = []
74
+ self.new_paths = set()
75
+ self.stale_paths = set()
76
+ self._search_job = None
77
+ self._loading = False
78
+ self.api = None
79
+
80
+ # Model 上传相关状态
81
+ self.local_folder_var = tk.StringVar()
82
+ self.remote_dir_var = tk.StringVar()
83
+
84
+ # 全局 Token
85
+ self.token_var = tk.StringVar(value=self._load_token())
86
+
87
+ # ============ 初始化 UI ============
88
+ self._build_global_ui()
89
+ self._build_json_tab()
90
+ self._build_model_tab()
91
+ self._build_dataset_tab() # <--- 新增这一行
92
+
93
+ self._refresh_api()
94
+ self._setup_dnd()
95
+ self.log_json("程序启动")
96
+
97
+ # ==========================================================
98
+ # ===================== 全局基础函数 =======================
99
+ # ==========================================================
100
+ def log_json(self, msg):
101
+ self._log_to_tab(msg, getattr(self, 'log_json_text', None))
102
+
103
+ def log_model(self, msg):
104
+ self._log_to_tab(msg, getattr(self, 'log_model_text', None))
105
+
106
+ def _log_to_tab(self, msg, target_text_widget):
107
+ if not target_text_widget: return
108
+ ts = time.strftime("%H:%M:%S")
109
+ line = f"[{ts}] {msg}"
110
+ try:
111
+ with open(LOG_FILE, "a", encoding="utf-8") as f:
112
+ f.write(line + "\n")
113
+ except Exception: pass
114
+
115
+ def _append():
116
+ try:
117
+ target_text_widget.configure(state="normal")
118
+ target_text_widget.insert("end", line + "\n")
119
+ target_text_widget.see("end")
120
+ target_text_widget.configure(state="disabled")
121
+ except Exception: pass
122
+
123
+ if threading.current_thread() is threading.main_thread():
124
+ _append()
125
+ else:
126
+ self.after(0, _append)
127
+
128
+ def log_exc(self, prefix="", is_model=False):
129
+ tb = traceback.format_exc()
130
+ msg = f"{prefix}异常:\n{tb}"
131
+ if is_model:
132
+ self.log_model(msg)
133
+ else:
134
+ self.log_json(msg)
135
+
136
+ def _load_token(self):
137
+ env = os.environ.get("HF_TOKEN", "").strip()
138
+ if env: return env
139
+ try:
140
+ with open(TOKEN_FILE, "r", encoding="utf-8") as f:
141
+ return f.read().strip()
142
+ except FileNotFoundError:
143
+ return ""
144
+
145
+ def _save_token(self, token):
146
+ if token:
147
+ with open(TOKEN_FILE, "w", encoding="utf-8") as f: f.write(token)
148
+ else:
149
+ try: os.remove(TOKEN_FILE)
150
+ except FileNotFoundError: pass
151
+
152
+ def _refresh_api(self):
153
+ token = self.token_var.get().strip()
154
+ self.api = HfApi(token=token) if token else HfApi()
155
+
156
+ def _ensure_token(self):
157
+ token = self.token_var.get().strip()
158
+ if not token:
159
+ messagebox.showwarning("需要 Token", "请先在上方填写 HF Token 再操作。")
160
+ self.log_json("操作中止:未填写 Token")
161
+ return None
162
+ self._refresh_api()
163
+ return token
164
+
165
+ def _build_global_ui(self):
166
+ # 1. 顶部 Token 行
167
+ token_row = ttk.Frame(self)
168
+ token_row.pack(fill="x", padx=8, pady=(8, 2))
169
+ ttk.Label(token_row, text="HF Token:").pack(side="left")
170
+ ttk.Entry(token_row, textvariable=self.token_var, width=60, show="*").pack(side="left", padx=6)
171
+ ttk.Button(token_row, text="📥 保存 Token", command=self._on_save_token).pack(side="left")
172
+ ttk.Button(token_row, text="❌ 清除", command=self._on_clear_token).pack(side="left", padx=4)
173
+
174
+ # 1. 恢复 Windows 默认主题(看起来最舒服)
175
+ style = ttk.Style()
176
+ # 如果你在 Windows 11 上,可以用 'vista' 或 'winnative',如果想要更现代可以试试 'xpnative'
177
+ if sys.platform.startswith("win"):
178
+ style.theme_use('vista')
179
+
180
+ # 2. 强制扁平化 Tab 的渲染结构(屏蔽掉系统的边框和虚线焦点)
181
+ style.layout("TNotebook.Tab", [
182
+ ('Notebook.tab', {'sticky': 'nswe', 'children': [
183
+ ('Notebook.padding', {'side': 'top', 'sticky': 'nswe', 'children': [
184
+ ('Notebook.label', {'side': 'top', 'sticky': ''})
185
+ ]})
186
+ ]})
187
+ ])
188
+
189
+ # 3. 基础设置
190
+ style.configure("TNotebook.Tab",
191
+ padding=[12, 6],
192
+ font=("Microsoft YaHei", 9),
193
+ borderwidth=0,
194
+ background="#f0f0f0",
195
+ foreground="#333333"
196
+ )
197
+
198
+ # 4. 锁定所有状态,防止任何像素级跳动
199
+ style.map("TNotebook.Tab",
200
+ padding=[("selected", [12, 6]), ("active", [12, 6]), ("!selected", [12, 6])],
201
+ borderwidth=[("selected", 0), ("active", 0), ("!selected", 0)],
202
+ # 焦点框设为空,去掉点击时的虚线框
203
+ focuscolor=[("selected", "none"), ("active", "none"), ("!selected", "none")],
204
+ # 选中和悬停时的颜色
205
+ background=[("selected", "#e1f5fe"), ("active", "#f5f5f5")],
206
+ foreground=[("selected", "#0277bd")]
207
+ )
208
+
209
+ # 3. 创建 Notebook 和 Tab 页面
210
+ self.notebook = ttk.Notebook(self)
211
+ self.notebook.pack(fill="both", expand=True, padx=8, pady=4)
212
+
213
+ self.tab_json = ttk.Frame(self.notebook)
214
+ self.tab_model = ttk.Frame(self.notebook)
215
+ self.tab_dataset = ttk.Frame(self.notebook)
216
+
217
+ # 4. 🌟 添加 Tab(Emoji + 文字)
218
+ self.notebook.add(self.tab_json, text=" 🚀 Space - JSON 管理 ")
219
+ self.notebook.add(self.tab_model, text=" 🧊 Models - 模型上传 ")
220
+ self.notebook.add(self.tab_dataset, text=" 📚 Dataset - 数据集上传 ")
221
+
222
+ def _on_save_token(self):
223
+ self._save_token(self.token_var.get().strip())
224
+ self._refresh_api()
225
+ self.log_json("Token 已保存")
226
+ messagebox.showinfo("完成", "Token 已保存到本地")
227
+
228
+ def _on_clear_token(self):
229
+ self.token_var.set("")
230
+ self._save_token("")
231
+ self._refresh_api()
232
+ self.log_json("Token 已清除")
233
+ messagebox.showinfo("完成", "Token 已清除")
234
+
235
+ def open_log_file(self):
236
+ try: os.startfile(LOG_FILE)
237
+ except Exception:
238
+ try:
239
+ import subprocess
240
+ subprocess.Popen(["notepad", LOG_FILE])
241
+ except Exception as e:
242
+ messagebox.showerror("错误", f"无法打开日志: {e}")
243
+
244
+ def _setup_dnd(self):
245
+ if not _HAS_DND: return
246
+ try:
247
+ TkinterDnD._require(self)
248
+ self.tree.drop_target_register(DND_FILES)
249
+ self.tree.dnd_bind("<<Drop>>", self._on_drop_json)
250
+ except Exception as e:
251
+ self.log_json(f"DnD 初始化失败: {e}")
252
+
253
+
254
+ # ==========================================================
255
+ # ==================== 区域 1: JSON 管理 ===================
256
+ # ==========================================================
257
+ def _build_json_tab(self):
258
+ parent = self.tab_json
259
+
260
+ top = ttk.Frame(parent)
261
+ top.pack(fill="x", padx=8, pady=4)
262
+ self.btn_load_remote = ttk.Button(top, text="🔄 拉取 models.json", command=self.load_remote)
263
+ self.btn_load_remote.pack(side="left")
264
+ ttk.Button(top, text="📂 加载本地 models.json", command=self.load_local).pack(side="left", padx=6)
265
+ ttk.Button(top, text="💾 保存 models.json 到本地", command=self.save_local).pack(side="left")
266
+ ttk.Button(top, text="🢁 上传 models.json", command=self.upload_models_json).pack(side="left", padx=6)
267
+ # 🌟 新增:配置同步按钮
268
+ ttk.Button(top, text="🌐 打开 Dataset Repo", compound="left", command=self.open_dataset_repo).pack(side="right", padx=6)
269
+ ttk.Button(top, text="⚙️ 编辑 site_config", command=self.open_site_config_editor).pack(side="right", padx=12)
270
+
271
+
272
+ filt = ttk.Frame(parent)
273
+ filt.pack(fill="x", padx=8)
274
+ ttk.Label(filt, text="🔍 搜索:").pack(side="left")
275
+ self.search_var = tk.StringVar()
276
+ self.search_var.trace_add("write", lambda *_: self._debounced_refresh())
277
+ ttk.Entry(filt, textvariable=self.search_var, width=30).pack(side="left", padx=4)
278
+
279
+ self.only_new = tk.BooleanVar(value=False)
280
+ ttk.Checkbutton(filt, text="只看新增", variable=self.only_new, command=self.refresh_tree).pack(side="left", padx=6)
281
+ self.only_empty_trigger = tk.BooleanVar(value=False)
282
+ ttk.Checkbutton(filt, text="只看 trigger 为空", variable=self.only_empty_trigger, command=self.refresh_tree).pack(side="left")
283
+
284
+ body = ttk.Panedwindow(parent, orient="vertical")
285
+ body.pack(fill="both", expand=True, padx=8, pady=6)
286
+
287
+ tree_frame = ttk.Frame(body)
288
+ cols = ("title", "trigger", "dataset")
289
+ self.tree = ttk.Treeview(tree_frame, columns=cols, show="tree headings", height=20)
290
+ self.tree.heading("#0", text="路径")
291
+ self.tree.heading("title", text="title")
292
+ self.tree.heading("trigger", text="trigger")
293
+ self.tree.heading("dataset", text="dataset")
294
+ self.tree.column("#0", width=380, stretch=True)
295
+ self.tree.column("title", width=180)
296
+ self.tree.column("trigger", width=260)
297
+ self.tree.column("dataset", width=240)
298
+ vsb = ttk.Scrollbar(tree_frame, orient="vertical", command=self.tree.yview)
299
+ self.tree.configure(yscrollcommand=vsb.set)
300
+ self.tree.pack(side="left", fill="both", expand=True)
301
+ vsb.pack(side="left", fill="y")
302
+ self.tree.tag_configure("new", background="#d4f5d4")
303
+ self.tree.tag_configure("empty_trigger", background="#fff3c4")
304
+ self.tree.tag_configure("stale", foreground="#999999")
305
+ self.tree.bind("<Double-1>", self.on_double_click)
306
+
307
+ log_frame = ttk.LabelFrame(body, text="JSON 操作日志")
308
+ self.log_json_text = ScrolledText(log_frame, height=10, state="disabled", wrap="word")
309
+ self.log_json_text.pack(fill="both", expand=True)
310
+
311
+ body.add(tree_frame, weight=3)
312
+ body.add(log_frame, weight=1)
313
+
314
+ def open_dataset_repo(self):
315
+ import webbrowser
316
+ webbrowser.open(f"https://huggingface.co/datasets/{DATASET_REPO_ID}/tree/main")
317
+
318
+ def _debounced_refresh(self):
319
+ if self._search_job: self.after_cancel(self._search_job)
320
+ self._search_job = self.after(250, self.refresh_tree)
321
+
322
+ def load_local(self):
323
+ self.log_json(f"加载本地 {OUTPUT_FILE}")
324
+ try:
325
+ with open(OUTPUT_FILE, "r", encoding="utf-8") as f:
326
+ data = json.load(f)
327
+ except FileNotFoundError:
328
+ messagebox.showwarning("提示", f"本地没有 {OUTPUT_FILE}")
329
+ return
330
+ except Exception as e:
331
+ self.log_json(f"读取失败: {e}")
332
+ return
333
+
334
+ self.items = data.get("items", {})
335
+ if not self.items and "titles" in data:
336
+ self.items = {k: {"title": v, "trigger": "", "dataset": ""} for k, v in data["titles"].items()}
337
+ self.new_paths = set()
338
+ self.stale_paths = set()
339
+ self.refresh_tree()
340
+ self.log_json(f"本地加载完成,{len(self.items)} 条")
341
+
342
+ def load_remote(self):
343
+ if self._loading: return
344
+ self._loading = True
345
+ self.btn_load_remote.configure(state="disabled")
346
+ token = self.token_var.get().strip()
347
+ self.log_json(f"开始拉取远端…")
348
+ threading.Thread(target=self._load_remote_worker, args=(token,), daemon=True).start()
349
+
350
+ def _load_remote_worker(self, token):
351
+ try:
352
+ api = HfApi(token=token) if token else HfApi()
353
+ t0 = time.time()
354
+ all_files = list(api.list_repo_files(repo_id=MODEL_REPO_ID, repo_type="model"))
355
+ remote_files = [f for f in all_files if f.endswith(".safetensors")]
356
+ self.log_json(f"步骤1: 远端文件 {len(all_files)},safetensors {len(remote_files)},{time.time()-t0:.2f}s")
357
+
358
+ space_subdomain = SPACE_ID.replace("/", "-").replace("_", "-").lower()
359
+ url = f"https://{space_subdomain}.static.hf.space/models.json"
360
+ remote_data = None
361
+ try:
362
+ req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
363
+ with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as response:
364
+ remote_data = json.loads(response.read().decode())
365
+ self.log_json(f"步骤2: JSON 获取成功")
366
+ except Exception as e:
367
+ self.log_json(f"步骤2 失败: {type(e).__name__}: {e}")
368
+
369
+ existing_items = {}
370
+ if remote_data:
371
+ existing_items = remote_data.get("items", {})
372
+ if not existing_items and "titles" in remote_data:
373
+ existing_items = {k: {"title": v, "trigger": "", "dataset": ""} for k, v in remote_data["titles"].items()}
374
+
375
+ merged, new_paths = {}, set()
376
+ remote_set = set(remote_files)
377
+ for path in remote_files:
378
+ if path in existing_items:
379
+ merged[path] = existing_items[path]
380
+ else:
381
+ merged[path] = {"title": "", "trigger": "", "dataset": ""}
382
+ new_paths.add(path)
383
+ stale_paths = set(existing_items.keys()) - remote_set
384
+ for p in stale_paths:
385
+ merged[p] = existing_items[p]
386
+
387
+ items = {k: merged[k] for k in sorted(merged.keys(), key=custom_sort_key)}
388
+ self.log_json(f"步骤3: 合并 {len(items)} 条,新增 {len(new_paths)},失效 {len(stale_paths)}")
389
+
390
+ self.after(0, lambda: self._apply_remote(items, remote_files, new_paths, stale_paths))
391
+ except Exception:
392
+ self.log_exc("拉取失败")
393
+ self.after(0, lambda: messagebox.showerror("错误", "拉取失败,详见日志"))
394
+ finally:
395
+ self.after(0, self._finish_loading)
396
+
397
+ def _apply_remote(self, items, remote_files, new_paths, stale_paths):
398
+ self.items = items
399
+ self.remote_files = remote_files
400
+ self.new_paths = new_paths
401
+ self.stale_paths = stale_paths
402
+ self.refresh_tree()
403
+
404
+ def _finish_loading(self):
405
+ self._loading = False
406
+ self.btn_load_remote.configure(state="normal")
407
+
408
+ def refresh_tree(self):
409
+ self.tree.delete(*self.tree.get_children())
410
+ kw = self.search_var.get().strip().lower()
411
+ only_new = self.only_new.get()
412
+ only_empty = self.only_empty_trigger.get()
413
+
414
+ for path, meta in self.items.items():
415
+ title = meta.get("title", "")
416
+ trigger = meta.get("trigger", "")
417
+ if kw and kw not in path.lower() and kw not in str(title).lower(): continue
418
+ if only_new and path not in self.new_paths: continue
419
+ if only_empty and trigger not in ("", [], None): continue
420
+
421
+ tags = []
422
+ if path in self.new_paths: tags.append("new")
423
+ elif path in self.stale_paths: tags.append("stale")
424
+ if trigger in ("", [], None): tags.append("empty_trigger")
425
+
426
+ trig_display = trigger if isinstance(trigger, str) else " | ".join(map(str, trigger))
427
+ self.tree.insert("", "end", iid=path, text=path, values=(title, trig_display, meta.get("dataset", "")), tags=tags)
428
+
429
+ def on_double_click(self, event):
430
+ sel = self.tree.focus()
431
+ if sel and sel in self.items:
432
+ self.open_manager(sel)
433
+
434
+ def open_manager(self, path):
435
+ meta = self.items[path]
436
+ win = tk.Toplevel(self)
437
+ win.title(f"编辑: {path}")
438
+ win.geometry("720x560")
439
+ win.transient(self); win.grab_set()
440
+
441
+ ttk.Label(win, text="路径").pack(anchor="w", padx=10, pady=(10, 0))
442
+ p_entry = ttk.Entry(win, width=90)
443
+ p_entry.insert(0, path); p_entry.configure(state="readonly"); p_entry.pack(padx=10, fill="x")
444
+
445
+ ttk.Label(win, text="title").pack(anchor="w", padx=10, pady=(8, 0))
446
+ title_var = tk.StringVar(value=meta.get("title", ""))
447
+ ttk.Entry(win, textvariable=title_var, width=90).pack(padx=10, fill="x")
448
+
449
+ ttk.Label(win, text="trigger").pack(anchor="w", padx=10, pady=(8, 0))
450
+ trig_text = tk.Text(win, height=8, wrap="word"); trig_text.pack(padx=10, fill="both")
451
+ trig = meta.get("trigger", "")
452
+ trig_text.insert("1.0", "\n".join(map(str, trig)) if isinstance(trig, list) else trig)
453
+
454
+ ttk.Label(win, text="dataset").pack(anchor="w", padx=10, pady=(8, 0))
455
+ ds_var = tk.StringVar(value=meta.get("dataset", ""))
456
+ ttk.Entry(win, textvariable=ds_var, width=90).pack(padx=10, fill="x")
457
+
458
+ def auto_dataset():
459
+ parts = path.split("/")
460
+ if len(parts) >= 3:
461
+ top, sub = parts[0], parts[1]
462
+ base = Path(parts[2]).stem
463
+ for noise in ("_onetrainer", "_Flux_512", "_Flux", "_512"):
464
+ base = base.replace(noise, "")
465
+ ds_var.set(f"{DATASET_BASE}/{top}/{sub}/{base}")
466
+
467
+ ttk.Button(win, text="自动生成 dataset", command=auto_dataset).pack(anchor="w", padx=10, pady=4)
468
+
469
+ def save_and_close():
470
+ title = title_var.get().strip()
471
+ raw = trig_text.get("1.0", "end").strip()
472
+ trigger_val = ([l.strip() for l in raw.splitlines() if l.strip()] if "\n" in raw else raw)
473
+ self.items[path] = {"title": title, "trigger": trigger_val, "dataset": ds_var.get().strip()}
474
+ self.refresh_tree()
475
+ win.destroy()
476
+
477
+ btn = ttk.Frame(win); btn.pack(fill="x", pady=10, padx=10)
478
+ ttk.Button(btn, text="保存", command=save_and_close).pack(side="right")
479
+ ttk.Button(btn, text="取消", command=win.destroy).pack(side="right", padx=6)
480
+
481
+ def save_local(self):
482
+ data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
483
+ sorted_items = {k: data_items[k] for k in sorted(data_items.keys(), key=custom_sort_key)}
484
+ with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
485
+ json.dump({"items": sorted_items}, f, indent=2, ensure_ascii=False)
486
+ self.log_json(f"已保存本地 {OUTPUT_FILE},{len(sorted_items)} 条")
487
+ messagebox.showinfo("完成", f"已保存到 {OUTPUT_FILE}")
488
+
489
+ def upload_models_json(self):
490
+ if not self._ensure_token(): return
491
+ if not messagebox.askyesno("确认", f"上传 models.json 到 {SPACE_ID}?"): return
492
+
493
+ data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
494
+ sorted_items = {k: data_items[k] for k in sorted(data_items.keys(), key=custom_sort_key)}
495
+
496
+ tmp = "models.upload.json"
497
+ with open(tmp, "w", encoding="utf-8") as f:
498
+ json.dump({"items": sorted_items}, f, indent=2, ensure_ascii=False)
499
+
500
+ self.log_json(f"开始上传 JSON ({len(sorted_items)} 条)")
501
+ api = self.api
502
+
503
+ def work():
504
+ try:
505
+ api.upload_file(path_or_fileobj=tmp, path_in_repo="models.json", repo_id=SPACE_ID, repo_type="space")
506
+ self.log_json("models.json 上传成功")
507
+ self.after(0, lambda: messagebox.showinfo("完成", "JSON 上传成功"))
508
+ except Exception:
509
+ self.log_exc("JSON 上传失败")
510
+ self.after(0, lambda: messagebox.showerror("失败", "详见日志"))
511
+ finally:
512
+ try: os.remove(tmp)
513
+ except OSError: pass
514
+
515
+ threading.Thread(target=work, daemon=True).start()
516
+
517
+ # ==========================================================
518
+ # ================= 新增:site_config 管理 =================
519
+ # ==========================================================
520
+ def open_site_config_editor(self):
521
+ if not self._ensure_token(): return
522
+
523
+ self.log_json("正在从 Space 拉取 site_config.json...")
524
+
525
+ def fetch_worker():
526
+ try:
527
+ # 直接从 Space 文件仓库下载
528
+ file_path = self.api.hf_hub_download(
529
+ repo_id=SPACE_ID,
530
+ filename=SITE_CONFIG_FILE,
531
+ repo_type="space"
532
+ )
533
+ with open(file_path, "r", encoding="utf-8") as f:
534
+ config_data = json.load(f)
535
+
536
+ self.after(0, lambda: self._show_config_window(config_data))
537
+ except Exception as e:
538
+ self.log_json(f"拉取 site_config 失败: {e}")
539
+ # 如果远端没有,就初始化一个空的
540
+ self.after(0, lambda: self._show_config_window({"models": [], "categories": []}))
541
+
542
+ threading.Thread(target=fetch_worker, daemon=True).start()
543
+
544
+ def _show_config_window(self, config_data):
545
+ win = tk.Toplevel(self)
546
+ win.title("编辑 site_config.json (保存后自动上传)")
547
+ win.geometry("600x550")
548
+ win.transient(self); win.grab_set()
549
+
550
+ ttk.Label(win, text="💡 提示:修改顺序即可改变前端左侧导航栏的展示顺序。", foreground="gray").pack(pady=5)
551
+
552
+ # 分成两个框:Models 和 Categories
553
+ frame_models = ttk.LabelFrame(win, text="Base Models (按顺序)")
554
+ frame_models.pack(fill="both", expand=True, padx=10, pady=5)
555
+
556
+ text_models = tk.Text(frame_models, height=10, wrap="none")
557
+ text_models.pack(fill="both", expand=True, padx=5, pady=5)
558
+ # 将 JSON 格式化为易读的字符串填入
559
+ text_models.insert("1.0", json.dumps(config_data.get("models", []), indent=2, ensure_ascii=False))
560
+
561
+ frame_cats = ttk.LabelFrame(win, text="Categories (按顺序)")
562
+ frame_cats.pack(fill="both", expand=True, padx=10, pady=5)
563
+
564
+ text_cats = tk.Text(frame_cats, height=10, wrap="none")
565
+ text_cats.pack(fill="both", expand=True, padx=5, pady=5)
566
+ text_cats.insert("1.0", json.dumps(config_data.get("categories", []), indent=2, ensure_ascii=False))
567
+
568
+ def save_and_upload():
569
+ try:
570
+ # 解析用户修改后的文本
571
+ new_models = json.loads(text_models.get("1.0", "end").strip())
572
+ new_cats = json.loads(text_cats.get("1.0", "end").strip())
573
+
574
+ new_config = {
575
+ "models": new_models,
576
+ "categories": new_cats
577
+ }
578
+
579
+ # 保存到本地临时文件
580
+ with open(SITE_CONFIG_FILE, "w", encoding="utf-8") as f:
581
+ json.dump(new_config, f, indent=2, ensure_ascii=False)
582
+
583
+ self.log_json("正在上传 site_config.json 到 Space...")
584
+ self.api.upload_file(
585
+ path_or_fileobj=SITE_CONFIG_FILE,
586
+ path_in_repo=SITE_CONFIG_FILE,
587
+ repo_id=SPACE_ID,
588
+ repo_type="space"
589
+ )
590
+ self.log_json("🎉 site_config.json 上传成功!")
591
+ messagebox.showinfo("成功", "配置已更新并上传到 Space!")
592
+ win.destroy()
593
+ except json.JSONDecodeError as e:
594
+ messagebox.showerror("JSON 格式错误", f"请检查括号、逗号是否漏了。\n报错信息: {e}")
595
+ except Exception as e:
596
+ self.log_json(f"上传失败: {e}")
597
+ messagebox.showerror("错误", f"上传失败: {e}")
598
+
599
+ btn_frame = ttk.Frame(win)
600
+ btn_frame.pack(fill="x", pady=10, padx=10)
601
+ ttk.Button(btn_frame, text="保存并上传", command=save_and_upload).pack(side="right")
602
+ ttk.Button(btn_frame, text="取消", command=win.destroy).pack(side="right", padx=6)
603
+
604
+ def _on_drop_json(self, event):
605
+ raw = event.data.strip("{}")
606
+ p = Path(raw)
607
+ if p.is_dir():
608
+ self.notebook.select(self.tab_model)
609
+ self.local_folder_var.set(str(p))
610
+
611
+
612
+ # ==========================================================
613
+ # ==================== 区域 2: 模型上传 ====================
614
+ # ==========================================================
615
+ def _build_model_tab(self):
616
+ parent = self.tab_model
617
+
618
+ # 1. 输入区
619
+ frame = ttk.Frame(parent)
620
+ frame.pack(fill="x", padx=10, pady=10)
621
+
622
+ # 本地文件夹
623
+ ttk.Label(frame, text="本地文件夹:").grid(row=0, column=0, sticky="w", pady=4)
624
+ ttk.Entry(frame, textvariable=self.local_folder_var, width=70).grid(row=0, column=1, padx=6, sticky="we")
625
+ # 🌟 加入 width=18,让按钮变宽
626
+ ttk.Button(frame, text="📁 浏览...", width=18, command=self._browse_local).grid(row=0, column=2)
627
+
628
+ # 远端目录
629
+ ttk.Label(frame, text="上传到 HF 目录:").grid(row=1, column=0, sticky="w", pady=4)
630
+ self.remote_combo = ttk.Combobox(frame, textvariable=self.remote_dir_var, width=67)
631
+ self.remote_combo.grid(row=1, column=1, padx=6, sticky="we")
632
+ # 🌟 加入 width=18
633
+ ttk.Button(frame, text="🔄 拉取 Models 目录", width=18, command=self._fetch_hf_dirs).grid(row=1, column=2)
634
+
635
+ # 按钮区
636
+ btn_row = ttk.Frame(frame)
637
+ btn_row.grid(row=2, column=0, columnspan=3, pady=15, sticky="w")
638
+ ttk.Button(btn_row, text="🢁 开始上传", command=self._start_model_upload).pack(side="left")
639
+ ttk.Button(btn_row, text="📄 打开日志文件", command=self.open_log_file).pack(side="left", padx=10)
640
+
641
+ self.status_model_var = tk.StringVar(value="就绪")
642
+ ttk.Label(btn_row, textvariable=self.status_model_var).pack(side="left", padx=10)
643
+
644
+ frame.columnconfigure(1, weight=1)
645
+
646
+ # 2. 日志区
647
+ log_frame = ttk.LabelFrame(parent, text="模型上传日志")
648
+ log_frame.pack(fill="both", expand=True, padx=10, pady=6)
649
+ self.log_model_text = ScrolledText(log_frame, height=15, state="disabled", wrap="word")
650
+ self.log_model_text.pack(fill="both", expand=True)
651
+
652
+
653
+ def _browse_local(self):
654
+ folder = filedialog.askdirectory(title="选择包含 .safetensors 的文件夹")
655
+ if folder:
656
+ self.local_folder_var.set(folder)
657
+
658
+ def _fetch_hf_dirs(self):
659
+ if not self._ensure_token(): return
660
+ self.status_model_var.set("正在读取远端目录...")
661
+ self.log_model("正在读取远端仓库目录列表...")
662
+
663
+ def work():
664
+ try:
665
+ api = self.api
666
+ all_files = list(api.list_repo_files(repo_id=MODEL_REPO_ID, repo_type="model"))
667
+ dirs = set()
668
+ for f in all_files:
669
+ d = str(Path(f).parent)
670
+ if d != ".":
671
+ dirs.add(d)
672
+
673
+ sorted_dirs = sorted(list(dirs))
674
+ self.after(0, lambda: self._update_remote_combo(sorted_dirs))
675
+ except Exception:
676
+ self.log_exc("读取远端目录失败", is_model=True)
677
+ finally:
678
+ self.after(0, lambda: self.status_model_var.set("就绪"))
679
+
680
+ threading.Thread(target=work, daemon=True).start()
681
+
682
+ def _update_remote_combo(self, dirs):
683
+ self.remote_combo['values'] = dirs
684
+ if dirs and not self.remote_dir_var.get():
685
+ pass # 留空或默认选第一个
686
+ self.log_model(f"已加载 {len(dirs)} 个远端目录")
687
+
688
+ def _start_model_upload(self):
689
+ local_dir = self.local_folder_var.get().strip()
690
+ remote_dir = self.remote_dir_var.get().strip().strip("/")
691
+
692
+ if not local_dir or not os.path.exists(local_dir):
693
+ messagebox.showwarning("错误", "请选择有效的本地文件夹。")
694
+ return
695
+ if not remote_dir:
696
+ if not messagebox.askyesno("确认", "远端目录为空,将上传到仓库根目录,继续?"):
697
+ return
698
+ if not self._ensure_token(): return
699
+
700
+ self.status_model_var.set("正在扫描并比对文件...")
701
+ self.log_model(f"开始扫描: {local_dir} -> {MODEL_REPO_ID}/{remote_dir}")
702
+
703
+ def scan_worker():
704
+ try:
705
+ api = self.api
706
+
707
+ # 1. 清洗本地图片 (png/webp -> jpg)
708
+ local_files = [f for f in Path(local_dir).iterdir() if f.is_file()]
709
+ for f in local_files:
710
+ if f.suffix.lower() in [".png", ".webp"]:
711
+ jpg_path = f.with_suffix(".jpg")
712
+ try:
713
+ self.log_model(f"清洗图片并转码: {f.name} -> {jpg_path.name}")
714
+ with Image.open(f) as img:
715
+ if img.mode in ("RGBA", "P"):
716
+ img = img.convert("RGB")
717
+ img.save(jpg_path, format="JPEG", quality=95)
718
+ os.remove(f) # 清理原图
719
+ except Exception as e:
720
+ self.log_model(f"转码失败 {f.name}: {e}")
721
+
722
+ # 2. 重新读取清洗后的文件列表
723
+ local_files = [f for f in Path(local_dir).iterdir() if f.is_file()]
724
+ if not local_files:
725
+ self.after(0, lambda: messagebox.showwarning("提示", "本地文件夹是空的。"))
726
+ return
727
+
728
+ # 3. 拉取远端文件用于比对
729
+ remote_all = set(api.list_repo_files(repo_id=MODEL_REPO_ID, repo_type="model"))
730
+
731
+ to_upload = []
732
+ for f in local_files:
733
+ if f.suffix.lower() not in [".safetensors"] + list(IMG_EXTS):
734
+ continue
735
+
736
+ remote_path = f"{remote_dir}/{f.name}" if remote_dir else f.name
737
+ if remote_path not in remote_all:
738
+ to_upload.append((f, remote_path))
739
+
740
+ if not to_upload:
741
+ self.after(0, lambda: messagebox.showinfo("完成", "远端已存在所有文件,无需上传。"))
742
+ self.after(0, lambda: self.status_model_var.set("无需上传"))
743
+ self.log_model("扫描完毕,无新增文件。")
744
+ return
745
+
746
+ msg = f"本地清理完毕。找到 {len(to_upload)} 个新文件待上传。\n\n目标目录: {remote_dir}\n\n继续?"
747
+ self.after(0, lambda: self._do_model_upload_confirm(msg, to_upload, api))
748
+
749
+ except Exception:
750
+ self.log_exc("扫描失败", is_model=True)
751
+ self.after(0, lambda: self.status_model_var.set("扫描失败"))
752
+
753
+ threading.Thread(target=scan_worker, daemon=True).start()
754
+
755
+ def _do_model_upload_confirm(self, msg, to_upload, api):
756
+ if not messagebox.askyesno("确认上传", msg):
757
+ self.status_model_var.set("已取消")
758
+ return
759
+
760
+ def upload_worker():
761
+ try:
762
+ from huggingface_hub import CommitOperationAdd
763
+
764
+ # 1. 准备操作列表
765
+ operations = []
766
+ for local_file, remote_path in to_upload:
767
+ # 极度关键:防止 Windows 反斜杠导致远端多建文件夹
768
+ safe_remote_path = remote_path.replace("\\", "/")
769
+ operations.append(
770
+ CommitOperationAdd(
771
+ path_or_fileobj=str(local_file),
772
+ path_in_repo=safe_remote_path
773
+ )
774
+ )
775
+
776
+ # 2. 准备 commit message
777
+ remote_dir = self.remote_dir_var.get().strip().replace("\\", "/").strip("/")
778
+ commit_msg = f"Upload files to {remote_dir}" if remote_dir else "Upload files to root"
779
+
780
+ self.log_model(f"开始打包 {len(operations)} 个文件,一次性 Commit...")
781
+ self.log_model(f"Commit message: {commit_msg}")
782
+
783
+ # 3. 一次性提交多个文件 (避免零碎 commit)
784
+ api.create_commit(
785
+ repo_id=MODEL_REPO_ID,
786
+ repo_type="model",
787
+ operations=operations,
788
+ commit_message=commit_msg
789
+ )
790
+
791
+ self.log_model("打包上传成功!")
792
+ self.after(0, lambda: messagebox.showinfo("完成", "上传成功!"))
793
+ self.after(0, lambda: self.status_model_var.set("上传完成"))
794
+ self.after(0, self.load_remote) # 自动刷新 JSON
795
+
796
+ except Exception as e:
797
+ self.log_exc("打包上传失败", is_model=True)
798
+ self.after(0, lambda: messagebox.showerror("失败", "详见日志"))
799
+ self.after(0, lambda: self.status_model_var.set("上传失败"))
800
+
801
+ threading.Thread(target=upload_worker, daemon=True).start()
802
+
803
+ # ==========================================================
804
+ # ==================== 区域 3: Dataset 上传 =================
805
+ # ==========================================================
806
+ def _build_dataset_tab(self):
807
+ parent = self.tab_dataset
808
+
809
+ frame = ttk.Frame(parent)
810
+ frame.pack(fill="x", padx=10, pady=10)
811
+
812
+ # 1. 本地母文件夹
813
+ ttk.Label(frame, text="本地母文件夹:").grid(row=0, column=0, sticky="w", pady=4)
814
+ self.ds_local_folder_var = tk.StringVar()
815
+ ttk.Entry(frame, textvariable=self.ds_local_folder_var, width=70).grid(row=0, column=1, columnspan=2, padx=6, sticky="we")
816
+ ttk.Button(frame, text="📁 浏览...", width=18, command=self._browse_ds_local).grid(row=0, column=3)
817
+
818
+ # 2. 地区/分类前缀 (动态读取改造)
819
+ ttk.Label(frame, text="Region 前缀:").grid(row=1, column=0, sticky="w", pady=8)
820
+
821
+ prefix_frame = ttk.Frame(frame)
822
+ prefix_frame.grid(row=1, column=1, columnspan=3, sticky="w", padx=6, pady=8)
823
+
824
+ # 核心变量初始化
825
+ self.ds_region_var = tk.StringVar(value="KR")
826
+ self.ds_custom_region_var = tk.StringVar() # 必须保留这个变量用于输入框
827
+
828
+ # 按钮:刷新 Dataset 目录
829
+ ttk.Button(prefix_frame, text="🔄 拉取 datasets 目录", command=self._fetch_dataset_dirs).pack(side="left", padx=(0, 10))
830
+
831
+ # 用子 Frame 专门放单选框,刷新时只清空这里的控件
832
+ self.ds_radio_container = ttk.Frame(prefix_frame)
833
+ self.ds_radio_container.pack(side="left")
834
+
835
+ # 初始默认放几个,防止还没拉取时界面太难看
836
+ self._render_dataset_radios(["CN", "KR", "JP", "Fictional", "WE"])
837
+
838
+ # 3. 按钮区
839
+ btn_row = ttk.Frame(frame)
840
+ btn_row.grid(row=2, column=0, columnspan=4, pady=15, sticky="w")
841
+ self.btn_ds_upload = ttk.Button(btn_row, text="🢁 开始 5合1 处理并上传", command=self._start_dataset_pipeline)
842
+ self.btn_ds_upload.pack(side="left")
843
+
844
+ self.status_ds_var = tk.StringVar(value="就绪")
845
+ ttk.Label(btn_row, textvariable=self.status_ds_var).pack(side="left", padx=10)
846
+
847
+ frame.columnconfigure(1, weight=1)
848
+ frame.columnconfigure(2, weight=1)
849
+
850
+ # 4. 日志区
851
+ log_frame = ttk.LabelFrame(parent, text="Dataset 自动化流水线日志")
852
+ log_frame.pack(fill="both", expand=True, padx=10, pady=6)
853
+ self.log_dataset_text = ScrolledText(log_frame, height=15, state="disabled", wrap="word")
854
+ self.log_dataset_text.pack(fill="both", expand=True)
855
+
856
+ def _fetch_dataset_dirs(self):
857
+ if not self._ensure_token(): return
858
+ self.log_dataset("正在读取 Dataset 仓库根目录...")
859
+
860
+ def work():
861
+ try:
862
+ files = self.api.list_repo_files(repo_id=DATASET_REPO_ID, repo_type="dataset")
863
+ dirs = set()
864
+ for f in files:
865
+ parts = f.split("/")
866
+ # 过滤掉 tool 和根目录文件,只保留文件夹
867
+ if len(parts) > 1 and parts[0] not in ["tool", ".git"]:
868
+ dirs.add(parts[0])
869
+
870
+ sorted_dirs = sorted(list(dirs))
871
+
872
+ # 🌟 新增:在主线程更新 UI 和打印日志
873
+ def update_ui():
874
+ self._render_dataset_radios(sorted_dirs)
875
+ self.log_dataset(f"✅ 远端目录刷新成功!共发现 {len(sorted_dirs)} 个分类: {', '.join(sorted_dirs)}")
876
+
877
+ self.after(0, update_ui)
878
+
879
+ except Exception as e:
880
+ self.log_exc("读取 Dataset 目录失败", is_model=True)
881
+ # 🌟 新增:报错时更新状态栏
882
+ self.after(0, lambda: self.status_ds_var.set("读取目录失败"))
883
+
884
+ threading.Thread(target=work, daemon=True).start()
885
+
886
+ def _render_dataset_radios(self, dirs):
887
+ # 清空旧的单选框和输入框
888
+ for widget in self.ds_radio_container.winfo_children():
889
+ widget.destroy()
890
+
891
+ # 保持原本选中的值,如果不存在了,默认选第一个
892
+ current_val = self.ds_region_var.get()
893
+ if current_val not in dirs and current_val != "Custom" and dirs:
894
+ self.ds_region_var.set(dirs[0])
895
+
896
+ # 动态生成分类单选框
897
+ for region in dirs:
898
+ ttk.Radiobutton(self.ds_radio_container, text=region, variable=self.ds_region_var, value=region).pack(side="left", padx=(0, 10))
899
+
900
+ # 加上自定义部分 (注意:父级是 self.ds_radio_container)
901
+ ttk.Radiobutton(self.ds_radio_container, text="自定义:", variable=self.ds_region_var, value="Custom").pack(side="left", padx=(10, 2))
902
+ self.ds_custom_entry = ttk.Entry(self.ds_radio_container, textvariable=self.ds_custom_region_var, width=15, state="disabled")
903
+ self.ds_custom_entry.pack(side="left")
904
+
905
+ # 🌟 重新绑定输入框激活逻辑 (因为输入框是刚刚重新创建的)
906
+ def on_region_change(*args):
907
+ if self.ds_region_var.get() == "Custom":
908
+ self.ds_custom_entry.configure(state="normal")
909
+ else:
910
+ self.ds_custom_entry.configure(state="disabled")
911
+
912
+ # 清理之前的 trace,防止多次刷新后绑定重复导致卡顿或报错
913
+ if hasattr(self, '_region_trace_id'):
914
+ try:
915
+ self.ds_region_var.trace_remove("write", self._region_trace_id)
916
+ except Exception:
917
+ pass
918
+
919
+ self._region_trace_id = self.ds_region_var.trace_add("write", on_region_change)
920
+
921
+
922
+ def log_dataset(self, msg):
923
+ self._log_to_tab(msg, getattr(self, 'log_dataset_text', None))
924
+
925
+ def _browse_ds_local(self):
926
+ folder = filedialog.askdirectory(title="选择包含子文件夹的数据集母目录")
927
+ if folder:
928
+ self.ds_local_folder_var.set(folder)
929
+
930
+ def _start_dataset_pipeline(self):
931
+ local_dir = self.ds_local_folder_var.get().strip()
932
+
933
+ # ==== 修改这里:获取前缀逻辑 ====
934
+ selected_region = self.ds_region_var.get()
935
+ if selected_region == "Custom":
936
+ region_prefix = self.ds_custom_region_var.get().strip().replace(" ", "_").replace("-", "_")
937
+ if not region_prefix:
938
+ messagebox.showwarning("错误", "您选择了自定义前缀,但输入框为空。")
939
+ return
940
+ else:
941
+ region_prefix = selected_region
942
+ # ===============================
943
+
944
+ if not local_dir or not os.path.exists(local_dir):
945
+ messagebox.showwarning("错误", "请选择有效的本地母文件夹。")
946
+ return
947
+ if not self._ensure_token(): return
948
+
949
+ if not messagebox.askyesno("确认", f"即将对以下目录执行 5合1 流水线:\n{local_dir}\n前缀:{region_prefix}\n\n继续?"):
950
+ return
951
+
952
+ self.btn_ds_upload.configure(state="disabled")
953
+ self.status_ds_var.set("处理中...")
954
+ self.log_dataset(f"================ 开始 5合1 流水线 ================")
955
+ self.log_dataset(f"目标目录: {local_dir} | 前缀: {region_prefix}")
956
+
957
+ def worker():
958
+ api = self.api
959
+ try:
960
+ # 1. 清理 txt 换行符
961
+ self._clean_txt_newlines(local_dir)
962
+
963
+ # 2. 清理 Mask 元数据
964
+ self._remove_mask_metadata(local_dir)
965
+
966
+ # 3. 转换 PNG -> JPG
967
+ self._convert_png_to_jpg(local_dir)
968
+
969
+ # 4. 打包 Parquet 并更新 YAML
970
+ temp_readme_path, valid_folders = self._generate_parquet_and_update_yaml(api, local_dir, region_prefix)
971
+
972
+ # 5. 上传
973
+ self._upload_files_and_readme(api, local_dir, region_prefix, temp_readme_path, valid_folders)
974
+
975
+ self.log_dataset("🎉 5合1 全流程处理完毕!")
976
+ self.after(0, lambda: messagebox.showinfo("完成", "Dataset 流水线执行成功!"))
977
+ self.after(0, lambda: self.status_ds_var.set("上传完成"))
978
+
979
+ # 刷新 JSON 的拉取(可选,如果需要马上更新界面的话)
980
+ self.after(0, self.load_remote)
981
+
982
+ except Exception as e:
983
+ self.log_exc("Dataset 流水线异常", is_model=True)
984
+ self.after(0, lambda: self.status_ds_var.set("执行失败"))
985
+ finally:
986
+ self.after(0, lambda: self.btn_ds_upload.configure(state="normal"))
987
+
988
+ threading.Thread(target=worker, daemon=True).start()
989
+
990
+ # ---------------- 原有脚本的函数改造 ----------------
991
+
992
+ def _clean_txt_newlines(self, current_dir):
993
+ import glob
994
+ self.log_dataset("🧹 [任务 1/5] 正在清理所有 TXT 文件中的换行符...")
995
+ txt_files = glob.glob(os.path.join(current_dir, "**", "*.txt"), recursive=True)
996
+ if not txt_files:
997
+ self.log_dataset(" ✅ 没有找到需要处理的 TXT 文件。")
998
+ return
999
+
1000
+ cleaned_count = 0
1001
+ for txt_path in txt_files:
1002
+ if os.path.dirname(txt_path) == current_dir:
1003
+ continue
1004
+ try:
1005
+ with open(txt_path, 'r', encoding='utf-8') as f:
1006
+ content = ''.join(line.strip() for line in f)
1007
+ with open(txt_path, 'w', encoding='utf-8') as f:
1008
+ f.write(content)
1009
+ cleaned_count += 1
1010
+ except Exception as e:
1011
+ self.log_dataset(f" ⚠️ 处理 {os.path.basename(txt_path)} 失败: {e}")
1012
+ self.log_dataset(f" 🎉 TXT 清理完成!共处理了 {cleaned_count} 个文件。")
1013
+
1014
+ def _remove_mask_metadata(self, current_dir):
1015
+ import glob
1016
+ self.log_dataset("🛡️ [任务 2/5] 正在剥离 Mask 蒙版图的元数据...")
1017
+ png_files = glob.glob(os.path.join(current_dir, "**", "*.png"), recursive=True)
1018
+ mask_files = [f for f in png_files if "masklabel" in os.path.basename(f).lower()]
1019
+
1020
+ if not mask_files:
1021
+ self.log_dataset(" ✅ 没有找到 masklabel.png 文件,跳过此步骤。")
1022
+ return
1023
+
1024
+ success_count = 0
1025
+ for file_path in mask_files:
1026
+ try:
1027
+ with Image.open(file_path) as img:
1028
+ data = list(img.getdata())
1029
+ clean_img = Image.new(img.mode, img.size)
1030
+ clean_img.putdata(data)
1031
+ clean_img.save(file_path, "PNG")
1032
+ success_count += 1
1033
+ except Exception as e:
1034
+ self.log_dataset(f" ⚠️ 处理失败 [{os.path.basename(file_path)}]: {e}")
1035
+ self.log_dataset(f" 🎉 元数据清理完成!共处理了 {success_count} 张 Mask 图。")
1036
+
1037
+ def _convert_png_to_jpg(self, current_dir):
1038
+ import glob
1039
+ self.log_dataset("🎬 [任务 3/5] 正在批量转换常规 PNG 为 JPG...")
1040
+ png_files = glob.glob(os.path.join(current_dir, "**", "*.png"), recursive=True)
1041
+ if not png_files:
1042
+ self.log_dataset(" ✅ 没有找到需要转换的 PNG 文件。")
1043
+ return
1044
+
1045
+ converted_count = 0
1046
+ for png_path in png_files:
1047
+ if "masklabel" in png_path.lower():
1048
+ continue
1049
+ jpg_path = os.path.splitext(png_path)[0] + ".jpg"
1050
+ try:
1051
+ subprocess.run(
1052
+ ["ffmpeg", "-y", "-i", png_path, "-q:v", "2", jpg_path],
1053
+ stdout=subprocess.DEVNULL,
1054
+ stderr=subprocess.DEVNULL,
1055
+ check=True
1056
+ )
1057
+ os.remove(png_path)
1058
+ converted_count += 1
1059
+ self.log_dataset(f" 🔄 转换: {os.path.basename(png_path)} -> {os.path.basename(jpg_path)}")
1060
+ except subprocess.CalledProcessError:
1061
+ self.log_dataset(f" ⚠️ 转换失败: {png_path}")
1062
+ except FileNotFoundError:
1063
+ self.log_dataset(" ❌ 错误: 找不到 ffmpeg!请确保已安装并配置了环境变量。")
1064
+ return
1065
+ self.log_dataset(f" 🎉 转换完成!共转换 {converted_count} 张图片。")
1066
+
1067
+ def _generate_parquet_and_update_yaml(self, api, current_dir, region_prefix):
1068
+ import glob, tempfile, pandas as pd
1069
+ from datasets import Dataset, Features, Image as HFImage, Value
1070
+ from huggingface_hub import DatasetCard
1071
+ from PIL import Image as PILImage
1072
+
1073
+ self.log_dataset("📦 [任务 4/5] 正在本地打包 Parquet 并更新 YAML 配置...")
1074
+ local_folder_name = os.path.basename(current_dir).replace(" ", "_").replace("-", "_")
1075
+
1076
+ self.log_dataset(f" ☁️ 正在读取云端 {DATASET_REPO_ID} 的现有配置...")
1077
+ try:
1078
+ # 注意这里用 DATASET_REPO_ID 替换了原脚本的 REPO_ID
1079
+ remote_readme_path = api.hf_hub_download(repo_id=DATASET_REPO_ID, filename="README.md", repo_type="dataset")
1080
+ with open(remote_readme_path, "r", encoding="utf-8") as f:
1081
+ content = f.read()
1082
+ card = DatasetCard(content)
1083
+ self.log_dataset(" ✅ 成功读取云端 README。")
1084
+ except Exception as e:
1085
+ self.log_dataset(f" ⚠️ 找不到云端 README,将新建: {e}")
1086
+ card = DatasetCard("")
1087
+
1088
+ yaml_data = card.data if card.data else {}
1089
+ if "configs" not in yaml_data: yaml_data["configs"] = []
1090
+
1091
+ yaml_data["dataset_info"] = {
1092
+ "features": [
1093
+ {"name": "image", "dtype": "image"},
1094
+ {"name": "file_name", "dtype": "string"},
1095
+ {"name": "text", "dtype": "string"}
1096
+ ]
1097
+ }
1098
+
1099
+ existing_config_names = [c.get("config_name") for c in yaml_data["configs"]]
1100
+ new_configs_added = False
1101
+ valid_folders = []
1102
+
1103
+ for sub_folder_name in os.listdir(current_dir):
1104
+ sub_folder_path = os.path.join(current_dir, sub_folder_name)
1105
+ if os.path.isdir(sub_folder_path) and not sub_folder_name.startswith('.'):
1106
+ jpg_files = glob.glob(os.path.join(sub_folder_path, "*.jpg")) + glob.glob(os.path.join(sub_folder_path, "*.jpeg"))
1107
+ if not jpg_files:
1108
+ self.log_dataset(f" ⚠️ 跳过子文件夹 [{sub_folder_name}]: 未找到任何 .jpg 图片。")
1109
+ continue
1110
+
1111
+ self.log_dataset(f" 📦 正在打包: {sub_folder_name}")
1112
+ data_rows = []
1113
+
1114
+ for jpg_path in jpg_files:
1115
+ file_name = os.path.basename(jpg_path)
1116
+ base_name = os.path.splitext(file_name)[0]
1117
+ txt_path = os.path.join(sub_folder_path, f"{base_name}.txt")
1118
+
1119
+ text_content = ""
1120
+ if os.path.exists(txt_path):
1121
+ with open(txt_path, "r", encoding="utf-8") as f:
1122
+ text_content = f.read().strip()
1123
+
1124
+ try:
1125
+ with open(jpg_path, "rb") as f:
1126
+ image_bytes = {"bytes": f.read(), "path": None}
1127
+ except Exception as e:
1128
+ self.log_dataset(f" ⚠️ 跳过损坏图片 {file_name}: {e}")
1129
+ continue
1130
+
1131
+ data_rows.append({
1132
+ "image": image_bytes,
1133
+ "file_name": file_name,
1134
+ "text": text_content
1135
+ })
1136
+
1137
+ if data_rows:
1138
+ df = pd.DataFrame(data_rows)
1139
+ features = Features({
1140
+ "image": HFImage(),
1141
+ "file_name": Value("string"),
1142
+ "text": Value("string")
1143
+ })
1144
+ ds = Dataset.from_pandas(df, features=features)
1145
+ ds.to_parquet(os.path.join(sub_folder_path, "train.parquet"))
1146
+ valid_folders.append(sub_folder_name)
1147
+ self.log_dataset(f" ✅ 生成 train.parquet ({len(data_rows)} 条数据)")
1148
+
1149
+ config_name = f"{local_folder_name}_{sub_folder_name}".lower()
1150
+ path_str = f"{region_prefix}/{local_folder_name}/{sub_folder_name}/train.parquet"
1151
+
1152
+ if config_name not in existing_config_names:
1153
+ yaml_data["configs"].append({
1154
+ "config_name": config_name,
1155
+ "data_files": [{"split": "train", "path": path_str}]
1156
+ })
1157
+ new_configs_added = True
1158
+ self.log_dataset(f" ➕ 已添加配置: {config_name}")
1159
+
1160
+ def get_priority(config):
1161
+ data_files = config.get("data_files", [])
1162
+ path = data_files[0].get("path", "") if data_files else ""
1163
+ priority_dict = {"CN": 1, "KR": 2, "JP": 3, "Fictional": 4, "WE": 5}
1164
+ path_prefix = path.split('/')[0] if path else ""
1165
+ return priority_dict.get(path_prefix, 99)
1166
+
1167
+ yaml_data["configs"] = sorted(yaml_data["configs"], key=lambda c: (get_priority(c), c.get("config_name", "")))
1168
+ self.log_dataset(" 🔄 已根据 [CN > KR > JP > Fictional > WE] 的顺序重排 YAML。")
1169
+
1170
+ if new_configs_added:
1171
+ card.data = yaml_data
1172
+ with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', suffix='.md', delete=False) as temp_readme:
1173
+ temp_readme.write(str(card))
1174
+ return temp_readme.name, valid_folders
1175
+ return None, valid_folders
1176
+
1177
+ def _upload_files_and_readme(self, api, current_dir, region_prefix, temp_readme_path, valid_folders):
1178
+ self.log_dataset("🚀 [任务 5/5] 正在上传文件到 Hugging Face...")
1179
+ local_folder_name = os.path.basename(current_dir).replace(" ", "_").replace("-", "_")
1180
+
1181
+ for sub_folder_name in valid_folders:
1182
+ sub_folder_path = os.path.join(current_dir, sub_folder_name)
1183
+ path_in_repo = f"{region_prefix}/{local_folder_name}/{sub_folder_name}"
1184
+
1185
+ self.log_dataset(f" 📦 开始上传: {sub_folder_name} -> {path_in_repo}")
1186
+ try:
1187
+ api.upload_folder(
1188
+ folder_path=sub_folder_path,
1189
+ repo_id=DATASET_REPO_ID,
1190
+ repo_type="dataset",
1191
+ path_in_repo=path_in_repo,
1192
+ commit_message=f"Update files for {path_in_repo}",
1193
+ allow_patterns=["*.jpg", "*.jpeg", "*.txt", "*.png", "*.parquet"]
1194
+ )
1195
+ self.log_dataset(f" 🎉 {sub_folder_name} 上传完成!")
1196
+ except Exception as e:
1197
+ self.log_dataset(f" ❌ {sub_folder_name} 上传失败: {e}")
1198
+
1199
+ if temp_readme_path:
1200
+ self.log_dataset("\n 📝 正在推送最新的 README.md...")
1201
+ try:
1202
+ api.upload_file(
1203
+ path_or_fileobj=temp_readme_path,
1204
+ path_in_repo="README.md",
1205
+ repo_id=DATASET_REPO_ID,
1206
+ repo_type="dataset",
1207
+ commit_message="Auto-update README YAML configs"
1208
+ )
1209
+ self.log_dataset(" 🎉 README.md 已成功更新到云端!")
1210
+ except Exception as e:
1211
+ self.log_dataset(f" ❌ README.md 上传失败: {e}")
1212
+ finally:
1213
+ os.remove(temp_readme_path)
1214
+
1215
+ if __name__ == "__main__":
1216
+ app = ModelsManager()
1217
+ app.mainloop()