photon-models / scripts /organize_models.py
Sucial's picture
Upload folder using huggingface_hub
fac7535 verified
Raw History Blame Contribute Delete
4.86 kB
#!/usr/bin/env python3
"""
organize_models.py
==================
把平铺的模型文件整理成「一个模型一个文件夹」。
整理前 整理后
------------------------------------- -----------------------------------------
safetensors/ safetensors/
indoor_dedup_v01.safetensors indoor_dedup_v01/
indoor_dedup_v01_blobs/ indoor_dedup_v01.safetensors
voice2stem_v03_fp16c_t2.safetensors blobs/ <- 原 *_blobs 目录
voice2stem_v03_fp16c_t2_blobs/ onnx-union-graph-v1.onnx
... ...
voice2stem_v03_fp16c_t2/
voice2stem_v03_fp16c_t2.safetensors
blobs/
ckpt/ 同理:
indoor_dedup_v01.ckpt -> ckpt/indoor_dedup_v01/indoor_dedup_v01.ckpt
indoor_dedup_v01.metadata.json -> ckpt/indoor_dedup_v01/metadata.json
同盘移动是重命名,不会复制数据,秒完成;已整理过的会被跳过,可重复运行。
用法
----
python organize_models.py # 整理 ./safetensors 和 ./ckpt
python organize_models.py --dry-run # 只看计划,不动文件
python organize_models.py --dirs safetensors
"""
from __future__ import annotations
import argparse
import os
import shutil
import sys
def unique(dst: str) -> str:
if not os.path.exists(dst):
return dst
stem, ext = os.path.splitext(dst)
i = 1
while os.path.exists(f"{stem}.{i}{ext}"):
i += 1
return f"{stem}.{i}{ext}"
def organize(root: str, apply: bool) -> tuple[int, int]:
"""返回 (移动的文件数, 移动的目录数)。"""
if not os.path.isdir(root):
print(f" 跳过(目录不存在):{root}")
return 0, 0
entries = os.listdir(root)
n_files = n_dirs = 0
# 1. 先移动 *_blobs 目录(要在文件移动前解析 stem)
for name in sorted(entries):
p = os.path.join(root, name)
if not (os.path.isdir(p) and name.endswith("_blobs")):
continue
stem = name[: -len("_blobs")]
target_parent = os.path.join(root, stem)
final = os.path.join(target_parent, "blobs")
if os.path.exists(final):
final = unique(final)
print(f" DIR {name} -> {os.path.relpath(final, root)}")
if apply:
os.makedirs(target_parent, exist_ok=True)
shutil.move(p, final)
n_dirs += 1
# 2. 模型主文件
for name in sorted(entries):
p = os.path.join(root, name)
if not os.path.isfile(p):
continue
if name.endswith(".safetensors"):
stem = name[: -len(".safetensors")]
target_dir = os.path.join(root, stem)
final = os.path.join(target_dir, name)
elif name.endswith(".ckpt"):
stem = name[: -len(".ckpt")]
target_dir = os.path.join(root, stem)
final = os.path.join(target_dir, name)
elif name.endswith(".metadata.json"): # 平铺模式留下的元数据
stem = name[: -len(".metadata.json")]
target_dir = os.path.join(root, stem)
final = os.path.join(target_dir, "metadata.json")
else:
continue
if os.path.exists(final):
print(f" SKIP {name}(目标已存在)")
continue
print(f" FILE {name} -> {os.path.relpath(final, root)}")
if apply:
os.makedirs(target_dir, exist_ok=True)
shutil.move(p, final)
n_files += 1
return n_files, n_dirs
def main() -> int:
here = os.path.dirname(os.path.abspath(__file__))
ap = argparse.ArgumentParser(description="把模型整理成一个模型一个文件夹")
ap.add_argument("--dirs", nargs="+", default=["raw/safetensors", "raw/ckpt"],
help="要整理的目录(相对脚本所在目录)")
ap.add_argument("--base", default=here, help="基准目录")
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不移动")
args = ap.parse_args()
apply = not args.dry_run
print(f"{'执行' if apply else '演练(--dry-run)'}:{args.base}\n")
tf = td = 0
for d in args.dirs:
root = d if os.path.isabs(d) else os.path.join(args.base, d)
print(f"[{d}]")
f, dd = organize(root, apply)
tf += f
td += dd
print()
print(f"合计:{tf} 个文件、{td} 个目录"
f"{'(未改动)' if not apply else ''}")
return 0
if __name__ == "__main__":
sys.exit(main())