File size: 4,861 Bytes
fac7535 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | #!/usr/bin/env python3
"""
organize_models.py
==================
把平铺的模型文件整理成「一个模型一个文件夹」。
整理前 整理后
------------------------------------- -----------------------------------------
safetensors/ safetensors/
indoor_dedup_v01.safetensors indoor_dedup_v01/
indoor_dedup_v01_blobs/ indoor_dedup_v01.safetensors
voice2stem_v03_fp16c_t2.safetensors blobs/ <- 原 *_blobs 目录
voice2stem_v03_fp16c_t2_blobs/ onnx-union-graph-v1.onnx
... ...
voice2stem_v03_fp16c_t2/
voice2stem_v03_fp16c_t2.safetensors
blobs/
ckpt/ 同理:
indoor_dedup_v01.ckpt -> ckpt/indoor_dedup_v01/indoor_dedup_v01.ckpt
indoor_dedup_v01.metadata.json -> ckpt/indoor_dedup_v01/metadata.json
同盘移动是重命名,不会复制数据,秒完成;已整理过的会被跳过,可重复运行。
用法
----
python organize_models.py # 整理 ./safetensors 和 ./ckpt
python organize_models.py --dry-run # 只看计划,不动文件
python organize_models.py --dirs safetensors
"""
from __future__ import annotations
import argparse
import os
import shutil
import sys
def unique(dst: str) -> str:
if not os.path.exists(dst):
return dst
stem, ext = os.path.splitext(dst)
i = 1
while os.path.exists(f"{stem}.{i}{ext}"):
i += 1
return f"{stem}.{i}{ext}"
def organize(root: str, apply: bool) -> tuple[int, int]:
"""返回 (移动的文件数, 移动的目录数)。"""
if not os.path.isdir(root):
print(f" 跳过(目录不存在):{root}")
return 0, 0
entries = os.listdir(root)
n_files = n_dirs = 0
# 1. 先移动 *_blobs 目录(要在文件移动前解析 stem)
for name in sorted(entries):
p = os.path.join(root, name)
if not (os.path.isdir(p) and name.endswith("_blobs")):
continue
stem = name[: -len("_blobs")]
target_parent = os.path.join(root, stem)
final = os.path.join(target_parent, "blobs")
if os.path.exists(final):
final = unique(final)
print(f" DIR {name} -> {os.path.relpath(final, root)}")
if apply:
os.makedirs(target_parent, exist_ok=True)
shutil.move(p, final)
n_dirs += 1
# 2. 模型主文件
for name in sorted(entries):
p = os.path.join(root, name)
if not os.path.isfile(p):
continue
if name.endswith(".safetensors"):
stem = name[: -len(".safetensors")]
target_dir = os.path.join(root, stem)
final = os.path.join(target_dir, name)
elif name.endswith(".ckpt"):
stem = name[: -len(".ckpt")]
target_dir = os.path.join(root, stem)
final = os.path.join(target_dir, name)
elif name.endswith(".metadata.json"): # 平铺模式留下的元数据
stem = name[: -len(".metadata.json")]
target_dir = os.path.join(root, stem)
final = os.path.join(target_dir, "metadata.json")
else:
continue
if os.path.exists(final):
print(f" SKIP {name}(目标已存在)")
continue
print(f" FILE {name} -> {os.path.relpath(final, root)}")
if apply:
os.makedirs(target_dir, exist_ok=True)
shutil.move(p, final)
n_files += 1
return n_files, n_dirs
def main() -> int:
here = os.path.dirname(os.path.abspath(__file__))
ap = argparse.ArgumentParser(description="把模型整理成一个模型一个文件夹")
ap.add_argument("--dirs", nargs="+", default=["raw/safetensors", "raw/ckpt"],
help="要整理的目录(相对脚本所在目录)")
ap.add_argument("--base", default=here, help="基准目录")
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不移动")
args = ap.parse_args()
apply = not args.dry_run
print(f"{'执行' if apply else '演练(--dry-run)'}:{args.base}\n")
tf = td = 0
for d in args.dirs:
root = d if os.path.isabs(d) else os.path.join(args.base, d)
print(f"[{d}]")
f, dd = organize(root, apply)
tf += f
td += dd
print()
print(f"合计:{tf} 个文件、{td} 个目录"
f"{'(未改动)' if not apply else ''}")
return 0
if __name__ == "__main__":
sys.exit(main())
|