File size: 4,861 Bytes
fac7535
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
#!/usr/bin/env python3
"""

organize_models.py

==================



把平铺的模型文件整理成「一个模型一个文件夹」。



整理前                                  整理后

-------------------------------------   -----------------------------------------

safetensors/                            safetensors/

  indoor_dedup_v01.safetensors            indoor_dedup_v01/

  indoor_dedup_v01_blobs/                   indoor_dedup_v01.safetensors

  voice2stem_v03_fp16c_t2.safetensors       blobs/            <- 原 *_blobs 目录

  voice2stem_v03_fp16c_t2_blobs/              onnx-union-graph-v1.onnx

  ...                                         ...

                                          voice2stem_v03_fp16c_t2/

                                            voice2stem_v03_fp16c_t2.safetensors

                                            blobs/



ckpt/ 同理:

  indoor_dedup_v01.ckpt          ->  ckpt/indoor_dedup_v01/indoor_dedup_v01.ckpt

  indoor_dedup_v01.metadata.json ->  ckpt/indoor_dedup_v01/metadata.json



同盘移动是重命名,不会复制数据,秒完成;已整理过的会被跳过,可重复运行。



用法

----

    python organize_models.py              # 整理 ./safetensors 和 ./ckpt

    python organize_models.py --dry-run    # 只看计划,不动文件

    python organize_models.py --dirs safetensors

"""
from __future__ import annotations

import argparse
import os
import shutil
import sys


def unique(dst: str) -> str:
    if not os.path.exists(dst):
        return dst
    stem, ext = os.path.splitext(dst)
    i = 1
    while os.path.exists(f"{stem}.{i}{ext}"):
        i += 1
    return f"{stem}.{i}{ext}"


def organize(root: str, apply: bool) -> tuple[int, int]:
    """返回 (移动的文件数, 移动的目录数)。"""
    if not os.path.isdir(root):
        print(f"  跳过(目录不存在):{root}")
        return 0, 0

    entries = os.listdir(root)
    n_files = n_dirs = 0

    # 1. 先移动 *_blobs 目录(要在文件移动前解析 stem)
    for name in sorted(entries):
        p = os.path.join(root, name)
        if not (os.path.isdir(p) and name.endswith("_blobs")):
            continue
        stem = name[: -len("_blobs")]
        target_parent = os.path.join(root, stem)
        final = os.path.join(target_parent, "blobs")
        if os.path.exists(final):
            final = unique(final)
        print(f"  DIR  {name}  ->  {os.path.relpath(final, root)}")
        if apply:
            os.makedirs(target_parent, exist_ok=True)
            shutil.move(p, final)
        n_dirs += 1

    # 2. 模型主文件
    for name in sorted(entries):
        p = os.path.join(root, name)
        if not os.path.isfile(p):
            continue
        if name.endswith(".safetensors"):
            stem = name[: -len(".safetensors")]
            target_dir = os.path.join(root, stem)
            final = os.path.join(target_dir, name)
        elif name.endswith(".ckpt"):
            stem = name[: -len(".ckpt")]
            target_dir = os.path.join(root, stem)
            final = os.path.join(target_dir, name)
        elif name.endswith(".metadata.json"):          # 平铺模式留下的元数据
            stem = name[: -len(".metadata.json")]
            target_dir = os.path.join(root, stem)
            final = os.path.join(target_dir, "metadata.json")
        else:
            continue
        if os.path.exists(final):
            print(f"  SKIP {name}(目标已存在)")
            continue
        print(f"  FILE {name}  ->  {os.path.relpath(final, root)}")
        if apply:
            os.makedirs(target_dir, exist_ok=True)
            shutil.move(p, final)
        n_files += 1

    return n_files, n_dirs


def main() -> int:
    here = os.path.dirname(os.path.abspath(__file__))
    ap = argparse.ArgumentParser(description="把模型整理成一个模型一个文件夹")
    ap.add_argument("--dirs", nargs="+", default=["raw/safetensors", "raw/ckpt"],
                    help="要整理的目录(相对脚本所在目录)")
    ap.add_argument("--base", default=here, help="基准目录")
    ap.add_argument("--dry-run", action="store_true", help="只打印计划,不移动")
    args = ap.parse_args()

    apply = not args.dry_run
    print(f"{'执行' if apply else '演练(--dry-run)'}:{args.base}\n")
    tf = td = 0
    for d in args.dirs:
        root = d if os.path.isabs(d) else os.path.join(args.base, d)
        print(f"[{d}]")
        f, dd = organize(root, apply)
        tf += f
        td += dd
        print()
    print(f"合计:{tf} 个文件、{td} 个目录"
          f"{'(未改动)' if not apply else ''}")
    return 0


if __name__ == "__main__":
    sys.exit(main())