photon-models / scripts /to_pymss.py
Sucial's picture
Upload folder using huggingface_hub
fac7535 verified
Raw History Blame Contribute Delete
15 kB
#!/usr/bin/env python3
"""
to_pymss.py
===========
把 iZotope `.model4` 解出来的 ckpt 转换成 **pymss 可直接加载** 的模型 + yaml。
pymss 的加载路径(`E:\\vs\\pymss-core\\pymss_core\\checkpoint.py`):
load_checkpoint -> unwrap_state_dict({"state"|"state_dict"|"model_state_dict"})
model.load_state_dict(sd, strict=True)
所以输出的 .ckpt 里放 `{"state_dict": <pymss 参数名>}`,yaml 里写 `model_type` 和超参,
pymss 就能按 yaml 建模型再 strict 加载。
键名映射(iZotope 规范名 / CoreML 别名 -> pymss BSRoformer):
band_split.{i}.norm.weight -> band_split.to_features.{i}.0.gamma
band_split.{i}.linear.weight/bias -> band_split.to_features.{i}.1.weight/bias
mask_est.{s}.freq.{i}.mlp.{j}.bias -> mask_estimators.{s}.to_freqs.{i}.0.{j}.bias
coreml...model_mask_estimators_*_weight_to_fp16
-> mask_estimators.{s}.to_freqs.{i}.0.{j}.weight
blk.{d}.{axis}_attn_norm.weight -> layers.{d}.{ax}.layers.0.0.norm.gamma
blk.{d}.{axis}_attn_qkv.weight -> layers.{d}.{ax}.layers.0.0.to_qkv.weight
blk.{d}.{axis}_attn_gate.* -> layers.{d}.{ax}.layers.0.0.to_gates.*
blk.{d}.{axis}_attn_out.weight -> layers.{d}.{ax}.layers.0.0.to_out.0.weight
blk.{d}.{axis}_ff_norm.weight -> layers.{d}.{ax}.layers.0.1.net.0.gamma
blk.{d}.{axis}_ff_in.* -> layers.{d}.{ax}.layers.0.1.net.1.*
blk.{d}.{axis}_ff_out.* -> layers.{d}.{ax}.layers.0.1.net.4.*
final_norm.weight -> final_norm.gamma
linear_62_bias_0 / linear_64_bias_0 -> 同名(use_shared_bias)
`rotary_embed.freqs` 是确定性 buffer,pymss 会自己生成,不拷贝。
用法
----
python to_pymss.py # 转换 + yaml + 逐模型 strict 校验
python to_pymss.py --only drumsep
python to_pymss.py --verify-only
"""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
import torch
import model_io
HERE = os.path.dirname(os.path.abspath(__file__))
SRC_DIR = os.path.join(HERE, "..", "raw")
OUT_DIR = os.path.join(HERE, "..", "torch_models")
PYMSS_CORE = r"E:\vs\pymss-core"
# 由截图 + 权重头数推出的音轨名
STEMS = {
"voice2stem_v03_fp16c_t2": {
"title": "Voice - 2 Stems", "model_type": "bs_roformer",
"instruments": ["vocals"], "note": "mask 数=1,其余为残差(other)"},
"voicesep2stem_fp16c_t2": {
"title": "Voice Separator", "model_type": "bs_roformer",
"instruments": ["stem_1", "stem_2"], "note": "2 个 mask 头,界面标 6 Stems 依赖 redimnet 迭代"},
"drumsep_1.3": {
"title": "Drums - 5 Stems", "model_type": "bs_roformer",
"instruments": ["kick", "snare", "toms", "hihat", "cymbals"], "note": ""},
"music4stemhq_fp16c_t2": {
"title": "Music - 4 Stems Ultra", "model_type": "bs_roformer",
"instruments": ["drums", "bass", "vocals", "other"], "note": ""},
"musicnew4stem_fp16c_t2": {
"title": "Music - 4 Stems", "model_type": "scnet",
"instruments": ["drums", "bass", "vocals", "other"], "note": ""},
"cinema4stem_fp16c_t2": {
"title": "Cinema - 4 Stems", "model_type": "bandit",
"instruments": ["dialogue", "music", "sfx"], "note": "ModuleDict 键为 music/sfx/speech,+other 构成 4 stem"},
}
# --------------------------------------------------------------------------- #
def load_canon(path):
"""返回 (canon, aliases)。aliases 里带 CoreML 扁平名。"""
sd = model_io.load_any(path)
canon, aliases = {}, {}
for k, v in sd.items():
if k.startswith(("__blob__", "__onnx__")):
continue
if k.startswith("coreml.source."):
aliases[re.sub(r"^coreml\.source\.[0-9a-f]+\.", "", k)] = v
elif k.startswith("coreml.fp16."):
aliases[k[len("coreml.fp16."):]] = v
elif k.startswith("onnx.source."):
continue
else:
canon[k] = v
return canon, aliases
def coreml_flat_weight(aliases, name):
"""CoreML 扁平名取权重:model_mask_estimators_0_to_freqs_3_0_2_weight_to_fp16"""
key = name.replace(".", "_") + "_to_fp16"
return aliases.get(key)
def map_bs_roformer(canon, aliases):
"""-> (state_dict, params_info, unmapped)"""
out, unmapped = {}, []
max_band = -1
PYMSS_PREFIX = ("band_split.to_features", "mask_estimators.", "layers.", "final_norm", "linear_6")
for k, v in canon.items():
if k.startswith(PYMSS_PREFIX):
if k == "final_norm.weight":
out["final_norm.gamma"] = v
elif k.startswith("coreml."):
pass
else:
out[k.replace(".branch.", ".")] = v
continue
m = re.fullmatch(r"band_split\.(\d+)\.(norm|linear)\.(weight|bias)", k)
if m:
i, mod, w = m.groups()
max_band = max(max_band, int(i))
if mod == "norm":
out[f"band_split.to_features.{i}.0.gamma"] = v
else:
out[f"band_split.to_features.{i}.1.{w}"] = v
continue
m = re.fullmatch(r"mask_est\.(\d+)\.freq\.(\d+)\.mlp\.(\d+)\.(weight|bias)", k)
if m:
s, i, j, w = m.groups()
out[f"mask_estimators.{s}.to_freqs.{i}.0.{j}.{w}"] = v
continue
m = re.fullmatch(r"blk\.(\d+)\.(time|freq)_(attn_norm|attn_qkv|attn_gate|attn_out"
r"|ff_norm|ff_in|ff_out)\.(weight|bias)", k)
if m:
d, axis, mod, w = m.groups()
ax = 0 if axis == "time" else 1
base = f"layers.{d}.{ax}.layers.0"
if mod in ("attn_norm", "ff_norm"):
net = "0" if mod == "attn_norm" else "1"
tail = f"{net}.norm.gamma" if mod == "attn_norm" else "1.net.0.gamma"
out[f"{base}.{tail}"] = v
else:
tail = {"attn_qkv": "0.to_qkv", "attn_gate": "0.to_gates", "attn_out": "0.to_out.0",
"ff_in": "1.net.1", "ff_out": "1.net.4"}[mod]
out[f"{base}.{tail}.{w}"] = v
continue
if k == "final_norm.weight":
out["final_norm.gamma"] = v
continue
if k in ("linear_62_bias_0", "linear_64_bias_0"):
out[k] = v
continue
if k.startswith("buffer_") or "rotary_embed_freqs" in k:
continue # 确定性 buffer,下面统一补齐
unmapped.append(k)
# final_norm 在某些模型里只以 CoreML 别名存在
if "final_norm.gamma" not in out:
for pat in ("model_final_norm_gamma_to_fp16", "final_norm_gamma_to_fp16"):
if pat in aliases:
out["final_norm.gamma"] = aliases[pat]
break
# 共享 bias / mask 权重来自 CoreML 别名
for k, v in aliases.items():
m = re.fullmatch(r"(?:model_)?linear_6(\d)_bias_0_to_fp16", k)
if m:
out[f"linear_6{m.group(1)}_bias_0"] = v
continue
m = re.fullmatch(r"(?:model_)?mask_estimators_(\d+)_to_freqs_(\d+)_0_(\d+)_(weight|bias)_to_fp16", k)
if m:
s, i, j, w = m.groups()
out[f"mask_estimators.{s}.to_freqs.{i}.0.{j}.{w}"] = v
continue
n_stems = len({k.split(".")[1] for k in out if k.startswith("mask_estimators.")})
dim = int(out["final_norm.gamma"].shape[0])
gate = next((v for k, v in out.items() if k.endswith("to_gates.weight")), None)
if gate is None:
return out, {"unsupported": "找不到 to_gates.weight(厂商改了注意力)"}, unmapped
heads = int(gate.shape[0])
depth = len({int(m.group(1)) for k in out
if (m := re.match(r"layers\.(\d+)\.", k))})
qkv_out = int(out["layers.0.0.layers.0.0.to_qkv.weight"].shape[0]) // 3
params = dict(dim=dim, depth=depth, heads=heads, dim_head=qkv_out // heads,
num_stems=n_stems, bands=max_band + 1,
use_shared_bias="linear_62_bias_0" in out,
ff_mult=int(out["layers.0.0.layers.0.1.net.1.weight"].shape[0]) // dim,
mlp_expansion_factor=int(out["mask_estimators.0.to_freqs.0.0.0.weight"].shape[0]) // dim)
# ---- 补齐 pymss state_dict 里必然存在、但我们没单独存的条目 ----
# 1) rotary_embed.freqs 是确定性 buffer,pymss 每个 (layer, axis) 各有一份引用
rot = next((v for k, v in canon.items() if "rotary_embed_freqs" in k), None)
if rot is None:
rot = next((v for k, v in aliases.items() if "rotary_embed_freqs" in k), None)
if rot is not None:
pass
# 2) use_shared_bias 时 pymss 的 to_qkv.bias / to_out.0.bias 与共享参数是同一对象,
# state_dict 里会同时出现两份名字,必须都写
for d in range(depth):
for ax in range(2):
base = f"layers.{d}.{ax}.layers.0"
if rot is not None:
out[f"{base}.0.rotary_embed.freqs"] = rot
if params["use_shared_bias"]:
out[f"{base}.0.to_qkv.bias"] = out["linear_62_bias_0"]
out[f"{base}.0.to_out.0.bias"] = out["linear_64_bias_0"]
pass
return out, params, unmapped
# --------------------------------------------------------------------------- #
def yaml_bs_roformer(p, stems):
freqs = p["freqs_per_bands"]
fp = "\n".join(f" - {f}" for f in freqs)
ins = "\n".join(f" - {s}" for s in stems)
return f"""audio:
chunk_size: 352800
dim_f: 1024
dim_t: 801
hop_length: 512
min_mean_abs: 0.0
n_fft: 2048
num_channels: 2
sample_rate: 44100
inference:
batch_size: 1
dim_t: 2048
num_overlap: 2
model:
attn_dropout: 0.0
depth: {p['depth']}
dim: {p['dim']}
dim_freqs_in: 1025
dim_head: {p['dim_head']}
ff_dropout: 0.0
flash_attn: true
freq_transformer_depth: 1
freqs_per_bands: !!python/tuple
{fp}
heads: {p['heads']}
mask_estimator_depth: 2
mlp_expansion_factor: {p['mlp_expansion_factor']}
num_stems: {p['num_stems']}
skip_connection: false
stereo: true
stft_hop_length: 512
stft_n_fft: 2048
stft_normalized: false
stft_win_length: 2048
time_transformer_depth: 1
use_shared_bias: {'true' if p['use_shared_bias'] else 'false'}
model_type: bs_roformer
training:
instruments:
{ins}
"""
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--src", default=SRC_DIR)
ap.add_argument("--out", default=OUT_DIR)
ap.add_argument("--only", default=None)
ap.add_argument("--verify-only", action="store_true")
args = ap.parse_args()
sys.path.insert(0, PYMSS_CORE)
from pymss_core.modules.bs_roformer.bs_roformer import BSRoformer
from pymss_core.modules.bs_roformer.common import DEFAULT_FREQS_PER_BANDS
os.makedirs(args.out, exist_ok=True)
report = []
for name, meta in STEMS.items():
if name not in STEMS or (args.only and args.only not in name):
continue
ckpt = model_io.model_path_for(name) or os.path.join(args.src, name, name + ".ckpt")
if not os.path.exists(ckpt):
report.append({"model": name, "status": "missing ckpt"})
continue
if meta["model_type"] != "bs_roformer":
report.append({"model": name, "status": f"skipped ({meta['model_type']} 暂未实现映射)"})
print(f"[skip] {name}: {meta['model_type']} 暂未实现")
continue
canon, aliases = load_canon(ckpt)
sd, p, unmapped = map_bs_roformer(canon, aliases)
p["freqs_per_bands"] = list(DEFAULT_FREQS_PER_BANDS)
if "unsupported" in p:
print(f"[skip] {name}: {p['unsupported']}")
report.append({"model": name, "status": p["unsupported"]})
continue
d = os.path.join(args.out, name)
os.makedirs(d, exist_ok=True)
dst = os.path.join(d, name + ".ckpt")
with open(os.path.join(d, "config.yaml"), "w", encoding="utf-8") as f:
f.write(yaml_bs_roformer(p, meta["instruments"]))
if not args.verify_only:
torch.save({"state_dict": sd}, dst)
# strict 校验:按 yaml 参数建 pymss 模型再加载
model = BSRoformer(dim=p["dim"], depth=p["depth"], stereo=True, num_stems=p["num_stems"],
time_transformer_depth=1, freq_transformer_depth=1,
dim_head=p["dim_head"], heads=p["heads"],
freqs_per_bands=DEFAULT_FREQS_PER_BANDS,
stft_n_fft=2048, stft_hop_length=512, stft_win_length=2048,
mask_estimator_depth=2,
mlp_expansion_factor=p["mlp_expansion_factor"],
use_shared_bias=p["use_shared_bias"])
ref_keys = set(model.state_dict())
dropped = [k for k in sd if k not in ref_keys]
if dropped:
print(f" 丢弃 {len(dropped)} 个 pymss 无对应参数的键,如 {dropped[:2]}")
sd = {k: v for k, v in sd.items() if k in ref_keys}
if not args.verify_only:
torch.save({"state_dict": sd}, dst)
try:
model.load_state_dict(torch.load(dst, map_location="cpu", weights_only=True)["state_dict"],
strict=True)
status = "OK(strict)"
except Exception as e:
missing = re.findall(r"Missing key\(s\) in state_dict: (.+)", str(e))
unexpected = re.findall(r"Unexpected key\(s\) in state_dict: (.+)", str(e))
status = (f"strict 失败: 缺 {len(missing[0].split(',')) if missing else 0} 个, "
f"多 {len(unexpected[0].split(',')) if unexpected else 0} 个")
if unexpected:
print(" 多余键:", unexpected[0][:200])
report.append({"model": name, "type": p, "title": meta["title"],
"instruments": meta["instruments"], "unmapped": unmapped[:10],
"n_unmapped": len(unmapped), "status": status})
print(f"[{name}] {status} dim={p['dim']} depth={p['depth']} heads={p['heads']} "
f"stems={p['num_stems']} shared_bias={p['use_shared_bias']} "
f"unmapped={len(unmapped)}")
with open(os.path.join(args.out, "_report.json"), "w", encoding="utf-8") as f:
json.dump(report, f, indent=2, ensure_ascii=False)
return 0
if __name__ == "__main__":
sys.exit(main())