photon-models / scripts /pymss_adapters.py
Sucial's picture
Upload folder using huggingface_hub
fac7535 verified
Raw History Blame Contribute Delete
11.3 kB
#!/usr/bin/env python3
"""
pymss_adapters.py
=================
**不改 pymss 库代码**,在外部用适配类把 iZotope `.model4` 解出来的 ckpt 装进 pymss 模型。
适配思路
--------
pymss 的 `load_state_dict` 只认参数名,所以适配层要做两件事:
1. **补全张量**:iZotope 容器把一部分权重放在 `coreml.f16.*` / `coreml.source.*` 的扁平别名里
(例如 `model_mask_estimators_0_to_freqs_3_0_2_weight_to_fp16`),只从规范名取会缺一大半。
2. **对名**:把 iZotope 的命名规范化(去掉 `model.` `/` `.branch.`,`.`→`_`),
和 pymss 模型 `state_dict()` 的每个键做匹配;名字对不上的再用形状 + 出现顺序兜底。
三个模型各自的处理
------------------
* `music4stemhq` : 架构与 bs_roformer 相同,只是参数多套了一层 `.branch.`(`BranchedRoformer`)
* `musicnew4stem` : SCNet,命名只差一个 `model.` 前缀(`ScnetAdapter`)
* `cinema4stem` : Bandit v2,`mask_estim` 是 ModuleDict(`BanditAdapter`)
用法
----
python pymss_adapters.py # 三个模型全部转换 + strict 校验
"""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
import torch
import model_io
HERE = os.path.dirname(os.path.abspath(__file__))
SRC_DIR = os.path.join(HERE, "..", "raw")
OUT_DIR = os.path.join(HERE, "..", "torch_models")
PYMSS_CORE = r"E:\vs\pymss-core"
# --------------------------------------------------------------------------- #
# 张量收集:规范名 + 别名
# --------------------------------------------------------------------------- #
def collect_tensors(path):
"""返回 {规范化名: tensor},兼容规范名 / coreml.source / coreml.fp16 三种来源。"""
sd = model_io.load_any(path)
raw = {}
for k, v in sd.items():
if k.startswith(("__blob__", "__onnx__")):
continue
k = re.sub(r"^(coreml|onnx)\.source\.[0-9a-f]+\.", "", k)
k = re.sub(r"^coreml\.fp16\.", "", k)
if k.startswith(("buffer_", "/Constant", "Constant_")):
continue
if k.endswith("_to_fp16"):
k = k[: -len("_to_fp16")]
raw[k] = v
out = {}
for k, v in raw.items():
n = k.replace(".", "_").replace("/", "_").replace("_branch_", "_")
n = re.sub(r"^model_", "", n)
n = re.sub(r"_+", "_", n).strip("_")
out[n] = v
return out
def norm_key(k: str) -> str:
n = k.replace(".", "_").replace("/", "_").replace("_branch_", "_")
n = re.sub(r"^model_", "", n)
n = re.sub(r"_+", "_", n).strip("_")
return n
def match_state_dict(model, tensors):
"""把收集到的张量按规范化名对到 model 的 state_dict 上。"""
ref = model.state_dict()
by_name = {}
unmatched = dict(tensors)
for k in ref:
n = norm_key(k)
if n in unmatched:
by_name[k] = unmatched.pop(n)
# rotary_embed.freqs 是共享的确定性 buffer,广播到所有 (layer, axis)
rot = next((v for k, v in by_name.items() if k.endswith("rotary_embed.freqs")), None)
if rot is None:
rot = next((v for nk, v in unmatched.items() if nk.endswith("rotary_embed_freqs")), None)
if rot is not None:
for k in ref:
if k.endswith("rotary_embed.freqs") and k not in by_name:
by_name[k] = rot
# 名字没对上的,按形状 + 同名字段兜底
missing = [k for k in ref if k not in by_name]
for k in list(missing):
want = tuple(ref[k].shape)
cand = [nk for nk, v in unmatched.items()
if tuple(v.shape) == want and nk.split("_")[-1] == k.split(".")[-1]]
if len(cand) == 1:
by_name[k] = unmatched.pop(cand[0])
return by_name, [k for k in ref if k not in by_name], list(unmatched)
# --------------------------------------------------------------------------- #
# 适配类
# --------------------------------------------------------------------------- #
class Adapter:
"""外部适配基类:持有 pymss 模型,负责建模型 / 装权重。"""
name = "adapter"
def __init__(self):
self.model = None
self.meta = {}
def build(self): # pragma: no cover - 子类实现
raise NotImplementedError
def load(self, ckpt):
tensors = collect_tensors(ckpt)
sd, missing, leftover = match_state_dict(self.model, tensors)
# 直接改名装进去(不减 pymss 代码)
st = self.model.state_dict()
diff = [k for k, v in sd.items() if tuple(v.shape) != tuple(st[k].shape)]
status = "OK" if not missing and not diff else f"缺 {len(missing)} / 形状不符 {len(diff)}"
if not missing and not diff:
self.model.load_state_dict(sd, strict=True)
self.meta = {"model": self.name, "status": status,
"matched": len(sd), "missing": missing[:8],
"shape_mismatch": [f"{k} {tuple(st[k].shape)} vs {tuple(sd[k].shape)}"
for k in diff[:8]],
"leftover": len(leftover)}
return self.meta
class BranchedRoformer(Adapter):
"""music4stemhq:与 bs_roformer 同构,参数多一层 `.branch.`,由规范化名抹平。"""
name = "music4stemhq_fp16c_t2"
def build(self):
from pymss_core.modules.bs_roformer.bs_roformer import BSRoformer
from pymss_core.modules.bs_roformer.common import DEFAULT_FREQS_PER_BANDS
self.model = BSRoformer(dim=256, depth=12, stereo=True, num_stems=4,
time_transformer_depth=1, freq_transformer_depth=1,
dim_head=64, heads=8, freqs_per_bands=DEFAULT_FREQS_PER_BANDS,
stft_n_fft=2048, stft_hop_length=512, stft_win_length=2048,
mask_estimator_depth=2, mlp_expansion_factor=4,
use_shared_bias=False)
return self
def yaml(self):
from to_pymss import yaml_bs_roformer
from pymss_core.modules.bs_roformer.common import DEFAULT_FREQS_PER_BANDS
return yaml_bs_roformer({"dim": 256, "depth": 12, "heads": 8, "dim_head": 64,
"num_stems": 4, "mlp_expansion_factor": 4,
"use_shared_bias": False,
"freqs_per_bands": list(DEFAULT_FREQS_PER_BANDS)},
["drums", "bass", "vocals", "other"])
class ScnetAdapter(Adapter):
"""musicnew4stem:SCNet,命名只差 `model.` 前缀。"""
name = "musicnew4stem_fp16c_t2"
def build(self):
from pymss_core.modules.scnet.scnet import SCNet
self.model = SCNet(dims=[4, 64, 128, 256],
sources=["drums", "bass", "vocals", "other"], audio_channels=2,
band_SR=[0.225, 0.372, 0.403], band_stride=[1, 4, 16],
band_kernel=[3, 4, 16], conv_depths=[3, 2, 1], conv_kernel=3,
num_dplayer=6, expand=1, compress=4, hop_size=1024,
nfft=4096, win_size=4096, normalized=True)
return self
def yaml(self):
return """audio:
chunk_size: 352800
min_mean_abs: 0.0
num_channels: 2
sample_rate: 44100
inference:
batch_size: 1
dim_t: 256
normalize: false
num_overlap: 2
model:
audio_channels: 2
band_SR:
- 0.225
- 0.372
- 0.403
band_kernel:
- 3
- 4
- 16
band_stride:
- 1
- 4
- 16
compress: 4
conv_depths:
- 3
- 2
- 1
conv_kernel: 3
dims:
- 4
- 64
- 128
- 256
expand: 1
hop_size: 1024
nfft: 4096
normalized: true
num_dplayer: 6
sources:
- drums
- bass
- other
- vocals
win_size: 4096
model_type: scnet
training:
instruments:
- drums
- bass
- vocals
- other
"""
class BanditAdapter(Adapter):
"""cinema4stem:Bandit v2,mask_estim 是 ModuleDict(music/sfx/speech)。"""
name = "cinema4stem_fp16c_t2"
def build(self):
from pymss_core.modules.bandit_v2.bandit import Bandit
self.model = Bandit(in_channels=2, stems=["music", "sfx", "speech"], fs=44100,
band_type="musical", n_bands=64, n_sqm_modules=12,
emb_dim=128, rnn_dim=256, bidirectional=True, rnn_type="GRU",
mlp_dim=512, hidden_activation="Tanh",
complex_mask=True, use_freq_weights=True,
n_fft=2048, win_length=2048, hop_length=512)
return self
def yaml(self):
return """audio:
chunk_size: 352800
min_mean_abs: 0.0
num_channels: 2
sample_rate: 44100
inference:
batch_size: 1
dim_t: 256
num_overlap: 2
model:
band_type: musical
bidirectional: true
complex_mask: true
emb_dim: 128
fs: 44100
hidden_activation: Tanh
in_channels: 2
mlp_dim: 512
n_bands: 64
n_fft: 2048
n_sqm_modules: 12
hop_length: 512
win_length: 2048
rnn_dim: 256
rnn_type: GRU
stems:
- music
- sfx
- speech
use_freq_weights: true
model_type: bandit
training:
instruments:
- dialogue
- music
- sfx
"""
ADAPTERS = [BranchedRoformer, ScnetAdapter, BanditAdapter]
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--src", default=SRC_DIR)
ap.add_argument("--out", default=OUT_DIR)
args = ap.parse_args()
sys.path.insert(0, PYMSS_CORE)
sys.path.insert(0, HERE)
os.makedirs(args.out, exist_ok=True)
report = []
for cls in ADAPTERS:
a = cls().build()
ckpt = model_io.model_path_for(a.name) or os.path.join(args.src, a.name, a.name + ".ckpt")
if not os.path.exists(ckpt):
print(f"[{a.name}] 缺少 ckpt")
continue
meta = a.load(ckpt)
d = os.path.join(args.out, a.name)
os.makedirs(d, exist_ok=True)
with open(os.path.join(d, "config.yaml"), "w", encoding="utf-8") as f:
f.write(a.yaml())
if meta["status"] == "OK":
sd = a.model.state_dict()
bad = [k for k, v in sd.items() if "weight" not in k and "bias" not in k
and "gamma" not in k and "freqs" not in k]
torch.save({"state_dict": {k: v.clone() for k, v in sd.items()}},
os.path.join(d, a.name + ".ckpt"))
print(f"[{a.name}] {meta['status']} 匹配 {meta['matched']} 多余 {meta['leftover']}")
if meta["missing"]:
print(" 缺:", meta["missing"])
if meta["shape_mismatch"]:
print(" 形状不符:", meta["shape_mismatch"])
report.append(meta)
with open(os.path.join(args.out, "_adapters_report.json"), "w", encoding="utf-8") as f:
json.dump(report, f, indent=2, ensure_ascii=False)
return 0
if __name__ == "__main__":
sys.exit(main())