tinyVoxtral-Realtime-for-testing-gguf / make_tiny_voxtral_rt.py
ngxson's picture
ngxson HF Staff
initial upload
2b4a4c3 verified
Raw History Blame Contribute Delete
3.86 kB
#!/usr/bin/env python3
# build a tiny random-weight Voxtral Realtime for server tests: real tokenizer and audio front-end, tiny transformers
# usage: make_tiny_voxtral_rt.py <dir with config.json + tekken.json of mistralai/Voxtral-Mini-4B-Realtime-2602> <output dir>
import importlib.util
import json
import shutil
import sys
from pathlib import Path
# torchaudio in this venv does not match torch, transformers does not need it here
_find_spec = importlib.util.find_spec
importlib.util.find_spec = lambda name, *a, **k: None if name.split(".")[0] == "torchaudio" else _find_spec(name, *a, **k)
sys.modules["torchaudio"] = None # type: ignore
import torch
from transformers.models.voxtral_realtime.configuration_voxtral_realtime import VoxtralRealtimeConfig
from transformers.models.voxtral_realtime.modeling_voxtral_realtime import VoxtralRealtimeForConditionalGeneration
src, out = Path(sys.argv[1]), Path(sys.argv[2])
cfg = json.loads((src / "config.json").read_text())
# mel bins, sliding window, downsample factor and delay tokens stay real, they shape the streaming
cfg["audio_config"].update(hidden_size=64, intermediate_size=128, num_hidden_layers=3,
num_attention_heads=2, num_key_value_heads=2, head_dim=32)
cfg["text_config"].update(hidden_size=64, intermediate_size=128, num_hidden_layers=2,
num_attention_heads=2, num_key_value_heads=1, head_dim=32)
cfg["hidden_size"] = 64
# the tokenizer keeps its 1000 special tokens (incl. [STREAMING_PAD]) and the first ranks of the normal vocab (incl. the 256 bytes)
N_NORMAL_VOCAB = 3000
tekken = json.loads((src / "tekken.json").read_text())
n_special = tekken["config"]["default_num_special_tokens"]
tekken["vocab"] = tekken["vocab"][:N_NORMAL_VOCAB]
tekken["config"]["num_vocab_tokens"] = N_NORMAL_VOCAB
tekken["config"]["default_vocab_size"] = n_special + N_NORMAL_VOCAB
cfg["text_config"]["vocab_size"] = n_special + N_NORMAL_VOCAB
cfg["audio_config"]["vocab_size"] = n_special + N_NORMAL_VOCAB
torch.manual_seed(42)
config = VoxtralRealtimeConfig(**{k: v for k, v in cfg.items() if k not in ("architectures", "transformers_version")})
config.architectures = cfg["architectures"]
model = VoxtralRealtimeForConditionalGeneration(config).to(torch.bfloat16)
n_params = sum(p.numel() for p in model.parameters())
print(f"params: {n_params / 1e6:.2f}M")
out.mkdir(parents=True, exist_ok=True)
model.save_pretrained(out)
# recent transformers save the text model as language_model.model.model.*, the released checkpoint has language_model.model.*
from safetensors.torch import load_file, save_file
weights = load_file(out / "model.safetensors")
weights = {k.replace("language_model.model.model.", "language_model.model."): v for k, v in weights.items()}
save_file(weights, out / "model.safetensors", metadata={"format": "pt"})
for f in ["generation_config.json", "processor_config.json"]:
shutil.copy(src / f, out / f)
(out / "tekken.json").write_text(json.dumps(tekken, ensure_ascii=False))
(out / "README.md").write_text(f"""---
license: apache-2.0
base_model: mistralai/Voxtral-Mini-4B-Realtime-2602
tags: [testing]
---
# tinyVoxtral-Realtime-for-testing
Random weights with the architecture and audio front-end of [mistralai/Voxtral-Mini-4B-Realtime-2602](https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602): the encoder is shrunk to 3 layers and the text model to 2 layers, both of hidden size 64. The tokenizer keeps its 1000 special tokens and the first {N_NORMAL_VOCAB} tokens of the normal vocab (all 256 bytes included). The mel bins, the sliding window, the downsample factor and the delay tokens are unchanged.
Built with [make_tiny_voxtral_rt.py](make_tiny_voxtral_rt.py).
The output is not meaningful. Do not use this model for anything other than testing a loader or a runtime.
""")
print(f"saved to {out}")