File size: 3,857 Bytes
2b4a4c3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#!/usr/bin/env python3
# build a tiny random-weight Voxtral Realtime for server tests: real tokenizer and audio front-end, tiny transformers
# usage: make_tiny_voxtral_rt.py <dir with config.json + tekken.json of mistralai/Voxtral-Mini-4B-Realtime-2602> <output dir>
import importlib.util
import json
import shutil
import sys
from pathlib import Path

# torchaudio in this venv does not match torch, transformers does not need it here
_find_spec = importlib.util.find_spec
importlib.util.find_spec = lambda name, *a, **k: None if name.split(".")[0] == "torchaudio" else _find_spec(name, *a, **k)
sys.modules["torchaudio"] = None  # type: ignore

import torch
from transformers.models.voxtral_realtime.configuration_voxtral_realtime import VoxtralRealtimeConfig
from transformers.models.voxtral_realtime.modeling_voxtral_realtime import VoxtralRealtimeForConditionalGeneration

src, out = Path(sys.argv[1]), Path(sys.argv[2])
cfg = json.loads((src / "config.json").read_text())

# mel bins, sliding window, downsample factor and delay tokens stay real, they shape the streaming
cfg["audio_config"].update(hidden_size=64, intermediate_size=128, num_hidden_layers=3,
                           num_attention_heads=2, num_key_value_heads=2, head_dim=32)
cfg["text_config"].update(hidden_size=64, intermediate_size=128, num_hidden_layers=2,
                          num_attention_heads=2, num_key_value_heads=1, head_dim=32)
cfg["hidden_size"] = 64

# the tokenizer keeps its 1000 special tokens (incl. [STREAMING_PAD]) and the first ranks of the normal vocab (incl. the 256 bytes)
N_NORMAL_VOCAB = 3000
tekken = json.loads((src / "tekken.json").read_text())
n_special = tekken["config"]["default_num_special_tokens"]
tekken["vocab"] = tekken["vocab"][:N_NORMAL_VOCAB]
tekken["config"]["num_vocab_tokens"] = N_NORMAL_VOCAB
tekken["config"]["default_vocab_size"] = n_special + N_NORMAL_VOCAB
cfg["text_config"]["vocab_size"] = n_special + N_NORMAL_VOCAB
cfg["audio_config"]["vocab_size"] = n_special + N_NORMAL_VOCAB

torch.manual_seed(42)
config = VoxtralRealtimeConfig(**{k: v for k, v in cfg.items() if k not in ("architectures", "transformers_version")})
config.architectures = cfg["architectures"]
model = VoxtralRealtimeForConditionalGeneration(config).to(torch.bfloat16)
n_params = sum(p.numel() for p in model.parameters())
print(f"params: {n_params / 1e6:.2f}M")

out.mkdir(parents=True, exist_ok=True)
model.save_pretrained(out)

# recent transformers save the text model as language_model.model.model.*, the released checkpoint has language_model.model.*
from safetensors.torch import load_file, save_file
weights = load_file(out / "model.safetensors")
weights = {k.replace("language_model.model.model.", "language_model.model."): v for k, v in weights.items()}
save_file(weights, out / "model.safetensors", metadata={"format": "pt"})
for f in ["generation_config.json", "processor_config.json"]:
    shutil.copy(src / f, out / f)
(out / "tekken.json").write_text(json.dumps(tekken, ensure_ascii=False))
(out / "README.md").write_text(f"""---
license: apache-2.0
base_model: mistralai/Voxtral-Mini-4B-Realtime-2602
tags: [testing]
---

# tinyVoxtral-Realtime-for-testing

Random weights with the architecture and audio front-end of [mistralai/Voxtral-Mini-4B-Realtime-2602](https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602): the encoder is shrunk to 3 layers and the text model to 2 layers, both of hidden size 64. The tokenizer keeps its 1000 special tokens and the first {N_NORMAL_VOCAB} tokens of the normal vocab (all 256 bytes included). The mel bins, the sliding window, the downsample factor and the delay tokens are unchanged.

Built with [make_tiny_voxtral_rt.py](make_tiny_voxtral_rt.py).

The output is not meaningful. Do not use this model for anything other than testing a loader or a runtime.
""")
print(f"saved to {out}")