---
library_name: transformers
base_model:
- deepseek-ai/DeepSeek-V4-Pro
---
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [deepseek-ai/DeepSeek-V4-Pro](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro).
Note:
- This model is in bfloat16, not quantized to fp8/fp4.
- Does not contain MTP layer.
- Chat template from [this PR](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/discussions/146/files).
| File path | Size |
|------|------|
| model.safetensors | 30.5MB |
### Example usage:
```python
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
model_id = "yujiepan/deepseek-v4-bf16-tiny-random"
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
# Note: `.from_pretrained()` automatically casts some layers to float32, based on
# `_keep_in_fp32_modules_strict`, causing fp32 x bf16 cases.
# Seems like a bug. Here we force bf16 dtype as a workaround.
# Tested on transformers 5.13.0.
model = AutoModelForCausalLM.from_pretrained(
model_id,
dtype=torch.bfloat16,
trust_remote_code=True,
).to(device).to(torch.bfloat16)
model.eval()
inputs = tokenizer.apply_chat_template(
[
{"role": "user", "content": f"Hello! Counting 1 to 200: {list(range(1, 201))}"}
],
tools=[
{
"name": "count",
"description": "Count numbers",
"parameters": {
"type": "object",
"properties": {
"numbers": {"type": "array", "items": {"type": "integer"}},
},
"required": ["numbers"],
},
},
],
return_tensors="pt",
add_generation_prompt=True,
thinking_mode="thinking",
reasoning_effort="max",
).to(device)
outputs = model.generate(**inputs, max_new_tokens=32, max_length=1024)
print(tokenizer.decode(outputs[0], skip_special_tokens=False))
```
### Codes to create this repo:
Click to expand
```python
import heapq
import json
import jinja2
import torch
from huggingface_hub import file_exists, hf_hub_download
from transformers import (
AutoConfig,
AutoModelForCausalLM,
AutoTokenizer,
DeepseekV4Config,
GenerationConfig,
set_seed,
)
source_model_id = "deepseek-ai/DeepSeek-V4-Pro"
save_folder = "/tmp/yujiepan/deepseek-v4-bf16-tiny-random" # pyright: ignore[reportUnusedExpression]
tokenizer = AutoTokenizer.from_pretrained(
source_model_id, trust_remote_code=True,
)
if file_exists(filename="chat_template.jinja", repo_id=source_model_id, repo_type='model', revision="refs/pr/146"):
with open(hf_hub_download(
source_model_id,
filename="chat_template.jinja",
repo_type='model',
revision="refs/pr/146",
), 'r', encoding='utf-8') as f:
tokenizer.chat_template = f.read()
tokenizer.save_pretrained(save_folder)
with open(hf_hub_download(
source_model_id,
filename='config.json',
repo_type='model',
), 'r', encoding='utf-8') as f:
config_json = json.load(f)
# Modify config_json here before loading.
config_json.update({
"head_dim": 128,
"hidden_size": 8,
"index_head_dim": 128,
"index_n_heads": 4,
"moe_intermediate_size": 32,
"n_routed_experts": 256,
# "n_shared_experts": 1,
"num_attention_heads": 8,
"num_hidden_layers": 7,
"num_experts_per_tok": 6,
"num_hash_layers": 3,
"num_key_value_heads": 1,
"q_lora_rank": 128,
"o_lora_rank": 128,
"o_groups": 8,
"layer_types": [
"sliding_attention",
"sliding_attention",
"compressed_sparse_attention",
"heavily_compressed_attention",
"compressed_sparse_attention",
"heavily_compressed_attention",
"compressed_sparse_attention"
],
"compress_ratios": [0, 0, 4, 128, 4, 128, 4, 0],
})
del config_json['quantization_config']
del config_json['expert_dtype']
with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
json.dump(config_json, f, indent=2)
config = AutoConfig.from_pretrained(
save_folder,
trust_remote_code=True,
)
assert isinstance(config, DeepseekV4Config)
model = AutoModelForCausalLM.from_config(
config,
dtype=torch.bfloat16,
trust_remote_code=True,
)
if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
model.generation_config = GenerationConfig.from_pretrained(
source_model_id, trust_remote_code=True,
)
set_seed(42)
max_width = max(len(name) for name in model.state_dict().keys())
name2size = {name: tensor.numel() * tensor.element_size() for name, tensor in model.state_dict().items()}
print("Top 10 tensor sizes:")
for name, size in heapq.nlargest(10, name2size.items(), key=lambda x: x[1]):
print(
name.ljust(max_width),
f'{size / 1024**2:.2f}MB',
model.state_dict()[name].shape,
)
print("Initializing parameters:")
with torch.no_grad():
for name, p in sorted(model.named_parameters()):
torch.nn.init.normal_(p, 0, 0.3)
print(
name.ljust(max_width),
f'{name2size[name] / 1024**2:.2f}MB',
p.shape, p.dtype,
)
for name, b in model.named_buffers():
print(
name.ljust(max_width),
f'{b.numel() * b.element_size() / 1024**2:.2f}MB',
b.shape, b.dtype,
)
if name.endswith("e_score_correction_bias"):
torch.nn.init.normal_(b, 0, 0.3)
elif name.endswith("tid2eid"):
b.copy_(torch.rand(
b.shape[0], config.n_routed_experts, device=b.device,
).topk(b.shape[1], dim=-1).indices.to(b.dtype))
elif name.endswith("inv_freq"):
pass
else:
raise ValueError(f"Unknown buffer: {name}")
model.save_pretrained(save_folder)
with open(f"{save_folder}/config.json", "r", encoding="utf-8") as f:
saved_config_json = json.load(f)
# vLLM reads old config.json, so we need to modify it to make it compatible.
saved_config_json.pop("mlp_layer_types", None)
saved_config_json["num_hash_layers"] = config_json["num_hash_layers"]
saved_config_json["compress_ratios"] = [0, 0, 4, 128, 4, 128, 4, 0]
rope_parameters = saved_config_json.pop("rope_parameters", None)
saved_config_json["rope_scaling"] = {
"beta_fast": 32,
"beta_slow": 1,
"factor": 16,
"original_max_position_embeddings": 65536,
"type": "yarn"
}
saved_config_json["rope_theta"] = 10000
saved_config_json["compress_rope_theta"] = 160000
with open(f"{save_folder}/config.json", "w", encoding="utf-8") as f:
json.dump(saved_config_json, f, indent=2)
```
Click to expand
```text
DeepseekV4ForCausalLM(
(model): DeepseekV4Model(
(embed_tokens): Embedding(129280, 8)
(layers): ModuleList(
(0-1): 2 x DeepseekV4DecoderLayer(
(self_attn): DeepseekV4Attention(
(q_a_proj): Linear(in_features=8, out_features=128, bias=False)
(q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
(q_b_norm): DeepseekV4UnweightedRMSNorm()
(kv_proj): Linear(in_features=8, out_features=128, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
(o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
)
(mlp): DeepseekV4SparseMoeBlock(
(gate): DeepseekV4HashRouter(
(score_fn): SqrtSoftplusActivation()
)
(experts): DeepseekV4Experts(
(act_fn): SiLUActivation()
)
(shared_experts): DeepseekV4MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
)
(input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(attn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
(ffn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
)
(2): DeepseekV4DecoderLayer(
(self_attn): DeepseekV4Attention(
(q_a_proj): Linear(in_features=8, out_features=128, bias=False)
(q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
(q_b_norm): DeepseekV4UnweightedRMSNorm()
(kv_proj): Linear(in_features=8, out_features=128, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
(o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
(compressor): DeepseekV4CSACompressor(
(kv_proj): Linear(in_features=8, out_features=256, bias=False)
(gate_proj): Linear(in_features=8, out_features=256, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(rotary_emb): DeepseekV4RotaryEmbedding()
(indexer): DeepseekV4Indexer(
(kv_proj): Linear(in_features=8, out_features=256, bias=False)
(gate_proj): Linear(in_features=8, out_features=256, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=512, bias=False)
(rotary_emb): DeepseekV4RotaryEmbedding()
(scorer): DeepseekV4IndexerScorer(
(weights_proj): Linear(in_features=8, out_features=4, bias=False)
)
)
)
)
(mlp): DeepseekV4SparseMoeBlock(
(gate): DeepseekV4HashRouter(
(score_fn): SqrtSoftplusActivation()
)
(experts): DeepseekV4Experts(
(act_fn): SiLUActivation()
)
(shared_experts): DeepseekV4MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
)
(input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(attn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
(ffn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
)
(3): DeepseekV4DecoderLayer(
(self_attn): DeepseekV4Attention(
(q_a_proj): Linear(in_features=8, out_features=128, bias=False)
(q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
(q_b_norm): DeepseekV4UnweightedRMSNorm()
(kv_proj): Linear(in_features=8, out_features=128, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
(o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
(compressor): DeepseekV4HCACompressor(
(kv_proj): Linear(in_features=8, out_features=128, bias=False)
(gate_proj): Linear(in_features=8, out_features=128, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(rotary_emb): DeepseekV4RotaryEmbedding()
)
)
(mlp): DeepseekV4SparseMoeBlock(
(gate): DeepseekV4TopKRouter(
(score_fn): SqrtSoftplusActivation()
)
(experts): DeepseekV4Experts(
(act_fn): SiLUActivation()
)
(shared_experts): DeepseekV4MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
)
(input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(attn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
(ffn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
)
(4): DeepseekV4DecoderLayer(
(self_attn): DeepseekV4Attention(
(q_a_proj): Linear(in_features=8, out_features=128, bias=False)
(q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
(q_b_norm): DeepseekV4UnweightedRMSNorm()
(kv_proj): Linear(in_features=8, out_features=128, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
(o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
(compressor): DeepseekV4CSACompressor(
(kv_proj): Linear(in_features=8, out_features=256, bias=False)
(gate_proj): Linear(in_features=8, out_features=256, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(rotary_emb): DeepseekV4RotaryEmbedding()
(indexer): DeepseekV4Indexer(
(kv_proj): Linear(in_features=8, out_features=256, bias=False)
(gate_proj): Linear(in_features=8, out_features=256, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=512, bias=False)
(rotary_emb): DeepseekV4RotaryEmbedding()
(scorer): DeepseekV4IndexerScorer(
(weights_proj): Linear(in_features=8, out_features=4, bias=False)
)
)
)
)
(mlp): DeepseekV4SparseMoeBlock(
(gate): DeepseekV4TopKRouter(
(score_fn): SqrtSoftplusActivation()
)
(experts): DeepseekV4Experts(
(act_fn): SiLUActivation()
)
(shared_experts): DeepseekV4MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
)
(input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(attn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
(ffn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
)
(5): DeepseekV4DecoderLayer(
(self_attn): DeepseekV4Attention(
(q_a_proj): Linear(in_features=8, out_features=128, bias=False)
(q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
(q_b_norm): DeepseekV4UnweightedRMSNorm()
(kv_proj): Linear(in_features=8, out_features=128, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
(o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
(compressor): DeepseekV4HCACompressor(
(kv_proj): Linear(in_features=8, out_features=128, bias=False)
(gate_proj): Linear(in_features=8, out_features=128, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(rotary_emb): DeepseekV4RotaryEmbedding()
)
)
(mlp): DeepseekV4SparseMoeBlock(
(gate): DeepseekV4TopKRouter(
(score_fn): SqrtSoftplusActivation()
)
(experts): DeepseekV4Experts(
(act_fn): SiLUActivation()
)
(shared_experts): DeepseekV4MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
)
(input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(attn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
(ffn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
)
(6): DeepseekV4DecoderLayer(
(self_attn): DeepseekV4Attention(
(q_a_proj): Linear(in_features=8, out_features=128, bias=False)
(q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
(q_b_norm): DeepseekV4UnweightedRMSNorm()
(kv_proj): Linear(in_features=8, out_features=128, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
(o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
(compressor): DeepseekV4CSACompressor(
(kv_proj): Linear(in_features=8, out_features=256, bias=False)
(gate_proj): Linear(in_features=8, out_features=256, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(rotary_emb): DeepseekV4RotaryEmbedding()
(indexer): DeepseekV4Indexer(
(kv_proj): Linear(in_features=8, out_features=256, bias=False)
(gate_proj): Linear(in_features=8, out_features=256, bias=False)
(kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
(q_b_proj): Linear(in_features=128, out_features=512, bias=False)
(rotary_emb): DeepseekV4RotaryEmbedding()
(scorer): DeepseekV4IndexerScorer(
(weights_proj): Linear(in_features=8, out_features=4, bias=False)
)
)
)
)
(mlp): DeepseekV4SparseMoeBlock(
(gate): DeepseekV4TopKRouter(
(score_fn): SqrtSoftplusActivation()
)
(experts): DeepseekV4Experts(
(act_fn): SiLUActivation()
)
(shared_experts): DeepseekV4MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
)
(input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
(attn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
(ffn_hc): DeepseekV4HyperConnection(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
)
)
(norm): DeepseekV4RMSNorm((8,), eps=1e-06)
(rotary_emb): DeepseekV4RotaryEmbedding()
(hc_head): DeepseekV4HyperHead(
(input_norm): DeepseekV4UnweightedRMSNorm()
)
)
(lm_head): Linear(in_features=8, out_features=129280, bias=False)
)
```