Hugging Face's logo

shibatch
/
tinydeepseekv2-3m

Text Generation
Transformers
Safetensors
English
deepseek-v2
multi-head-latent-attention
mla
mixture-of-experts
Mixture of Experts
tinystories
tiny-model
validation
debug-model
Model card Files
xet
 Community
tinydeepseekv2-3m
File size: 5,544 Bytes
d093b93
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
{
  "model_type": "deepseek_v2_moe",
  "hf_model_type": "deepseek_v2",
  "parameter_count": 2928392,
  "tokenizer_size": 1003,
  "final_eval_loss": 1.4225675538182259,
  "final_eval_ppl": 4.1477563679496425,
  "final_train_lm_loss": 1.409641880750656,
  "router_usage": {
    "model.layers.1.mlp.gate": [
      0.2502,
      0.2487,
      0.2522,
      0.2488
    ],
    "model.layers.2.mlp.gate": [
      0.2538,
      0.2481,
      0.2487,
      0.2495
    ],
    "model.layers.3.mlp.gate": [
      0.2512,
      0.2454,
      0.2519,
      0.2514
    ],
    "model.layers.4.mlp.gate": [
      0.2465,
      0.2535,
      0.2542,
      0.2458
    ]
  },
  "args": {
    "train_files": [
      "data/train-00000-of-00004-2d5a1467fff1081b.parquet",
      "data/train-00001-of-00004-5852b56a2bd28fd9.parquet",
      "data/train-00002-of-00004-a26307300439e943.parquet",
      "data/train-00003-of-00004-d243063613e5a057.parquet"
    ],
    "output_dir": "runs/deepseek_v2_3m_fix2_100p_1epoch",
    "max_rows": 0,
    "data_fraction": 1.0,
    "vocab_size": 1024,
    "base_vocab_size": 1000,
    "min_frequency": 2,
    "legacy_tokenizer": true,
    "legacy_special_token_ids": true,
    "deepseek_arch": "v2",
    "hidden_size": 216,
    "intermediate_size": 432,
    "num_hidden_layers": 5,
    "num_attention_heads": 8,
    "num_key_value_heads": 8,
    "qk_nope_head_dim": 16,
    "qk_rope_head_dim": 16,
    "v_head_dim": 32,
    "q_lora_rank": -1,
    "kv_lora_rank": 64,
    "attention_bias": false,
    "attention_dropout": 0.0,
    "rms_norm_eps": 1e-06,
    "initializer_range": 0.02,
    "rope_theta": 10000,
    "first_k_dense_replace": 1,
    "n_routed_experts": 4,
    "n_shared_experts": 1,
    "num_experts_per_tok": 1,
    "n_group": 0,
    "topk_group": 0,
    "topk_method": "greedy",
    "norm_topk_prob": false,
    "routed_scaling_factor": 4.0,
    "moe_intermediate_size": 128,
    "router_aux_loss_alpha": 0.01,
    "official_router_init": true,
    "max_position_embeddings": 2048,
    "tie_word_embeddings": true,
    "use_cache": false,
    "mlp_bias": false,
    "block_size": 256,
    "batch_size": 16,
    "max_steps": 152568,
    "num_epochs": 1.0,
    "learning_rate": 0.0003,
    "weight_decay": 0.0,
    "grad_clip": 1.0,
    "warmup_steps": 2000,
    "min_lr_ratio": 0.1,
    "validation_fraction": 0.01,
    "log_steps": 500,
    "eval_steps": 10000,
    "eval_batches": 32,
    "save_steps": 10000,
    "resume_from_checkpoint": "",
    "seed": 1234,
    "dtype": "float32",
    "device": "cuda",
    "num_workers": 0,
    "generate_prompts": [
      "Once upon",
      "There was a little",
      "One day"
    ],
    "generate_new_tokens": 100,
    "generation_temperature": 0.8,
    "generation_top_p": 0.95,
    "generation_top_k": 40,
    "generation_repetition_penalty": 1.1,
    "do_sample": true
  },
  "config": {
    "transformers_version": "5.14.1",
    "architectures": [
      "DeepseekV2ForCausalLM"
    ],
    "output_hidden_states": false,
    "return_dict": true,
    "dtype": "float32",
    "chunk_size_feed_forward": 0,
    "is_encoder_decoder": false,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "problem_type": null,
    "vocab_size": 1024,
    "hidden_size": 216,
    "intermediate_size": 432,
    "num_hidden_layers": 5,
    "num_attention_heads": 8,
    "num_key_value_heads": 8,
    "hidden_act": "silu",
    "max_position_embeddings": 2048,
    "initializer_range": 0.02,
    "rms_norm_eps": 1e-06,
    "use_cache": false,
    "pad_token_id": 1000,
    "bos_token_id": 1000,
    "eos_token_id": 1001,
    "pretraining_tp": 1,
    "tie_word_embeddings": true,
    "rope_parameters": {
      "rope_type": "default",
      "rope_theta": 10000
    },
    "attention_bias": false,
    "attention_dropout": 0.0,
    "mlp_bias": false,
    "head_dim": 16,
    "first_k_dense_replace": 1,
    "kv_lora_rank": 64,
    "q_lora_rank": null,
    "n_group": null,
    "n_routed_experts": 4,
    "n_shared_experts": 1,
    "qk_nope_head_dim": 16,
    "qk_rope_head_dim": 16,
    "routed_scaling_factor": 4.0,
    "topk_group": null,
    "topk_method": "greedy",
    "norm_topk_prob": false,
    "v_head_dim": 32,
    "num_experts_per_tok": 1,
    "moe_intermediate_size": 128,
    "_name_or_path": "",
    "aux_loss_alpha": 0.01,
    "seq_aux": true,
    "model_type": "deepseek_v2",
    "output_attentions": false
  },
  "generations": {
    "Once upon": "Once upon a time, there was a little boy named Timmy. He loved to play in the sand and splash in the water. One day, he was playing with his toy car when it slipped on some rocks. His mom came and said, \"Oh no, I'm sorry! Let's bury something that is not ours so we can find out.\"\n\nTimmy didn't understand what \"oray\" meant, but",
    "There was a little": "There was a little girl named Lily who loved to explore. One day, she saw a big bush with lots of yummy peaches on it. She wanted to eat one, but her mom said no.\n\nLily didn't listen and kept trying to eat her peach. She tried and tried but the bush wouldn't eat. Finally, she ate all the peaches and started to feel sick. Her mom told her not to eat",
    "One day": "One day, the boy saw a palace in the park. He thought it was the most special palace he had ever seen! He wanted to play with it, so he called out to his friend, Alice.\n\nAlice asked the palace: \"Can I go in?\"\n\nHis friend replied: \"Yes, but you must be very kind and gentle. What do you think?\"\n\n"
  }
}