Hugging Face's logo

kavanmevada
/
eng-word-model-ds32

Text Generation
Transformers
TensorBoard
Safetensors
deepseek_v3
custom_code
text-generation-inference
fp8
Model card Files
xet
 Metrics Community
kavanmevada commited on
Commit
b1a9c67
·
verified ·
1 Parent(s): 5d2a828

Training in progress, step 10

Browse files
Files changed (14) hide show
  1. config.json +74 -0
  2. generation_config.json +8 -0
  3. model.safetensors +3 -0
  4. runs/Oct07_10-54-21_5dc35bd65435/events.out.tfevents.1759834469.5dc35bd65435.21425.0 +3 -0
  5. runs/Oct07_11-03-20_5dc35bd65435/events.out.tfevents.1759835008.5dc35bd65435.23843.0 +3 -0
  6. runs/Oct07_11-10-03_5dc35bd65435/events.out.tfevents.1759835410.5dc35bd65435.25554.0 +3 -0
  7. runs/Oct07_11-14-10_5dc35bd65435/events.out.tfevents.1759835658.5dc35bd65435.26771.0 +3 -0
  8. runs/Oct07_11-21-35_5dc35bd65435/events.out.tfevents.1759836103.5dc35bd65435.28770.0 +3 -0
  9. runs/Oct07_11-35-15_5dc35bd65435/events.out.tfevents.1759836923.5dc35bd65435.32181.0 +3 -0
  10. runs/Oct07_11-42-44_5dc35bd65435/events.out.tfevents.1759837372.5dc35bd65435.34232.0 +3 -0
  11. runs/Oct07_11-49-19_5dc35bd65435/events.out.tfevents.1759837768.5dc35bd65435.35924.0 +3 -0
  12. runs/Oct07_12-02-21_5dc35bd65435/events.out.tfevents.1759838552.5dc35bd65435.39594.0 +3 -0
  13. runs/Oct07_12-08-20_5dc35bd65435/events.out.tfevents.1759838908.5dc35bd65435.41221.0 +3 -0
  14. training_args.bin +3 -0
config.json ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DeepseekV3ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_deepseek.DeepseekV3Config",
9
+ "AutoModel": "modeling_deepseek.DeepseekV3Model",
10
+ "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
11
+ },
12
+ "bos_token_id": 0,
13
+ "dtype": "bfloat16",
14
+ "eos_token_id": 1,
15
+ "ep_size": 1,
16
+ "first_k_dense_replace": 2,
17
+ "head_dim": 64,
18
+ "hidden_act": "silu",
19
+ "hidden_size": 1012,
20
+ "initializer_range": 0.02,
21
+ "intermediate_size": 4056,
22
+ "kv_lora_rank": 64,
23
+ "max_position_embeddings": 2048,
24
+ "model_type": "deepseek_v3",
25
+ "moe_intermediate_size": 2048,
26
+ "moe_layer_freq": 1,
27
+ "n_group": 8,
28
+ "n_routed_experts": 16,
29
+ "n_shared_experts": 1,
30
+ "norm_topk_prob": true,
31
+ "num_attention_heads": 16,
32
+ "num_experts_per_tok": 2,
33
+ "num_hidden_layers": 20,
34
+ "num_key_value_heads": 16,
35
+ "num_nextn_predict_layers": 1,
36
+ "pad_token_id": 0,
37
+ "pretraining_tp": 1,
38
+ "q_lora_rank": 128,
39
+ "qk_head_dim": 192,
40
+ "qk_nope_head_dim": 128,
41
+ "qk_rope_head_dim": 64,
42
+ "quantization_config": {
43
+ "activation_scheme": "dynamic",
44
+ "fmt": "e4m3",
45
+ "quant_method": "fp8",
46
+ "scale_fmt": "ue8m0",
47
+ "weight_block_size": [
48
+ 128,
49
+ 128
50
+ ]
51
+ },
52
+ "rms_norm_eps": 1e-06,
53
+ "rope_interleave": true,
54
+ "rope_scaling": {
55
+ "beta_fast": 32.0,
56
+ "beta_slow": 1.0,
57
+ "factor": 40.0,
58
+ "mscale": 1.0,
59
+ "mscale_all_dim": 1.0,
60
+ "original_max_position_embeddings": 4096,
61
+ "rope_type": "yarn",
62
+ "type": "yarn"
63
+ },
64
+ "rope_theta": 10000,
65
+ "routed_scaling_factor": 2.5,
66
+ "scoring_func": "sigmoid",
67
+ "tie_word_embeddings": false,
68
+ "topk_group": 1,
69
+ "topk_method": "noaux_tc",
70
+ "transformers_version": "4.57.0.dev0",
71
+ "use_cache": false,
72
+ "v_head_dim": 16,
73
+ "vocab_size": 64000
74
+ }
generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.57.0.dev0",
7
+ "use_cache": false
8
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c926480da0d9471c7b339a0ecd629e0fdd763b453e0ce511bbd2fa82eb53f8a3
3
+ size 4156744752
runs/Oct07_10-54-21_5dc35bd65435/events.out.tfevents.1759834469.5dc35bd65435.21425.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a2305f575abe3e490fe283bcac1845735cbac0b13ca06e0dcfded4ef5ace71d
3
+ size 5927
runs/Oct07_11-03-20_5dc35bd65435/events.out.tfevents.1759835008.5dc35bd65435.23843.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d19805628b56cd38d519ad3d64d9a136fa0b4805aa3ca16f6aca4ca1e84b9ff
3
+ size 6134
runs/Oct07_11-10-03_5dc35bd65435/events.out.tfevents.1759835410.5dc35bd65435.25554.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18adda3adef7b518d45e5fc20f0eea2025cdc845750b94128af05cb3a9e200ac
3
+ size 4184
runs/Oct07_11-14-10_5dc35bd65435/events.out.tfevents.1759835658.5dc35bd65435.26771.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3bc6f221129895774aa2979fb4603ef58c18a500ffcec1b828712e82e66cf40e
3
+ size 6134
runs/Oct07_11-21-35_5dc35bd65435/events.out.tfevents.1759836103.5dc35bd65435.28770.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df83174a1d336b08936babb4cc757f3463f9ba42fb8b5db03ba7c64a4d59bcdc
3
+ size 6134
runs/Oct07_11-35-15_5dc35bd65435/events.out.tfevents.1759836923.5dc35bd65435.32181.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62fc8a9d53a1bfb05f3f33cd51299a8516c00fc30b4af47f2f2b6883b20220d8
3
+ size 6133
runs/Oct07_11-42-44_5dc35bd65435/events.out.tfevents.1759837372.5dc35bd65435.34232.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:138b0158abeaa3436286eb295efb2586e63fb7ee48cc174f866438f2cd023c29
3
+ size 6133
runs/Oct07_11-49-19_5dc35bd65435/events.out.tfevents.1759837768.5dc35bd65435.35924.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d041d021f517d7d052aa51eda2409214c2b652531d08e435c22c5c631236547
3
+ size 6133
runs/Oct07_12-02-21_5dc35bd65435/events.out.tfevents.1759838552.5dc35bd65435.39594.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2119f4f140d9c19d65f147d28285c658c716c6fd12f8cc07c8a8412e1d6253d9
3
+ size 5927
runs/Oct07_12-08-20_5dc35bd65435/events.out.tfevents.1759838908.5dc35bd65435.41221.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af30947ddcbaf31f0e5cf7aac280113c94a8bdc08feca99a75b4f4047f468acc
3
+ size 7997
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb08b65ddf8cfe9535a7296485be97b38a5d3726d26656631c74af48caeb158c
3
+ size 5649