miceworks commited on
Commit
185e8ff
·
verified ·
1 Parent(s): 771ed82

Sync MarkV3 HF AutoModel loader for 12h trained weights (configuration_mark_v3.py)

Browse files
Files changed (1) hide show
  1. configuration_mark_v3.py +4 -11
configuration_mark_v3.py CHANGED
@@ -1,9 +1,4 @@
1
- """
2
- Hugging Face Transformers Configuration for Mark-SLM-V3-150M
3
- """
4
-
5
- from transformers.configuration_utils import PretrainedConfig
6
-
7
 
8
  class MarkV3Config(PretrainedConfig):
9
  model_type = "deepseek2"
@@ -11,9 +6,9 @@ class MarkV3Config(PretrainedConfig):
11
 
12
  def __init__(
13
  self,
14
- vocab_size: int = 32032,
15
  hidden_size: int = 576,
16
- num_hidden_layers: int = 15,
17
  num_attention_heads: int = 9,
18
  num_key_value_heads: int = 9,
19
  intermediate_size: int = 1536,
@@ -26,8 +21,7 @@ class MarkV3Config(PretrainedConfig):
26
  num_experts_per_tok: int = 2,
27
  n_shared_experts: int = 1,
28
  moe_intermediate_size: int = 256,
29
- max_position_embeddings: int = 8192,
30
- original_max_position_embeddings: int = 4096,
31
  rope_theta: float = 10000.0,
32
  rope_scaling_factor: float = 2.0,
33
  rms_norm_eps: float = 1e-6,
@@ -60,7 +54,6 @@ class MarkV3Config(PretrainedConfig):
60
  self.n_shared_experts = n_shared_experts
61
  self.moe_intermediate_size = moe_intermediate_size
62
  self.max_position_embeddings = max_position_embeddings
63
- self.original_max_position_embeddings = original_max_position_embeddings
64
  self.rope_theta = rope_theta
65
  self.rope_scaling_factor = rope_scaling_factor
66
  self.rms_norm_eps = rms_norm_eps
 
1
+ from transformers import PretrainedConfig
 
 
 
 
 
2
 
3
  class MarkV3Config(PretrainedConfig):
4
  model_type = "deepseek2"
 
6
 
7
  def __init__(
8
  self,
9
+ vocab_size: int = 49152,
10
  hidden_size: int = 576,
11
+ num_hidden_layers: int = 14,
12
  num_attention_heads: int = 9,
13
  num_key_value_heads: int = 9,
14
  intermediate_size: int = 1536,
 
21
  num_experts_per_tok: int = 2,
22
  n_shared_experts: int = 1,
23
  moe_intermediate_size: int = 256,
24
+ max_position_embeddings: int = 4096,
 
25
  rope_theta: float = 10000.0,
26
  rope_scaling_factor: float = 2.0,
27
  rms_norm_eps: float = 1e-6,
 
54
  self.n_shared_experts = n_shared_experts
55
  self.moe_intermediate_size = moe_intermediate_size
56
  self.max_position_embeddings = max_position_embeddings
 
57
  self.rope_theta = rope_theta
58
  self.rope_scaling_factor = rope_scaling_factor
59
  self.rms_norm_eps = rms_norm_eps