Update modeling_qwen.py

modeling_qwen.py

@@ -154,7 +154,7 @@ class QWenAttention(nn.Module):
             if self.rotary_ndims is not None
             else self.hidden_size_per_attention_head
         )
-        self.rotary_emb = RotaryEmbedding(dim, base=config.rotary_emb_base)
+        self.rotary_emb = QWenRotaryEmbedding(dim, base=config.rotary_emb_base)
 
         self.use_dynamic_ntk = config.use_dynamic_ntk
         self.use_logn_attn = config.use_logn_attn
@@ -386,12 +386,12 @@ class QWenBlock(nn.Module):
         hidden_size = config.hidden_size
         self.bf16 = config.bf16
 
-        self.ln_1 = RMSNorm(
+        self.ln_1 = QWenRMSNorm(
             hidden_size,
             eps=config.layer_norm_epsilon,
         )
         self.attn = QWenAttention(config)
-        self.ln_2 = RMSNorm(
+        self.ln_2 = QWenRMSNorm(
             hidden_size,
             eps=config.layer_norm_epsilon,
         )
@@ -460,7 +460,7 @@ class QWenPreTrainedModel(PreTrainedModel):
             module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
             if module.padding_idx is not None:
                 module.weight.data[module.padding_idx].zero_()
-        elif isinstance(module, RMSNorm):
+        elif isinstance(module, QWenRMSNorm):
             module.weight.data.fill_(1.0)
 
         for name, p in module.named_parameters():
@@ -500,7 +500,7 @@ class QWenModel(QWenPreTrainedModel):
                 for i in range(config.num_hidden_layers)
             ]
         )
-        self.ln_f = RMSNorm(
+        self.ln_f = QWenRMSNorm(
             self.embed_dim,
             eps=config.layer_norm_epsilon,
         )
@@ -1041,7 +1041,7 @@ class QWenLMHeadModel(QWenPreTrainedModel):
         )
 
 
-class RotaryEmbedding(torch.nn.Module):
+class QWenRotaryEmbedding(torch.nn.Module):
     def __init__(self, dim, base=10000):
         super().__init__()
         self.dim = dim
@@ -1104,7 +1104,7 @@ def apply_rotary_pos_emb(t, freqs):
         return torch.cat((t_, t_pass_), dim=-1).type_as(t)
 
 
-class RMSNorm(torch.nn.Module):
+class QWenRMSNorm(torch.nn.Module):
     def __init__(self, dim: int, eps: float = 1e-6):
         super().__init__()
         self.eps = eps