Update modeling_transnormer.py

modeling_transnormer.py

@@ -734,43 +734,6 @@ class TransnormerModel(TransnormerPreTrainedModel):
             slope_rate = slope_rates[idx]
             slope_rate = slope_rate * (1 - idx / (self.num_layers - 1) + 1e-5)
             mask = linear_attn_mask
-
-            # if self.gradient_checkpointing and self.training:
-
-            #     def create_custom_forward(module):
-            #         def custom_forward(*inputs):
-            #             # None for past_key_value
-            #             return module(*inputs, output_attentions, None)
-
-            #         return custom_forward
-
-            #     # layer_outputs = torch.utils.checkpoint.checkpoint(
-            #     #     create_custom_forward(layer),
-            #     #     hidden_states,
-            #     #     mask,
-            #     #     linear_attn_padding_mask,
-            #     #     None,
-            #     # )
-            #     layer_outputs = torch.utils.checkpoint.checkpoint(
-            #         create_custom_forward(layer),
-            #         hidden_states,
-            #         mask,
-            #         linear_attn_padding_mask,
-            #         None,
-            #         output_attentions,
-            #         use_cache,
-            #         slope_rate,
-            #     )
-            # else:
-            #     layer_outputs = layer(
-            #         hidden_states,
-            #         attn_mask=mask,
-            #         attn_padding_mask=linear_attn_padding_mask,
-            #         past_key_value=past_key_value,
-            #         output_attentions=output_attentions,
-            #         use_cache=use_cache,
-            #         slope_rate=slope_rate,
-            #     )
                 
             layer_outputs = layer(
                 hidden_states,
@@ -789,9 +752,6 @@ class TransnormerModel(TransnormerPreTrainedModel):
 
             if output_attentions:
                 all_self_attns += (layer_outputs[1],)
-            
-            # if idx == 0:
-            #     break
 
         hidden_states = self.final_norm(hidden_states)