Upload modeling_vit.py

modeling_vit.py

@@ -234,10 +234,11 @@ class ViTSelfAttention(nn.Module):
             ssm_eps=None,
             tau=None,
             skip_attn=False,
-            attn_gate_type=AttentionGateType.none,
-            attn_gate_init=None,
+            attn_gate_type=AttentionGateType.conditional_per_token,
+            attn_gate_init=0.25,
             attn_gate_mlp=False,
             attn_gate_mlp2=False,
+            max_seq_length=None,
             attn_gate_linear_all_features=False) -> None:
         super().__init__()
         if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):