Upload modeling_bert.py

modeling_bert.py

@@ -21,6 +21,7 @@ import warnings
 from dataclasses import dataclass
 from typing import List, Optional, Tuple, Union
 from functools import partial
+from enum import Flag, auto
 import torch
 import torch.utils.checkpoint
 from packaging import version
@@ -56,6 +57,18 @@ from transformers.utils import (
 )
 from .configuration_bert import BertConfig
 
+class BaseEnumOptions(Flag):
+    def __str__(self):
+        return self.name
+
+    @classmethod
+    def list_names(cls):
+        return [m.name for m in cls]
+class AttentionGateType(BaseEnumOptions):
+    none = 0
+    unconditional_per_head = 1
+    conditional_per_head = 2
+    conditional_per_token = 3
 
 def softmax_n_shifted_zeros(input: torch.Tensor, n: int, dim=-1) -> torch.Tensor:
     """
@@ -91,6 +104,7 @@ def clipped_softmax(data, dim=1, eta=1.1, gamma=-0.1, **kw):
     return torch.clip(stretched_out, 0, 1)
 
 
+
 def clipped_softmax1(data, dim=1, eta=1.1, gamma=-0.1, **kw):
     sm_out = softmax_1(data, dim=dim, **kw)
     stretched_out = sm_out * (eta - gamma) + gamma