Update modeling_bert.py
Browse files- modeling_bert.py +1 -1
modeling_bert.py
CHANGED
|
@@ -286,7 +286,7 @@ class BertSelfAttention(nn.Module):
|
|
| 286 |
self,
|
| 287 |
config,
|
| 288 |
position_embedding_type=None,
|
| 289 |
-
softmax_fn=partial(
|
| 290 |
max_seq_length = 128,
|
| 291 |
alpha=None,
|
| 292 |
ssm_eps=None,
|
|
|
|
| 286 |
self,
|
| 287 |
config,
|
| 288 |
position_embedding_type=None,
|
| 289 |
+
softmax_fn=partial(clipped_softmax1, gamma=-0.025, eta=1.0),
|
| 290 |
max_seq_length = 128,
|
| 291 |
alpha=None,
|
| 292 |
ssm_eps=None,
|