Fix

Fix for numerical stability

spear_modules.py

@@ -1547,7 +1547,8 @@ class SwooshROnnx(torch.nn.Module):
 # ActivationDropoutAndLinearFunction.
 def SwooshLForward(x: Tensor):
     x_offset = x - 4.0
-    log_sum = (1.0 + x_offset.exp()).log().to(x.dtype)
+    # log_sum = (1.0 + x_offset.exp()).log().to(x.dtype)
+    log_sum = torch.nn.functional.softplus(x_offset)
     log_sum = torch.where(log_sum == float("inf"), x_offset, log_sum)
     return log_sum - 0.08 * x - 0.035
 
@@ -1556,7 +1557,8 @@ def SwooshLForward(x: Tensor):
 # ActivationDropoutAndLinearFunction.
 def SwooshRForward(x: Tensor):
     x_offset = x - 1.0
-    log_sum = (1.0 + x_offset.exp()).log().to(x.dtype)
+    # log_sum = (1.0 + x_offset.exp()).log().to(x.dtype)
+    log_sum = torch.nn.functional.softplus(x_offset)
     log_sum = torch.where(log_sum == float("inf"), x_offset, log_sum)
     return log_sum - 0.08 * x - 0.313261687
 
@@ -1962,4 +1964,4 @@ if __name__ == "__main__":
     _test_double_swish_deriv()
     _test_swooshr_deriv()
     _test_swooshl_deriv()
-    _test_activation_dropout_and_linear()
+    _test_activation_dropout_and_linear()