Update src/loss.py
Browse files- src/loss.py +2 -2
src/loss.py
CHANGED
|
@@ -116,8 +116,8 @@ class LossSchedulerModel(torch.nn.Module):
|
|
| 116 |
jitter=A.matrix_generator.forward(t)
|
| 117 |
for(D,E,J)in zip(B,A.we[t],jitter[t]):
|
| 118 |
dynamic_scale = torch.tanh(A.decay_rate * torch.tensor(t)).to(D.device)
|
| 119 |
-
|
| 120 |
-
C +=
|
| 121 |
return C.to(xT.dtype) #C.to(xT.dtype)
|
| 122 |
|
| 123 |
class LossScheduler:
|
|
|
|
| 116 |
jitter=A.matrix_generator.forward(t)
|
| 117 |
for(D,E,J)in zip(B,A.we[t],jitter[t]):
|
| 118 |
dynamic_scale = torch.tanh(A.decay_rate * torch.tensor(t)).to(D.device)
|
| 119 |
+
correction=D*(E+J.to(D.device))
|
| 120 |
+
C += correction * dynamic_scale
|
| 121 |
return C.to(xT.dtype) #C.to(xT.dtype)
|
| 122 |
|
| 123 |
class LossScheduler:
|