manbeast3b commited on
Commit
db5589c
·
verified ·
1 Parent(s): a0053a7

Update src/loss.py

Browse files
Files changed (1) hide show
  1. src/loss.py +2 -2
src/loss.py CHANGED
@@ -116,8 +116,8 @@ class LossSchedulerModel(torch.nn.Module):
116
  jitter=A.matrix_generator.forward(t)
117
  for(D,E,J)in zip(B,A.we[t],jitter[t]):
118
  dynamic_scale = torch.tanh(A.decay_rate * torch.tensor(t)).to(D.device)
119
- C+=D*(E+J.to(D.device))
120
- C += C * dynamic_scale
121
  return C.to(xT.dtype) #C.to(xT.dtype)
122
 
123
  class LossScheduler:
 
116
  jitter=A.matrix_generator.forward(t)
117
  for(D,E,J)in zip(B,A.we[t],jitter[t]):
118
  dynamic_scale = torch.tanh(A.decay_rate * torch.tensor(t)).to(D.device)
119
+ correction=D*(E+J.to(D.device))
120
+ C += correction * dynamic_scale
121
  return C.to(xT.dtype) #C.to(xT.dtype)
122
 
123
  class LossScheduler: