anlp-a2-models / src /part2 /optimizers.py
Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
10 kB
import torch
from torch.optim import Optimizer
import math
class CustomAdamW(Optimizer):
def __init__(self, params, lr = 1e-3, betas = (0.9, 0.95), eps = 1e-8, weight_decay = 0.1):
defaults = dict(lr = lr, betas = betas, eps = eps, weight_decay = weight_decay)
super().__init__(params, defaults)
@torch.no_grad()
def step(self, closure = None):
loss = None
if closure is not None:
with torch.enable_grad():
loss = closure()
for group in self.param_groups:
lr = group["lr"]
beta1, beta2 = group["betas"]
eps = group["eps"]
weight_decay = group["weight_decay"]
for p in group["params"]:
if p.grad is None:
continue
grad = p.grad
if grad.is_sparse:
raise RuntimeError("CustomAdamW does not support sparse gradients")
state = self.state[p]
if len(state) == 0:
state["step"] = 0
if "m" not in state:
state["m"] = torch.zeros_like(p.data)
if "v" not in state:
state["v"] = torch.zeros_like(p.data)
state["step"] += 1
step = state["step"]
## Moment Updation
state["m"] = beta1 * state["m"] + (1 - beta1) * grad
state["v"] = beta2 * state["v"] + (1 - beta2) * (grad ** 2)
## Bias Correction
bias_correction1 = 1.0 - beta1 ** step
m_hat = state["m"] / bias_correction1
bias_correction2 = 1.0 - beta2 ** step
v_hat = state["v"] / bias_correction2
## Weight Update
p.data = p.data - lr * m_hat / (torch.sqrt(v_hat) + eps) - lr * p.data * weight_decay
return loss
class CustomCautious(Optimizer):
def __init__(self, params, lr = 1e-3, betas = (0.9, 0.95), eps = 1e-8, weight_decay = 0.1):
defaults = dict(lr = lr, betas = betas, eps = eps, weight_decay = weight_decay)
super().__init__(params, defaults)
@torch.no_grad()
def step(self, closure = None):
loss = None
if closure is not None:
with torch.enable_grad():
loss = closure()
for group in self.param_groups:
lr = group["lr"]
beta1, beta2 = group["betas"]
eps = group["eps"]
weight_decay = group["weight_decay"]
for p in group["params"]:
if p.grad is None:
continue
grad = p.grad
if grad.is_sparse:
raise RuntimeError("CustomAdamW does not support sparse gradients")
state = self.state[p]
if len(state) == 0:
state["step"] = 0
if "m" not in state:
state["m"] = torch.zeros_like(p.data)
if "v" not in state:
state["v"] = torch.zeros_like(p.data)
state["step"] += 1
step = state["step"]
## Moment Updation
state["m"] = beta1 * state["m"] + (1 - beta1) * grad
state["v"] = beta2 * state["v"] + (1 - beta2) * (grad ** 2)
## Bias Correction
bias_correction1 = 1.0 - beta1 ** step
m_hat = state["m"] / bias_correction1
bias_correction2 = 1.0 - beta2 ** step
v_hat = state["v"] / bias_correction2
## Calculate u
u = m_hat / (torch.sqrt(v_hat) + eps)
## Calculate s
s = (u * grad > 0).float()
## Calculate u_hat
u_hat = (u * s) / torch.clamp(torch.mean(s), min = 1e-8)
## Weight Update
p.data = p.data - lr * u_hat - lr * weight_decay * p.data
return loss
class CustomLion(Optimizer):
def __init__(self, params, lr = 2e-4, betas = (0.9, 0.99), weight_decay = 0.5):
defaults = dict(lr = lr, betas = betas, weight_decay = weight_decay)
super().__init__(params, defaults)
@torch.no_grad()
def step(self, closure = None):
loss = None
if closure is not None:
with torch.enable_grad():
loss = closure()
for group in self.param_groups:
lr = group["lr"]
beta1, beta2 = group["betas"]
weight_decay = group["weight_decay"]
for p in group["params"]:
if p.grad is None:
continue
grad = p.grad
if grad.is_sparse:
raise RuntimeError("CustomAdamW does not support sparse gradients")
state = self.state[p]
if len(state) == 0:
state["step"] = 0
if "m" not in state:
state["m"] = torch.zeros_like(p.data)
state["step"] += 1
## Calculate m_hat
m_hat = beta1 * state["m"] + (1 - beta1) * grad
## Moment Updation
state["m"] = beta2 * state["m"] + (1 - beta2) * grad
## Weight update
p.data = p.data - lr * torch.sign(m_hat) - lr * weight_decay * p.data
return loss
def newtonschulz_orthogonalization(X, steps = 5, eps = 1e-7):
if X.ndim != 2:
raise ValueError(f"expected 2D matrix, but got a tensor with {X.ndim} dimensions")
rows, cols = X.shape
is_transposed = False
if rows > cols:
X = X.T
is_transposed = True
norm = torch.linalg.matrix_norm(X)
X = X / (norm + eps)
a, b, c = (3.4445, -4.7750, 2.0315)
for _ in range(1, steps + 1):
first_term = a * X
A = X @ X.T
second_term = ((b * A) + c * (A @ A)) @ X
X = first_term + second_term
if is_transposed:
X = X.T
return X
class CustomScion(Optimizer):
def __init__(self, params, lr = 1e-3, beta = 0.9, weight_decay = 0.01, steps = 5, eps = 1e-7):
defaults = dict(lr = lr, beta = beta, weight_decay = weight_decay, steps = steps, eps = eps)
super().__init__(params, defaults)
@torch.no_grad()
def step(self, closure = None):
loss = None
if closure is not None:
with torch.enable_grad():
loss = closure()
for group in self.param_groups:
lr = group["lr"]
beta = group["beta"]
weight_decay = group["weight_decay"]
steps = group["steps"]
eps = group["eps"]
for p in group["params"]:
if p.grad is None:
continue
grad = p.grad
if grad.is_sparse:
raise RuntimeError("CustomScion does not support sparse gradients")
state = self.state[p]
if len(state) == 0:
state["step"] = 0
if "m" not in state:
state["m"] = torch.zeros_like(p.data)
state["step"] += 1
## Momentum update
state["m"] = beta * state["m"] + grad
## Calculate u
u = beta * state["m"] + grad
## Apply NewtonSchulz Orthogonalization
if p.ndim == 2:
u = newtonschulz_orthogonalization(u, steps = steps, eps = eps)
rows, cols = p.data.shape
s = math.sqrt(max(1, rows / cols))
u = u * s
else:
u = torch.sign(u)
## Weight update
p.data = p.data - lr * u - lr * weight_decay * p.data
return loss
class CustomSophia(Optimizer):
def __init__(self, params, lr = 1e-4, betas = (0.965, 0.99), rho = 0.04, weight_decay = 0.1, eps = 1e-12):
defaults = dict(lr = lr, betas = betas, rho = rho, weight_decay = weight_decay, eps = eps)
super().__init__(params, defaults)
@torch.no_grad()
def update_hessian(self):
for group in self.param_groups:
beta1, beta2 = group["betas"]
for p in group["params"]:
if p.grad is None:
continue
state = self.state[p]
grad = p.grad
if "h" not in state:
state["h"] = torch.zeros_like(p.data)
state["h"] = beta2 * state["h"] + (1 - beta2) * (grad ** 2)
@torch.no_grad()
def step(self, closure = None):
loss = None
if closure is not None:
with torch.enable_grad():
loss = closure()
for group in self.param_groups:
lr = group["lr"]
beta1, beta2 = group["betas"]
rho = group["rho"]
weight_decay = group["weight_decay"]
eps = group["eps"]
for p in group["params"]:
if p.grad is None:
continue
grad = p.grad
if grad.is_sparse:
raise RuntimeError("CustomSophia does not support sparse gradients")
state = self.state[p]
if "step" not in state:
state["step"] = 0
if "m" not in state:
state["m"] = torch.zeros_like(p.data)
if "h" not in state:
state["h"] = torch.zeros_like(p.data)
state["step"] += 1
state["m"] = beta1 * state["m"] + (1 - beta1) * grad
denom = torch.clamp(rho * state["h"], min = eps)
ratio = torch.clamp(state["m"] / denom, min = -1.0, max = 1.0)
p.data = p.data - lr * ratio - lr * weight_decay * p.data
return loss