import torch from torch.optim import Optimizer import math class CustomAdamW(Optimizer): def __init__(self, params, lr = 1e-3, betas = (0.9, 0.95), eps = 1e-8, weight_decay = 0.1): defaults = dict(lr = lr, betas = betas, eps = eps, weight_decay = weight_decay) super().__init__(params, defaults) @torch.no_grad() def step(self, closure = None): loss = None if closure is not None: with torch.enable_grad(): loss = closure() for group in self.param_groups: lr = group["lr"] beta1, beta2 = group["betas"] eps = group["eps"] weight_decay = group["weight_decay"] for p in group["params"]: if p.grad is None: continue grad = p.grad if grad.is_sparse: raise RuntimeError("CustomAdamW does not support sparse gradients") state = self.state[p] if len(state) == 0: state["step"] = 0 if "m" not in state: state["m"] = torch.zeros_like(p.data) if "v" not in state: state["v"] = torch.zeros_like(p.data) state["step"] += 1 step = state["step"] ## Moment Updation state["m"] = beta1 * state["m"] + (1 - beta1) * grad state["v"] = beta2 * state["v"] + (1 - beta2) * (grad ** 2) ## Bias Correction bias_correction1 = 1.0 - beta1 ** step m_hat = state["m"] / bias_correction1 bias_correction2 = 1.0 - beta2 ** step v_hat = state["v"] / bias_correction2 ## Weight Update p.data = p.data - lr * m_hat / (torch.sqrt(v_hat) + eps) - lr * p.data * weight_decay return loss class CustomCautious(Optimizer): def __init__(self, params, lr = 1e-3, betas = (0.9, 0.95), eps = 1e-8, weight_decay = 0.1): defaults = dict(lr = lr, betas = betas, eps = eps, weight_decay = weight_decay) super().__init__(params, defaults) @torch.no_grad() def step(self, closure = None): loss = None if closure is not None: with torch.enable_grad(): loss = closure() for group in self.param_groups: lr = group["lr"] beta1, beta2 = group["betas"] eps = group["eps"] weight_decay = group["weight_decay"] for p in group["params"]: if p.grad is None: continue grad = p.grad if grad.is_sparse: raise RuntimeError("CustomAdamW does not support sparse gradients") state = self.state[p] if len(state) == 0: state["step"] = 0 if "m" not in state: state["m"] = torch.zeros_like(p.data) if "v" not in state: state["v"] = torch.zeros_like(p.data) state["step"] += 1 step = state["step"] ## Moment Updation state["m"] = beta1 * state["m"] + (1 - beta1) * grad state["v"] = beta2 * state["v"] + (1 - beta2) * (grad ** 2) ## Bias Correction bias_correction1 = 1.0 - beta1 ** step m_hat = state["m"] / bias_correction1 bias_correction2 = 1.0 - beta2 ** step v_hat = state["v"] / bias_correction2 ## Calculate u u = m_hat / (torch.sqrt(v_hat) + eps) ## Calculate s s = (u * grad > 0).float() ## Calculate u_hat u_hat = (u * s) / torch.clamp(torch.mean(s), min = 1e-8) ## Weight Update p.data = p.data - lr * u_hat - lr * weight_decay * p.data return loss class CustomLion(Optimizer): def __init__(self, params, lr = 2e-4, betas = (0.9, 0.99), weight_decay = 0.5): defaults = dict(lr = lr, betas = betas, weight_decay = weight_decay) super().__init__(params, defaults) @torch.no_grad() def step(self, closure = None): loss = None if closure is not None: with torch.enable_grad(): loss = closure() for group in self.param_groups: lr = group["lr"] beta1, beta2 = group["betas"] weight_decay = group["weight_decay"] for p in group["params"]: if p.grad is None: continue grad = p.grad if grad.is_sparse: raise RuntimeError("CustomAdamW does not support sparse gradients") state = self.state[p] if len(state) == 0: state["step"] = 0 if "m" not in state: state["m"] = torch.zeros_like(p.data) state["step"] += 1 ## Calculate m_hat m_hat = beta1 * state["m"] + (1 - beta1) * grad ## Moment Updation state["m"] = beta2 * state["m"] + (1 - beta2) * grad ## Weight update p.data = p.data - lr * torch.sign(m_hat) - lr * weight_decay * p.data return loss def newtonschulz_orthogonalization(X, steps = 5, eps = 1e-7): if X.ndim != 2: raise ValueError(f"expected 2D matrix, but got a tensor with {X.ndim} dimensions") rows, cols = X.shape is_transposed = False if rows > cols: X = X.T is_transposed = True norm = torch.linalg.matrix_norm(X) X = X / (norm + eps) a, b, c = (3.4445, -4.7750, 2.0315) for _ in range(1, steps + 1): first_term = a * X A = X @ X.T second_term = ((b * A) + c * (A @ A)) @ X X = first_term + second_term if is_transposed: X = X.T return X class CustomScion(Optimizer): def __init__(self, params, lr = 1e-3, beta = 0.9, weight_decay = 0.01, steps = 5, eps = 1e-7): defaults = dict(lr = lr, beta = beta, weight_decay = weight_decay, steps = steps, eps = eps) super().__init__(params, defaults) @torch.no_grad() def step(self, closure = None): loss = None if closure is not None: with torch.enable_grad(): loss = closure() for group in self.param_groups: lr = group["lr"] beta = group["beta"] weight_decay = group["weight_decay"] steps = group["steps"] eps = group["eps"] for p in group["params"]: if p.grad is None: continue grad = p.grad if grad.is_sparse: raise RuntimeError("CustomScion does not support sparse gradients") state = self.state[p] if len(state) == 0: state["step"] = 0 if "m" not in state: state["m"] = torch.zeros_like(p.data) state["step"] += 1 ## Momentum update state["m"] = beta * state["m"] + grad ## Calculate u u = beta * state["m"] + grad ## Apply NewtonSchulz Orthogonalization if p.ndim == 2: u = newtonschulz_orthogonalization(u, steps = steps, eps = eps) rows, cols = p.data.shape s = math.sqrt(max(1, rows / cols)) u = u * s else: u = torch.sign(u) ## Weight update p.data = p.data - lr * u - lr * weight_decay * p.data return loss class CustomSophia(Optimizer): def __init__(self, params, lr = 1e-4, betas = (0.965, 0.99), rho = 0.04, weight_decay = 0.1, eps = 1e-12): defaults = dict(lr = lr, betas = betas, rho = rho, weight_decay = weight_decay, eps = eps) super().__init__(params, defaults) @torch.no_grad() def update_hessian(self): for group in self.param_groups: beta1, beta2 = group["betas"] for p in group["params"]: if p.grad is None: continue state = self.state[p] grad = p.grad if "h" not in state: state["h"] = torch.zeros_like(p.data) state["h"] = beta2 * state["h"] + (1 - beta2) * (grad ** 2) @torch.no_grad() def step(self, closure = None): loss = None if closure is not None: with torch.enable_grad(): loss = closure() for group in self.param_groups: lr = group["lr"] beta1, beta2 = group["betas"] rho = group["rho"] weight_decay = group["weight_decay"] eps = group["eps"] for p in group["params"]: if p.grad is None: continue grad = p.grad if grad.is_sparse: raise RuntimeError("CustomSophia does not support sparse gradients") state = self.state[p] if "step" not in state: state["step"] = 0 if "m" not in state: state["m"] = torch.zeros_like(p.data) if "h" not in state: state["h"] = torch.zeros_like(p.data) state["step"] += 1 state["m"] = beta1 * state["m"] + (1 - beta1) * grad denom = torch.clamp(rho * state["h"], min = eps) ratio = torch.clamp(state["m"] / denom, min = -1.0, max = 1.0) p.data = p.data - lr * ratio - lr * weight_decay * p.data return loss