Download src/part2/optimizers.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/optimizers.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part2/optimizers.py
-
curl -L -o optimizers.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/optimizers.py
10 kB
| import torch | |
| from torch.optim import Optimizer | |
| import math | |
| class CustomAdamW(Optimizer): | |
| def __init__(self, params, lr = 1e-3, betas = (0.9, 0.95), eps = 1e-8, weight_decay = 0.1): | |
| defaults = dict(lr = lr, betas = betas, eps = eps, weight_decay = weight_decay) | |
| super().__init__(params, defaults) | |
| def step(self, closure = None): | |
| loss = None | |
| if closure is not None: | |
| with torch.enable_grad(): | |
| loss = closure() | |
| for group in self.param_groups: | |
| lr = group["lr"] | |
| beta1, beta2 = group["betas"] | |
| eps = group["eps"] | |
| weight_decay = group["weight_decay"] | |
| for p in group["params"]: | |
| if p.grad is None: | |
| continue | |
| grad = p.grad | |
| if grad.is_sparse: | |
| raise RuntimeError("CustomAdamW does not support sparse gradients") | |
| state = self.state[p] | |
| if len(state) == 0: | |
| state["step"] = 0 | |
| if "m" not in state: | |
| state["m"] = torch.zeros_like(p.data) | |
| if "v" not in state: | |
| state["v"] = torch.zeros_like(p.data) | |
| state["step"] += 1 | |
| step = state["step"] | |
| ## Moment Updation | |
| state["m"] = beta1 * state["m"] + (1 - beta1) * grad | |
| state["v"] = beta2 * state["v"] + (1 - beta2) * (grad ** 2) | |
| ## Bias Correction | |
| bias_correction1 = 1.0 - beta1 ** step | |
| m_hat = state["m"] / bias_correction1 | |
| bias_correction2 = 1.0 - beta2 ** step | |
| v_hat = state["v"] / bias_correction2 | |
| ## Weight Update | |
| p.data = p.data - lr * m_hat / (torch.sqrt(v_hat) + eps) - lr * p.data * weight_decay | |
| return loss | |
| class CustomCautious(Optimizer): | |
| def __init__(self, params, lr = 1e-3, betas = (0.9, 0.95), eps = 1e-8, weight_decay = 0.1): | |
| defaults = dict(lr = lr, betas = betas, eps = eps, weight_decay = weight_decay) | |
| super().__init__(params, defaults) | |
| def step(self, closure = None): | |
| loss = None | |
| if closure is not None: | |
| with torch.enable_grad(): | |
| loss = closure() | |
| for group in self.param_groups: | |
| lr = group["lr"] | |
| beta1, beta2 = group["betas"] | |
| eps = group["eps"] | |
| weight_decay = group["weight_decay"] | |
| for p in group["params"]: | |
| if p.grad is None: | |
| continue | |
| grad = p.grad | |
| if grad.is_sparse: | |
| raise RuntimeError("CustomAdamW does not support sparse gradients") | |
| state = self.state[p] | |
| if len(state) == 0: | |
| state["step"] = 0 | |
| if "m" not in state: | |
| state["m"] = torch.zeros_like(p.data) | |
| if "v" not in state: | |
| state["v"] = torch.zeros_like(p.data) | |
| state["step"] += 1 | |
| step = state["step"] | |
| ## Moment Updation | |
| state["m"] = beta1 * state["m"] + (1 - beta1) * grad | |
| state["v"] = beta2 * state["v"] + (1 - beta2) * (grad ** 2) | |
| ## Bias Correction | |
| bias_correction1 = 1.0 - beta1 ** step | |
| m_hat = state["m"] / bias_correction1 | |
| bias_correction2 = 1.0 - beta2 ** step | |
| v_hat = state["v"] / bias_correction2 | |
| ## Calculate u | |
| u = m_hat / (torch.sqrt(v_hat) + eps) | |
| ## Calculate s | |
| s = (u * grad > 0).float() | |
| ## Calculate u_hat | |
| u_hat = (u * s) / torch.clamp(torch.mean(s), min = 1e-8) | |
| ## Weight Update | |
| p.data = p.data - lr * u_hat - lr * weight_decay * p.data | |
| return loss | |
| class CustomLion(Optimizer): | |
| def __init__(self, params, lr = 2e-4, betas = (0.9, 0.99), weight_decay = 0.5): | |
| defaults = dict(lr = lr, betas = betas, weight_decay = weight_decay) | |
| super().__init__(params, defaults) | |
| def step(self, closure = None): | |
| loss = None | |
| if closure is not None: | |
| with torch.enable_grad(): | |
| loss = closure() | |
| for group in self.param_groups: | |
| lr = group["lr"] | |
| beta1, beta2 = group["betas"] | |
| weight_decay = group["weight_decay"] | |
| for p in group["params"]: | |
| if p.grad is None: | |
| continue | |
| grad = p.grad | |
| if grad.is_sparse: | |
| raise RuntimeError("CustomAdamW does not support sparse gradients") | |
| state = self.state[p] | |
| if len(state) == 0: | |
| state["step"] = 0 | |
| if "m" not in state: | |
| state["m"] = torch.zeros_like(p.data) | |
| state["step"] += 1 | |
| ## Calculate m_hat | |
| m_hat = beta1 * state["m"] + (1 - beta1) * grad | |
| ## Moment Updation | |
| state["m"] = beta2 * state["m"] + (1 - beta2) * grad | |
| ## Weight update | |
| p.data = p.data - lr * torch.sign(m_hat) - lr * weight_decay * p.data | |
| return loss | |
| def newtonschulz_orthogonalization(X, steps = 5, eps = 1e-7): | |
| if X.ndim != 2: | |
| raise ValueError(f"expected 2D matrix, but got a tensor with {X.ndim} dimensions") | |
| rows, cols = X.shape | |
| is_transposed = False | |
| if rows > cols: | |
| X = X.T | |
| is_transposed = True | |
| norm = torch.linalg.matrix_norm(X) | |
| X = X / (norm + eps) | |
| a, b, c = (3.4445, -4.7750, 2.0315) | |
| for _ in range(1, steps + 1): | |
| first_term = a * X | |
| A = X @ X.T | |
| second_term = ((b * A) + c * (A @ A)) @ X | |
| X = first_term + second_term | |
| if is_transposed: | |
| X = X.T | |
| return X | |
| class CustomScion(Optimizer): | |
| def __init__(self, params, lr = 1e-3, beta = 0.9, weight_decay = 0.01, steps = 5, eps = 1e-7): | |
| defaults = dict(lr = lr, beta = beta, weight_decay = weight_decay, steps = steps, eps = eps) | |
| super().__init__(params, defaults) | |
| def step(self, closure = None): | |
| loss = None | |
| if closure is not None: | |
| with torch.enable_grad(): | |
| loss = closure() | |
| for group in self.param_groups: | |
| lr = group["lr"] | |
| beta = group["beta"] | |
| weight_decay = group["weight_decay"] | |
| steps = group["steps"] | |
| eps = group["eps"] | |
| for p in group["params"]: | |
| if p.grad is None: | |
| continue | |
| grad = p.grad | |
| if grad.is_sparse: | |
| raise RuntimeError("CustomScion does not support sparse gradients") | |
| state = self.state[p] | |
| if len(state) == 0: | |
| state["step"] = 0 | |
| if "m" not in state: | |
| state["m"] = torch.zeros_like(p.data) | |
| state["step"] += 1 | |
| ## Momentum update | |
| state["m"] = beta * state["m"] + grad | |
| ## Calculate u | |
| u = beta * state["m"] + grad | |
| ## Apply NewtonSchulz Orthogonalization | |
| if p.ndim == 2: | |
| u = newtonschulz_orthogonalization(u, steps = steps, eps = eps) | |
| rows, cols = p.data.shape | |
| s = math.sqrt(max(1, rows / cols)) | |
| u = u * s | |
| else: | |
| u = torch.sign(u) | |
| ## Weight update | |
| p.data = p.data - lr * u - lr * weight_decay * p.data | |
| return loss | |
| class CustomSophia(Optimizer): | |
| def __init__(self, params, lr = 1e-4, betas = (0.965, 0.99), rho = 0.04, weight_decay = 0.1, eps = 1e-12): | |
| defaults = dict(lr = lr, betas = betas, rho = rho, weight_decay = weight_decay, eps = eps) | |
| super().__init__(params, defaults) | |
| def update_hessian(self): | |
| for group in self.param_groups: | |
| beta1, beta2 = group["betas"] | |
| for p in group["params"]: | |
| if p.grad is None: | |
| continue | |
| state = self.state[p] | |
| grad = p.grad | |
| if "h" not in state: | |
| state["h"] = torch.zeros_like(p.data) | |
| state["h"] = beta2 * state["h"] + (1 - beta2) * (grad ** 2) | |
| def step(self, closure = None): | |
| loss = None | |
| if closure is not None: | |
| with torch.enable_grad(): | |
| loss = closure() | |
| for group in self.param_groups: | |
| lr = group["lr"] | |
| beta1, beta2 = group["betas"] | |
| rho = group["rho"] | |
| weight_decay = group["weight_decay"] | |
| eps = group["eps"] | |
| for p in group["params"]: | |
| if p.grad is None: | |
| continue | |
| grad = p.grad | |
| if grad.is_sparse: | |
| raise RuntimeError("CustomSophia does not support sparse gradients") | |
| state = self.state[p] | |
| if "step" not in state: | |
| state["step"] = 0 | |
| if "m" not in state: | |
| state["m"] = torch.zeros_like(p.data) | |
| if "h" not in state: | |
| state["h"] = torch.zeros_like(p.data) | |
| state["step"] += 1 | |
| state["m"] = beta1 * state["m"] + (1 - beta1) * grad | |
| denom = torch.clamp(rho * state["h"], min = eps) | |
| ratio = torch.clamp(state["m"] / denom, min = -1.0, max = 1.0) | |
| p.data = p.data - lr * ratio - lr * weight_decay * p.data | |
| return loss | |