Upload 5 files

optimizer/Use_Kohya-sd-script.txt

@@ -0,0 +1,46 @@
+Kohya-sd-script での使用法
+
+これら Emoシリーズ を Kohya-sd-script で簡単につかうには、
+このフォルダをこのまま Kohya-sd-script の "sd-script" フォルダに配置してください
+
+sd-script/optimizer
+
+この配置にした場合、
+
+--optimizer_type=optimizer.emonavi.EmoNavi
+--optimizer_type=optimizer.emofact.EmoFact
+--optimizer_type=optimizer.emolynx.EmoLynx
+
+このように指定するだけで各Optimizerを利用できます(いずれかひとつを指定してください)
+---
+Kohya-sd-script の柔軟な構成により、これらをすぐ試せます
+Kohya-sd-script の開発者と協力者の皆さまに深く感謝します
+Kohya-sd-script: https://github.com/kohya-ss/sd-scripts
+
+fact は、Adafactor を参考にしました
+Lynx は、Lion と Tiger を参考にしました
+Emoシリーズはこれまでの様々なOptimizerの成果に学び完成しました
+すべての開発者の皆さまに感謝します
+
+
+Usage with Kohya-sd-script
+
+To easily use these Emo series with Kohya-sd-script,
+simply place this folder as-is into the "sd-scripts" folder within your Kohya-sd-script installation:
+
+sd-scripts/optimizer
+
+With this setup,
+
+--optimizer_type=optimizer.emonavi.EmoNavi
+--optimizer_type=optimizer.emofact.EmoFact
+--optimizer_type=optimizer.emolynx.EmoLynx
+
+You can utilize each optimizer by simply specifying one of the above.
+
+Thanks to the flexible configuration of Kohya-sd-script, you can try these out right away. We extend our deepest gratitude to the developers and contributors of Kohya-sd-script:
+Kohya-sd-script: https://github.com/kohya-ss/sd-scripts
+
+Fact was inspired by Adafactor.
+Lynx was inspired by Lion and Tiger.
+The Emo series was completed by learning from the achievements of various optimizers developed to date. We are grateful to all developers.

optimizer/emofact.py

@@ -0,0 +1,117 @@
+import torch
+from torch.optim import Optimizer
+import math
+
+class EmoFact(Optimizer):
+    # クラス定義＆初期化
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999),
+                 eps=1e-8, weight_decay=0.01):
+        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
+        super().__init__(params, defaults)
+
+    # 感情EMA更新(緊張と安静)
+    def _update_ema(self, state, loss_val):
+        ema = state.setdefault('ema', {})
+        ema['short'] = 0.3 * loss_val + 0.7 * ema.get('short', loss_val)
+        ema['long'] = 0.01 * loss_val + 0.99 * ema.get('long', loss_val)
+        return ema
+
+    # 感情スカラー値生成(EMA差分、滑らかな非線形スカラー、tanh 5 * diff で鋭敏さ強調)
+    def _compute_scalar(self, ema):
+        diff = ema['short'] - ema['long']
+        return math.tanh(5 * diff)
+
+    # Shadow混合比率(> 0.6：70〜90%、 < 0.6：10%、 > 0.3：30%、 平時：0%)
+    def _decide_ratio(self, scalar):
+        if scalar > 0.6:
+            return 0.7 + 0.2 * scalar
+        elif scalar < -0.6:
+            return 0.1
+        elif abs(scalar) > 0.3:
+            return 0.3
+        return 0.0
+
+    # 損失取得(損失値 loss_val を数値化、感情判定に使用、存在しないパラメータ(更新不要)はスキップ)
+    @torch.no_grad()
+    def step(self, closure=None):
+        loss = closure() if closure is not None else None
+        loss_val = loss.item() if loss is not None else 0.0
+
+        for group in self.param_groups:
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+
+                grad = p.grad.data
+                state = self.state[p]
+
+                # 感情EMA更新・スカラー生成 (既存ロジックを維持)
+                ema = self._update_ema(state, loss_val)
+                scalar = self._compute_scalar(ema)
+                ratio = self._decide_ratio(scalar)
+
+                # shadow_param：必要時のみ更新 (既存ロジックを維持)
+                if ratio > 0:
+                    if 'shadow' not in state:
+                        state['shadow'] = p.data.clone()
+                    else:
+                        p.data.mul_(1 - ratio).add_(state['shadow'], alpha=ratio)
+                        state['shadow'].lerp_(p.data, 0.05)
+                
+                # --- 新しい勾配補正ロジック ---
+                # 行列の形状が2次元以上の場合、分散情報ベースのAB近似を使用
+                if grad.dim() >= 2:
+                    # 行と列の2乗平均を計算 (分散の軽量な近似)
+                    r_sq = torch.mean(grad * grad, dim=tuple(range(1, grad.dim())), keepdim=True).add_(group['eps'])
+                    c_sq = torch.mean(grad * grad, dim=0, keepdim=True).add_(group['eps'])
+
+                    # 分散情報から勾配の近似行列を生成
+                    # AB行列として見立てたものを直接生成し更新項を計算する
+                    # A = sqrt(r_sq), B = sqrt(c_sq) とすることでAB行列の近似を再現
+                    # これをEMAで平滑化する
+                    beta1, beta2 = group['betas']
+                    
+                    state.setdefault('exp_avg_r', torch.zeros_like(r_sq)).mul_(beta1).add_(torch.sqrt(r_sq), alpha=1 - beta1)
+                    state.setdefault('exp_avg_c', torch.zeros_like(c_sq)).mul_(beta1).add_(torch.sqrt(c_sq), alpha=1 - beta1)
+                    
+                    # 再構築した近似勾配の平方根の積で正規化
+                    # これにより2次モーメントのような役割を果たす
+                    denom = torch.sqrt(state['exp_avg_r'] * state['exp_avg_c']).add_(group['eps'])
+                    
+                    # 最終的な更新項を計算
+                    update_term = grad / denom
+
+                # 1次元(ベクトル)の勾配補正(decoupled weight decay 構造に近い)
+                else:
+                    exp_avg = state.setdefault('exp_avg', torch.zeros_like(p.data))
+                    exp_avg_sq = state.setdefault('exp_avg_sq', torch.zeros_like(p.data))
+                    beta1, beta2 = group['betas']
+                    exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+                    exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+                    denom = exp_avg_sq.sqrt().add_(group['eps'])
+                    update_term = exp_avg / denom
+
+                # 最終的なパラメータ更新 (decoupled weight decayも適用)
+                p.data.add_(p.data, alpha=-group['weight_decay'] * group['lr'])
+                p.data.add_(update_term, alpha=-group['lr'])
+
+                # --- Early Stop ロジック (既存ロジックを維持) ---
+                hist = self.state.setdefault('scalar_hist', [])
+                hist.append(scalar)
+                if len(hist) > 32:
+                    hist.pop(0)
+
+        # Early Stop判断
+        if len(self.state['scalar_hist']) >= 32:
+            buf = self.state['scalar_hist']
+            avg_abs = sum(abs(s) for s in buf) / len(buf)
+            std = sum((s - sum(buf)/len(buf))**2 for s in buf) / len(buf)
+            if avg_abs < 0.05 and std < 0.005:
+                self.should_stop = True
+
+        return loss
+
+"""
+Fact is inspired by Adafactor,  
+and its VRAM-friendly design is something everyone loves.
+"""

optimizer/emolynx.py

@@ -0,0 +1,129 @@
+import torch
+from torch.optim import Optimizer
+import math
+from typing import Tuple, Callable, Union
+
+# Helper function (Lynx)
+def exists(val):
+    return val is not None
+
+class EmoLynx(Optimizer):
+    # クラス定義＆初期化
+    def __init__(self, params: Union[list, torch.nn.Module], lr=1e-3, betas=(0.9, 0.99), 
+    # lynx用ベータ･互換性の追加(lynx用beta1･beta2)
+                 eps=1e-8, weight_decay=0.01, decoupled_weight_decay: bool = False): 
+
+        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
+        super().__init__(params, defaults)
+        
+        # lynxに応じてウェイト減衰のため保存
+        self._init_lr = lr
+        self.decoupled_wd = decoupled_weight_decay
+        self.should_stop = False # 停止フラグの初期化
+
+    # 感情EMA更新(緊張と安静)
+    def _update_ema(self, state, loss_val):
+        ema = state.setdefault('ema', {})
+        ema['short'] = 0.3 * loss_val + 0.7 * ema.get('short', loss_val)
+        ema['long']  = 0.01 * loss_val + 0.99 * ema.get('long', loss_val)
+        return ema
+
+    # 感情スカラー値生成(EMA差分、滑らかな非線形スカラー、tanh 5 * diff で鋭敏さ強調)
+    def _compute_scalar(self, ema):
+        diff = ema['short'] - ema['long']
+        return math.tanh(5 * diff)
+
+    # Shadow混合比率(> 0.6：70〜90%、 < 0.6：10%、 > 0.3：30%、 平時：0%)
+    def _decide_ratio(self, scalar):
+        if scalar > 0.6:
+            return 0.7 + 0.2 * scalar
+        elif scalar < -0.6:
+            return 0.1
+        elif abs(scalar) > 0.3:
+            return 0.3
+        return 0.0
+
+    # 損失取得(損失値 loss_val を数値化、感情判定に使用、存在しないパラメータ(更新不要)はスキップ)
+    @torch.no_grad()
+    def step(self, closure: Callable | None = None): # クロージャの型ヒントを追加
+        loss = None
+        if exists(closure): # 一貫性のためにexistsヘルパーを使う
+            with torch.enable_grad():
+                loss = closure()
+        loss_val = loss.item() if loss is not None else 0.0
+
+        for group in self.param_groups:
+            # リンクス共通パラメータ抽出
+            lr, wd, beta1, beta2 = group['lr'], group['weight_decay'], *group['betas']
+            
+            # ウェイト減衰の処理を分離 (from lynx)
+            _wd_actual = wd
+            if self.decoupled_wd:
+                _wd_actual /= self._init_lr # 非連結時ウェイト減衰調整
+
+            for p in filter(lambda p: exists(p.grad), group['params']): # PGチェックにフィルタ
+
+                grad = p.grad # PG直接使用(計算に".data"不要)
+                state = self.state[p]
+
+                # EMA更新・スカラー生成(EMA差分からスカラーを生成しスパイク比率を決定)
+                ema = self._update_ema(state, loss_val)
+                scalar = self._compute_scalar(ema)
+                ratio = self._decide_ratio(scalar)
+
+                # shadow_param：必要時のみ更新(スパイク部分に現在値を5%ずつ追従させる動的履歴)
+                if ratio > 0:
+                    if 'shadow' not in state:
+                        state['shadow'] = p.data.clone()
+                    else:
+                        p.data.mul_(1 - ratio).add_(state['shadow'], alpha=ratio) 
+                        state['shadow'].lerp_(p.data, 0.05) 
+                        # lynx更新前 p.data で shadow 更新(現在値を5%ずつ追従)
+                        # p.data.mul_(1 - ratio).add_(state['shadow'], alpha=ratio) 
+                        # EmoNavi: p.data = p.data * (1-ratio) + shadow * ratio
+
+                # --- Start Lynx Gradient Update Logic ---
+                
+                # lynx初期化(exp_avg_sq)
+                if 'exp_avg' not in state:
+                    state['exp_avg'] = torch.zeros_like(p)
+                exp_avg = state['exp_avg']
+
+                # Stepweight decay (from lynx): p.data = p.data * (1 - lr * wd)
+                # decoupled_wd 考慮 _wd_actual 使用(EmoNaviのwdは最後に適用)
+                p.data.mul_(1. - lr * _wd_actual)
+
+                # 勾配ブレンド
+                # m_t = beta1 * exp_avg_prev + (1 - beta1) * grad
+                blended_grad = grad.mul(1. - beta1).add_(exp_avg, alpha=beta1)
+                
+                # p: p.data = p.data - lr * sign(blended_grad)
+                p.data.add_(blended_grad.sign_(), alpha = -lr)
+
+                # exp_avg = beta2 * exp_avg + (1 - beta2) * grad
+                exp_avg.mul_(beta2).add_(grad, alpha = 1. - beta2)
+
+                # --- End Lynx Gradient Update Logic ---
+
+                # Early Stop用 scalar記録(バッファ共通で管理/最大32件保持/動静評価)
+                # この部分は p.state ではなく self.state ���アクセスする
+                hist = self.state.setdefault('scalar_hist', [])
+                hist.append(scalar)
+                if len(hist) > 32:
+                    hist.pop(0)
+
+        # Early Stop判断(静けさの合図) - This part is outside the inner loop
+        if len(self.state['scalar_hist']) >= 32:
+            buf = self.state['scalar_hist']
+            avg_abs = sum(abs(s) for s in buf) / len(buf)
+            std = sum((s - sum(buf)/len(buf))**2 for s in buf) / len(buf)
+            if avg_abs < 0.05 and std < 0.005:
+                self.should_stop = True # 💡 外部からこれを見て判断可
+
+        return loss
+
+"""
+Lynx was developed with inspiration from Lion and Tiger, 
+which we deeply respect for their lightweight and intelligent design.  
+Lynx also integrates EmoNAVI to enhance its capabilities.
+"""

optimizer/emonavi.py

@@ -0,0 +1,96 @@
+import torch
+from torch.optim import Optimizer
+import math
+
+class EmoNavi(Optimizer):
+    # クラス定義＆初期化
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999),
+                 eps=1e-8, weight_decay=0.01):
+        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
+        super().__init__(params, defaults)
+    # 感情EMA更新(緊張と安静)
+    def _update_ema(self, state, loss_val):
+        ema = state.setdefault('ema', {})
+        ema['short'] = 0.3 * loss_val + 0.7 * ema.get('short', loss_val)
+        ema['long']  = 0.01 * loss_val + 0.99 * ema.get('long', loss_val)
+        return ema
+    # 感情スカラー値生成(EMA差分、滑らかな非線形スカラー、tanh 5 * diff で鋭敏さ強調)
+    def _compute_scalar(self, ema):
+        diff = ema['short'] - ema['long']
+        return math.tanh(5 * diff)
+    # Shadow混合比率(> 0.6：70〜90%、 < 0.6：10%、 > 0.3：30%、 平時：0%)
+    def _decide_ratio(self, scalar):
+        if scalar > 0.6:
+            return 0.7 + 0.2 * scalar
+        elif scalar < -0.6:
+            return 0.1
+        elif abs(scalar) > 0.3:
+            return 0.3
+        return 0.0
+    # 損失取得(損失値 loss_val を数値化、感情判定に使用、存在しないパラメータ(更新不要)はスキップ)
+    @torch.no_grad()
+    def step(self, closure=None):
+        loss = closure() if closure is not None else None
+        loss_val = loss.item() if loss is not None else 0.0
+
+        for group in self.param_groups:
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+
+                grad = p.grad.data
+                state = self.state[p]
+
+                # EMA更新・スカラー生成(EMA差分からスカラーを生成しスパイク比率を決定)
+                ema = self._update_ema(state, loss_val)
+                scalar = self._compute_scalar(ema)
+                ratio = self._decide_ratio(scalar)
+
+                # shadow_param：必要時のみ更新(スパイク部分に現在値を5%ずつ追従させる動的履歴)
+                if ratio > 0:
+                    if 'shadow' not in state:
+                        state['shadow'] = p.data.clone()
+                    else:
+                        p.data.mul_(1 - ratio).add_(state['shadow'], alpha=ratio)
+                        state['shadow'].lerp_(p.data, 0.05)
+                
+                # スカラー生成：短期と長期EMAの差分から信号を得る(高ぶりの強さ)
+                # 混合比率：スカラーが閾値を超える場合にのみ計算される(信頼できる感情信号かどうかの選別)
+                # → スカラー値が小さい場合は ratio = 0 となり、shadow混合は行われない
+                # → 信頼できる強い差分のときのみ感情機構が発動する(暗黙の信頼度判定)                
+                
+                # 1次・2次モーメントを使った勾配補正(decoupled weight decay 構造に近い)
+                exp_avg = state.setdefault('exp_avg', torch.zeros_like(p.data))
+                exp_avg_sq = state.setdefault('exp_avg_sq', torch.zeros_like(p.data))
+                beta1, beta2 = group['betas']
+                exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+                denom = exp_avg_sq.sqrt().add_(group['eps'])
+
+                step_size = group['lr']
+                if group['weight_decay']:
+                    p.data.add_(p.data, alpha=-group['weight_decay'] * step_size)
+                p.data.addcdiv_(exp_avg, denom, value=-step_size)
+
+        # 感情機構の発火が収まり"十分に安定"していることを外部伝達できる(自動停止ロジックではない)
+                # Early Stop用 scalar 記録(バッファ共通で管理/最大32件保持/動静評価)
+                hist = self.state.setdefault('scalar_hist', [])
+                hist.append(scalar)
+                if len(hist) > 32:
+                    hist.pop(0)
+
+        # Early Stop判断(静けさの合図)
+        if len(self.state['scalar_hist']) >= 32:
+            buf = self.state['scalar_hist']
+            avg_abs = sum(abs(s) for s in buf) / len(buf)
+            std = sum((s - sum(buf)/len(buf))**2 for s in buf) / len(buf)
+            if avg_abs < 0.05 and std < 0.005:
+                self.should_stop = True  # 💡 外部からこれを見て判断可
+
+        # 32ステップ分のスカラー値の静かな条件を満たした時"フラグ" should_stop = True になるだけ
+
+        return loss
+
+#    https://github.com/muooon/EmoNavi
+#    An emotion-driven optimizer that feels loss and navigates accordingly.
+#    Don't think. Feel. Don't stop. Keep running. Believe in what's beyond.