"""Original rank16/alpha32 language-linear LoRA, no new package installation. The complete standalone export retains full base weights and explicit adapters. It does not merge adapters into a BF16 weight and claim identical rounding. """ import math from contract import need TARGETS = ('q_proj', 'k_proj', 'v_proj', 'o_proj', 'in_proj_qkv', 'in_proj_z', 'out_proj', 'up_proj', 'down_proj', 'gate_proj') def linear_class(): import torch from torch import nn from torch.nn import functional as F class LanguageLoRALinear(nn.Module): def __init__(self, base, rank=16, alpha=32, seed=20260920): super().__init__() need(isinstance(base, nn.Linear), 'Only exact language nn.Linear targets') self.base = base self.rank = rank self.alpha = alpha self.scale = alpha / rank # Same A=Kaiming-uniform(a=sqrt(5)), B=zeros convention as PEFT. generator = torch.Generator(device='cpu').manual_seed(seed) initial = torch.empty(rank, base.in_features, dtype=torch.float32) initial.uniform_(-1 / math.sqrt(base.in_features), 1 / math.sqrt(base.in_features), generator=generator) self.lora_A = nn.Parameter(initial.to(base.weight.device)) self.lora_B = nn.Parameter(torch.zeros(base.out_features, rank, dtype=torch.float32, device=base.weight.device)) self.base.requires_grad_(False) @property def weight(self): return self.base.weight @property def bias(self): return self.base.bias def forward(self, value): base = self.base(value) # FSDP BF16 and standalone native BF16 use exactly the same casts, # two linear operations, scalar multiplication and final addition. a = self.lora_A.to(value.dtype) b = self.lora_B.to(value.dtype) delta = F.linear(F.linear(value, a), b) * self.scale return base + delta.to(base.dtype) return LanguageLoRALinear def install(backbone, recipe, seed): import hashlib import torch need(recipe['rank'] == 16 and recipe['alpha'] == 32 and recipe['dropout'] == 0 and recipe['bias'] == 'none' and tuple(recipe['targets']) == TARGETS, 'Exact inherited language LoRA recipe') need(recipe['only_prefix'] == 'language_model.', 'No visual/embedding targets') cls = linear_class() names = [(name, module) for name, module in backbone.named_modules() if name.startswith('language_model.') and name.rsplit('.', 1)[-1] in TARGETS] need(names, 'Actual language projection targets must exist') if 'expected_module_names' in recipe: need(sorted(n for n, _ in names) == recipe['expected_module_names'], 'Exact published language target coverage differs') report = [] for name, module in names: need(isinstance(module, torch.nn.Linear), 'A target is not nn.Linear') parent_name, leaf = name.rsplit('.', 1) parent = backbone.get_submodule(parent_name) unique_seed = seed + int(hashlib.sha256(name.encode()).hexdigest()[:8], 16) adapter = cls(module, recipe['rank'], recipe['alpha'], unique_seed) setattr(parent, leaf, adapter) report.append({'module': name, 'in_features': module.in_features, 'out_features': module.out_features, 'rank': recipe['rank'], 'alpha': recipe['alpha'], 'B_initialized_zero': True}) return sorted(report, key=lambda v: v['module']) def configure_trainable(model, mode): if mode == 'full': model.requires_grad_(True) need(not any('lora_' in n for n, _ in model.named_parameters()), 'Full arm cannot contain adapters') return need(mode == 'language_lora', 'Unknown declared trainable range') model.requires_grad_(False) for name, p in model.named_parameters(): if (name.startswith('backbone.language_model.') and name.endswith(('.lora_A', '.lora_B'))) or name == 'readout.weight': p.requires_grad_(True) actual = [n for n, p in model.named_parameters() if p.requires_grad] need(actual and 'readout.weight' in actual and any(n.endswith('.lora_B') for n in actual), 'Actual language adapters and original saved head required') need(all(n == 'readout.weight' or n.startswith('backbone.language_model.') and n.endswith(('.lora_A', '.lora_B')) for n in actual), 'Unexpected trainable body layer') def split_export_body(body, mode): """Return complete original base keys plus unmerged FP32 A/B parameters.""" if mode == 'full': need(not any('lora_' in k for k in body), 'No adapter in full export') return body, {} canonical, adapter = {}, {} for name, value in body.items(): if name.endswith(('.lora_A', '.lora_B')): need(name.startswith('language_model.'), 'No visual adapter export') adapter[name] = value else: # Only the exact wrapped target leaf owns a .base.weight/.base.bias. if '.base.' in name: prefix, suffix = name.rsplit('.base.', 1) need(prefix.startswith('language_model.') and prefix.rsplit('.', 1)[-1] in TARGETS and suffix in ('weight', 'bias'), 'Unknown wrapped base key') name = prefix + '.' + suffix need(name not in canonical, 'Duplicate canonical base key') canonical[name] = value need(adapter, 'Actual adapters required for LoRA export') return canonical, adapter def load_export_adapter(backbone, checkpoint): import json import torch from pathlib import Path from safetensors.torch import load_file path = Path(checkpoint) config = json.loads((path / 'adaptation_config.json').read_text()) need(config['mode'] == 'language_lora' and config['merged'] is False, 'Preserve native unmerged adapter execution') report = install(backbone, config['recipe'], config['seed']) need(report == config['actual_module_layout'], 'Exact full actual adapter module coverage') values = load_file(str(path / 'language_adapter.safetensors')) params = {n: p for n, p in backbone.named_parameters() if n.endswith(('.lora_A', '.lora_B'))} need(set(params) == set(values), 'Complete standalone adapter state coverage') with torch.no_grad(): for name, p in params.items(): v = values[name] need(v.dtype == torch.float32 and v.shape == p.shape, 'Actual FP32 adapter shape/dtype') p.copy_(v.to(p.device)) return report