Download native_runtime/language_lora.py from tinnel123/VisionJEV-4B: direct link, hf CLI and curl.
- Browser
- Download file 6.87 kB
-
https://huggingface.co/tinnel123/VisionJEV-4B/resolve/main/native_runtime/language_lora.py
- Command line
-
hf download hf://tinnel123/VisionJEV-4B/native_runtime/language_lora.py
-
curl -L -o language_lora.py https://huggingface.co/tinnel123/VisionJEV-4B/resolve/main/native_runtime/language_lora.py
6.87 kB
| """Original rank16/alpha32 language-linear LoRA, no new package installation. | |
| The complete standalone export retains full base weights and explicit adapters. | |
| It does not merge adapters into a BF16 weight and claim identical rounding. | |
| """ | |
| import math | |
| from contract import need | |
| TARGETS = ('q_proj', 'k_proj', 'v_proj', 'o_proj', 'in_proj_qkv', 'in_proj_z', | |
| 'out_proj', 'up_proj', 'down_proj', 'gate_proj') | |
| def linear_class(): | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class LanguageLoRALinear(nn.Module): | |
| def __init__(self, base, rank=16, alpha=32, seed=20260920): | |
| super().__init__() | |
| need(isinstance(base, nn.Linear), 'Only exact language nn.Linear targets') | |
| self.base = base | |
| self.rank = rank | |
| self.alpha = alpha | |
| self.scale = alpha / rank | |
| # Same A=Kaiming-uniform(a=sqrt(5)), B=zeros convention as PEFT. | |
| generator = torch.Generator(device='cpu').manual_seed(seed) | |
| initial = torch.empty(rank, base.in_features, dtype=torch.float32) | |
| initial.uniform_(-1 / math.sqrt(base.in_features), 1 / math.sqrt(base.in_features), generator=generator) | |
| self.lora_A = nn.Parameter(initial.to(base.weight.device)) | |
| self.lora_B = nn.Parameter(torch.zeros(base.out_features, rank, dtype=torch.float32, | |
| device=base.weight.device)) | |
| self.base.requires_grad_(False) | |
| def weight(self): | |
| return self.base.weight | |
| def bias(self): | |
| return self.base.bias | |
| def forward(self, value): | |
| base = self.base(value) | |
| # FSDP BF16 and standalone native BF16 use exactly the same casts, | |
| # two linear operations, scalar multiplication and final addition. | |
| a = self.lora_A.to(value.dtype) | |
| b = self.lora_B.to(value.dtype) | |
| delta = F.linear(F.linear(value, a), b) * self.scale | |
| return base + delta.to(base.dtype) | |
| return LanguageLoRALinear | |
| def install(backbone, recipe, seed): | |
| import hashlib | |
| import torch | |
| need(recipe['rank'] == 16 and recipe['alpha'] == 32 and recipe['dropout'] == 0 | |
| and recipe['bias'] == 'none' and tuple(recipe['targets']) == TARGETS, | |
| 'Exact inherited language LoRA recipe') | |
| need(recipe['only_prefix'] == 'language_model.', 'No visual/embedding targets') | |
| cls = linear_class() | |
| names = [(name, module) for name, module in backbone.named_modules() | |
| if name.startswith('language_model.') and name.rsplit('.', 1)[-1] in TARGETS] | |
| need(names, 'Actual language projection targets must exist') | |
| if 'expected_module_names' in recipe: | |
| need(sorted(n for n, _ in names) == recipe['expected_module_names'], 'Exact published language target coverage differs') | |
| report = [] | |
| for name, module in names: | |
| need(isinstance(module, torch.nn.Linear), 'A target is not nn.Linear') | |
| parent_name, leaf = name.rsplit('.', 1) | |
| parent = backbone.get_submodule(parent_name) | |
| unique_seed = seed + int(hashlib.sha256(name.encode()).hexdigest()[:8], 16) | |
| adapter = cls(module, recipe['rank'], recipe['alpha'], unique_seed) | |
| setattr(parent, leaf, adapter) | |
| report.append({'module': name, 'in_features': module.in_features, | |
| 'out_features': module.out_features, 'rank': recipe['rank'], | |
| 'alpha': recipe['alpha'], 'B_initialized_zero': True}) | |
| return sorted(report, key=lambda v: v['module']) | |
| def configure_trainable(model, mode): | |
| if mode == 'full': | |
| model.requires_grad_(True) | |
| need(not any('lora_' in n for n, _ in model.named_parameters()), 'Full arm cannot contain adapters') | |
| return | |
| need(mode == 'language_lora', 'Unknown declared trainable range') | |
| model.requires_grad_(False) | |
| for name, p in model.named_parameters(): | |
| if (name.startswith('backbone.language_model.') and name.endswith(('.lora_A', '.lora_B'))) or name == 'readout.weight': | |
| p.requires_grad_(True) | |
| actual = [n for n, p in model.named_parameters() if p.requires_grad] | |
| need(actual and 'readout.weight' in actual and any(n.endswith('.lora_B') for n in actual), | |
| 'Actual language adapters and original saved head required') | |
| need(all(n == 'readout.weight' or n.startswith('backbone.language_model.') | |
| and n.endswith(('.lora_A', '.lora_B')) for n in actual), 'Unexpected trainable body layer') | |
| def split_export_body(body, mode): | |
| """Return complete original base keys plus unmerged FP32 A/B parameters.""" | |
| if mode == 'full': | |
| need(not any('lora_' in k for k in body), 'No adapter in full export') | |
| return body, {} | |
| canonical, adapter = {}, {} | |
| for name, value in body.items(): | |
| if name.endswith(('.lora_A', '.lora_B')): | |
| need(name.startswith('language_model.'), 'No visual adapter export') | |
| adapter[name] = value | |
| else: | |
| # Only the exact wrapped target leaf owns a .base.weight/.base.bias. | |
| if '.base.' in name: | |
| prefix, suffix = name.rsplit('.base.', 1) | |
| need(prefix.startswith('language_model.') and prefix.rsplit('.', 1)[-1] in TARGETS | |
| and suffix in ('weight', 'bias'), 'Unknown wrapped base key') | |
| name = prefix + '.' + suffix | |
| need(name not in canonical, 'Duplicate canonical base key') | |
| canonical[name] = value | |
| need(adapter, 'Actual adapters required for LoRA export') | |
| return canonical, adapter | |
| def load_export_adapter(backbone, checkpoint): | |
| import json | |
| import torch | |
| from pathlib import Path | |
| from safetensors.torch import load_file | |
| path = Path(checkpoint) | |
| config = json.loads((path / 'adaptation_config.json').read_text()) | |
| need(config['mode'] == 'language_lora' and config['merged'] is False, | |
| 'Preserve native unmerged adapter execution') | |
| report = install(backbone, config['recipe'], config['seed']) | |
| need(report == config['actual_module_layout'], 'Exact full actual adapter module coverage') | |
| values = load_file(str(path / 'language_adapter.safetensors')) | |
| params = {n: p for n, p in backbone.named_parameters() if n.endswith(('.lora_A', '.lora_B'))} | |
| need(set(params) == set(values), 'Complete standalone adapter state coverage') | |
| with torch.no_grad(): | |
| for name, p in params.items(): | |
| v = values[name] | |
| need(v.dtype == torch.float32 and v.shape == p.shape, 'Actual FP32 adapter shape/dtype') | |
| p.copy_(v.to(p.device)) | |
| return report | |