VisionJEV-4B / native_runtime /language_lora.py
tinnel123's picture
Release VisionJEV 1.0.0 (4B): exact complete native checkpoint and inference
1120183 verified
Raw History Blame Contribute Delete
6.87 kB
"""Original rank16/alpha32 language-linear LoRA, no new package installation.
The complete standalone export retains full base weights and explicit adapters.
It does not merge adapters into a BF16 weight and claim identical rounding.
"""
import math
from contract import need
TARGETS = ('q_proj', 'k_proj', 'v_proj', 'o_proj', 'in_proj_qkv', 'in_proj_z',
'out_proj', 'up_proj', 'down_proj', 'gate_proj')
def linear_class():
import torch
from torch import nn
from torch.nn import functional as F
class LanguageLoRALinear(nn.Module):
def __init__(self, base, rank=16, alpha=32, seed=20260920):
super().__init__()
need(isinstance(base, nn.Linear), 'Only exact language nn.Linear targets')
self.base = base
self.rank = rank
self.alpha = alpha
self.scale = alpha / rank
# Same A=Kaiming-uniform(a=sqrt(5)), B=zeros convention as PEFT.
generator = torch.Generator(device='cpu').manual_seed(seed)
initial = torch.empty(rank, base.in_features, dtype=torch.float32)
initial.uniform_(-1 / math.sqrt(base.in_features), 1 / math.sqrt(base.in_features), generator=generator)
self.lora_A = nn.Parameter(initial.to(base.weight.device))
self.lora_B = nn.Parameter(torch.zeros(base.out_features, rank, dtype=torch.float32,
device=base.weight.device))
self.base.requires_grad_(False)
@property
def weight(self):
return self.base.weight
@property
def bias(self):
return self.base.bias
def forward(self, value):
base = self.base(value)
# FSDP BF16 and standalone native BF16 use exactly the same casts,
# two linear operations, scalar multiplication and final addition.
a = self.lora_A.to(value.dtype)
b = self.lora_B.to(value.dtype)
delta = F.linear(F.linear(value, a), b) * self.scale
return base + delta.to(base.dtype)
return LanguageLoRALinear
def install(backbone, recipe, seed):
import hashlib
import torch
need(recipe['rank'] == 16 and recipe['alpha'] == 32 and recipe['dropout'] == 0
and recipe['bias'] == 'none' and tuple(recipe['targets']) == TARGETS,
'Exact inherited language LoRA recipe')
need(recipe['only_prefix'] == 'language_model.', 'No visual/embedding targets')
cls = linear_class()
names = [(name, module) for name, module in backbone.named_modules()
if name.startswith('language_model.') and name.rsplit('.', 1)[-1] in TARGETS]
need(names, 'Actual language projection targets must exist')
if 'expected_module_names' in recipe:
need(sorted(n for n, _ in names) == recipe['expected_module_names'], 'Exact published language target coverage differs')
report = []
for name, module in names:
need(isinstance(module, torch.nn.Linear), 'A target is not nn.Linear')
parent_name, leaf = name.rsplit('.', 1)
parent = backbone.get_submodule(parent_name)
unique_seed = seed + int(hashlib.sha256(name.encode()).hexdigest()[:8], 16)
adapter = cls(module, recipe['rank'], recipe['alpha'], unique_seed)
setattr(parent, leaf, adapter)
report.append({'module': name, 'in_features': module.in_features,
'out_features': module.out_features, 'rank': recipe['rank'],
'alpha': recipe['alpha'], 'B_initialized_zero': True})
return sorted(report, key=lambda v: v['module'])
def configure_trainable(model, mode):
if mode == 'full':
model.requires_grad_(True)
need(not any('lora_' in n for n, _ in model.named_parameters()), 'Full arm cannot contain adapters')
return
need(mode == 'language_lora', 'Unknown declared trainable range')
model.requires_grad_(False)
for name, p in model.named_parameters():
if (name.startswith('backbone.language_model.') and name.endswith(('.lora_A', '.lora_B'))) or name == 'readout.weight':
p.requires_grad_(True)
actual = [n for n, p in model.named_parameters() if p.requires_grad]
need(actual and 'readout.weight' in actual and any(n.endswith('.lora_B') for n in actual),
'Actual language adapters and original saved head required')
need(all(n == 'readout.weight' or n.startswith('backbone.language_model.')
and n.endswith(('.lora_A', '.lora_B')) for n in actual), 'Unexpected trainable body layer')
def split_export_body(body, mode):
"""Return complete original base keys plus unmerged FP32 A/B parameters."""
if mode == 'full':
need(not any('lora_' in k for k in body), 'No adapter in full export')
return body, {}
canonical, adapter = {}, {}
for name, value in body.items():
if name.endswith(('.lora_A', '.lora_B')):
need(name.startswith('language_model.'), 'No visual adapter export')
adapter[name] = value
else:
# Only the exact wrapped target leaf owns a .base.weight/.base.bias.
if '.base.' in name:
prefix, suffix = name.rsplit('.base.', 1)
need(prefix.startswith('language_model.') and prefix.rsplit('.', 1)[-1] in TARGETS
and suffix in ('weight', 'bias'), 'Unknown wrapped base key')
name = prefix + '.' + suffix
need(name not in canonical, 'Duplicate canonical base key')
canonical[name] = value
need(adapter, 'Actual adapters required for LoRA export')
return canonical, adapter
def load_export_adapter(backbone, checkpoint):
import json
import torch
from pathlib import Path
from safetensors.torch import load_file
path = Path(checkpoint)
config = json.loads((path / 'adaptation_config.json').read_text())
need(config['mode'] == 'language_lora' and config['merged'] is False,
'Preserve native unmerged adapter execution')
report = install(backbone, config['recipe'], config['seed'])
need(report == config['actual_module_layout'], 'Exact full actual adapter module coverage')
values = load_file(str(path / 'language_adapter.safetensors'))
params = {n: p for n, p in backbone.named_parameters() if n.endswith(('.lora_A', '.lora_B'))}
need(set(params) == set(values), 'Complete standalone adapter state coverage')
with torch.no_grad():
for name, p in params.items():
v = values[name]
need(v.dtype == torch.float32 and v.shape == p.shape, 'Actual FP32 adapter shape/dtype')
p.copy_(v.to(p.device))
return report