Fibo-2.0-dev-OrbitQuant-W4A8 / fibo2_orbitquant.py

Fibo-2.0-dev-OrbitQuant-W4A8: OrbitQuant W4A8 transformer and text encoder

e290fd7 verified 1 day ago

8.47 kB

"""Load Fibo 2.0 with OrbitQuant W4A8 components into diffusers' ``BriaFibo2Pipeline``."""

from __future__ import annotations

import dataclasses
import json
import os
from pathlib import Path

import torch
from torch import nn

GROUPS = ("qkv", "attn_out", "ff_in", "ff_out", "xq", "xkv", "xout")
# The deepest Qwen3-VL hidden state the pipeline reads is 35, the output of layer 34.
TEXT_LAYERS = 35

# Negative captions of the original model card: text-to-image steers away from soft, featureless, artifact-laden
# images; edits use the empty caption the model was trained with.
NEGATIVE_T2I = '{"short_description":"","objects":[{"description":"the main subject","texture":"smooth, featureless, plastic-like, airbrushed, no fine surface detail","appearance_details":"soft blurred edges, simplified shapes, no fine detail","skin_tone_and_texture":"waxy airbrushed skin, poreless, plastic smooth"}],"background_setting":"a plain seamless studio backdrop, perfectly smooth gradient, blurred and featureless, no visible texture","lighting":{},"aesthetics":{"artifacts":["oversaturated colours","neon colour cast","overexposed highlights","lens flare","slight blur","motion blur","haze"]},"photographic_characteristics":{"depth_of_field":"shallow, heavy bokeh obscuring most of the frame","focus":"soft, slightly out of focus, lacking fine texture"},"style_medium":"","context":"","artistic_style":""}'
NEGATIVE_EDIT = '{"short_description":"","objects":[],"background_setting":"","lighting":{},"aesthetics":{},"photographic_characteristics":{},"style_medium":"","context":"","artistic_style":""}'


def _allocator(device: str) -> None:
    # Growable segments return the transient blocks of one stage to the next. An explicit setting is left untouched.
    if not device.startswith("cuda") or {"PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_ALLOC_CONF"} & set(os.environ):
        return
    if not torch.cuda.is_initialized():
        os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
    else:
        setter = getattr(torch._C, "_accelerator_setAllocatorSettings", None)
        (setter or torch.cuda.memory._set_allocator_settings)("expandable_segments:True")


def _load_transformer(path: Path, device: str, fused: bool, attention: str, activation_bits: int):
    import orbitquant  # noqa: F401 -- registers the OrbitQuant layers
    from diffusers import BriaFibo2Transformer2DModel as Transformer
    from orbitquant.artifacts.manifest import OrbitQuantManifest
    from orbitquant.config import OrbitQuantConfig
    from orbitquant.layers import OrbitQuantLinear
    from orbitquant.modeling import _parent_and_child, _set_child
    from safetensors.torch import load_file

    config = OrbitQuantConfig.from_dict(json.loads((path / "quantization_config.json").read_text()))
    manifest = OrbitQuantManifest.from_dict(json.loads((path / "orbitquant_manifest.json").read_text()))
    with torch.device("meta"):
        model = Transformer.from_config(Transformer.load_config(str(path)))
    for name in manifest.quantized_modules:
        bits = manifest.module_bits.get(name)
        layer_config = dataclasses.replace(config, weight_bits=bits) if bits and bits != config.weight_bits else config
        parent, child = _parent_and_child(model, name)
        _set_child(parent, child, OrbitQuantLinear.empty_from_linear(
            model.get_submodule(name), config=layer_config, module_name=name))
    model.load_state_dict(load_file(str(path / "model.safetensors"), device=device), strict=True, assign=True)
    model = model.to(device).eval()
    if fused:
        from orbitquant.fused import fuse

        fuse(model, config=config, attention=attention, activation_bits=dict.fromkeys(GROUPS, activation_bits))
    return model


def _park(model: nn.Module) -> None:
    """Move a component to host memory right after its GPU build, so the next build does not stack on it."""
    from orbitquant.fused import release

    release(model)
    model.to("cpu")
    torch.cuda.empty_cache()


class HostEmbedding(nn.Module):
    """Token embedding kept in host memory: a prompt reads a few hundred of its 152k rows, so the 0.8 GB table stays
    off the GPU. The table is not a registered parameter, so moving the text encoder does not move it."""

    def __init__(self, embedding: nn.Embedding):
        super().__init__()
        self.__dict__["table"] = embedding.weight.detach().to("cpu")
        self.num_embeddings, self.embedding_dim = self.table.shape

    @property
    def weight(self):
        return self.table

    def forward(self, ids: torch.Tensor) -> torch.Tensor:
        return nn.functional.embedding(ids.cpu(), self.table).to(ids.device, non_blocking=True)


def _load_text_encoder(path: Path, device: str, fused: bool, activation_bits: int, host_embedding: bool):
    """Qwen3-VL without the LM head and the unused last decoder layer. The final norm becomes the identity, so the
    last recorded hidden state is the raw output of layer 34, as in the full model; the vision tower stays BF16 for
    the images to edit and the drawn caption texts."""
    import orbitquant  # noqa: F401 -- registers the OrbitQuant backend with Transformers
    from transformers import Qwen3VLForConditionalGeneration

    full = Qwen3VLForConditionalGeneration.from_pretrained(str(path), dtype=torch.bfloat16, device_map={"": device})
    model = full.model.eval()
    language = model.language_model
    language.layers = language.layers[:TEXT_LAYERS]
    language.config.num_hidden_layers = TEXT_LAYERS
    if getattr(language.config, "layer_types", None):
        language.config.layer_types = language.config.layer_types[:TEXT_LAYERS]
    language.norm = nn.Identity()
    del full
    if fused:
        from orbitquant.fused import fuse_linears

        fuse_linears(model, activation_bits=activation_bits)
    if host_embedding:
        language.embed_tokens = HostEmbedding(language.embed_tokens)
    torch.cuda.empty_cache()
    return model


def load_pipeline(path: str | Path, device: str = "cuda", attention: str = "cudnn", offload: bool = False,
                  fused: bool | None = None, activation_bits: int = 8, host_embedding: bool = True):
    """Fibo 2.0 pipeline with the W4A8 transformer and text encoder; call it as the original ``BriaFibo2Pipeline``.
    ``attention`` is ``cudnn`` (default), ``flash``, ``sdpa`` or ``int8`` (faster, changes the image slightly).
    ``offload`` keeps the components in host memory and moves each to the GPU only while it runs. ``fused=None``
    fuses the projections on CUDA devices with compute capability 8.0 or newer (needed for 8-bit activations)."""
    path = Path(path)
    _allocator(device)
    from diffusers import AutoencoderKLFlux2, BriaFibo2Pipeline, FlowMatchEulerDiscreteScheduler
    from transformers import Qwen3VLProcessor

    if fused is None:
        fused = device.startswith("cuda") and torch.cuda.get_device_capability(device) >= (8, 0)
    # The transformer is fused on the GPU even with offload: on the host the group build runs many small parallel
    # copies, which stall for minutes when a container's CPU quota is far below its visible core count.
    transformer = _load_transformer(path / "transformer", device, fused, attention, activation_bits)
    if offload:
        _park(transformer)
    text_encoder = _load_text_encoder(path / "text_encoder", device, fused, activation_bits, host_embedding)
    if offload:
        _park(text_encoder)
    vae = AutoencoderKLFlux2.from_pretrained(str(path / "vae"), dtype=torch.bfloat16)
    vae = vae.to("cpu" if offload else device).eval()
    pipeline = BriaFibo2Pipeline(
        transformer=transformer,
        scheduler=FlowMatchEulerDiscreteScheduler.from_pretrained(str(path / "scheduler")),
        vae=vae,
        text_encoder=text_encoder,
        processor=Qwen3VLProcessor.from_pretrained(str(path / "processor")),
    )
    if offload:
        pipeline.enable_model_cpu_offload(device=device)
    torch.cuda.empty_cache()
    return pipeline


def fonts(path: str | Path) -> list[str]:
    """The fonts Fibo 2.0's training drew caption texts with (pass them as ``fonts=``)."""
    return [str(Path(path) / "fonts" / name) for name in ("Aileron-Regular.otf", "DejaVuSans.ttf")]


def sampling(path: str | Path) -> dict:
    """Default steps and guidance of the release (Dev: 50 and 5.0; Edit Turbo: 4 and 1.0)."""
    return json.loads((Path(path) / "manifest.json").read_text())["sampling"]

	"""Load Fibo 2.0 with OrbitQuant W4A8 components into diffusers' ``BriaFibo2Pipeline``."""

	from __future__ import annotations

	import dataclasses
	import json
	import os
	from pathlib import Path

	import torch
	from torch import nn

	GROUPS = ("qkv", "attn_out", "ff_in", "ff_out", "xq", "xkv", "xout")
	# The deepest Qwen3-VL hidden state the pipeline reads is 35, the output of layer 34.
	TEXT_LAYERS = 35

	# Negative captions of the original model card: text-to-image steers away from soft, featureless, artifact-laden
	# images; edits use the empty caption the model was trained with.
	NEGATIVE_T2I = '{"short_description":"","objects":[{"description":"the main subject","texture":"smooth, featureless, plastic-like, airbrushed, no fine surface detail","appearance_details":"soft blurred edges, simplified shapes, no fine detail","skin_tone_and_texture":"waxy airbrushed skin, poreless, plastic smooth"}],"background_setting":"a plain seamless studio backdrop, perfectly smooth gradient, blurred and featureless, no visible texture","lighting":{},"aesthetics":{"artifacts":["oversaturated colours","neon colour cast","overexposed highlights","lens flare","slight blur","motion blur","haze"]},"photographic_characteristics":{"depth_of_field":"shallow, heavy bokeh obscuring most of the frame","focus":"soft, slightly out of focus, lacking fine texture"},"style_medium":"","context":"","artistic_style":""}'
	NEGATIVE_EDIT = '{"short_description":"","objects":[],"background_setting":"","lighting":{},"aesthetics":{},"photographic_characteristics":{},"style_medium":"","context":"","artistic_style":""}'


	def _allocator(device: str) -> None:
	# Growable segments return the transient blocks of one stage to the next. An explicit setting is left untouched.
	if not device.startswith("cuda") or {"PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_ALLOC_CONF"} & set(os.environ):
	return
	if not torch.cuda.is_initialized():
	os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
	else:
	setter = getattr(torch._C, "_accelerator_setAllocatorSettings", None)
	(setter or torch.cuda.memory._set_allocator_settings)("expandable_segments:True")


	def _load_transformer(path: Path, device: str, fused: bool, attention: str, activation_bits: int):
	import orbitquant # noqa: F401 -- registers the OrbitQuant layers
	from diffusers import BriaFibo2Transformer2DModel as Transformer
	from orbitquant.artifacts.manifest import OrbitQuantManifest
	from orbitquant.config import OrbitQuantConfig
	from orbitquant.layers import OrbitQuantLinear
	from orbitquant.modeling import _parent_and_child, _set_child
	from safetensors.torch import load_file

	config = OrbitQuantConfig.from_dict(json.loads((path / "quantization_config.json").read_text()))
	manifest = OrbitQuantManifest.from_dict(json.loads((path / "orbitquant_manifest.json").read_text()))
	with torch.device("meta"):
	model = Transformer.from_config(Transformer.load_config(str(path)))
	for name in manifest.quantized_modules:
	bits = manifest.module_bits.get(name)
	layer_config = dataclasses.replace(config, weight_bits=bits) if bits and bits != config.weight_bits else config
	parent, child = _parent_and_child(model, name)
	_set_child(parent, child, OrbitQuantLinear.empty_from_linear(
	model.get_submodule(name), config=layer_config, module_name=name))
	model.load_state_dict(load_file(str(path / "model.safetensors"), device=device), strict=True, assign=True)
	model = model.to(device).eval()
	if fused:
	from orbitquant.fused import fuse

	fuse(model, config=config, attention=attention, activation_bits=dict.fromkeys(GROUPS, activation_bits))
	return model


	def _park(model: nn.Module) -> None:
	"""Move a component to host memory right after its GPU build, so the next build does not stack on it."""
	from orbitquant.fused import release

	release(model)
	model.to("cpu")
	torch.cuda.empty_cache()


	class HostEmbedding(nn.Module):
	"""Token embedding kept in host memory: a prompt reads a few hundred of its 152k rows, so the 0.8 GB table stays
	off the GPU. The table is not a registered parameter, so moving the text encoder does not move it."""

	def __init__(self, embedding: nn.Embedding):
	super().__init__()
	self.__dict__["table"] = embedding.weight.detach().to("cpu")
	self.num_embeddings, self.embedding_dim = self.table.shape

	@property
	def weight(self):
	return self.table

	def forward(self, ids: torch.Tensor) -> torch.Tensor:
	return nn.functional.embedding(ids.cpu(), self.table).to(ids.device, non_blocking=True)


	def _load_text_encoder(path: Path, device: str, fused: bool, activation_bits: int, host_embedding: bool):
	"""Qwen3-VL without the LM head and the unused last decoder layer. The final norm becomes the identity, so the
	last recorded hidden state is the raw output of layer 34, as in the full model; the vision tower stays BF16 for
	the images to edit and the drawn caption texts."""
	import orbitquant # noqa: F401 -- registers the OrbitQuant backend with Transformers
	from transformers import Qwen3VLForConditionalGeneration

	full = Qwen3VLForConditionalGeneration.from_pretrained(str(path), dtype=torch.bfloat16, device_map={"": device})
	model = full.model.eval()
	language = model.language_model
	language.layers = language.layers[:TEXT_LAYERS]
	language.config.num_hidden_layers = TEXT_LAYERS
	if getattr(language.config, "layer_types", None):
	language.config.layer_types = language.config.layer_types[:TEXT_LAYERS]
	language.norm = nn.Identity()
	del full
	if fused:
	from orbitquant.fused import fuse_linears

	fuse_linears(model, activation_bits=activation_bits)
	if host_embedding:
	language.embed_tokens = HostEmbedding(language.embed_tokens)
	torch.cuda.empty_cache()
	return model


	def load_pipeline(path: str \| Path, device: str = "cuda", attention: str = "cudnn", offload: bool = False,
	fused: bool \| None = None, activation_bits: int = 8, host_embedding: bool = True):
	"""Fibo 2.0 pipeline with the W4A8 transformer and text encoder; call it as the original ``BriaFibo2Pipeline``.

	``attention`` is ``cudnn`` (default), ``flash``, ``sdpa`` or ``int8`` (faster, changes the image slightly).
	``offload`` keeps the components in host memory and moves each to the GPU only while it runs. ``fused=None``
	fuses the projections on CUDA devices with compute capability 8.0 or newer (needed for 8-bit activations)."""
	path = Path(path)
	_allocator(device)
	from diffusers import AutoencoderKLFlux2, BriaFibo2Pipeline, FlowMatchEulerDiscreteScheduler
	from transformers import Qwen3VLProcessor

	if fused is None:
	fused = device.startswith("cuda") and torch.cuda.get_device_capability(device) >= (8, 0)
	# The transformer is fused on the GPU even with offload: on the host the group build runs many small parallel
	# copies, which stall for minutes when a container's CPU quota is far below its visible core count.
	transformer = _load_transformer(path / "transformer", device, fused, attention, activation_bits)
	if offload:
	_park(transformer)
	text_encoder = _load_text_encoder(path / "text_encoder", device, fused, activation_bits, host_embedding)
	if offload:
	_park(text_encoder)
	vae = AutoencoderKLFlux2.from_pretrained(str(path / "vae"), dtype=torch.bfloat16)
	vae = vae.to("cpu" if offload else device).eval()
	pipeline = BriaFibo2Pipeline(
	transformer=transformer,
	scheduler=FlowMatchEulerDiscreteScheduler.from_pretrained(str(path / "scheduler")),
	vae=vae,
	text_encoder=text_encoder,
	processor=Qwen3VLProcessor.from_pretrained(str(path / "processor")),
	)
	if offload:
	pipeline.enable_model_cpu_offload(device=device)
	torch.cuda.empty_cache()
	return pipeline


	def fonts(path: str \| Path) -> list[str]:
	"""The fonts Fibo 2.0's training drew caption texts with (pass them as ``fonts=``)."""
	return [str(Path(path) / "fonts" / name) for name in ("Aileron-Regular.otf", "DejaVuSans.ttf")]


	def sampling(path: str \| Path) -> dict:
	"""Default steps and guidance of the release (Dev: 50 and 5.0; Edit Turbo: 4 and 1.0)."""
	return json.loads((Path(path) / "manifest.json").read_text())["sampling"]