Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+vision_encoder/siglip/redux.png filter=lfs diff=lfs merge=lfs -text
+text_encoder/Glyph-SDXL-v2/assets/Arial.ttf filter=lfs diff=lfs merge=lfs -text

text_encoder/Glyph-SDXL-v2/assets/Arial.ttf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35c0f3559d8db569e36c31095b8a60d441643d95f59139de40e23fada819b833
+size 275572

text_encoder/Glyph-SDXL-v2/checkpoints/byt5_mapper.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d5911cf07328d949eff29cc08ca7637dc4fe5312a8fa351ca4bec07d357b1c5
+size 301553807

text_encoder/Glyph-SDXL-v2/checkpoints/byt5_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca8c97c89136f767d4534449bbf3f25296d390574e0af1cc16f09774a901d6db
+size 877308845

text_encoder/Glyph-SDXL-v2/checkpoints/unet_inserted_attn.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b6af4376281be262f3b52ca0b16b0244099161693f65a7db352f53878481767
+size 908

text_encoder/Glyph-SDXL-v2/checkpoints/unet_lora.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47ae2328a9c4892a24c4a66f25780ab61a55cbd8eb693a1966cc99e674e832be
+size 743590514

text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/__init__.py

@@ -0,0 +1,2 @@
+from .pipelines import *
+from .models import *

text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/models/__init__.py

@@ -0,0 +1,3 @@
+from .cross_attn_insert_transformer_blocks import CrossAttnInsertBasicTransformerBlock
+
+__all__ = ['CrossAttnInsertBasicTransformerBlock']

text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/models/cross_attn_insert_transformer_blocks.py

@@ -0,0 +1,377 @@
+from typing import Optional, Dict, Any
+import copy
+
+import torch
+import torch.nn as nn
+
+from diffusers.models.attention import (
+    BasicTransformerBlock,
+    SinusoidalPositionalEmbedding,
+    AdaLayerNorm,
+    AdaLayerNormZero,
+    AdaLayerNormContinuous,
+    Attention,
+    FeedForward,
+    GatedSelfAttentionDense,
+    GELU,
+    GEGLU,
+    ApproximateGELU,
+    _chunked_feed_forward,
+)
+
+class CrossAttnInsertBasicTransformerBlock(BasicTransformerBlock):
+    def __init__(
+        self,
+        dim: int,
+        num_attention_heads: int,
+        attention_head_dim: int,
+        dropout=0.0,
+        cross_attention_dim: Optional[int] = None,
+        glyph_cross_attention_dim: Optional[int] = None,
+        activation_fn: str = "geglu",
+        num_embeds_ada_norm: Optional[int] = None,
+        attention_bias: bool = False,
+        only_cross_attention: bool = False,
+        double_self_attention: bool = False,
+        upcast_attention: bool = False,
+        norm_elementwise_affine: bool = True,
+        norm_type: str = "layer_norm",  # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single', 'layer_norm_i2vgen'
+        norm_eps: float = 1e-5,
+        final_dropout: bool = False,
+        attention_type: str = "default",
+        positional_embeddings: Optional[str] = None,
+        num_positional_embeddings: Optional[int] = None,
+        ada_norm_continous_conditioning_embedding_dim: Optional[int] = None,
+        ada_norm_bias: Optional[int] = None,
+        ff_inner_dim: Optional[int] = None,
+        ff_bias: bool = True,
+        attention_out_bias: bool = True,
+    ):
+        super(BasicTransformerBlock, self).__init__()
+        self.only_cross_attention = only_cross_attention
+
+        if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
+            raise ValueError(
+                f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
+                f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."
+            )
+
+        self.norm_type = norm_type
+        self.num_embeds_ada_norm = num_embeds_ada_norm
+
+        if positional_embeddings and (num_positional_embeddings is None):
+            raise ValueError(
+                "If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined."
+            )
+
+        if positional_embeddings == "sinusoidal":
+            self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings)
+        else:
+            self.pos_embed = None
+
+        # Define 3 blocks. Each block has its own normalization layer.
+        # 1. Self-Attn
+        if norm_type == "ada_norm":
+            self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm)
+        elif norm_type == "ada_norm_zero":
+            self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm)
+        elif norm_type == "ada_norm_continuous":
+            self.norm1 = AdaLayerNormContinuous(
+                dim,
+                ada_norm_continous_conditioning_embedding_dim,
+                norm_elementwise_affine,
+                norm_eps,
+                ada_norm_bias,
+                "rms_norm",
+            )
+        else:
+            self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
+
+        self.attn1 = Attention(
+            query_dim=dim,
+            heads=num_attention_heads,
+            dim_head=attention_head_dim,
+            dropout=dropout,
+            bias=attention_bias,
+            cross_attention_dim=cross_attention_dim if only_cross_attention else None,
+            upcast_attention=upcast_attention,
+            out_bias=attention_out_bias,
+        )
+
+        # 2. Cross-Attn
+        if cross_attention_dim is not None or double_self_attention:
+            # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
+            # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
+            # the second cross attention block.
+            if norm_type == "ada_norm":
+                self.norm2 = AdaLayerNorm(dim, num_embeds_ada_norm)
+            elif norm_type == "ada_norm_continuous":
+                self.norm2 = AdaLayerNormContinuous(
+                    dim,
+                    ada_norm_continous_conditioning_embedding_dim,
+                    norm_elementwise_affine,
+                    norm_eps,
+                    ada_norm_bias,
+                    "rms_norm",
+                )
+            else:
+                self.norm2 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine)
+
+            self.attn2 = Attention(
+                query_dim=dim,
+                cross_attention_dim=cross_attention_dim if not double_self_attention else None,
+                heads=num_attention_heads,
+                dim_head=attention_head_dim,
+                dropout=dropout,
+                bias=attention_bias,
+                upcast_attention=upcast_attention,
+                out_bias=attention_out_bias,
+            )  # is self-attn if encoder_hidden_states is none
+        else:
+            self.norm2 = None
+            self.attn2 = None
+        
+        # 3. Feed-forward
+        if norm_type == "ada_norm_continuous":
+            self.norm3 = AdaLayerNormContinuous(
+                dim,
+                ada_norm_continous_conditioning_embedding_dim,
+                norm_elementwise_affine,
+                norm_eps,
+                ada_norm_bias,
+                "layer_norm",
+            )
+
+        elif norm_type in ["ada_norm_zero", "ada_norm", "layer_norm", "ada_norm_continuous"]:
+            self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine)
+        elif norm_type == "layer_norm_i2vgen":
+            self.norm3 = None
+
+        self.ff = FeedForward(
+            dim,
+            dropout=dropout,
+            activation_fn=activation_fn,
+            final_dropout=final_dropout,
+            inner_dim=ff_inner_dim,
+            bias=ff_bias,
+        )
+
+        # 4. Fuser
+        if attention_type == "gated" or attention_type == "gated-text-image":
+            self.fuser = GatedSelfAttentionDense(dim, cross_attention_dim, num_attention_heads, attention_head_dim)
+
+        # 5. Scale-shift for PixArt-Alpha.
+        if norm_type == "ada_norm_single":
+            self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5)
+
+        # let chunk size default to None
+        self._chunk_size = None
+        self._chunk_dim = 0
+    
+    def get_inserted_modules(self):
+        return ()
+    
+    def get_inserted_modules_names(self):
+        return ()
+    
+    def get_origin_modules(self):
+        inserted_modules = self.get_inserted_modules()
+        origin_modules = []
+        for module in self.children():
+            if module not in inserted_modules:
+                origin_modules.append(module)
+        return tuple(origin_modules)
+        
+    
+    @classmethod
+    def from_transformer_block(
+            cls, 
+            transformer_block,
+            glyph_cross_attention_dim,
+        ):
+        inner_dim = transformer_block.attn1.query_dim
+        num_attention_heads = transformer_block.attn1.heads
+        attention_head_dim = transformer_block.attn1.inner_dim // num_attention_heads
+        dropout = transformer_block.attn1.dropout
+        cross_attention_dim = transformer_block.attn2.cross_attention_dim
+        if isinstance(transformer_block.ff.net[0], GELU):
+            if transformer_block.ff.net[0].approximate == "tanh":
+                activation_fn = "gelu-approximate"
+            else:
+                activation_fn = "gelu"
+        elif isinstance(transformer_block.ff.net[0], GEGLU):
+            activation_fn = "geglu"
+        elif isinstance(transformer_block.ff.net[0], ApproximateGELU):
+            activation_fn = "geglu-approximate"
+        num_embeds_ada_norm = transformer_block.num_embeds_ada_norm
+        attention_bias = transformer_block.attn1.to_q.bias is not None
+        only_cross_attention = transformer_block.only_cross_attention
+        double_self_attention = transformer_block.attn2.cross_attention_dim is None
+        upcast_attention = transformer_block.attn1.upcast_attention
+        norm_type = transformer_block.norm_type
+        assert isinstance(transformer_block.norm1, nn.LayerNorm)
+        norm_elementwise_affine = transformer_block.norm1.elementwise_affine
+        norm_eps = transformer_block.norm1.eps
+        assert getattr(transformer_block, 'fuser', None) is None
+        attention_type = "default"
+        model = cls(
+            inner_dim,
+            num_attention_heads,
+            attention_head_dim,
+            dropout=dropout,
+            cross_attention_dim=cross_attention_dim,
+            glyph_cross_attention_dim=glyph_cross_attention_dim,
+            activation_fn=activation_fn,
+            num_embeds_ada_norm=num_embeds_ada_norm,
+            attention_bias=attention_bias,
+            only_cross_attention=only_cross_attention,
+            double_self_attention=double_self_attention,
+            upcast_attention=upcast_attention,
+            norm_type=norm_type,
+            norm_elementwise_affine=norm_elementwise_affine,
+            norm_eps=norm_eps,
+            attention_type=attention_type,
+        )
+        missing_keys, unexpected_keys = model.load_state_dict(
+            transformer_block.state_dict(),
+            strict=False,
+        )
+        assert len(unexpected_keys) == 0
+        assert all(i.startswith('glyph') for i in missing_keys)
+        
+        return model
+
+    def forward(
+        self,
+        hidden_states: torch.FloatTensor,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.FloatTensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        timestep: Optional[torch.LongTensor] = None,
+        cross_attention_kwargs: Dict[str, Any] = None,
+        class_labels: Optional[torch.LongTensor] = None,
+        added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
+    ) -> torch.FloatTensor:
+        # Notice that normalization is always applied before the real computation in the following blocks.
+        # 0. Self-Attention
+        batch_size = hidden_states.shape[0]
+
+        if self.norm_type == "ada_norm":
+            norm_hidden_states = self.norm1(hidden_states, timestep)
+        elif self.norm_type == "ada_norm_zero":
+            norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(
+                hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype
+            )
+        elif self.norm_type in ["layer_norm", "layer_norm_i2vgen"]:
+            norm_hidden_states = self.norm1(hidden_states)
+        elif self.norm_type == "ada_norm_continuous":
+            norm_hidden_states = self.norm1(hidden_states, added_cond_kwargs["pooled_text_emb"])
+        elif self.norm_type == "ada_norm_single":
+            shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
+                self.scale_shift_table[None] + timestep.reshape(batch_size, 6, -1)
+            ).chunk(6, dim=1)
+            norm_hidden_states = self.norm1(hidden_states)
+            norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa
+            norm_hidden_states = norm_hidden_states.squeeze(1)
+        else:
+            raise ValueError("Incorrect norm used")
+
+        if self.pos_embed is not None:
+            norm_hidden_states = self.pos_embed(norm_hidden_states)
+
+        # 1. Retrieve lora scale.
+        lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0
+
+        # 2. Prepare GLIGEN inputs
+        cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {}
+        gligen_kwargs = cross_attention_kwargs.pop("gligen", None)
+        
+        glyph_encoder_hidden_states = cross_attention_kwargs.pop("glyph_encoder_hidden_states", None)
+        # a dict. visual_feat_len: tensor(b, visual_feat_len，text—_feat_len)
+        glyph_attn_mask = cross_attention_kwargs.pop("glyph_attn_masks_dict", None)
+        bg_attn_mask = cross_attention_kwargs.pop("bg_attn_masks_dict", None)
+        if glyph_attn_mask is not None:
+            glyph_attn_mask = glyph_attn_mask[hidden_states.shape[1]]
+        if bg_attn_mask is not None:
+            bg_attn_mask = bg_attn_mask[hidden_states.shape[1]]
+        assert encoder_attention_mask is None, "encoder_attention_mask is not supported in this block."
+
+        attn_output = self.attn1(
+            norm_hidden_states,
+            encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,
+            attention_mask=attention_mask,
+            **cross_attention_kwargs,
+        )
+        if self.norm_type == "ada_norm_zero":
+            attn_output = gate_msa.unsqueeze(1) * attn_output
+        elif self.norm_type == "ada_norm_single":
+            attn_output = gate_msa * attn_output
+
+        hidden_states = attn_output + hidden_states
+        if hidden_states.ndim == 4:
+            hidden_states = hidden_states.squeeze(1)
+
+        # 2.5 GLIGEN Control
+        if gligen_kwargs is not None:
+            hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"])
+
+        # 3. Cross-Attention
+        if self.attn2 is not None:
+            if self.norm_type == "ada_norm":
+                norm_hidden_states = self.norm2(hidden_states, timestep)
+            elif self.norm_type in ["ada_norm_zero", "layer_norm", "layer_norm_i2vgen"]:
+                norm_hidden_states = self.norm2(hidden_states)
+            elif self.norm_type == "ada_norm_single":
+                # For PixArt norm2 isn't applied here:
+                # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L70C1-L76C103
+                norm_hidden_states = hidden_states
+            elif self.norm_type == "ada_norm_continuous":
+                norm_hidden_states = self.norm2(hidden_states, added_cond_kwargs["pooled_text_emb"])
+            else:
+                raise ValueError("Incorrect norm")
+
+            if self.pos_embed is not None and self.norm_type != "ada_norm_single":
+                norm_hidden_states = self.pos_embed(norm_hidden_states)
+
+            attn_output = self.attn2(
+                norm_hidden_states,
+                encoder_hidden_states=torch.cat([encoder_hidden_states, glyph_encoder_hidden_states], dim=1),
+                attention_mask=torch.cat([bg_attn_mask, glyph_attn_mask], dim=-1),
+                **cross_attention_kwargs,
+            )
+
+            hidden_states = attn_output + hidden_states
+
+        # 4. Feed-forward
+        # i2vgen doesn't have this norm 🤷‍♂️
+        if self.norm_type == "ada_norm_continuous":
+            norm_hidden_states = self.norm3(hidden_states, added_cond_kwargs["pooled_text_emb"])
+        elif not self.norm_type == "ada_norm_single":
+            norm_hidden_states = self.norm3(hidden_states)
+
+        if self.norm_type == "ada_norm_zero":
+            norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
+
+        if self.norm_type == "ada_norm_single":
+            norm_hidden_states = self.norm2(hidden_states)
+            norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp
+
+        if self._chunk_size is not None:
+            # "feed_forward_chunk_size" can be used to save memory
+            ff_output = _chunked_feed_forward(
+                self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size, lora_scale=lora_scale
+            )
+        else:
+            ff_output = self.ff(norm_hidden_states, scale=lora_scale)
+
+        if self.norm_type == "ada_norm_zero":
+            ff_output = gate_mlp.unsqueeze(1) * ff_output
+        elif self.norm_type == "ada_norm_single":
+            ff_output = gate_mlp * ff_output
+
+        hidden_states = ff_output + hidden_states
+        if hidden_states.ndim == 4:
+            hidden_states = hidden_states.squeeze(1)
+
+        return hidden_states
+    

text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/pipelines/__init__.py

@@ -0,0 +1,5 @@
+from .pipeline_stable_diffusion_glyph_xl import StableDiffusionGlyphXLPipeline
+
+__all__ = [
+    'StableDiffusionGlyphXLPipeline',
+]

text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/pipelines/pipeline_stable_diffusion_glyph_xl.py

@@ -0,0 +1,922 @@
+
+from typing import Optional, List, Union, Dict, Tuple, Callable, Any
+import torch
+
+from transformers import T5EncoderModel, T5Tokenizer
+import torch.nn.functional as F
+
+from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import (
+    StableDiffusionXLPipeline,
+    AutoencoderKL,
+    CLIPTextModel,
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+    UNet2DConditionModel,
+    KarrasDiffusionSchedulers,
+    CLIPVisionModelWithProjection,
+    CLIPImageProcessor,
+    VaeImageProcessor,
+    is_invisible_watermark_available,
+    StableDiffusionXLLoraLoaderMixin,
+    PipelineImageInput,
+    adjust_lora_scale_text_encoder,
+    scale_lora_layers,
+    unscale_lora_layers,
+    USE_PEFT_BACKEND,
+    StableDiffusionXLPipelineOutput,
+    ImageProjection,
+    logging,
+    rescale_noise_cfg,
+    retrieve_timesteps,
+    deprecate,
+)
+import numpy as np
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+from diffusers.pipelines.stable_diffusion_xl.watermark import StableDiffusionXLWatermarker
+
+class StableDiffusionGlyphXLPipeline(StableDiffusionXLPipeline):
+    model_cpu_offload_seq = "text_encoder->text_encoder_2->byt5_text_encoder->image_encoder->unet->byt5_mapper->vae"
+    _optional_components = [
+        "tokenizer",
+        "tokenizer_2",
+        "byt5_tokenizer",
+        "text_encoder",
+        "text_encoder_2",
+        "byt5_text_encoder",
+        "byt5_mapper",
+        "image_encoder",
+        "feature_extractor",
+    ]
+    _callback_tensor_inputs = [
+        "latents",
+        "prompt_embeds",
+        "negative_prompt_embeds",
+        "add_text_embeds",
+        "add_time_ids",
+        "negative_pooled_prompt_embeds",
+        "negative_add_time_ids",
+    ]
+    def __init__(
+        self,
+        vae: AutoencoderKL,
+        text_encoder: CLIPTextModel,
+        text_encoder_2: CLIPTextModelWithProjection,
+        byt5_text_encoder: T5EncoderModel,
+        tokenizer: CLIPTokenizer,
+        tokenizer_2: CLIPTokenizer,
+        byt5_tokenizer: T5Tokenizer,
+        byt5_mapper,
+        unet: UNet2DConditionModel,
+        scheduler: KarrasDiffusionSchedulers,
+        byt5_max_length: int = 512,
+        image_encoder: CLIPVisionModelWithProjection = None,
+        feature_extractor: CLIPImageProcessor = None,
+        force_zeros_for_empty_prompt: bool = True,
+        add_watermarker: Optional[bool] = None,
+    ):
+        super(StableDiffusionXLPipeline, self).__init__()
+
+        self.register_modules(
+            vae=vae,
+            text_encoder=text_encoder,
+            text_encoder_2=text_encoder_2,
+            byt5_text_encoder=byt5_text_encoder,
+            tokenizer=tokenizer,
+            tokenizer_2=tokenizer_2,
+            byt5_tokenizer=byt5_tokenizer,
+            byt5_mapper=byt5_mapper,
+            unet=unet,
+            scheduler=scheduler,
+            image_encoder=image_encoder,
+            feature_extractor=feature_extractor,
+        )
+        self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
+        self.register_to_config(byt5_max_length=byt5_max_length)
+        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
+        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
+        self.byt5_max_length = byt5_max_length
+
+        self.default_sample_size = self.unet.config.sample_size
+
+        add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()
+
+        if add_watermarker:
+            self.watermark = StableDiffusionXLWatermarker()
+        else:
+            self.watermark = None
+
+    def encode_prompt(
+        self,
+        prompt: str,
+        prompt_2: Optional[str] = None,
+        text_prompt = None,
+        device: Optional[torch.device] = None,
+        num_images_per_prompt: int = 1,
+        do_classifier_free_guidance: bool = True,
+        negative_prompt: Optional[str] = None,
+        negative_prompt_2: Optional[str] = None,
+        prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_prompt_embeds: Optional[torch.FloatTensor] = None,
+        pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        lora_scale: Optional[float] = None,
+        clip_skip: Optional[int] = None,
+        text_attn_mask: Optional[torch.LongTensor] = None,
+        byt5_prompt_embeds: Optional[torch.FloatTensor] = None,
+    ):
+        r"""
+        Encodes the prompt into text encoder hidden states.
+
+        Args:
+            prompt (`str` or `List[str]`, *optional*):
+                prompt to be encoded
+            prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
+                used in both text-encoders
+            device: (`torch.device`):
+                torch device
+            num_images_per_prompt (`int`):
+                number of images that should be generated per prompt
+            do_classifier_free_guidance (`bool`):
+                whether to use classifier free guidance or not
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. If not defined, one has to pass
+                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
+                less than `1`).
+            negative_prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
+                `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
+            prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+            negative_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
+                argument.
+            pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+                If not provided, pooled text embeddings will be generated from `prompt` input argument.
+            negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
+                input argument.
+            lora_scale (`float`, *optional*):
+                A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
+            clip_skip (`int`, *optional*):
+                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
+                the output of the pre-final layer will be used for computing the prompt embeddings.
+        """
+        device = device or self._execution_device
+
+        # set lora scale so that monkey patched LoRA
+        # function of text encoder can correctly access it
+        if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
+            self._lora_scale = lora_scale
+
+            # dynamically adjust the LoRA scale
+            if self.text_encoder is not None:
+                if not USE_PEFT_BACKEND:
+                    adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
+                else:
+                    scale_lora_layers(self.text_encoder, lora_scale)
+
+            if self.text_encoder_2 is not None:
+                if not USE_PEFT_BACKEND:
+                    adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
+                else:
+                    scale_lora_layers(self.text_encoder_2, lora_scale)
+
+        prompt = [prompt] if isinstance(prompt, str) else prompt
+
+        if prompt is not None:
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        # Define tokenizers and text encoders
+        tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
+        text_encoders = (
+            [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
+        )
+
+        if prompt_embeds is None:
+            assert len(prompt) == 1
+            prompt_2 = prompt_2 or prompt
+            prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
+            
+            text_prompt = [text_prompt] if isinstance(text_prompt, str) else text_prompt
+
+            # textual inversion: procecss multi-vector tokens if necessary
+            prompt_embeds_list = []
+            prompts = [prompt, prompt_2]
+            text_input_id_batchs = []
+            for prompt, tokenizer in zip(prompts, tokenizers):
+                pad_token = tokenizer.pad_token_id
+                total_tokens = tokenizer(prompt, truncation=False)['input_ids'][0]
+                bos = total_tokens[0]
+                eos = total_tokens[-1]
+                total_tokens = total_tokens[1:-1]
+                new_total_tokens = []
+                empty_flag = True
+                while len(total_tokens) >= 75:
+                    head_75_tokens = [total_tokens.pop(0) for _ in range(75)]
+                    temp_77_token_ids = [bos] + head_75_tokens + [eos]
+                    new_total_tokens.append(temp_77_token_ids)
+                    empty_flag = False
+                if len(total_tokens) > 0 or empty_flag:
+                    padding_len = 75 - len(total_tokens)
+                    temp_77_token_ids = [bos] + total_tokens + [eos] + [pad_token] * padding_len
+                    new_total_tokens.append(temp_77_token_ids)
+                # 1,segment_len, 77
+                new_total_tokens = torch.tensor(new_total_tokens, dtype=torch.long).unsqueeze(0)
+                text_input_id_batchs.append(new_total_tokens)
+            if text_input_id_batchs[0].shape[1] > text_input_id_batchs[1].shape[1]:
+                tokenizer = tokenizers[1]
+                pad_token = tokenizer.pad_token_id
+                bos = tokenizer.bos_token_id
+                eos = tokenizer.eos_token_id
+                padding_len = text_input_id_batchs[0].shape[1] - text_input_id_batchs[1].shape[1]
+                # padding_len, 77
+                padding_part = torch.tensor([[bos] + [eos] + [pad_token] * 75 for _ in range(padding_len)])
+                # 1, padding_len, 77
+                padding_part = padding_part.unsqueeze(0)
+                text_input_id_batchs[1] = torch.cat((text_input_id_batchs[1],padding_part), dim=1)
+            elif text_input_id_batchs[0].shape[1] < text_input_id_batchs[1].shape[1]:
+                tokenizer = tokenizers[0]
+                pad_token = tokenizer.pad_token_id
+                bos = tokenizer.bos_token_id
+                eos = tokenizer.eos_token_id
+                padding_len = text_input_id_batchs[1].shape[1] - text_input_id_batchs[0].shape[1]
+                # padding_len, 77
+                padding_part = torch.tensor([[bos] + [eos] + [pad_token] * 75 for _ in range(padding_len)])
+                # 1, padding_len, 77
+                padding_part = padding_part.unsqueeze(0)
+                text_input_id_batchs[0] = torch.cat((text_input_id_batchs[0],padding_part), dim=1)
+            
+            embeddings = []
+            for segment_idx in range(text_input_id_batchs[0].shape[1]):
+                prompt_embeds_list = []
+                for i, text_encoder in enumerate(text_encoders):
+                    # 1, segment_len, sequence_len
+                    text_input_ids = text_input_id_batchs[i].to(text_encoder.device)
+                    # 1, sequence_len, dim
+                    prompt_embeds = text_encoder(
+                        text_input_ids[:, segment_idx],
+                        output_hidden_states=True,
+                    )
+
+                    # We are only ALWAYS interested in the pooled output of the final text encoder
+                    temp_pooled_prompt_embeds = prompt_embeds[0]
+                    if clip_skip is None:
+                        prompt_embeds = prompt_embeds.hidden_states[-2]
+                    else:
+                        prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]
+                    bs_embed, seq_len, _ = prompt_embeds.shape
+                    prompt_embeds = prompt_embeds.view(bs_embed, seq_len, -1)
+                    prompt_embeds_list.append(prompt_embeds)
+                # b, sequence_len, dim
+                prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
+                embeddings.append(prompt_embeds)
+                if segment_idx == 0:
+                    # use the first segment's pooled prompt embeddings as 
+                    # the pooled prompt embeddings
+                    # b, dim->b, dim
+                    pooled_prompt_embeds = temp_pooled_prompt_embeds.view(bs_embed, -1)
+            # b, segment_len * sequence_len, dim
+            prompt_embeds = torch.cat(embeddings, dim=1)
+            
+            if byt5_prompt_embeds is None:
+                byt5_text_inputs = self.byt5_tokenizer(
+                    text_prompt,
+                    padding="max_length",
+                    max_length=self.byt5_max_length,
+                    truncation=True,
+                    add_special_tokens=True,
+                    return_tensors="pt",
+                )
+                byt5_text_input_ids = byt5_text_inputs.input_ids
+                byt5_attention_mask = byt5_text_inputs.attention_mask.to(self.byt5_text_encoder.device) if text_attn_mask is None else text_attn_mask.to(self.byt5_text_encoder.device, dtype=byt5_text_inputs.attention_mask.dtype)
+                with torch.cuda.amp.autocast(enabled=False):
+                    byt5_prompt_embeds = self.byt5_text_encoder(
+                        byt5_text_input_ids.to(self.byt5_text_encoder.device),
+                        attention_mask=byt5_attention_mask.float(),
+                    )
+                    byt5_prompt_embeds = byt5_prompt_embeds[0]
+                    byt5_prompt_embeds = self.byt5_mapper(byt5_prompt_embeds, byt5_attention_mask)
+
+        # get unconditional embeddings for classifier free guidance
+        zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
+        if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
+            negative_prompt_embeds = torch.zeros_like(prompt_embeds)
+            negative_byt5_prompt_embeds = torch.zeros_like(byt5_prompt_embeds)
+            negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
+        elif do_classifier_free_guidance and negative_prompt_embeds is None:
+            raise NotImplementedError
+
+        if self.text_encoder_2 is not None:
+            prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
+        else:
+            prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device)
+
+        bs_embed, seq_len, _ = prompt_embeds.shape
+        # duplicate text embeddings for each generation per prompt, using mps friendly method
+        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
+        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
+
+        if do_classifier_free_guidance:
+            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
+            seq_len = negative_prompt_embeds.shape[1]
+            byt5_seq_len = negative_byt5_prompt_embeds.shape[1]
+
+            if self.text_encoder_2 is not None:
+                negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
+            else:
+                negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device)
+            negative_byt5_prompt_embeds = negative_byt5_prompt_embeds.to(dtype=self.byt5_text_encoder.dtype, device=device)
+
+            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
+            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
+            negative_byt5_prompt_embeds = negative_byt5_prompt_embeds.repeat(1, num_images_per_prompt, 1)
+            negative_byt5_prompt_embeds = negative_byt5_prompt_embeds.view(batch_size * num_images_per_prompt, byt5_seq_len, -1)
+
+        pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
+            bs_embed * num_images_per_prompt, -1
+        )
+        if do_classifier_free_guidance:
+            negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
+                bs_embed * num_images_per_prompt, -1
+            )
+
+        if self.text_encoder is not None:
+            if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
+                # Retrieve the original scale by scaling back the LoRA layers
+                unscale_lora_layers(self.text_encoder, lora_scale)
+
+        if self.text_encoder_2 is not None:
+            if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
+                # Retrieve the original scale by scaling back the LoRA layers
+                unscale_lora_layers(self.text_encoder_2, lora_scale)
+
+        return (
+            prompt_embeds, 
+            negative_prompt_embeds, 
+            pooled_prompt_embeds,
+            negative_pooled_prompt_embeds,
+            byt5_prompt_embeds, 
+            negative_byt5_prompt_embeds,
+        )
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]] = None,
+        prompt_2: Optional[Union[str, List[str]]] = None,
+        text_prompt = None,
+        texts = None,
+        bboxes = None,
+        height: Optional[int] = None,
+        width: Optional[int] = None,
+        num_inference_steps: int = 50,
+        timesteps: List[int] = None,
+        denoising_end: Optional[float] = None,
+        guidance_scale: float = 5.0,
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        negative_prompt_2: Optional[Union[str, List[str]]] = None,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_prompt_embeds: Optional[torch.FloatTensor] = None,
+        pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+        ip_adapter_image: Optional[PipelineImageInput] = None,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+        guidance_rescale: float = 0.0,
+        original_size: Optional[Tuple[int, int]] = None,
+        crops_coords_top_left: Tuple[int, int] = (0, 0),
+        target_size: Optional[Tuple[int, int]] = None,
+        negative_original_size: Optional[Tuple[int, int]] = None,
+        negative_crops_coords_top_left: Tuple[int, int] = (0, 0),
+        negative_target_size: Optional[Tuple[int, int]] = None,
+        clip_skip: Optional[int] = None,
+        callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
+        callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+        text_attn_mask: torch.LongTensor = None,
+        denoising_start: Optional[float] = None,
+        byt5_prompt_embeds: Optional[torch.FloatTensor] = None,
+        **kwargs,
+    ):
+        r"""
+        Function invoked when calling the pipeline for generation.
+
+        Args:
+            prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
+                instead.
+            prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
+                used in both text-encoders
+            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+                The height in pixels of the generated image. This is set to 1024 by default for the best results.
+                Anything below 512 pixels won't work well for
+                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
+                and checkpoints that are not specifically fine-tuned on low resolutions.
+            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+                The width in pixels of the generated image. This is set to 1024 by default for the best results.
+                Anything below 512 pixels won't work well for
+                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
+                and checkpoints that are not specifically fine-tuned on low resolutions.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            timesteps (`List[int]`, *optional*):
+                Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
+                in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
+                passed will be used. Must be in descending order.
+            denoising_end (`float`, *optional*):
+                When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
+                completed before it is intentionally prematurely terminated. As a result, the returned sample will
+                still retain a substantial amount of noise as determined by the discrete timesteps selected by the
+                scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
+                "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
+                Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
+            guidance_scale (`float`, *optional*, defaults to 5.0):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. If not defined, one has to pass
+                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
+                less than `1`).
+            negative_prompt_2 (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
+                `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
+                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
+                to make generation deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+            negative_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
+                argument.
+            pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+                If not provided, pooled text embeddings will be generated from `prompt` input argument.
+            negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+                Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
+                input argument.
+            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead
+                of a plain tuple.
+            cross_attention_kwargs (`dict`, *optional*):
+                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+                `self.processor` in
+                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+            guidance_rescale (`float`, *optional*, defaults to 0.0):
+                Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
+                Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of
+                [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).
+                Guidance rescale factor should fix overexposure when using zero terminal SNR.
+            original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
+                `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
+                explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
+                `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
+                `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
+                `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                For most cases, `target_size` should be set to the desired height and width of the generated image. If
+                not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
+                section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
+            negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                To negatively condition the generation process based on a specific image resolution. Part of SDXL's
+                micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
+                To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
+                micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
+                To negatively condition the generation process based on a target image resolution. It should be as same
+                as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
+                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
+                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
+            callback_on_step_end (`Callable`, *optional*):
+                A function that calls at the end of each denoising steps during the inference. The function is called
+                with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
+                callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
+                `callback_on_step_end_tensor_inputs`.
+            callback_on_step_end_tensor_inputs (`List`, *optional*):
+                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
+                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
+                `._callback_tensor_inputs` attribute of your pipeline class.
+
+        Examples:
+
+        Returns:
+            [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
+            `tuple`. When returning a tuple, the first element is a list with the generated images.
+        """
+
+        callback = kwargs.pop("callback", None)
+        callback_steps = kwargs.pop("callback_steps", None)
+
+        if callback is not None:
+            deprecate(
+                "callback",
+                "1.0.0",
+                "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
+            )
+        if callback_steps is not None:
+            deprecate(
+                "callback_steps",
+                "1.0.0",
+                "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
+            )
+
+        # 0. Default height and width to unet
+        height = height or self.default_sample_size * self.vae_scale_factor
+        width = width or self.default_sample_size * self.vae_scale_factor
+
+        original_size = original_size or (height, width)
+        target_size = target_size or (height, width)
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(
+            prompt,
+            prompt_2,
+            height,
+            width,
+            callback_steps,
+            negative_prompt,
+            negative_prompt_2,
+            prompt_embeds,
+            negative_prompt_embeds,
+            pooled_prompt_embeds,
+            negative_pooled_prompt_embeds,
+            callback_on_step_end_tensor_inputs,
+        )
+
+        self._guidance_scale = guidance_scale
+        self._guidance_rescale = guidance_rescale
+        self._clip_skip = clip_skip
+        self._cross_attention_kwargs = cross_attention_kwargs
+        self._denoising_end = denoising_end
+        self._interrupt = False
+
+        # 2. Define call parameters
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        device = self._execution_device
+
+        # 3. Encode input prompt
+        lora_scale = (
+            self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
+        )
+
+        (
+            prompt_embeds,
+            negative_prompt_embeds,
+            pooled_prompt_embeds,
+            negative_pooled_prompt_embeds,
+            byt5_prompt_embeds,
+            negative_byt5_prompt_embeds,
+        ) = self.encode_prompt(
+            prompt=prompt,
+            prompt_2=prompt_2,
+            text_prompt=text_prompt,
+            device=device,
+            num_images_per_prompt=num_images_per_prompt,
+            do_classifier_free_guidance=self.do_classifier_free_guidance,
+            negative_prompt=negative_prompt,
+            negative_prompt_2=negative_prompt_2,
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            lora_scale=lora_scale,
+            clip_skip=self.clip_skip,
+            text_attn_mask=text_attn_mask,
+            byt5_prompt_embeds=byt5_prompt_embeds,
+        )
+
+        # 4. Prepare timesteps
+        timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps)
+
+        # 5. Prepare latent variables
+        num_channels_latents = self.unet.config.in_channels
+        latents = self.prepare_latents(
+            batch_size * num_images_per_prompt,
+            num_channels_latents,
+            height,
+            width,
+            prompt_embeds.dtype,
+            device,
+            generator,
+            latents,
+        )
+
+        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        # 7. Prepare added time ids & embeddings
+        add_text_embeds = pooled_prompt_embeds
+        if self.text_encoder_2 is None:
+            text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
+        else:
+            text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
+
+        add_time_ids = self._get_add_time_ids(
+            original_size,
+            crops_coords_top_left,
+            target_size,
+            dtype=prompt_embeds.dtype,
+            text_encoder_projection_dim=text_encoder_projection_dim,
+        )
+        if negative_original_size is not None and negative_target_size is not None:
+            negative_add_time_ids = self._get_add_time_ids(
+                negative_original_size,
+                negative_crops_coords_top_left,
+                negative_target_size,
+                dtype=prompt_embeds.dtype,
+                text_encoder_projection_dim=text_encoder_projection_dim,
+            )
+        else:
+            negative_add_time_ids = add_time_ids
+
+        if self.do_classifier_free_guidance:
+            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
+            byt5_prompt_embeds = torch.cat([negative_byt5_prompt_embeds, byt5_prompt_embeds], dim=0)
+
+            add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
+            add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)
+
+        prompt_embeds = prompt_embeds.to(device)
+        byt5_prompt_embeds = byt5_prompt_embeds.to(device)
+        add_text_embeds = add_text_embeds.to(device)
+        add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
+
+        if ip_adapter_image is not None:
+            output_hidden_state = False if isinstance(self.unet.encoder_hid_proj, ImageProjection) else True
+            image_embeds, negative_image_embeds = self.encode_image(
+                ip_adapter_image, device, num_images_per_prompt, output_hidden_state
+            )
+            if self.do_classifier_free_guidance:
+                image_embeds = torch.cat([negative_image_embeds, image_embeds])
+                image_embeds = image_embeds.to(device)
+
+        # 8. Denoising loop
+        num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+
+        # 8.1 Apply denoising_end
+        if (
+            self.denoising_end is not None
+            and isinstance(self.denoising_end, float)
+            and self.denoising_end > 0
+            and self.denoising_end < 1
+        ):
+            discrete_timestep_cutoff = int(
+                round(
+                    self.scheduler.config.num_train_timesteps
+                    - (self.denoising_end * self.scheduler.config.num_train_timesteps)
+                )
+            )
+            num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
+            timesteps = timesteps[:num_inference_steps]
+
+        # 9. Optionally get Guidance Scale Embedding
+        timestep_cond = None
+        if self.unet.config.time_cond_proj_dim is not None:
+            guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
+            timestep_cond = self.get_guidance_scale_embedding(
+                guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
+            ).to(device=device, dtype=latents.dtype)
+        
+        assert batch_size == 1, "batch_size > 1 is not supported"
+        if texts is not None:
+            glyph_attn_mask = self.get_glyph_attn_mask(texts, bboxes)
+            # h,w
+            bg_attn_mask = glyph_attn_mask.sum(-1) == 0
+            # 1,h,w,byt5_max_len
+            glyph_attn_masks = glyph_attn_mask.unsqueeze(0).to(device)
+            # 1,h,w
+            bg_attn_masks = bg_attn_mask.unsqueeze(0).to(glyph_attn_masks.dtype).to(device)
+            
+            # b, h, w, text_feat_len
+            glyph_attn_masks = (1 - glyph_attn_masks) * -10000.0
+            # b, h, w
+            bg_attn_masks = (1 - bg_attn_masks) * -10000.0
+            num_down_sample = sum(1 if i == 'CrossAttnDownBlock2D' else 0 for i in self.unet.config['down_block_types']) - 1
+            initial_resolution = self.default_sample_size
+            initial_resolution = initial_resolution // 2**sum(1 if i == 'DownBlock2D' else 0 for i in self.unet.config['down_block_types'])
+            resolution_list = [initial_resolution] + [initial_resolution // 2**i for i in range(1, num_down_sample + 1)]
+            glyph_attn_masks_dict = dict()
+            bg_attn_masks_dict = dict()
+            # b, text_fet_len, h, w
+            glyph_attn_masks = glyph_attn_masks.permute(0, 3, 1, 2)
+            # b, 1, h, w
+            bg_attn_masks = bg_attn_masks.unsqueeze(1)
+            for mask_resolution in resolution_list:
+                down_scaled_glyph_attn_masks = F.interpolate(
+                    glyph_attn_masks, size=(mask_resolution, mask_resolution), mode='nearest',
+                )
+                # b, text_fet_len, h, w->b, h, w, text_fet_len->b, h*w, text_fet_len
+                down_scaled_glyph_attn_masks = down_scaled_glyph_attn_masks.permute(0, 2, 3, 1).flatten(1, 2)
+                glyph_attn_masks_dict[mask_resolution * mask_resolution] = down_scaled_glyph_attn_masks
+
+                down_scaled_bg_attn_masks = F.interpolate(
+                    bg_attn_masks, size=(mask_resolution, mask_resolution), mode='nearest',
+                )
+                # b,1,h,w->b,h,w->b,h,w,1->b,h*w,1->b,h*w,clip_feat_len
+                down_scaled_bg_attn_masks = down_scaled_bg_attn_masks.squeeze(1).unsqueeze(-1)
+                down_scaled_bg_attn_masks = down_scaled_bg_attn_masks.flatten(1, 2)
+                down_scaled_bg_attn_masks = down_scaled_bg_attn_masks.repeat(1, 1, prompt_embeds.shape[1])
+                bg_attn_masks_dict[mask_resolution * mask_resolution] = down_scaled_bg_attn_masks
+            if self.do_classifier_free_guidance:
+                for key in glyph_attn_masks_dict:
+                    glyph_attn_masks_dict[key] = torch.cat([
+                        torch.zeros_like(glyph_attn_masks_dict[key]), 
+                        glyph_attn_masks_dict[key]], 
+                    dim=0)
+                for key in bg_attn_masks_dict:
+                    bg_attn_masks_dict[key] = torch.cat([
+                        torch.zeros_like(bg_attn_masks_dict[key]), 
+                        bg_attn_masks_dict[key]], 
+                    dim=0)
+        else:
+            glyph_attn_masks_dict = None
+            bg_attn_masks_dict = None
+
+        self._num_timesteps = len(timesteps)
+        with self.progress_bar(total=num_inference_steps) as progress_bar:
+            for i, t in enumerate(timesteps):
+                if self.interrupt:
+                    continue
+
+                # expand the latents if we are doing classifier free guidance
+                latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
+
+                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+                # predict the noise residual
+                added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
+                if ip_adapter_image is not None:
+                    added_cond_kwargs["image_embeds"] = image_embeds
+                if self.cross_attention_kwargs is None:
+                    cross_attention_kwargs = {}
+                else:
+                    cross_attention_kwargs = self.cross_attention_kwargs
+                cross_attention_kwargs['glyph_encoder_hidden_states'] = byt5_prompt_embeds
+                cross_attention_kwargs['glyph_attn_masks_dict'] = glyph_attn_masks_dict
+                cross_attention_kwargs['bg_attn_masks_dict'] = bg_attn_masks_dict
+                
+                noise_pred = self.unet(
+                    latent_model_input,
+                    t,
+                    encoder_hidden_states=prompt_embeds,
+                    timestep_cond=timestep_cond,
+                    cross_attention_kwargs=cross_attention_kwargs,
+                    added_cond_kwargs=added_cond_kwargs,
+                    return_dict=False,
+                )[0]
+
+                # perform guidance
+                if self.do_classifier_free_guidance:
+                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                    noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+                if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
+                    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
+                    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)
+
+                # compute the previous noisy sample x_t -> x_t-1
+                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
+
+                if callback_on_step_end is not None:
+                    callback_kwargs = {}
+                    for k in callback_on_step_end_tensor_inputs:
+                        callback_kwargs[k] = locals()[k]
+                    callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+                    latents = callback_outputs.pop("latents", latents)
+                    prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+                    negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
+                    add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
+                    negative_pooled_prompt_embeds = callback_outputs.pop(
+                        "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds
+                    )
+                    add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
+                    negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids)
+
+                # call the callback, if provided
+                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+                    progress_bar.update()
+                    if callback is not None and i % callback_steps == 0:
+                        step_idx = i // getattr(self.scheduler, "order", 1)
+                        callback(step_idx, t, latents)
+
+        if not output_type == "latent":
+            # make sure the VAE is in float32 mode, as it overflows in float16
+            needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
+
+            if needs_upcasting:
+                self.upcast_vae()
+                latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
+
+            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
+
+            # cast back to fp16 if needed
+            if needs_upcasting:
+                self.vae.to(dtype=torch.float16)
+        else:
+            image = latents
+
+        if not output_type == "latent":
+            # apply watermark if available
+            if self.watermark is not None:
+                image = self.watermark.apply_watermark(image)
+
+            image = self.image_processor.postprocess(image, output_type=output_type)
+
+        # Offload all models
+        self.maybe_free_model_hooks()
+
+        if not return_dict:
+            return (image,)
+
+        return StableDiffusionXLPipelineOutput(images=image)
+
+    def get_glyph_attn_mask(self, texts, bboxes):
+        resolution = self.default_sample_size
+        text_idx_list = self.get_text_start_pos(texts)
+        mask_tensor = torch.zeros(
+            resolution, resolution, self.byt5_max_length,
+        )
+        for idx, bbox in enumerate(bboxes):
+            # box is in [x, y, w, h] format
+            # area of [y:y+h, x:x+w]
+            bbox = [int(v * resolution + 0.5) for v in bbox]
+            bbox[2] = max(bbox[2], 1)
+            bbox[3] = max(bbox[3], 1)
+            bbox[0: 2] =  np.clip(bbox[0: 2], 0, resolution - 1).tolist()
+            bbox[2: 4] = np.clip(bbox[2: 4], 1, resolution).tolist()
+            mask_tensor[
+                bbox[1]: bbox[1] + bbox[3], 
+                bbox[0]: bbox[0] + bbox[2], 
+                text_idx_list[idx]: text_idx_list[idx + 1]
+            ] = 1
+        return mask_tensor
+
+    def get_text_start_pos(self, texts):
+        prompt = "".encode('utf-8')
+        '''
+        Text "{text}" in {color}, {type}.
+        '''
+        pos_list = []
+        for text in texts:
+            pos_list.append(len(prompt))
+            text_prompt = f'Text "{text}"'
+
+            attr_list = ['0', '1']
+
+            attr_suffix = ", ".join(attr_list)
+            text_prompt += " in " + attr_suffix
+            text_prompt += ". "
+            text_prompt = text_prompt.encode('utf-8')
+
+            prompt = prompt + text_prompt
+        pos_list.append(len(prompt))
+        return pos_list

text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/__init__.py

@@ -0,0 +1,7 @@
+from .simple_byt5_mapper import ByT5Mapper
+from .byt5_block_byt5_mapper import T5EncoderBlockByT5Mapper
+
+__all__ = [
+    'ByT5Mapper',
+    'T5EncoderBlockByT5Mapper',
+]

text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/byt5_block_byt5_mapper.py

@@ -0,0 +1,151 @@
+import torch
+import torch.nn as nn
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import warnings
+
+import logging
+from torch import Tensor
+from diffusers import ModelMixin
+from transformers.models.t5.modeling_t5 import T5LayerSelfAttention, T5LayerFF, T5LayerNorm
+
+logger = logging.getLogger(__name__)
+
+class T5EncoderBlock(nn.Module):
+    def __init__(self, config, has_relative_attention_bias=False):
+        super().__init__()
+        self.layer = nn.ModuleList()
+        self.layer.append(T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias))
+        self.layer.append(T5LayerFF(config))
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        position_bias=None,
+        layer_head_mask=None,
+        output_attentions=False,
+    ):
+        self_attn_past_key_value, cross_attn_past_key_value = None, None
+
+        self_attention_outputs = self.layer[0](
+            hidden_states,
+            attention_mask=attention_mask,
+            position_bias=position_bias,
+            layer_head_mask=layer_head_mask,
+            past_key_value=self_attn_past_key_value,
+            use_cache=False,
+            output_attentions=output_attentions,
+        )
+        hidden_states, present_key_value_state = self_attention_outputs[:2]
+        attention_outputs = self_attention_outputs[2:]  # Keep self-attention outputs and relative position weights
+
+        # clamp inf values to enable fp16 training
+        if hidden_states.dtype == torch.float16:
+            clamp_value = torch.where(
+                torch.isinf(hidden_states).any(),
+                torch.finfo(hidden_states.dtype).max - 1000,
+                torch.finfo(hidden_states.dtype).max,
+            )
+            hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
+
+        # Apply Feed Forward layer
+        hidden_states = self.layer[-1](hidden_states)
+
+        # clamp inf values to enable fp16 training
+        if hidden_states.dtype == torch.float16:
+            clamp_value = torch.where(
+                torch.isinf(hidden_states).any(),
+                torch.finfo(hidden_states.dtype).max - 1000,
+                torch.finfo(hidden_states.dtype).max,
+            )
+            hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
+
+        outputs = (hidden_states,) + attention_outputs
+
+        return outputs  # hidden-states, present_key_value_states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights)
+
+class T5EncoderBlockByT5Mapper(ModelMixin):
+    def __init__(self, byt5_config, num_layers, sdxl_channels=None):
+        super().__init__()
+        if num_layers > 0:
+            self.blocks = nn.ModuleList(
+                [
+                    T5EncoderBlock(
+                        byt5_config, 
+                        has_relative_attention_bias=bool(i == 0)) 
+                    for i in range(num_layers)
+                ]
+            )
+        else:
+            self.blocks = None
+        self.layer_norm = T5LayerNorm(byt5_config.d_model, eps=byt5_config.layer_norm_epsilon)
+        if sdxl_channels is not None:
+            self.channel_mapper = nn.Linear(byt5_config.d_model, sdxl_channels)
+            self.final_layer_norm = T5LayerNorm(sdxl_channels, eps=byt5_config.layer_norm_epsilon)
+        else:
+            self.channel_mapper = None
+            self.final_layer_norm = None
+            
+    def get_extended_attention_mask(
+        self, attention_mask: Tensor, input_shape: Tuple[int], device: torch.device = None, dtype: torch.float = None
+    ) -> Tensor:
+        """
+        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
+
+        Arguments:
+            attention_mask (`torch.Tensor`):
+                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
+            input_shape (`Tuple[int]`):
+                The shape of the input to the model.
+
+        Returns:
+            `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
+        """
+        if dtype is None:
+            dtype = self.dtype
+
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        if attention_mask.dim() == 3:
+            extended_attention_mask = attention_mask[:, None, :, :]
+        elif attention_mask.dim() == 2:
+            # Provided a padding mask of dimensions [batch_size, seq_length]
+            # - if the model is a decoder, apply a causal mask in addition to the padding mask
+            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
+            extended_attention_mask = attention_mask[:, None, None, :]
+        else:
+            raise ValueError(
+                f"Wrong shape for input_ids (shape {input_shape}) or attention_mask (shape {attention_mask.shape})"
+            )
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and the dtype's smallest value for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+        extended_attention_mask = extended_attention_mask.to(dtype=dtype)  # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * torch.finfo(dtype).min
+        return extended_attention_mask
+
+    
+    def forward(self, inputs_embeds, attention_mask):
+        input_shape = inputs_embeds.size()[:-1]
+        extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+        
+        hidden_states = inputs_embeds
+        position_bias = None
+        
+        if self.blocks is not None:
+            for layer_module in self.blocks:
+                layer_outputs = layer_module(
+                    hidden_states,
+                    attention_mask=extended_attention_mask,
+                    position_bias=position_bias,
+                )
+                hidden_states, position_bias = layer_outputs
+        hidden_states = self.layer_norm(hidden_states)
+        if self.channel_mapper is not None:
+            hidden_states = self.channel_mapper(hidden_states)
+            hidden_states = self.final_layer_norm(hidden_states)
+        return hidden_states

text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/simple_byt5_mapper.py

@@ -0,0 +1,16 @@
+from diffusers import ModelMixin
+import torch.nn as nn
+
+class ByT5Mapper(ModelMixin):
+    def __init__(self, byt5_output_dim, sdxl_text_dim):
+        super().__init__()
+        self.mapper = nn.Sequential(
+                nn.LayerNorm(byt5_output_dim),
+                nn.Linear(byt5_output_dim, sdxl_text_dim),
+                nn.ReLU(),
+                nn.Linear(sdxl_text_dim, sdxl_text_dim)
+        )
+    
+    def forward(self, byt5_embedding):
+        return self.mapper(byt5_embedding)
+    

text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/__init__.py

@@ -0,0 +1,23 @@
+from .parse_config import parse_config
+from .constants import (
+    UNET_CKPT_NAME,
+    BYT5_CKPT_NAME,
+    BYT5_MAPPER_CKPT_NAME,
+    INSERTED_ATTN_CKPT_NAME,
+    huggingface_cache_dir,
+)
+from .load_pretrained_byt5 import load_byt5_and_byt5_tokenizer
+from .format_prompt import PromptFormat, MultilingualPromptFormat
+
+__all__ = [
+    'parse_config', 
+    'UNET_CKPT_NAME',
+    'BYT5_CKPT_NAME',
+    'BYT5_MAPPER_CKPT_NAME',
+    'huggingface_cache_dir',
+    'load_byt5_and_byt5_tokenizer',
+    'INSERTED_ATTN_CKPT_NAME',
+    'PromptFormat',
+    'MultilingualPromptFormat',
+]
+

text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/constants.py

@@ -0,0 +1,5 @@
+UNET_CKPT_NAME = "unet_lora.pt"
+INSERTED_ATTN_CKPT_NAME = "unet_inserted_attn.pt"
+BYT5_CKPT_NAME = "byt5_model.pt"
+BYT5_MAPPER_CKPT_NAME = "byt5_mapper.pt"
+huggingface_cache_dir = None

text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/format_prompt.py

@@ -0,0 +1,113 @@
+import json
+import webcolors
+
+
+def closest_color(requested_color):  
+    min_colors = {}  
+    for key, name in webcolors.CSS3_HEX_TO_NAMES.items():  
+        r_c, g_c, b_c = webcolors.hex_to_rgb(key)  
+        rd = (r_c - requested_color[0]) ** 2  
+        gd = (g_c - requested_color[1]) ** 2  
+        bd = (b_c - requested_color[2]) ** 2  
+        min_colors[(rd + gd + bd)] = name  
+    return min_colors[min(min_colors.keys())]
+
+def convert_rgb_to_names(rgb_tuple):  
+    try:  
+        color_name = webcolors.rgb_to_name(rgb_tuple)  
+    except ValueError:  
+        color_name = closest_color(rgb_tuple)  
+    return color_name
+
+class PromptFormat():
+    def __init__(
+        self,
+        font_path: str = 'assets/font_idx_512.json',
+        color_path: str = 'assets/color_idx.json',
+    ):
+        with open(font_path, 'r') as f:
+            self.font_dict = json.load(f)
+        with open(color_path, 'r') as f:
+            self.color_dict = json.load(f)
+
+    def format_checker(self, texts, styles):
+        assert len(texts) == len(styles), 'length of texts must be equal to length of styles'
+        for style in styles:
+            assert style['font-family'] in self.font_dict, f"invalid font-family: {style['font-family']}"
+            rgb_color = webcolors.hex_to_rgb(style['color'])
+            color_name = convert_rgb_to_names(rgb_color)
+            assert color_name in self.color_dict, f"invalid color hex {color_name}"
+
+    def format_prompt(self, texts, styles):
+        self.format_checker(texts, styles)
+
+        prompt = ""
+        '''
+        Text "{text}" in {color}, {type}.
+        '''
+        for text, style in zip(texts, styles):
+            text_prompt = f'Text "{text}"'
+
+            attr_list = []
+
+            # format color
+            hex_color = style["color"]
+            rgb_color = webcolors.hex_to_rgb(hex_color)
+            color_name = convert_rgb_to_names(rgb_color)
+            attr_list.append(f"<color-{self.color_dict[color_name]}>")
+
+            # format font
+            attr_list.append(f"<font-{self.font_dict[style['font-family']]}>")
+            attr_suffix = ", ".join(attr_list)
+            text_prompt += " in " + attr_suffix
+            text_prompt += ". "
+
+            prompt = prompt + text_prompt
+        return prompt
+
+
+class MultilingualPromptFormat():
+    def __init__(
+        self,
+        font_path: str = 'assets/multilingual_10-lang_idx.json',
+        color_path: str = 'assets/color_idx.json',
+    ):
+        with open(font_path, 'r') as f:
+            self.font_dict = json.load(f)
+        with open(color_path, 'r') as f:
+            self.color_dict = json.load(f)
+
+    def format_checker(self, texts, styles):
+        assert len(texts) == len(styles), 'length of texts must be equal to length of styles'
+        for style in styles:
+            assert style['font-family'] in self.font_dict, f"invalid font-family: {style['font-family']}"
+            rgb_color = webcolors.hex_to_rgb(style['color'])
+            color_name = convert_rgb_to_names(rgb_color)
+            assert color_name in self.color_dict, f"invalid color hex {color_name}"
+
+    def format_prompt(self, texts, styles):
+        self.format_checker(texts, styles)
+
+        prompt = ""
+        '''
+        Text "{text}" in {color}, {type}.
+        '''
+        for text, style in zip(texts, styles):
+            text_prompt = f'Text "{text}"'
+
+            attr_list = []
+
+            # format color
+            hex_color = style["color"]
+            rgb_color = webcolors.hex_to_rgb(hex_color)
+            color_name = convert_rgb_to_names(rgb_color)
+            attr_list.append(f"<color-{self.color_dict[color_name]}>")
+
+            # format font
+            attr_list.append(f"<{style['font-family'][:2]}-font-{self.font_dict[style['font-family']]}>")
+            attr_suffix = ", ".join(attr_list)
+            text_prompt += " in " + attr_suffix
+            text_prompt += ". "
+
+            prompt = prompt + text_prompt
+        return prompt

text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/load_pretrained_byt5.py

@@ -0,0 +1,60 @@
+import json
+
+from transformers import AutoTokenizer, T5ForConditionalGeneration
+from diffusers.utils import logging
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+def add_special_token(tokenizer, text_encoder, add_color, add_font, color_ann_path, font_ann_path, multilingual=False):
+    with open(font_ann_path, 'r') as f:
+        idx_font_dict = json.load(f)
+    with open(color_ann_path, 'r') as f:
+        idx_color_dict = json.load(f)
+
+    if multilingual:
+        font_token = []
+        for font_code in idx_font_dict:
+            prefix = font_code[:2]
+            font_token.append(f'<{prefix}-font-{idx_font_dict[font_code]}>')
+    else:
+        font_token = [f'<font-{i}>' for i in range(len(idx_font_dict))]
+    color_token = [f'<color-{i}>' for i in range(len(idx_color_dict))]
+    additional_special_tokens = []
+    if add_color:
+        additional_special_tokens += color_token
+    if add_font:
+        additional_special_tokens += font_token
+    tokenizer.add_tokens(additional_special_tokens, special_tokens=True)
+    text_encoder.resize_token_embeddings(len(tokenizer))
+
+def load_byt5_and_byt5_tokenizer(
+    byt5_name='google/byt5-small', 
+    special_token=False, 
+    color_special_token=False,
+    font_special_token=False,
+    color_ann_path='assets/color_idx.json',
+    font_ann_path='assets/font_idx_512.json',
+    huggingface_cache_dir=None,
+    multilingual=False,
+):
+    byt5_tokenizer = AutoTokenizer.from_pretrained(
+        byt5_name, cache_dir=huggingface_cache_dir,
+    )
+    byt5_text_encoder = T5ForConditionalGeneration.from_pretrained(
+        byt5_name, cache_dir=huggingface_cache_dir,
+    ).get_encoder()
+
+    if special_token:
+        add_special_token(
+            byt5_tokenizer, 
+            byt5_text_encoder, 
+            add_color=color_special_token, 
+            add_font=font_special_token, 
+            color_ann_path=color_ann_path,
+            font_ann_path=font_ann_path,
+            multilingual=multilingual,
+        )
+
+    logger.info(f'Loaded original byt5 weight')
+    
+    return byt5_text_encoder, byt5_tokenizer

text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/parse_config.py

@@ -0,0 +1,17 @@
+import argparse
+import os
+import os.path as osp
+from mmengine.config import Config
+
+
+def parse_config(path=None):
+    if path is None:
+        parser = argparse.ArgumentParser()
+        parser.add_argument('config_dir', type=str)
+        args = parser.parse_args()
+        path = args.config_dir
+    config = Config.fromfile(path)
+    
+    config.config_dir = path
+
+    return config

text_encoder/byt5-small/flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3aafee96d60e98aa18b3c7f73a2c5a2360f1f2f6df79361190a4c9e05c5ab21
+size 1198558445

text_encoder/byt5-small/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c5aaf56299d6f2d4eaadad550a40765198828ead4d74f0a15f91cbe0961931a
+size 1198627927

text_encoder/byt5-small/tf_model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f97320dd5eb49cb2323a21d584cef7c1cfc9a0976efa978fcef438676b952bc2
+size 1198900664

text_encoder/llm/model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e97b877e47fde53a6c6e77aafb36e58e91ee9d95c4a3eeac6f1b5c0e6a1c986e
+size 3900233256

text_encoder/llm/model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9a300a43b4724eee2abe7c18ceb26768d0ab011eb0cad19d9bfd2476a24d024
+size 3864726320

text_encoder/llm/model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:111223d173e00bbee81cba1216fad28668df3476706b7fd26f4d5b50f8b3a507
+size 3864726424

text_encoder/llm/model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef47f634fa57d46ee134edcc09f34085a47da1e16c12a2abe0d67118be6d72ed
+size 3864733680

text_encoder/llm/model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c859795ad3a627a9b95bcb762e059d5b768a4a36fdd4affeff269d93fdecc67
+size 1089994880

transformer/1080p_sr_distilled/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": false,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "1080p",
+  "ideal_task": null,
+  "in_channels": 98,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": true,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/1080p_sr_distilled/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:691dc1b81b49d942e2eb95e6d61b91321e17b868536eaa4e843db6e406390411
+size 33325793672

transformer/480p_i2v/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": true,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "480p",
+  "ideal_task": "i2v",
+  "in_channels": 32,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": false,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/480p_i2v/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4d7d3e61404f5c742b57260f1b6a3bc41bb12fc880438252bf37913487dec56
+size 33306632192

transformer/480p_i2v_distilled/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": true,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "480p",
+  "ideal_task": "i2v",
+  "in_channels": 32,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": false,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/480p_i2v_distilled/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f51fe1c4302be44e25afcd1a9186385606482da8a77e2ee7793b0e8385b9cd57
+size 33306632192

transformer/480p_t2v/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": true,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "480p",
+  "ideal_task": "t2v",
+  "in_channels": 32,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": false,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/480p_t2v/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71f9affa1115fef2b14bd41fba30eab966fe80c9ed98e0fcba495dbc6d8fff86
+size 33306632192

transformer/480p_t2v_distilled/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": true,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "480p",
+  "ideal_task": "t2v",
+  "in_channels": 32,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": false,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/480p_t2v_distilled/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f35dc10a4037a618b22fef4ee20f8a9d972b4cf2e684764ed8f442fc7a2583f
+size 33306632192

transformer/720p_i2v/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": true,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "720p",
+  "ideal_task": "i2v",
+  "in_channels": 32,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": false,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/720p_i2v/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ffd6e2e1c2de585fd011ace1a64105804830aa331ddb25a2fb4a32497f159a4
+size 33306632192

transformer/720p_i2v_distilled/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": true,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "720p",
+  "ideal_task": "i2v",
+  "in_channels": 32,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": false,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/720p_i2v_distilled/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:542a9a4367ebfe5c584d925308f63f11070d044e3615b07af95edd4e0500240a
+size 33306632192

transformer/720p_i2v_distilled_sparse/config.json

@@ -0,0 +1,67 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flex-block-attn",
+  "attn_param": {
+    "attn_mask_share_within_head": 0,
+    "attn_pad_type": "zero",
+    "attn_sparse_type": "ssta",
+    "attn_use_text_mask": 1,
+    "ssta_adaptive_pool": null,
+    "ssta_lambda": 0.7,
+    "ssta_sampling_type": "importance",
+    "ssta_threshold": 0.0,
+    "ssta_topk": 64,
+    "tile_size": [
+      6,
+      8,
+      8
+    ],
+    "win_ratio": 10,
+    "win_size": [
+      [
+        3,
+        3,
+        3
+      ]
+    ],
+    "win_type": "fixed"
+  },
+  "concat_condition": true,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "720p",
+  "ideal_task": "i2v",
+  "in_channels": 32,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": false,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/720p_i2v_distilled_sparse/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47345844cc15df1e64a38cc9715fa31471c2c1c68a2857404eda027d60f355a6
+size 33306632192

transformer/720p_sr_distilled/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": false,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "720p",
+  "ideal_task": null,
+  "in_channels": 98,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": true,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/720p_sr_distilled/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fcab24a2404731a5e42d5f4adf7731a69771a3b2fc7786f90233f599947794a
+size 33325793672

transformer/720p_t2v/config.json

@@ -0,0 +1,43 @@
+{
+  "_class_name": "HunyuanVideo_1_5_DiffusionTransformer",
+  "_diffusers_version": "0.35.0",
+  "attn_mode": "flash",
+  "attn_param": null,
+  "concat_condition": true,
+  "glyph_byT5_v2": true,
+  "guidance_embed": false,
+  "heads_num": 16,
+  "hidden_size": 2048,
+  "ideal_resolution": "720p",
+  "ideal_task": "t2v",
+  "in_channels": 32,
+  "is_reshape_temporal_channels": false,
+  "mlp_act_type": "gelu_tanh",
+  "mlp_width_ratio": 4,
+  "mm_double_blocks_depth": 54,
+  "mm_single_blocks_depth": 0,
+  "out_channels": 32,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "qk_norm": true,
+  "qk_norm_type": "rms",
+  "qkv_bias": true,
+  "rope_dim_list": [
+    16,
+    56,
+    56
+  ],
+  "rope_theta": 256,
+  "text_pool_type": null,
+  "text_projection": "single_refiner",
+  "text_states_dim": 3584,
+  "text_states_dim_2": null,
+  "use_attention_mask": true,
+  "use_cond_type_embedding": true,
+  "use_meanflow": false,
+  "vision_projection": "linear",
+  "vision_states_dim": 1152
+}

transformer/720p_t2v/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acb0a23ccd0b6c662a22bcc9783544fd917418227a5bdf5e2cbecb22a142c3cc
+size 33306632192