""" Gemma-2-2B text encoder for PixelDiT. Handles chi_prompt prefix + select_index to match training exactly. Usage: from pixeldit.text_encoder_gemma import GemmaEncoder enc = GemmaEncoder() cond = enc.encode(["a dragon at sunset"]) # [1, 300, 2304] null = enc.encode_null(1) # [1, 300, 2304] """ import torch from transformers import AutoTokenizer, AutoModelForCausalLM _GEMMA_ID = "Efficient-Large-Model/gemma-2-2b-it" _GEMMA_DIM = 2304 _TXT_MAX = 300 _CHI_PROMPT = "\n".join([ 'Given a user prompt, generate an "Enhanced prompt" that provides detailed visual descriptions suitable for image generation. Evaluate the level of detail in the user prompt:', '- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, and spatial relationships to create vivid and concrete scenes.', '- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating.', 'Here are examples of how to transform or refine prompts:', '- User Prompt: A cat sleeping -> Enhanced: A small, fluffy white cat curled up in a round shape, sleeping peacefully on a warm sunny windowsill, surrounded by pots of blooming red flowers.', '- User Prompt: A busy city street -> Enhanced: A bustling city street scene at dusk, featuring glowing street lamps, a diverse crowd of people in colorful clothing, and a double-decker bus passing by towering glass skyscrapers.', 'Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations:', 'User Prompt: ', ]) _SELECT_IDX = [0] + list(range(-(_TXT_MAX - 1), 0)) class GemmaEncoder: def __init__( self, model_id=_GEMMA_ID, output_device="cuda", output_dtype=torch.bfloat16, ): self.output_device = torch.device(output_device) self.output_dtype = output_dtype print(f"[GemmaEncoder] loading {model_id} (CPU)") self.tokenizer = AutoTokenizer.from_pretrained(model_id) self.tokenizer.padding_side = "right" self._model = ( AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32) .get_decoder().eval() ) self._num_chi_tokens = len(self.tokenizer.encode(_CHI_PROMPT)) print("[GemmaEncoder] ready") @torch.no_grad() def encode(self, texts: list[str]) -> torch.Tensor: """Returns [B, 300, 2304].""" texts_full = [_CHI_PROMPT + t for t in texts] max_len = self._num_chi_tokens + _TXT_MAX - 2 tok = self.tokenizer( texts_full, max_length=max_len, padding="max_length", truncation=True, return_tensors="pt", ) emb = self._model( input_ids=tok.input_ids, attention_mask=tok.attention_mask, ).last_hidden_state emb = emb[:, _SELECT_IDX, :] return emb.to(self.output_device).to(self.output_dtype) @torch.no_grad() def encode_null(self, batch_size: int) -> torch.Tensor: """Returns [B, 300, 2304] for empty string (CFG unconditional).""" tok = self.tokenizer( [""] * batch_size, max_length=_TXT_MAX, padding="max_length", truncation=True, return_tensors="pt", ) emb = self._model( input_ids=tok.input_ids, attention_mask=tok.attention_mask, ).last_hidden_state return emb.to(self.output_device).to(self.output_dtype)