Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import numpy as np | |
| import random | |
| import torch | |
| import spaces | |
| import os | |
| import json | |
| from PIL import Image | |
| from diffusers import QwenImageEditPipeline, FlowMatchEulerDiscreteScheduler | |
| from huggingface_hub import InferenceClient | |
| import math | |
| # Assuming optimization.py and qwenimage/ are in the same directory | |
| from optimization import optimize_pipeline_ | |
| from qwenimage.pipeline_qwen_image_edit import QwenImageEditPipeline as QwenImageEditPipelineCustom | |
| from qwenimage.transformer_qwen_image import QwenImageTransformer2DModel | |
| from qwenimage.qwen_fa3_processor import QwenDoubleStreamAttnProcessorFA3 | |
| # --- prompt enhancement using hugging face inferenceclient --- | |
| def polish_prompt_hf(original_prompt, system_prompt): | |
| """ | |
| Rewrites the prompt using a Hugging Face InferenceClient. | |
| """ | |
| api_key = os.environ.get("HF_TOKEN") # Changed to HF_TOKEN as per common practice | |
| if not api_key: | |
| print("Warning: HF_TOKEN not set. Falling back to original prompt.") | |
| return original_prompt | |
| try: | |
| client = InferenceClient( | |
| provider="cerebras", | |
| api_key=api_key, | |
| ) | |
| messages = [ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": original_prompt} | |
| ] | |
| completion = client.chat.completions.create( | |
| model="qwen/qwen3-235b-a22b-instruct-2507", | |
| messages=messages, | |
| ) | |
| result = completion.choices[0].message.content | |
| if '{"rewritten"' in result: | |
| try: | |
| result = result.replace('```json', '').replace('```', '') | |
| result_json = json.loads(result) | |
| polished_prompt = result_json.get('rewritten', result) | |
| except Exception: # Catch broader exception for JSON parsing | |
| polished_prompt = result | |
| else: | |
| polished_prompt = result | |
| polished_prompt = polished_prompt.strip().replace("\n", " ") | |
| return polished_prompt | |
| except Exception as e: # Catch broader exception for API calls | |
| print(f"Error during API call to Hugging Face: {e}") | |
| return original_prompt | |
| def polish_prompt(prompt, img): | |
| """ | |
| Main function to polish prompts for image editing using HF inference. | |
| """ | |
| system_prompt = ''' | |
| # EDIT INSTRUCTION REWRITER | |
| You are a professional edit instruction rewriter. Your task is to generate a precise, concise, and visually achievable professional-level edit instruction based on the user-provided instruction and the image to be edited. | |
| Please strictly follow the rewriting rules below: | |
| ## 1. GENERAL PRINCIPLES | |
| - Keep the rewritten prompt **concise**. Avoid overly long sentences and reduce unnecessary descriptive language. | |
| - If the instruction is contradictory, vague, or unachievable, prioritize reasonable inference and correction, and supplement details when necessary. | |
| - Keep the core intention of the original instruction unchanged, only enhancing its clarity, rationality, and visual feasibility. | |
| - All added objects or modifications must align with the logic and style of the edited input image's overall scene. | |
| ## 2. TASK TYPE HANDLING RULES | |
| ### 1. ADD, DELETE, REPLACE TASKS | |
| - If the instruction is clear (already includes task type, target entity, position, quantity, attributes), preserve the original intent and only refine the grammar. | |
| - If the description is vague, supplement with minimal but sufficient details (category, color, size, orientation, position, etc.). For example: | |
| > Original: "add an animal" | |
| > Rewritten: "add a light-gray cat in the bottom-right corner, sitting and facing the camera" | |
| - Remove meaningless instructions: e.g., "add 0 objects" should be ignored or flagged as invalid. | |
| - For replacement tasks, specify "replace Y with X" and briefly describe the key visual features of X. | |
| ### 2. TEXT EDITING TASKS | |
| - All text content must be enclosed in English double quotes " ". Do not translate or alter the original language of the text, and do not change the capitalization. | |
| - **For text replacement tasks, always use the fixed template:** | |
| - Replace "XX" to "YY". | |
| - Replace the XX bounding box to "YY". | |
| - If the user does not specify text content, infer and add concise text based on the instruction and the input image's context. For example: | |
| > Original: "add a line of text" (poster) | |
| > Rewritten: "add text "Limited Edition" at the top center with slight shadow" | |
| - Specify text position, color, and layout in a concise way. | |
| ### 3. HUMAN EDITING TASKS | |
| - Maintain the person's core visual consistency (ethnicity, gender, age, hairstyle, expression, outfit, etc.). | |
| - If modifying appearance (e.g., clothes, hairstyle), ensure the new element is consistent with the original style. | |
| - **For expression changes, they must be natural and subtle, never exaggerated.** - If deletion is not specifically emphasized, the most important subject in the original image (e.g., a person, an animal) should be preserved. | |
| - For background change tasks, emphasize maintaining subject consistency at first. | |
| - Example: | |
| > Original: "change the person's hat" | |
| > Rewritten: "replace the man's hat with a dark brown beret; keep smile, short hair, and gray jacket unchanged" | |
| ### 4. STYLE TRANSFORMATION OR ENHANCEMENT TASKS | |
| - If a style is specified, describe it concisely with key visual traits. For example: | |
| > Original: "disco style" | |
| > Rewritten: "1970s Disco: flashing lights, disco ball, mirrored walls, colorful tones" | |
| - If the instruction says "use reference style" or "keep current style," analyze the input image, extract main features (color, composition, texture, lighting, art style), and integrate them concisely. | |
| - **For coloring tasks, including restoring old photos, always use the fixed template:** "restore old photograph, remove scratches, reduce noise, enhance details, high resolution, realistic, natural skin tones, clear facial features, no distortion, vintage photo restoration" | |
| - If there are other changes, place the style description at the end. | |
| ## 3. RATIONALITY AND LOGIC CHECKS | |
| - Resolve contradictory instructions: e.g., "remove all trees but keep all trees" should be logically corrected. | |
| - Add missing key information: if position is unspecified, choose a reasonable area based on composition (near subject, empty space, center/edges). | |
| # OUTPUT FORMAT | |
| Return only the rewritten instruction text directly, without JSON formatting or any other wrapper. | |
| ''' | |
| full_prompt = f"{system_prompt}\n\nUser input: {prompt}\n\nRewritten prompt:" | |
| return polish_prompt_hf(full_prompt, system_prompt) | |
| # --- model loading --- | |
| dtype = torch.bfloat16 | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| # Scheduler configuration for lightning | |
| scheduler_config = { | |
| "base_image_seq_len": 256, | |
| "base_shift": math.log(3), | |
| "invert_sigmas": False, # Corrected boolean case | |
| "max_image_seq_len": 8192, | |
| "max_shift": math.log(3), | |
| "num_train_timesteps": 1000, | |
| "shift": 1.0, | |
| "shift_terminal": None, # Corrected None case | |
| "stochastic_sampling": False, # Corrected boolean case | |
| "time_shift_type": "exponential", | |
| "use_beta_sigmas": False, # Corrected boolean case | |
| "use_dynamic_shifting": True, # Corrected boolean case | |
| "use_exponential_sigmas": False, # Corrected boolean case | |
| "use_karras_sigmas": False, # Corrected boolean case | |
| } | |
| # Initialize scheduler with lightning config | |
| scheduler = FlowMatchEulerDiscreteScheduler.from_config(scheduler_config) | |
| # Load the edit pipeline with lightning scheduler | |
| pipe = QwenImageEditPipelineCustom.from_pretrained( # Corrected class name | |
| "qwen/qwen-image-edit", | |
| scheduler=scheduler, | |
| torch_dtype=dtype | |
| ).to(device) | |
| # Load lightning LoRA weights for acceleration | |
| try: | |
| pipe.load_lora_weights( | |
| "lightx2v/qwen-image-lightning", | |
| weight_name="qwen-image-lightning-8steps-v1.1.safetensors" | |
| ) | |
| pipe.fuse_lora() | |
| print("Successfully loaded lightning LoRA weights") | |
| except Exception as e: # Catch broader exception | |
| print(f"Warning: Could not load lightning LoRA weights: {e}") | |
| print("Continuing with base model...") | |
| # Apply the same optimizations from the first version | |
| pipe.transformer.__class__ = QwenImageTransformer2DModel # Corrected class name | |
| pipe.transformer.set_attn_processor(QwenDoubleStreamAttnProcessorFA3()) # Corrected class name | |
| # --- Ahead-of-time compilation --- | |
| # It's important that the dummy image for optimization has the expected dimensions (e.g., 1024x1024) | |
| optimize_pipeline_(pipe, image=Image.new("RGB", (1024, 1024)), prompt="prompt") | |
| # --- UI constants and helpers --- | |
| max_seed = np.iinfo(np.int32).max | |
| # --- Main inference function --- | |
| spaces.gpu(duration=60) | |
| def infer( | |
| image, | |
| prompt, | |
| seed=42, | |
| randomize_seed=False, # Corrected boolean case | |
| true_guidance_scale=1.0, | |
| num_inference_steps=8, # Default to 8 steps for fast inference | |
| rewrite_prompt=True, # Corrected boolean case | |
| output_size="Original (1024x1024)", # New parameter for output size | |
| progress=gr.Progress(track_tqdm=True), # Corrected class name | |
| ): | |
| """ | |
| Generates an edited image using the Qwen-Image-Edit pipeline with lightning acceleration, | |
| and optionally resizes the output. | |
| """ | |
| negative_prompt = " " | |
| if randomize_seed: | |
| seed = random.randint(0, max_seed) | |
| generator = torch.Generator(device=device).manual_seed(seed) # Corrected class name | |
| print(f"Original prompt: '{prompt}'") | |
| print(f"Negative prompt: '{negative_prompt}'") | |
| print(f"Seed: {seed}, Steps: {num_inference_steps}, Guidance: {true_guidance_scale}") | |
| if rewrite_prompt: | |
| prompt = polish_prompt(prompt, image) | |
| print(f"Rewritten prompt: {prompt}") | |
| try: | |
| images = pipe( | |
| image, | |
| prompt=prompt, | |
| negative_prompt=negative_prompt, | |
| num_inference_steps=num_inference_steps, | |
| generator=generator, | |
| true_cfg_scale=true_guidance_scale, | |
| num_images_per_prompt=1 | |
| ).images | |
| output_image = images[0] | |
| # Post-processing: Resize if a different output size is selected | |
| if output_size != "Original (1024x1024)": | |
| try: | |
| if output_size == "Small (512x512)": | |
| target_size = (512, 512) | |
| elif output_size == "Medium (768x768)": | |
| target_size = (768, 768) | |
| elif output_size == "Large (1536x1536)": | |
| target_size = (1536, 1536) | |
| else: # Custom size, parse it from "Custom (WxH)" | |
| width, height = map(int, output_size.split('(')[1][:-1].split('x')) | |
| target_size = (width, height) | |
| output_image = output_image.resize(target_size, Image.LANCZOS) # Use LANCZOS for high quality down/upscaling | |
| print(f"Resized output image to: {target_size[0]}x{target_size[1]}") | |
| except Exception as resize_e: | |
| print(f"Warning: Could not resize image to {output_size}: {resize_e}") | |
| print("Returning original size image.") | |
| return output_image, seed | |
| except Exception as e: | |
| print(f"Error during inference: {e}") | |
| raise e | |
| # --- Examples and UI layout --- | |
| examples = [ | |
| # Example for demonstration, replace with actual image paths | |
| # Ensure these paths are valid if running locally, or adjust for Hugging Face Spaces | |
| [Image.new("RGB", (1024, 1024), color = 'red'), "Change the color to blue"], | |
| [Image.new("RGB", (1024, 1024), color = 'green'), "Add a fluffy white cat sitting in the center"], | |
| ] | |
| css = """ | |
| #col-container { | |
| margin: 0 auto; | |
| max-width: 1024px; | |
| } | |
| #logo-title { | |
| text-align: center; | |
| } | |
| #logo-title img { | |
| width: 400px; | |
| } | |
| #edit_text{margin-top: -62px !important} | |
| """ | |
| with gr.Blocks(css=css) as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.HTML(""" | |
| <div id="logo-title"> | |
| <img src="https://qianwen-res.oss-cn-beijing.aliyuncs.com/qwen-image/qwen_image_edit_logo.png" alt="Qwen-Image Edit Logo" width="400" style="display: block; margin: 0 auto;"> | |
| <h2 style="font-style: italic;color: #5b47d1;margin-top: -27px !important;margin-left: 96px">Fast, 8-steps with Lightning LoRA</h2> | |
| </div> | |
| """) | |
| gr.Markdown(""" | |
| [Learn more](https://github.com/QwenLM/Qwen-Image) about the Qwen-Image series. | |
| This demo uses the [Qwen-Image-Lightning](https://huggingface.co/lightx2v/qwen-image-lightning) LoRA for accelerated inference. | |
| Try on [Qwen Chat](https://chat.qwen.ai/), or [download model](https://huggingface.co/Qwen/Qwen-Image-Edit) to run locally with ComfyUI or Diffusers. | |
| """) | |
| with gr.Row(): | |
| with gr.Column(): | |
| input_image = gr.Image( | |
| label="Input Image", | |
| show_label=True, | |
| type="pil" | |
| ) | |
| result = gr.Image( | |
| label="Result", | |
| show_label=True, | |
| type="pil" | |
| ) | |
| with gr.Row(): | |
| prompt = gr.Text( | |
| label="Edit Instruction", | |
| show_label=False, | |
| placeholder="Describe the edit instruction (e.g., 'replace the background with a sunset', 'add a red hat', 'remove the person')", | |
| container=False, | |
| ) | |
| run_button = gr.Button("Edit!", variant="primary") | |
| with gr.Accordion("Advanced Settings", open=False): | |
| seed = gr.Slider( | |
| label="Seed", | |
| minimum=0, | |
| maximum=max_seed, | |
| step=1, | |
| value=0, | |
| ) | |
| randomize_seed = gr.Checkbox(label="Randomize Seed", value=True) | |
| with gr.Row(): | |
| true_guidance_scale = gr.Slider( | |
| label="True Guidance Scale", | |
| minimum=1.0, | |
| maximum=10.0, | |
| step=0.1, | |
| value=1.0 | |
| ) | |
| num_inference_steps = gr.Slider( | |
| label="Number of Inference Steps", | |
| minimum=4, | |
| maximum=28, | |
| step=1, | |
| value=8 | |
| ) | |
| rewrite_prompt = gr.Checkbox( | |
| label="Enhance Prompt (using HF Inference)", | |
| value=True | |
| ) | |
| # New dropdown for output image size | |
| output_size = gr.Dropdown( | |
| label="Output Image Size", | |
| choices=["Original (1024x1024)", "Small (512x512)", "Medium (768x768)", "Large (1536x1536)"], | |
| value="Original (1024x1024)" | |
| ) | |
| gr.Examples(examples=examples, inputs=[input_image, prompt], outputs=[result, seed], fn=infer, cache_examples=False) # Changed to use the new example inputs/outputs | |
| gr.on( | |
| triggers=[run_button.click, prompt.submit], | |
| fn=infer, | |
| inputs=[ | |
| input_image, | |
| prompt, | |
| seed, | |
| randomize_seed, | |
| true_guidance_scale, | |
| num_inference_steps, | |
| rewrite_prompt, | |
| output_size, # Added output_size to inputs | |
| ], | |
| outputs=[result, seed], | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch() | |