Update app.py

app.py

@@ -3,318 +3,205 @@ import numpy as np
 import random
 import torch
 import spaces
-
 from PIL import Image
 from diffusers import QwenImageEditPipeline
-
 import os
 import base64
 import json
 from huggingface_hub import InferenceClient
 
-SYSTEM_PROMPT = '''
-# Edit Instruction Rewriter
-You are a professional edit instruction rewriter. Your task is to generate a precise, concise, and visually achievable professional-level edit instruction based on the user-provided instruction and the image to be edited.  
-
-Please strictly follow the rewriting rules below:
-
-## 1. General Principles
-- Keep the rewritten prompt **concise**. Avoid overly long sentences and reduce unnecessary descriptive language.  
-- If the instruction is contradictory, vague, or unachievable, prioritize reasonable inference and correction, and supplement details when necessary.  
-- Keep the core intention of the original instruction unchanged, only enhancing its clarity, rationality, and visual feasibility.  
-- All added objects or modifications must align with the logic and style of the edited input image’s overall scene.  
-
-## 2. Task Type Handling Rules
-### 1. Add, Delete, Replace Tasks
-- If the instruction is clear (already includes task type, target entity, position, quantity, attributes), preserve the original intent and only refine the grammar.  
-- If the description is vague, supplement with minimal but sufficient details (category, color, size, orientation, position, etc.). For example:  
-    > Original: "Add an animal"  
-    > Rewritten: "Add a light-gray cat in the bottom-right corner, sitting and facing the camera"  
-- Remove meaningless instructions: e.g., "Add 0 objects" should be ignored or flagged as invalid.  
-- For replacement tasks, specify "Replace Y with X" and briefly describe the key visual features of X.  
-
-### 2. Text Editing Tasks
-- All text content must be enclosed in English double quotes `" "`. Do not translate or alter the original language of the text, and do not change the capitalization.  
-- **For text replacement tasks, always use the fixed template:**
-    - `Replace "xx" to "yy"`.  
-    - `Replace the xx bounding box to "yy"`.  
-- If the user does not specify text content, infer and add concise text based on the instruction and the input image’s context. For example:  
-    > Original: "Add a line of text" (poster)  
-    > Rewritten: "Add text \"LIMITED EDITION\" at the top center with slight shadow"  
-- Specify text position, color, and layout in a concise way.  
-
-### 3. Human Editing Tasks
-- Maintain the person’s core visual consistency (ethnicity, gender, age, hairstyle, expression, outfit, etc.).  
-- If modifying appearance (e.g., clothes, hairstyle), ensure the new element is consistent with the original style.  
-- **For expression changes, they must be natural and subtle, never exaggerated.**  
-- If deletion is not specifically emphasized, the most important subject in the original image (e.g., a person, an animal) should be preserved.
-    - For background change tasks, emphasize maintaining subject consistency at first.  
-- Example:  
-    > Original: "Change the person’s hat"  
-    > Rewritten: "Replace the man’s hat with a dark brown beret; keep smile, short hair, and gray jacket unchanged"  
-
-### 4. Style Transformation or Enhancement Tasks
-- If a style is specified, describe it concisely with key visual traits. For example:  
-    > Original: "Disco style"  
-    > Rewritten: "1970s disco: flashing lights, disco ball, mirrored walls, colorful tones"  
-- If the instruction says "use reference style" or "keep current style," analyze the input image, extract main features (color, composition, texture, lighting, art style), and integrate them concisely.  
-- **For coloring tasks, including restoring old photos, always use the fixed template:** "Restore old photograph, remove scratches, reduce noise, enhance details, high resolution, realistic, natural skin tones, clear facial features, no distortion, vintage photo restoration"  
-- If there are other changes, place the style description at the end.
-
-## 3. Rationality and Logic Checks
-- Resolve contradictory instructions: e.g., "Remove all trees but keep all trees" should be logically corrected.  
-- Add missing key information: if position is unspecified, choose a reasonable area based on composition (near subject, empty space, center/edges).  
-
-# Output Format Example
-```json
-{
-   "Rewritten": "..."
-}
-'''
-
-# def polish_prompt(original_prompt, system_prompt):
-#     """
-#     Rewrites the prompt using a Hugging Face InferenceClient.
-#     """
-#     # Ensure HF_TOKEN is set
-#     api_key = os.environ.get("HF_TOKEN")
-#     if not api_key:
-#         raise EnvironmentError("HF_TOKEN is not set. Please set it in your environment.")
-
-#     # Initialize the client
-#     client = InferenceClient(
-#         provider="cerebras",
-#         api_key=api_key,
-#     )
-
-#     # Format the messages for the chat completions API
-#     messages = [
-#         {"role": "system", "content": system_prompt},
-#         {"role": "user", "content": original_prompt}
-#     ]
-
-#     try:
-#         # Call the API
-#         completion = client.chat.completions.create(
-#             model="Qwen/Qwen3-235B-A22B-Instruct-2507",
-#             messages=messages,
-#         )
-#         polished_prompt = completion.choices[0].message.content
-#         polished_prompt = polished_prompt.strip().replace("\n", " ")
-#         return polished_prompt
-#     except Exception as e:
-#         print(f"Error during API call to Hugging Face: {e}")
-#         # Fallback to original prompt if enhancement fails
-#         return original_prompt
-
-
-def polish_prompt(prompt, system_prompt):
+def get_caption_language(prompt):
+    """Detects if the prompt contains Chinese characters."""
+    ranges = [
+        ('\u4e00', '\u9fff'),  # CJK Unified Ideographs
+    ]
+    for char in prompt:
+        if any(start <= char <= end for start, end in ranges):
+            return 'zh'
+    return 'en'
+
+def polish_prompt(original_prompt, system_prompt, hf_token):
     """
     Rewrites the prompt using a Hugging Face InferenceClient.
+    Requires user-provided HF token for API access.
     """
-    # Ensure HF_TOKEN is set
-    api_key = os.environ.get("HF_TOKEN")
-    if not api_key:
-        raise EnvironmentError("HF_TOKEN is not set. Please set it in your environment.")
-    # Initialize the client
+    if not hf_token or not hf_token.strip():
+        gr.Warning("HF Token is required for prompt rewriting but was not provided!")
+        return original_prompt
+
     client = InferenceClient(
         provider="cerebras",
-        api_key=api_key,
+        api_key=hf_token,
     )
-    
-    prompt = f"{system_prompt}\n\nUser Input: {prompt}\n\nRewritten Prompt:"
-    success=False
-    while not success:
-        try:
-            completion = client.chat.completions.create(
-                model="Qwen/Qwen3-235B-A22B-Instruct-2507",
-                messages=messages,
-            )
-            # print(f"Result: {result}")
-            # print(f"Polished Prompt: {polished_prompt}")
-            if isinstance(result, str):
-                result = result.replace('```json','')
-                result = result.replace('```','')
-                result = json.loads(result)
-            else:
-                result = json.loads(result)
-
-            polished_prompt = result['Rewritten']
-            polished_prompt = polished_prompt.strip()
-            polished_prompt = polished_prompt.replace("\n", " ")
-            success = True
-        except Exception as e:
-            print(f"Error during API call to Hugging Face: {e}")
-            # Fallback to original prompt if enhancement fails
-            return prompt
-
-
-def encode_image(pil_image):
-    import io
-    buffered = io.BytesIO()
-    pil_image.save(buffered, format="PNG")
-    return base64.b64encode(buffered.getvalue()).decode("utf-8")
-
-
-
-
-def api(prompt, img_list, model="qwen-vl-max-latest", kwargs={}):
-    import dashscope
-    api_key = os.environ.get('DASH_API_KEY')
-    if not api_key:
-        raise EnvironmentError("DASH_API_KEY is not set")
-    assert model in ["qwen-vl-max-latest"], f"Not implemented model {model}"
-    sys_promot = "you are a helpful assistant, you should provide useful answers to users."
     messages = [
-        {"role": "system", "content": sys_promot},
-        {"role": "user", "content": []}]
-    for img in img_list:
-        messages[1]["content"].append(
-            {"image": f"data:image/png;base64,{encode_image(img)}"})
-    messages[1]["content"].append({"text": f"{prompt}"})
-
-    response_format = kwargs.get('response_format', None)
-
-    response = dashscope.MultiModalConversation.call(
-        api_key=api_key,
-        model=model, # For example, use qwen-plus here. You can change the model name as needed. Model list: https://help.aliyun.com/zh/model-studio/getting-started/models
-        messages=messages,
-        result_format='message',
-        response_format=response_format,
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": original_prompt}
+    ]
+    try:
+        completion = client.chat.completions.create(
+            model="Qwen/Qwen3-235B-A22B-Instruct-2507",
+            messages=messages,
+            max_tokens=2000,
         )
+        polished_prompt = completion.choices[0].message.content
+        polished_prompt = polished_prompt.strip().replace("\n", " ")
+        return polished_prompt
+    except Exception as e:
+        print(f"Error during Hugging Face API call: {e}")
+        gr.Warning("Failed to rewrite prompt. Using original.")
+        return original_prompt
+
+SYSTEM_PROMPT_EDIT = '''
+# Edit Instruction Rewriter
+You are a professional edit instruction rewriter. Your task is to generate a precise, concise, and visually achievable instruction based on the user's intent and the input image.
+## 1. General Principles
+- Keep the rewritten instruction **concise** and clear.
+- Avoid contradictions, vagueness, or unachievable instructions.
+- Maintain the core logic of the original instruction; only enhance clarity and feasibility.
+- Ensure new added elements or modifications align with the image's original context and art style.
+## 2. Task Types
+### Add, Delete, Replace:
+- When the input is detailed, only refine grammar and clarity.
+- For vague instructions, infer minimal but sufficient details.
+- For replacement, use the format: `"Replace X with Y"`.
+### Text Editing (e.g., text replacement):
+- Enclose text content in quotes, e.g., `Replace "abc" with "xyz"`.
+- Preserving the original structure and language—**do not translate** or alter style.
+### Human Editing (e.g., change a person’s face/hair):
+- Preserve core visual identity (gender, ethnic features).
+- Describe expressions in subtle and natural terms.
+- Maintain key clothing or styling details unless explicitly replaced.
+### Style Transformation:
+- If a style is specified, e.g., `Disco style`, rewrite it to encapsulate the essential visual traits.
+- Use a fixed template for **coloring/restoration**:  
+  `"Restore old photograph, remove scratches, reduce noise, enhance details, high resolution, realistic, natural skin tones, clear facial features, no distortion, vintage photo restoration"`  
+  if applicable.
+## 4. Output Format
+Please provide the rewritten instruction in a clean `json` format as:
+{
+  "Rewritten": "..."
+}
+'''
 
-    if response.status_code == 200:
-        return response.output.choices[0].message.content[0]['text']
-    else:
-        raise Exception(f'Failed to post: {response}')
-
-# --- Model Loading ---
 dtype = torch.bfloat16
 device = "cuda" if torch.cuda.is_available() else "cpu"
-
-# Load the model pipeline
 pipe = QwenImageEditPipeline.from_pretrained("Qwen/Qwen-Image-Edit", torch_dtype=dtype).to(device)
+pipe.load_lora_weights(
+    "lightx2v/Qwen-Image-Edit-Lightning",
+    weight_name="Qwen-Image-Edit-Lightning-8steps-V1.1.safetensors"
+)
+pipe.fuse_lora()
 
-# --- UI Constants and Helpers ---
-MAX_SEED = np.iinfo(np.int32).max
-
-# --- Main Inference Function (with hardcoded negative prompt) ---
-@spaces.GPU()
+@spaces.GPU(duration=60)
 def infer(
     image,
     prompt,
     seed=42,
     randomize_seed=False,
     true_guidance_scale=1.0,
-    num_inference_steps=20,
+    num_inference_steps=8,
     rewrite_prompt=False,
+    hf_token="",
     num_images_per_prompt=1,
     progress=gr.Progress(track_tqdm=True),
 ):
     """
-    Generates an image using the local Qwen-Image diffusers pipeline.
+    Requires user-provided HF token for prompt rewriting.
     """
-    # Hardcode the negative prompt as requested
     negative_prompt = " "
-    
     if randomize_seed:
         seed = random.randint(0, MAX_SEED)
-
-    # Set up the generator for reproducibility
     generator = torch.Generator(device=device).manual_seed(seed)
     
-    print(f"Calling pipeline with prompt: '{prompt}'")
-    print(f"Negative Prompt: '{negative_prompt}'")
-    print(f"Seed: {seed}, Steps: {num_inference_steps}, Guidance: {true_guidance_scale}")
     if rewrite_prompt:
-        prompt = polish_prompt(prompt, image)
-        print(f"Rewritten Prompt: {prompt}")
-
-    # Generate the image
-    image = pipe(
+        lang = get_caption_language(prompt)
+        system_prompt = SYSTEM_PROMPT_EDIT
+        polished_prompt = polish_prompt(prompt, system_prompt, hf_token)
+        print(f"Rewritten Prompt: {polished_prompt}")
+        prompt = polished_prompt
+    
+    edited_images = pipe(
         image,
         prompt=prompt,
         negative_prompt=negative_prompt,
         num_inference_steps=num_inference_steps,
         generator=generator,
         true_cfg_scale=true_guidance_scale,
-        num_images_per_prompt=num_images_per_prompt
+        num_images_per_prompt=num_images_per_prompt,
     ).images
+    
+    return edited_images, seed
 
-    return image, seed
-
-# --- Examples and UI Layout ---
-examples = []
-
-css = """
-#col-container {
-    margin: 0 auto;
-    max-width: 1024px;
-}
-#edit_text{margin-top: -62px !important}
-"""
-
-with gr.Blocks(css=css) as demo:
-    with gr.Column(elem_id="col-container"):
-        gr.HTML('<img src="https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/qwen_image_edit_logo.png" alt="Qwen-Image Logo" width="400" style="display: block; margin: 0 auto;">')
-        gr.Markdown("[Learn more](https://github.com/QwenLM/Qwen-Image) about the Qwen-Image series. Try on [Qwen Chat](https://chat.qwen.ai/), or [download model](https://huggingface.co/Qwen/Qwen-Image-Edit) to run locally with ComfyUI or diffusers.")
-        with gr.Row():
-            with gr.Column():
-                input_image = gr.Image(label="Input Image", show_label=False, type="pil")
-
-            # result = gr.Image(label="Result", show_label=False, type="pil")
-            result = gr.Gallery(label="Result", show_label=False, type="pil")
+MAX_SEED = np.iinfo(np.int32).max
+examples = [
+    "Replace the cat with a friendly golden retriever. Make it look happier, and add more background details.",
+    "Add text 'Qwen - AI for image editing' in Chinese at the bottom center with a small shadow.",
+    "Change the style to 1970s vintage, add old photo effect, restore any scratches on the wall or window.",
+    "Remove the blue sky and replace it with a dark night cityscape.",
+    """Replace "Qwen" with "通义" in the Image. Ensure Chinese font is used for "通义" and position it to the top left with a light heading-style font."""
+]
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Qwen-Image-Edit with Prompt Enhancement")
+    gr.Markdown("⚠️ **Prompt rewriting requires your own [Hugging Face token](https://huggingface.co/settings/tokens)**")
+    
+    with gr.Column():
+        input_image = gr.Image(label="Input Image", type="pil")
+        prompt = gr.Text(label="Edit Instruction", placeholder="e.g. Add a dog to the right side.")
+        run_button = gr.Button("Edit", variant="primary")
+        result = gr.Gallery(label="Output Images", show_label=False)
+    
+    with gr.Accordion("Advanced Settings", open=False):
+        seed = gr.Slider(
+            label="Seed",
+            minimum=0,
+            maximum=MAX_SEED,
+            step=1,
+            value=0
+        )
+        randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
+        
         with gr.Row():
-            prompt = gr.Text(
-                    label="Prompt",
-                    show_label=False,
-                    placeholder="describe the edit instruction",
-                    container=False,
+            true_guidance_scale = gr.Slider(
+                label="True Guidance Scale",
+                minimum=1.0,
+                maximum=5.0,
+                step=0.1,
+                value=4.0
             )
-            run_button = gr.Button("Edit!", variant="primary")
-
-        with gr.Accordion("Advanced Settings", open=False):
-            # Negative prompt UI element is removed here
-
-            seed = gr.Slider(
-                label="Seed",
-                minimum=0,
-                maximum=MAX_SEED,
+            num_inference_steps = gr.Slider(
+                label="Inference Steps (Fast 8-step mode)",
+                minimum=4,
+                maximum=8,
                 step=1,
-                value=0,
+                value=8
+            )
+            num_images_per_prompt = gr.Slider(
+                label="Images per Prompt",
+                minimum=1,
+                maximum=4,
+                step=1,
+                value=1
+            )
+        
+        with gr.Group():
+            rewrite_toggle = gr.Checkbox(label="Use Prompt Rewriter (Requires HF Token)", value=False, interactive=True)
+            hf_token_input = gr.Textbox(
+                label="Your Hugging Face Token",
+                type="password",
+                placeholder="hf_xxxxxxxxxxxxxxxx",
+                visible=False,
+                info="Required for prompt rewriting - get yours from [Hugging Face settings](https://huggingface.co/settings/tokens)"
+            )
+            
+            def toggle_token_visibility(checked):
+                return gr.update(visible=checked)
+            
+            rewrite_toggle.change(
+                toggle_token_visibility,
+                inputs=[rewrite_toggle],
+                outputs=[hf_token_input]
             )
-
-            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
-
-            with gr.Row():
-
-                true_guidance_scale = gr.Slider(
-                    label="True guidance scale",
-                    minimum=1.0,
-                    maximum=10.0,
-                    step=0.1,
-                    value=4.0
-                )
-
-                num_inference_steps = gr.Slider(
-                    label="Number of inference steps",
-                    minimum=1,
-                    maximum=50,
-                    step=1,
-                    value=50,
-                )
-                
-                num_images_per_prompt = gr.Slider(
-                    label="Number of images per prompt",
-                    minimum=1,
-                    maximum=4,
-                    step=1,
-                    value=1,
-                )
-                
-                rewrite_prompt = gr.Checkbox(label="Rewrite prompt", value=False, visible=False)
-
-        # gr.Examples(examples=examples, inputs=[prompt], outputs=[result, seed], fn=infer, cache_examples=False)
 
     gr.on(
         triggers=[run_button.click, prompt.submit],
@@ -326,8 +213,9 @@ with gr.Blocks(css=css) as demo:
             randomize_seed,
             true_guidance_scale,
             num_inference_steps,
-            rewrite_prompt,
-            num_images_per_prompt,
+            rewrite_toggle,
+            hf_token_input,
+            num_images_per_prompt
         ],
         outputs=[result, seed],
     )