Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,23 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+boy.jpg filter=lfs diff=lfs merge=lfs -text
+dog.jpg filter=lfs diff=lfs merge=lfs -text
+dragon.jpg filter=lfs diff=lfs merge=lfs -text
+dump.jpg filter=lfs diff=lfs merge=lfs -text
+egg.jpg filter=lfs diff=lfs merge=lfs -text
+elephant.jpg filter=lfs diff=lfs merge=lfs -text
+family.jpg filter=lfs diff=lfs merge=lfs -text
+fold.jpg filter=lfs diff=lfs merge=lfs -text
+fruit.jpg filter=lfs diff=lfs merge=lfs -text
+girl.jpg filter=lfs diff=lfs merge=lfs -text
+girl2.jpg filter=lfs diff=lfs merge=lfs -text
+laion.jpg filter=lfs diff=lfs merge=lfs -text
+lizard.jpg filter=lfs diff=lfs merge=lfs -text
+pole.jpg filter=lfs diff=lfs merge=lfs -text
+robot.jpg filter=lfs diff=lfs merge=lfs -text
+robot2.jpg filter=lfs diff=lfs merge=lfs -text
+robot3.jpg filter=lfs diff=lfs merge=lfs -text
+sky.jpg filter=lfs diff=lfs merge=lfs -text
+whale.jpg filter=lfs diff=lfs merge=lfs -text
+wood.jpg filter=lfs diff=lfs merge=lfs -text

aligner.py

@@ -1,6 +1,6 @@
 import torch
 from torch import nn
-#from refiner import Qwen2Connector
+from refiner import Qwen2Connector
 
 import torch
 import torch.nn as nn
@@ -425,11 +425,11 @@ class ConceptAligner(nn.Module):
         empty_pooled_clip = torch.load('empty_pooled_clip.pt', map_location='cpu')
         self.register_buffer('empty_pooled_clip', empty_pooled_clip)
 
-        test_eps = torch.randn([1, 300, per_dim], dtype=torch.bfloat16).to('cpu')*0.7
+        test_eps = torch.randn([1, 512, per_dim], dtype=torch.bfloat16).to('cpu')*0.7
         self.register_buffer('test_eps', test_eps)
 
-        self.init_proj = nn.Sequential(nn.Linear(768, 300*16), nn.SiLU())
-        self.proj = nn.Sequential(nn.Linear(16, 1024), nn.SiLU(),
+        self.init_proj = nn.Sequential(nn.Linear(768, 512*8), nn.SiLU())
+        self.proj = nn.Sequential(nn.Linear(8, 1024), nn.SiLU(),
                                   nn.Linear(1024, 1024), nn.SiLU())
         self.text_proj = nn.Sequential(nn.Linear(4096, 1024), nn.SiLU(),
                                    nn.Linear(1024, 1024), nn.SiLU())
@@ -465,7 +465,7 @@ class ConceptAligner(nn.Module):
         device = text_features.device
 
         if image_features is not None:
-            visual_hidden = self.proj(self.init_proj(image_features).view(len(image_features), 300, -1))
+            visual_hidden = self.proj(self.init_proj(image_features).view(len(image_features), text_features.size(1), -1))
             text_hidden = self.text_proj(text_features.detach())
             hidden = visual_hidden - text_hidden
             mu = self.proj_mu(hidden)
@@ -510,13 +510,14 @@ if __name__ == '__main__':
 
     dim = 4096
     num_heads = 32
+
     dtype = torch.bfloat16
     model = ConceptAligner().to('cuda').to(dtype)
-    x = torch.randn([5, 300, dim]).to('cuda').to(dtype)
-    y = torch.randn([5, 300, dim]).to('cuda').to(dtype)
+    x = torch.randn([5, 512, dim]).to('cuda').to(dtype)
+    y = torch.randn([5, 512, dim]).to('cuda').to(dtype)
     i = torch.randn([5,768]).to('cuda').to(dtype)
     y[1] = y[0]
-    m = torch.ones([5, 300]).to('cuda').to(dtype)
+    m = torch.ones([5, 512]).to('cuda').to(dtype)
     m[:3,:128] = 0
     prompt_embeds, _, pooled_prompt_embeds, text_ids, aux_info = model(x, i)
     print(prompt_embeds.shape, pooled_prompt_embeds.shape, text_ids.shape)
@@ -524,14 +525,15 @@ if __name__ == '__main__':
     for k in aux_info:
         print(k, aux_info[k].shape, aux_info[k].min(), aux_info[k].max(), aux_info[k].mean())
 
-    from text_encoder import LoraT5Embedder
+    from text_encoder import JacobianLoraT5Embedder
     from datasets import load_dataset
-    dataset = load_dataset("facebook/emu_edit_test_set", split='validation[:200]')
-    item = dataset[0:4]
-    another_item = dataset[0:4]
+    #dataset = load_dataset("facebook/emu_edit_test_set", split='validation[:200]')
+    #item = dataset[0:4]
+    #another_item = dataset[0:4]
+    image = Image.open('example512.jpg').convert('RGB')
     from diffusers.models.normalization import RMSNorm
     clip_processor = AutoProcessor.from_pretrained("./clip-vit-large-patch14")
-    clip_images = clip_processor(images=item['image'], return_tensors="pt").pixel_values.to('cuda:0').to(dtype)
+    clip_images = clip_processor(images=image, return_tensors="pt").pixel_values.to('cuda:0').to(dtype).repeat(4,1,1,1)
     texts = []
     texts.append("""A heartwarming 3D rendered scene of
     an elderly farmer and a tiny orange
@@ -567,8 +569,8 @@ if __name__ == '__main__':
     texts.append(
         """In the image, a fluffy white cat sits peacefully on a windowsill surrounded by potted green plants. Sunlight filters through sheer white curtains, casting soft golden patterns across its fur. The window reveals a clear blue sky outside, with the silhouettes of trees swaying gently in the distance. The cat’s posture is calm and elegant, its tail curled neatly around its paws. The atmosphere is serene and homey, capturing a tranquil afternoon moment of quiet observation.""")
 
-    text_encoder = LoraT5Embedder(device='cuda').to(dtype)
-    text_features, _, _, _, image_features, _ = text_encoder(texts, clip_images)
+    text_encoder = JacobianLoraT5Embedder(device='cuda', max_length=512, num_jacobian_samples=1).to(torch.bfloat16)
+    text_features, image_features, _, _ = text_encoder(texts, clip_images)
     print(text_features.shape, image_features.shape, ' >>>>>>>>> text input')
     images = []
     pipe = FluxPipeline.from_pretrained("./FLUX.1-dev", dtype=torch.bfloat16, text_encoder=None).to(torch.bfloat16)
@@ -576,7 +578,7 @@ if __name__ == '__main__':
 
     for txt_feat, img_feat in zip(text_features, image_features):
 
-        prompt_embeds, _, pooled_prompt_embeds, text_ids, aux_info = model(txt_feat.unsqueeze(0), img_feat.unsqueeze(0))
+        prompt_embeds, _, pooled_prompt_embeds, text_ids, aux_info = model(txt_feat.unsqueeze(0), None)
         image = pipe(
         prompt_embeds=prompt_embeds,
         pooled_prompt_embeds=pooled_prompt_embeds,
@@ -816,6 +818,7 @@ glow reminiscent of the glow of the moon. HD,
     # for (start_dim, end_dim) in [(0,4096), (1024,4096), (2048, 4096), (1024, 2048)]:
 
 
+
     for emb in ['floral', 'golden']:
         for temp in [2.5]:
             for thr in [-1, 0.5, 0.75, 0.85, 0.95]:

app.py

@@ -180,7 +180,7 @@ def reset_history():
 css = """
 #col-container {
     margin: 0 auto;
-    max-width: 1400px;
+    max-width: 1200px;
 }
 """
 
@@ -194,15 +194,17 @@ with gr.Blocks(css=css, title="ConceptAligner") as demo:
 
         with gr.Row():
             with gr.Column(scale=1):
-                prompt_input = gr.Textbox(
+                prompt_input = gr.Text(
                     label="Prompt",
-                    lines=8,
-                    placeholder="Describe your image in detail...",
+                    show_label=False,
+                    max_lines=3,
+                    placeholder="Describe your image...",
+                    container=False,
                 )
 
                 with gr.Row():
                     generate_btn = gr.Button("✨ Generate", variant="primary", scale=3)
-                    reset_btn = gr.Button("🔄 Clear History", variant="secondary", scale=1)
+                    reset_btn = gr.Button("🔄 Clear", variant="secondary", scale=1)
 
                 with gr.Accordion("⚙️ Settings", open=False):
                     seed = gr.Slider(
@@ -214,21 +216,19 @@ with gr.Blocks(css=css, title="ConceptAligner") as demo:
                     )
 
                     guidance_scale = gr.Slider(
-                        label="Guidance Scale",
+                        label="Creativity Level",
                         minimum=1.0,
                         maximum=10.0,
                         step=0.5,
                         value=3.5,
-                        info="Higher = follows prompt more closely (3-4 recommended)"
                     )
 
                     num_inference_steps = gr.Slider(
-                        label="Number of Steps",
+                        label="Quality (steps)",
                         minimum=10,
                         maximum=50,
                         step=1,
                         value=20,
-                        info="More steps = higher quality but slower"
                     )
 
                     with gr.Row():
@@ -263,17 +263,17 @@ with gr.Blocks(css=css, title="ConceptAligner") as demo:
                 with gr.Row():
                     with gr.Column():
                         gr.Markdown("**Previous**")
-                        prev_image = gr.Image(label="Previous", show_label=False, type="pil", height=450)
+                        prev_image = gr.Image(label="Previous", show_label=False, type="pil", height=400)
                         prev_prompt_display = gr.Textbox(
                             label="Previous Prompt",
-                            lines=3,
+                            lines=2,
                             interactive=False,
                             show_label=False
                         )
 
                     with gr.Column():
                         gr.Markdown("**Latest**")
-                        current_image = gr.Image(label="Current", show_label=False, type="pil", height=450)
+                        current_image = gr.Image(label="Current", show_label=False, type="pil", height=400)
 
         gr.Markdown("### 📝 Try This Example")
         gr.Examples(

requirements.txt

@@ -1,6 +1,6 @@
 torch==2.8.0
 torchvision==0.23.0
-gradio==3.50.2
+gradio==6.9.0
 diffusers==0.35.2
 transformers==4.57.1
 safetensors==0.6.2