adding lora

app.py

@@ -176,13 +176,32 @@ def adapters_textual_inversion_change(adapter_textual_inversion, config):
     
     return adapter_textual_inversion_description, in_adapters_textual_inversion_token, config, config, assemble_code(config)
 
+def adapters_lora_change(adapter_lora, config):
+    
+    if str(adapter_lora) != 'None' and str(adapter_lora) != 'null' and type(adapter_lora) != list:
+        
+        adapter_lora_description = adapters['lora'][adapter_lora]['description']
+        adapter_lora_token = adapters['lora'][adapter_lora]['token']
+        adapter_lora_weight = adapters['lora'][adapter_lora]['weight']
+        
+    else:
+        adapter_lora_description = ""
+        adapter_lora_token = ""
+        adapter_lora_weight = ""
+    
+    config = set_config(config, 'adapter_lora', adapter_lora)
+    config = set_config(config, 'adapter_lora_token', adapter_lora_token)
+    config = set_config(config, 'adapter_lora_weight', adapter_lora_weight)
+    
+    return adapter_lora_description, adapter_lora_token, adapter_lora_weight, config, config, assemble_code(config)
+
 def textual_inversion_token_change(adapter_textual_inversion_token, config):
     
     config = set_config(config, 'adapter_textual_inversion_token', adapter_textual_inversion_token)
 
     return config, config, assemble_code(config)
 
-def run_inference(config, config_history, progress=gr.Progress(track_tqdm=True)):
+def run_inference(config, config_history, pipeline, progress=gr.Progress(track_tqdm=True)):
     
     # str_config = str_config.replace("'", '"').replace('None', 'null').replace('False', 'false')
     # config = json.loads(str_config)
@@ -246,9 +265,16 @@ def run_inference(config, config_history, progress=gr.Progress(track_tqdm=True))
         if str(config["adapter_textual_inversion"]).lower() != 'none' and str(config["adapter_textual_inversion"]).lower() != 'null':
             pipeline.load_textual_inversion(config["adapter_textual_inversion"], token=config["adapter_textual_inversion_token"])
         
+        # LoRA
+        if str(config["adapter_lora"]).lower() != 'none' and str(config["adapter_lora"]).lower() != 'null':
+            if str(config["adapter_lora_weight"]).lower() != 'none':
+                pipeline.load_lora_weights(config["adapter_lora"], weight_name=config["adapter_lora_weight"], token=config["adapter_lora_token"])
+            else:
+                pipeline.load_lora_weights(config["adapter_lora"], token=config["adapter_lora_token"])
+            
         progress((3,3), desc="Creating the result...")
 
-        prompt = config["prompt"] + config["trigger_token"] + config["adapter_textual_inversion_token"]
+        prompt = config["prompt"] + config["trigger_token"] + config["adapter_textual_inversion_token"] + config["adapter_lora_token"]
         
         image = pipeline(
             prompt = prompt,
@@ -266,11 +292,11 @@ def run_inference(config, config_history, progress=gr.Progress(track_tqdm=True))
             
         config_history.append(config.copy())
 
-        return image[0], dict_list_to_markdown_table(config_history), config_history
+        return image[0], dict_list_to_markdown_table(config_history), config_history, pipeline
     
     else:
     
-        return "Please select a model AND a scheduler.", None, config_history
+        return "Please select a model AND a scheduler.", None, config_history, pipeline
 
 appConfig = load_app_config()
 models = appConfig.get("models", {})
@@ -294,6 +320,11 @@ js = '''function js(){
             // some things I just don't understand, this is one of them
             return [adapter_textual_inversion, null, adapter_textual_inversion, config, null]
         }
+        window.set_adapter_lora_cookie = function(adapter_lora, config){
+            document.cookie = 'adapter_lora='+ adapter_lora+'; Path=/; SameSite=Strict';
+            // some things I just don't understand, this is one of them
+            return [adapter_lora, null, null, adapter_lora, config, null]
+        }
         window.set_cookie_2 = function(key, value, config){
             document.cookie = key+'='+value+'; Path=/; SameSite=Strict';
             // some things I just don't understand, this is one of them
@@ -307,6 +338,7 @@ with gr.Blocks(analytics_enabled=False) as demo:
     
     config = gr.State(value=get_initial_config())
     config_history = gr.State(value=[])
+    pipeline = gr.State()
 
     gr.Markdown('''## Text-2-Image Playground
                 <small>by <a target="_blank" href="https://www.linkedin.com/in/nickyreinert/">Nicky Reinert</a> | 
@@ -364,6 +396,15 @@ with gr.Blocks(analytics_enabled=False) as demo:
         in_adapters_textual_inversion = gr.Dropdown(value="None", choices=list(adapters['textual_inversion'].keys()), label="Textual Inversion Adapter", info="leave empty to not use an adapter")
         in_adapters_textual_inversion_token = gr.Textbox(value="None", label="Token", info="required to activate the token, will be added to your prompt")
         out_adapters_textual_inversion_description = gr.Textbox(value="", label="Description")
+    with gr.Row():
+        gr.Markdown('#### LoRA')
+    with gr.Row():
+        gr.Markdown('(Low-Rank-Adaption is a performant fine tuning technique)')
+    with gr.Row():
+        in_adapters_lora = gr.Dropdown(value="None", choices=list(adapters['lora'].keys()), label="LoRA Adapter", info="leave empty to not use an adapter")
+        in_adapters_lora_token = gr.Textbox(value="None", label="Token", info="required to activate the token, will be added to your prompt")
+        in_adapters_lora_weight = gr.Textbox(value="", label="Description")
+        out_adapters_lora_description = gr.Textbox(value="", label="Description")
         
     gr.Markdown("### Inference settings")
     with gr.Row():
@@ -392,6 +433,7 @@ with gr.Blocks(analytics_enabled=False) as demo:
     in_schedulers.change(schedulers_change, inputs=[in_schedulers, config], outputs=[out_scheduler_description, config, out_config, out_code], js="(value, config) => set_cookie_2('scheduler', value, config)")
     in_auto_encoders.change(auto_encoders_change, inputs=[in_auto_encoders, config], outputs=[out_auto_encoder_description, config, out_config, out_code], js="(value, config) => set_cookie_2('auto_encoder', value, config)")
     in_adapters_textual_inversion.change(adapters_textual_inversion_change, inputs=[in_adapters_textual_inversion, config], outputs=[out_adapters_textual_inversion_description, in_adapters_textual_inversion_token, config, out_config, out_code], js="(adapter_textual_inversion, config) => set_adapter_textual_inversion_cookie(adapter_textual_inversion, config)")
+    in_adapters_lora.change(adapters_lora_change, inputs=[in_adapters_lora, config], outputs=[out_adapters_lora_description, in_adapters_lora_token, in_adapters_lora_weight, config, out_config, out_code], js="(adapter_lora, config) => set_adapter_lora_cookie(adapter_lora, config)")
     
     # `GENERIC` CHANGE LISTENERS, SAME INPUT, SAME OUTPUT STRUCTURE
     in_devices.change(fn=device_change, inputs=[in_devices, config], outputs=[config, out_config, out_code], js="(value, config) => set_cookie('device', value, config)")
@@ -412,7 +454,7 @@ with gr.Blocks(analytics_enabled=False) as demo:
     in_trigger_token.change(trigger_token_change, inputs=[in_trigger_token, config], outputs=[config, out_config, out_code], js="(value, config) => set_cookie('trigger_token', value, config)")
     in_negative_prompt.change(negative_prompt_change, inputs=[in_negative_prompt, config], outputs=[config, out_config, out_code], js="(value, config) => set_cookie('negative_prompt', value, config)")
     
-    ev_run_inference = btn_start_pipeline.click(run_inference, inputs=[config, config_history], outputs=[out_image, out_config_history, config_history])
+    ev_run_inference = btn_start_pipeline.click(run_inference, inputs=[config, config_history, pipeline], outputs=[out_image, out_config_history, config_history, pipeline])
     btn_stop_pipeline.click(fn=None, inputs=None, outputs=None, cancels=[ev_run_inference])
 
     # send current respect initial config to init_config to populate parameters to all relevant input fields
@@ -439,7 +481,11 @@ with gr.Blocks(analytics_enabled=False) as demo:
             in_inference_steps,
             in_manual_seed,
             in_guidance_scale,
-            in_adapters_textual_inversion
+            in_adapters_textual_inversion,
+            in_adapters_textual_inversion_token,
+            in_adapters_lora,
+            in_adapters_lora_token,
+            in_adapters_lora_weight
             ])
 
 demo.launch(show_error=True)

appConfig.json

@@ -85,6 +85,34 @@
                 "token": "<gta-artwork>",
                 "description": "see https://huggingface.co/sd-concepts-library/gta5-artwork"
             }
+        },
+        "lora": {
+            "None": {"token": "", "description": "", "weight": ""},
+            "nerijs/pixel-art-xl": {
+                "token": "pixel",
+                "weight": "pixel-art-xl.safetensors",
+                "description": "see https://huggingface.co/nerijs/pixel-art-xl"
+            },
+            "ByteDance/SDXL-Lightning": {
+                "token": "SDXLL",
+                "weight": "sdxl_lightning_4step_lora.safetensors",
+                "description": "see https://huggingface.co/ByteDance/SDXL-Lightning"
+            },
+            "super-cereal-sdxl-lora": {
+                "token": "cereals",
+                "weight": "cereal_box_sdxl_v1.safetensors",
+                "description": "see https://huggingface.co/ostris/super-cereal-sdxl-lora"
+            },
+            "CiroN2022/toy-face": {
+                "token": "toy face",
+                "weight": "toy_face_sdxl.safetensors",
+                "description": "see https://huggingface.co/CiroN2022/toy-face"
+            },
+            "latent-consistency/lcm-lora-sdxl": {
+                "token": "lora",
+                "weight": "None",
+                "description": "required base model is stabilityai/stable-diffusion-xl-base-1.0; required scheduler is LCMScheduler, Latent Consistency Models (LCM) enable quality image generation in typically 2-4 steps making it possible to use diffusion models in almost real-time settings; see, https://huggingface.co/docs/diffusers/using-diffusers/inference_with_lcm_lora and https://huggingface.co/latent-consistency/lcm-lora-sdxl"
+            }
         }
     },
     "schedulers": {

config.py

@@ -54,6 +54,9 @@ def get_initial_config():
         "guidance_scale": 5,
         "adapter_textual_inversion": None,
         "adapter_textual_inversion_token": None,
+        "adapter_lora": None,
+        "adapter_lora_token": None,
+        "adapter_lora_weight": None,
         "prompt": 'A white rabbit',
         "trigger_token": '',
         "negative_prompt": 'lowres, cropped, worst quality, low quality',
@@ -107,7 +110,10 @@ def get_config_from_url(initial_config, request: Request):
             return_config['manual_seed'],
             return_config['guidance_scale'],
             return_config['adapter_textual_inversion'],
-            return_config['adapter_textual_inversion_token']
+            return_config['adapter_textual_inversion_token'],
+            return_config['adapter_lora'],
+            return_config['adapter_lora_token'],
+            return_config['adapter_lora_weight']
             ]
         
 def load_app_config():
@@ -198,10 +204,16 @@ def assemble_code(str_config):
         code.append(f'manual_seed = {config["manual_seed"]}')
         code.append(f'generator = torch.manual_seed(manual_seed)')
 
+    # ADAPTER
     if str(config["adapter_textual_inversion"]).lower() != 'none':
         code.append(f'pipeline.load_textual_inversion("{config["adapter_textual_inversion"]}", token="{config["adapter_textual_inversion_token"]}")')
+    if str(config["adapter_lora"]).lower() != 'none':
+        if str(config["adapter_lora_weight"]).lower() != 'none':
+            code.append(f'pipeline.load_lora_weights("{config["adapter_lora"]}", weight_name="{config["adapter_lora_weight"]}", adapter_name="{config["adapter_lora_token"]}")')
+        else:
+            code.append(f'pipeline.load_lora_weights("{config["adapter_lora"]}", adapter_name="{config["adapter_lora_token"]}")')
 
-    code.append(f'prompt = "{config["prompt"]} {config["trigger_token"]} {config["adapter_textual_inversion_token"]}"')
+    code.append(f'prompt = "{config["prompt"]} {config["trigger_token"]} {config["adapter_textual_inversion_token"]} {config["adapter_lora_token"]}"')
     code.append(f'negative_prompt = "{config["negative_prompt"]}"')
     code.append(f'inference_steps = {config["inference_steps"]}')
     code.append(f'guidance_scale = {config["guidance_scale"]}')

requirements.txt

@@ -7,4 +7,5 @@ urllib3==1.26.6
 transformers
 gradio
 stripe>=9.0.0
-accelerate
+accelerate
+peft