Deploy minimal Fooocus to Hugging Face Space

.huggingface.yaml

@@ -0,0 +1,3 @@
+sdk: gradio
+python_version: 3.10
+app_file: entry_with_update.py

entry_with_update.py

@@ -0,0 +1,46 @@
+import os
+import sys
+
+
+root = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(root)
+os.chdir(root)
+
+
+try:
+    import pygit2
+    pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0)
+
+    repo = pygit2.Repository(os.path.abspath(os.path.dirname(__file__)))
+
+    branch_name = repo.head.shorthand
+
+    remote_name = 'origin'
+    remote = repo.remotes[remote_name]
+
+    remote.fetch()
+
+    local_branch_ref = f'refs/heads/{branch_name}'
+    local_branch = repo.lookup_reference(local_branch_ref)
+
+    remote_reference = f'refs/remotes/{remote_name}/{branch_name}'
+    remote_commit = repo.revparse_single(remote_reference)
+
+    merge_result, _ = repo.merge_analysis(remote_commit.id)
+
+    if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
+        print("Already up-to-date")
+    elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
+        local_branch.set_target(remote_commit.id)
+        repo.head.set_target(remote_commit.id)
+        repo.checkout_tree(repo.get(remote_commit.id))
+        repo.reset(local_branch.target, pygit2.GIT_RESET_HARD)
+        print("Fast-forward merge")
+    elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
+        print("Update failed - Did you modify any file?")
+except Exception as e:
+    print('Update failed.')
+    print(str(e))
+
+print('Update succeeded.')
+from launch import *

fooocus_version.py

@@ -0,0 +1 @@
+version = '2.5.5'

launch.py

@@ -0,0 +1,152 @@
+import os
+import ssl
+import sys
+
+print('[System ARGV] ' + str(sys.argv))
+
+root = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(root)
+os.chdir(root)
+
+os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+os.environ["PYTORCH_MPS_HIGH_WATERMARK_RATIO"] = "0.0"
+if "GRADIO_SERVER_PORT" not in os.environ:
+    os.environ["GRADIO_SERVER_PORT"] = "7865"
+
+ssl._create_default_https_context = ssl._create_unverified_context
+
+import platform
+import fooocus_version
+
+from build_launcher import build_launcher
+from modules.launch_util import is_installed, run, python, run_pip, requirements_met, delete_folder_content
+from modules.model_loader import load_file_from_url
+
+REINSTALL_ALL = False
+TRY_INSTALL_XFORMERS = False
+
+
+def prepare_environment():
+    torch_index_url = os.environ.get('TORCH_INDEX_URL', "https://download.pytorch.org/whl/cu121")
+    torch_command = os.environ.get('TORCH_COMMAND',
+                                   f"pip install torch==2.1.0 torchvision==0.16.0 --extra-index-url {torch_index_url}")
+    requirements_file = os.environ.get('REQS_FILE', "requirements_versions.txt")
+
+    print(f"Python {sys.version}")
+    print(f"Fooocus version: {fooocus_version.version}")
+
+    if REINSTALL_ALL or not is_installed("torch") or not is_installed("torchvision"):
+        run(f'"{python}" -m {torch_command}', "Installing torch and torchvision", "Couldn't install torch", live=True)
+
+    if TRY_INSTALL_XFORMERS:
+        if REINSTALL_ALL or not is_installed("xformers"):
+            xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.23')
+            if platform.system() == "Windows":
+                if platform.python_version().startswith("3.10"):
+                    run_pip(f"install -U -I --no-deps {xformers_package}", "xformers", live=True)
+                else:
+                    print("Installation of xformers is not supported in this version of Python.")
+                    print(
+                        "You can also check this and build manually: https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Xformers#building-xformers-on-windows-by-duckness")
+                    if not is_installed("xformers"):
+                        exit(0)
+            elif platform.system() == "Linux":
+                run_pip(f"install -U -I --no-deps {xformers_package}", "xformers")
+
+    if REINSTALL_ALL or not requirements_met(requirements_file):
+        run_pip(f"install -r \"{requirements_file}\"", "requirements")
+
+    return
+
+
+vae_approx_filenames = [
+    ('xlvaeapp.pth', 'https://huggingface.co/lllyasviel/misc/resolve/main/xlvaeapp.pth'),
+    ('vaeapp_sd15.pth', 'https://huggingface.co/lllyasviel/misc/resolve/main/vaeapp_sd15.pt'),
+    ('xl-to-v1_interposer-v4.0.safetensors',
+     'https://huggingface.co/mashb1t/misc/resolve/main/xl-to-v1_interposer-v4.0.safetensors')
+]
+
+
+def ini_args():
+    from args_manager import args
+    return args
+
+
+prepare_environment()
+build_launcher()
+args = ini_args()
+
+if args.gpu_device_id is not None:
+    os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_device_id)
+    print("Set device to:", args.gpu_device_id)
+
+if args.hf_mirror is not None:
+    os.environ['HF_MIRROR'] = str(args.hf_mirror)
+    print("Set hf_mirror to:", args.hf_mirror)
+
+from modules import config
+from modules.hash_cache import init_cache
+
+os.environ["U2NET_HOME"] = config.path_inpaint
+
+os.environ['GRADIO_TEMP_DIR'] = config.temp_path
+
+if config.temp_path_cleanup_on_launch:
+    print(f'[Cleanup] Attempting to delete content of temp dir {config.temp_path}')
+    result = delete_folder_content(config.temp_path, '[Cleanup] ')
+    if result:
+        print("[Cleanup] Cleanup successful")
+    else:
+        print(f"[Cleanup] Failed to delete content of temp dir.")
+
+
+def download_models(default_model, previous_default_models, checkpoint_downloads, embeddings_downloads, lora_downloads, vae_downloads):
+    from modules.util import get_file_from_folder_list
+
+    for file_name, url in vae_approx_filenames:
+        load_file_from_url(url=url, model_dir=config.path_vae_approx, file_name=file_name)
+
+    load_file_from_url(
+        url='https://huggingface.co/lllyasviel/misc/resolve/main/fooocus_expansion.bin',
+        model_dir=config.path_fooocus_expansion,
+        file_name='pytorch_model.bin'
+    )
+
+    if args.disable_preset_download:
+        print('Skipped model download.')
+        return default_model, checkpoint_downloads
+
+    if not args.always_download_new_model:
+        if not os.path.isfile(get_file_from_folder_list(default_model, config.paths_checkpoints)):
+            for alternative_model_name in previous_default_models:
+                if os.path.isfile(get_file_from_folder_list(alternative_model_name, config.paths_checkpoints)):
+                    print(f'You do not have [{default_model}] but you have [{alternative_model_name}].')
+                    print(f'Fooocus will use [{alternative_model_name}] to avoid downloading new models, '
+                          f'but you are not using the latest models.')
+                    print('Use --always-download-new-model to avoid fallback and always get new models.')
+                    checkpoint_downloads = {}
+                    default_model = alternative_model_name
+                    break
+
+    for file_name, url in checkpoint_downloads.items():
+        model_dir = os.path.dirname(get_file_from_folder_list(file_name, config.paths_checkpoints))
+        load_file_from_url(url=url, model_dir=model_dir, file_name=file_name)
+    for file_name, url in embeddings_downloads.items():
+        load_file_from_url(url=url, model_dir=config.path_embeddings, file_name=file_name)
+    for file_name, url in lora_downloads.items():
+        model_dir = os.path.dirname(get_file_from_folder_list(file_name, config.paths_loras))
+        load_file_from_url(url=url, model_dir=model_dir, file_name=file_name)
+    for file_name, url in vae_downloads.items():
+        load_file_from_url(url=url, model_dir=config.path_vae, file_name=file_name)
+
+    return default_model, checkpoint_downloads
+
+
+config.default_base_model_name, config.checkpoint_downloads = download_models(
+    config.default_base_model_name, config.previous_default_models, config.checkpoint_downloads,
+    config.embeddings_downloads, config.lora_downloads, config.vae_downloads)
+
+config.update_files()
+init_cache(config.model_filenames, config.paths_checkpoints, config.lora_filenames, config.paths_loras)
+
+from webui import *

models/configs/anything_v3.yaml

@@ -0,0 +1,73 @@
+model:
+  base_learning_rate: 1.0e-04
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+
+    scheduler_config: # 10000 warmup steps
+      target: ldm.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 10000 ]
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder
+      params:
+        layer: "hidden"
+        layer_idx: -2

models/configs/v1-inference.yaml

@@ -0,0 +1,70 @@
+model:
+  base_learning_rate: 1.0e-04
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+
+    scheduler_config: # 10000 warmup steps
+      target: ldm.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 10000 ]
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder

models/configs/v1-inference_clip_skip_2.yaml

@@ -0,0 +1,73 @@
+model:
+  base_learning_rate: 1.0e-04
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+
+    scheduler_config: # 10000 warmup steps
+      target: ldm.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 10000 ]
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder
+      params:
+        layer: "hidden"
+        layer_idx: -2

models/configs/v1-inference_clip_skip_2_fp16.yaml

@@ -0,0 +1,74 @@
+model:
+  base_learning_rate: 1.0e-04
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+
+    scheduler_config: # 10000 warmup steps
+      target: ldm.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 10000 ]
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_fp16: True
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder
+      params:
+        layer: "hidden"
+        layer_idx: -2

models/configs/v1-inference_fp16.yaml

@@ -0,0 +1,71 @@
+model:
+  base_learning_rate: 1.0e-04
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+
+    scheduler_config: # 10000 warmup steps
+      target: ldm.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 10000 ]
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_fp16: True
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder

models/configs/v1-inpainting-inference.yaml

@@ -0,0 +1,71 @@
+model:
+  base_learning_rate: 7.5e-05
+  target: ldm.models.diffusion.ddpm.LatentInpaintDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: hybrid   # important
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    finetune_keys: null
+
+    scheduler_config: # 10000 warmup steps
+      target: ldm.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 2500 ] # NOTE for resuming. use 10000 if starting from scratch
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 # unused
+        in_channels: 9  # 4 data + 4 downscaled image + 1 mask
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder
+

models/configs/v2-inference-v.yaml

@@ -0,0 +1,68 @@
+model:
+  base_learning_rate: 1.0e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    parameterization: "v"
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False # we set this to false because this is an inference only config
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        use_fp16: True
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"

models/configs/v2-inference-v_fp32.yaml

@@ -0,0 +1,68 @@
+model:
+  base_learning_rate: 1.0e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    parameterization: "v"
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False # we set this to false because this is an inference only config
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        use_fp16: False
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"

models/configs/v2-inference.yaml

@@ -0,0 +1,67 @@
+model:
+  base_learning_rate: 1.0e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False # we set this to false because this is an inference only config
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        use_fp16: True
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"

models/configs/v2-inference_fp32.yaml

@@ -0,0 +1,67 @@
+model:
+  base_learning_rate: 1.0e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False # we set this to false because this is an inference only config
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        use_fp16: False
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"

models/configs/v2-inpainting-inference.yaml

@@ -0,0 +1,158 @@
+model:
+  base_learning_rate: 5.0e-05
+  target: ldm.models.diffusion.ddpm.LatentInpaintDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: hybrid
+    scale_factor: 0.18215
+    monitor: val/loss_simple_ema
+    finetune_keys: null
+    use_ema: False
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        image_size: 32 # unused
+        in_channels: 9
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+            - 1
+            - 2
+            - 4
+            - 4
+          num_res_blocks: 2
+          attn_resolutions: [ ]
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"
+
+
+data:
+  target: ldm.data.laion.WebDataModuleFromConfig
+  params:
+    tar_base: null  # for concat as in LAION-A
+    p_unsafe_threshold: 0.1
+    filter_word_list: "data/filters.yaml"
+    max_pwatermark: 0.45
+    batch_size: 8
+    num_workers: 6
+    multinode: True
+    min_size: 512
+    train:
+      shards:
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-0/{00000..18699}.tar -"
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-1/{00000..18699}.tar -"
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-2/{00000..18699}.tar -"
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-3/{00000..18699}.tar -"
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-4/{00000..18699}.tar -"  #{00000-94333}.tar"
+      shuffle: 10000
+      image_key: jpg
+      image_transforms:
+      - target: torchvision.transforms.Resize
+        params:
+          size: 512
+          interpolation: 3
+      - target: torchvision.transforms.RandomCrop
+        params:
+          size: 512
+      postprocess:
+        target: ldm.data.laion.AddMask
+        params:
+          mode: "512train-large"
+          p_drop: 0.25
+    # NOTE use enough shards to avoid empty validation loops in workers
+    validation:
+      shards:
+        - "pipe:aws s3 cp s3://deep-floyd-s3/datasets/laion_cleaned-part5/{93001..94333}.tar - "
+      shuffle: 0
+      image_key: jpg
+      image_transforms:
+      - target: torchvision.transforms.Resize
+        params:
+          size: 512
+          interpolation: 3
+      - target: torchvision.transforms.CenterCrop
+        params:
+          size: 512
+      postprocess:
+        target: ldm.data.laion.AddMask
+        params:
+          mode: "512train-large"
+          p_drop: 0.25
+
+lightning:
+  find_unused_parameters: True
+  modelcheckpoint:
+    params:
+      every_n_train_steps: 5000
+
+  callbacks:
+    metrics_over_trainsteps_checkpoint:
+      params:
+        every_n_train_steps: 10000
+
+    image_logger:
+      target: main.ImageLogger
+      params:
+        enable_autocast: False
+        disabled: False
+        batch_frequency: 1000
+        max_images: 4
+        increase_log_steps: False
+        log_first_step: False
+        log_images_kwargs:
+          use_ema_scope: False
+          inpaint: False
+          plot_progressive_rows: False
+          plot_diffusion_rows: False
+          N: 4
+          unconditional_guidance_scale: 5.0
+          unconditional_guidance_label: [""]
+          ddim_steps: 50  # todo check these out for depth2img,
+          ddim_eta: 0.0   # todo check these out for depth2img,
+
+  trainer:
+    benchmark: True
+    val_check_interval: 5000000
+    num_sanity_val_steps: 0
+    accumulate_grad_batches: 1

models/prompt_expansion/fooocus_expansion/config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "gpt2",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "pad_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.23.0.dev0",
+  "use_cache": true,
+  "vocab_size": 50257
+}

models/prompt_expansion/fooocus_expansion/positive.txt

@@ -0,0 +1,642 @@
+abundant
+accelerated
+accepted
+accepting
+acclaimed
+accomplished
+acknowledged
+activated
+adapted
+adjusted
+admirable
+adorable
+adorned
+advanced
+adventurous
+advocated
+aesthetic
+affirmed
+affluent
+agile
+aimed
+aligned
+alive
+altered
+amazing
+ambient
+amplified
+analytical
+animated
+appealing
+applauded
+appreciated
+ardent
+aromatic
+arranged
+arresting
+articulate
+artistic
+associated
+assured
+astonishing
+astounding
+atmosphere
+attempted
+attentive
+attractive
+authentic
+authoritative
+awarded
+awesome
+backed
+background
+baked
+balance
+balanced
+balancing
+beaten
+beautiful
+beloved
+beneficial
+benevolent
+best
+bestowed
+blazing
+blended
+blessed
+boosted
+borne
+brave
+breathtaking
+brewed
+bright
+brilliant
+brought
+built
+burning
+calm
+calmed
+candid
+caring
+carried
+catchy
+celebrated
+celestial
+certain
+championed
+changed
+charismatic
+charming
+chased
+cheered
+cheerful
+cherished
+chic
+chosen
+cinematic
+clad
+classic
+classy
+clear
+coached
+coherent
+collected
+color
+colorful
+colors
+colossal
+combined
+comforting
+commanding
+committed
+compassionate
+compatible
+complete
+complex
+complimentary
+composed
+composition
+comprehensive
+conceived
+conferred
+confident
+connected
+considerable
+considered
+consistent
+conspicuous
+constructed
+constructive
+contemplated
+contemporary
+content
+contrasted
+conveyed
+cooked
+cool
+coordinated
+coupled
+courageous
+coveted
+cozy
+created
+creative
+credited
+crisp
+critical
+cultivated
+cured
+curious
+current
+customized
+cute
+daring
+darling
+dazzling
+decorated
+decorative
+dedicated
+deep
+defended
+definitive
+delicate
+delightful
+delivered
+depicted
+designed
+desirable
+desired
+destined
+detail
+detailed
+determined
+developed
+devoted
+devout
+diligent
+direct
+directed
+discovered
+dispatched
+displayed
+distilled
+distinct
+distinctive
+distinguished
+diverse
+divine
+dramatic
+draped
+dreamed
+driven
+dynamic
+earnest
+eased
+ecstatic
+educated
+effective
+elaborate
+elegant
+elevated
+elite
+eminent
+emotional
+empowered
+empowering
+enchanted
+encouraged
+endorsed
+endowed
+enduring
+energetic
+engaging
+enhanced
+enigmatic
+enlightened
+enormous
+enticing
+envisioned
+epic
+esteemed
+eternal
+everlasting
+evolved
+exalted
+examining
+excellent
+exceptional
+exciting
+exclusive
+exemplary
+exotic
+expansive
+exposed
+expressive
+exquisite
+extended
+extraordinary
+extremely
+fabulous
+facilitated
+fair
+faithful
+famous
+fancy
+fantastic
+fascinating
+fashionable
+fashioned
+favorable
+favored
+fearless
+fermented
+fertile
+festive
+fiery
+fine
+finest
+firm
+fixed
+flaming
+flashing
+flashy
+flavored
+flawless
+flourishing
+flowing
+focus
+focused
+formal
+formed
+fortunate
+fostering
+frank
+fresh
+fried
+friendly
+fruitful
+fulfilled
+full
+futuristic
+generous
+gentle
+genuine
+gifted
+gigantic
+glamorous
+glorious
+glossy
+glowing
+gorgeous
+graceful
+gracious
+grand
+granted
+grateful
+great
+grilled
+grounded
+grown
+guarded
+guided
+hailed
+handsome
+healing
+healthy
+heartfelt
+heavenly
+heroic
+highly
+historic
+holistic
+holy
+honest
+honored
+hoped
+hopeful
+iconic
+ideal
+illuminated
+illuminating
+illumination
+illustrious
+imaginative
+imagined
+immense
+immortal
+imposing
+impressive
+improved
+incredible
+infinite
+informed
+ingenious
+innocent
+innovative
+insightful
+inspirational
+inspired
+inspiring
+instructed
+integrated
+intense
+intricate
+intriguing
+invaluable
+invented
+investigative
+invincible
+inviting
+irresistible
+joined
+joyful
+keen
+kindly
+kinetic
+knockout
+laced
+lasting
+lauded
+lavish
+legendary
+lifted
+light
+limited
+linked
+lively
+located
+logical
+loved
+lovely
+loving
+loyal
+lucid
+lucky
+lush
+luxurious
+luxury
+magic
+magical
+magnificent
+majestic
+marked
+marvelous
+massive
+matched
+matured
+meaningful
+memorable
+merged
+merry
+meticulous
+mindful
+miraculous
+modern
+modified
+monstrous
+monumental
+motivated
+motivational
+moved
+moving
+mystical
+mythical
+naive
+neat
+new
+nice
+nifty
+noble
+notable
+noteworthy
+novel
+nuanced
+offered
+open
+optimal
+optimistic
+orderly
+organized
+original
+originated
+outstanding
+overwhelming
+paired
+palpable
+passionate
+peaceful
+perfect
+perfected
+perpetual
+persistent
+phenomenal
+pious
+pivotal
+placed
+planned
+pleasant
+pleased
+pleasing
+plentiful
+plotted
+plush
+poetic
+poignant
+polished
+positive
+praised
+precious
+precise
+premier
+premium
+presented
+preserved
+prestigious
+pretty
+priceless
+prime
+pristine
+probing
+productive
+professional
+profound
+progressed
+progressive
+prominent
+promoted
+pronounced
+propelled
+proportional
+prosperous
+protected
+provided
+provocative
+pure
+pursued
+pushed
+quaint
+quality
+questioning
+quiet
+radiant
+rare
+rational
+real
+reborn
+reclaimed
+recognized
+recovered
+refined
+reflected
+refreshed
+refreshing
+related
+relaxed
+relentless
+reliable
+relieved
+remarkable
+renewed
+renowned
+representative
+rescued
+resilient
+respected
+respectful
+restored
+retrieved
+revealed
+revealing
+revered
+revived
+rewarded
+rich
+roasted
+robust
+romantic
+royal
+sacred
+salient
+satisfied
+satisfying
+saturated
+saved
+scenic
+scientific
+select
+sensational
+serious
+set
+shaped
+sharp
+shielded
+shining
+shiny
+shown
+significant
+silent
+sincere
+singular
+situated
+sleek
+slick
+smart
+snug
+solemn
+solid
+soothing
+sophisticated
+sought
+sparkling
+special
+spectacular
+sped
+spirited
+spiritual
+splendid
+spread
+stable
+steady
+still
+stimulated
+stimulating
+stirred
+straightforward
+striking
+strong
+structured
+stunning
+sturdy
+stylish
+sublime
+successful
+sunny
+superb
+superior
+supplied
+supported
+supportive
+supreme
+sure
+surreal
+sweet
+symbolic
+symmetry
+synchronized
+systematic
+tailored
+taking
+targeted
+taught
+tempting
+tender
+terrific
+thankful
+theatrical
+thought
+thoughtful
+thrilled
+thrilling
+thriving
+tidy
+timeless
+touching
+tough
+trained
+tranquil
+transformed
+translucent
+transparent
+transported
+tremendous
+trendy
+tried
+trim
+true
+trustworthy
+unbelievable
+unconditional
+uncovered
+unified
+unique
+united
+universal
+unmatched
+unparalleled
+upheld
+valiant
+valued
+varied
+very
+vibrant
+virtuous
+vivid
+warm
+wealthy
+whole
+winning
+wished
+witty
+wonderful
+worshipped
+worthy

models/prompt_expansion/fooocus_expansion/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

models/prompt_expansion/fooocus_expansion/tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1024,
+  "name_or_path": "gpt2",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

modules/anisotropic.py

@@ -0,0 +1,200 @@
+import torch
+
+
+Tensor = torch.Tensor
+Device = torch.DeviceObjType
+Dtype = torch.Type
+pad = torch.nn.functional.pad
+
+
+def _compute_zero_padding(kernel_size: tuple[int, int] | int) -> tuple[int, int]:
+    ky, kx = _unpack_2d_ks(kernel_size)
+    return (ky - 1) // 2, (kx - 1) // 2
+
+
+def _unpack_2d_ks(kernel_size: tuple[int, int] | int) -> tuple[int, int]:
+    if isinstance(kernel_size, int):
+        ky = kx = kernel_size
+    else:
+        assert len(kernel_size) == 2, '2D Kernel size should have a length of 2.'
+        ky, kx = kernel_size
+
+    ky = int(ky)
+    kx = int(kx)
+    return ky, kx
+
+
+def gaussian(
+    window_size: int, sigma: Tensor | float, *, device: Device | None = None, dtype: Dtype | None = None
+) -> Tensor:
+
+    batch_size = sigma.shape[0]
+
+    x = (torch.arange(window_size, device=sigma.device, dtype=sigma.dtype) - window_size // 2).expand(batch_size, -1)
+
+    if window_size % 2 == 0:
+        x = x + 0.5
+
+    gauss = torch.exp(-x.pow(2.0) / (2 * sigma.pow(2.0)))
+
+    return gauss / gauss.sum(-1, keepdim=True)
+
+
+def get_gaussian_kernel1d(
+    kernel_size: int,
+    sigma: float | Tensor,
+    force_even: bool = False,
+    *,
+    device: Device | None = None,
+    dtype: Dtype | None = None,
+) -> Tensor:
+
+    return gaussian(kernel_size, sigma, device=device, dtype=dtype)
+
+
+def get_gaussian_kernel2d(
+    kernel_size: tuple[int, int] | int,
+    sigma: tuple[float, float] | Tensor,
+    force_even: bool = False,
+    *,
+    device: Device | None = None,
+    dtype: Dtype | None = None,
+) -> Tensor:
+
+    sigma = torch.Tensor([[sigma, sigma]]).to(device=device, dtype=dtype)
+
+    ksize_y, ksize_x = _unpack_2d_ks(kernel_size)
+    sigma_y, sigma_x = sigma[:, 0, None], sigma[:, 1, None]
+
+    kernel_y = get_gaussian_kernel1d(ksize_y, sigma_y, force_even, device=device, dtype=dtype)[..., None]
+    kernel_x = get_gaussian_kernel1d(ksize_x, sigma_x, force_even, device=device, dtype=dtype)[..., None]
+
+    return kernel_y * kernel_x.view(-1, 1, ksize_x)
+
+
+def _bilateral_blur(
+    input: Tensor,
+    guidance: Tensor | None,
+    kernel_size: tuple[int, int] | int,
+    sigma_color: float | Tensor,
+    sigma_space: tuple[float, float] | Tensor,
+    border_type: str = 'reflect',
+    color_distance_type: str = 'l1',
+) -> Tensor:
+
+    if isinstance(sigma_color, Tensor):
+        sigma_color = sigma_color.to(device=input.device, dtype=input.dtype).view(-1, 1, 1, 1, 1)
+
+    ky, kx = _unpack_2d_ks(kernel_size)
+    pad_y, pad_x = _compute_zero_padding(kernel_size)
+
+    padded_input = pad(input, (pad_x, pad_x, pad_y, pad_y), mode=border_type)
+    unfolded_input = padded_input.unfold(2, ky, 1).unfold(3, kx, 1).flatten(-2)  # (B, C, H, W, Ky x Kx)
+
+    if guidance is None:
+        guidance = input
+        unfolded_guidance = unfolded_input
+    else:
+        padded_guidance = pad(guidance, (pad_x, pad_x, pad_y, pad_y), mode=border_type)
+        unfolded_guidance = padded_guidance.unfold(2, ky, 1).unfold(3, kx, 1).flatten(-2)  # (B, C, H, W, Ky x Kx)
+
+    diff = unfolded_guidance - guidance.unsqueeze(-1)
+    if color_distance_type == "l1":
+        color_distance_sq = diff.abs().sum(1, keepdim=True).square()
+    elif color_distance_type == "l2":
+        color_distance_sq = diff.square().sum(1, keepdim=True)
+    else:
+        raise ValueError("color_distance_type only acceps l1 or l2")
+    color_kernel = (-0.5 / sigma_color**2 * color_distance_sq).exp()  # (B, 1, H, W, Ky x Kx)
+
+    space_kernel = get_gaussian_kernel2d(kernel_size, sigma_space, device=input.device, dtype=input.dtype)
+    space_kernel = space_kernel.view(-1, 1, 1, 1, kx * ky)
+
+    kernel = space_kernel * color_kernel
+    out = (unfolded_input * kernel).sum(-1) / kernel.sum(-1)
+    return out
+
+
+def bilateral_blur(
+    input: Tensor,
+    kernel_size: tuple[int, int] | int = (13, 13),
+    sigma_color: float | Tensor = 3.0,
+    sigma_space: tuple[float, float] | Tensor = 3.0,
+    border_type: str = 'reflect',
+    color_distance_type: str = 'l1',
+) -> Tensor:
+    return _bilateral_blur(input, None, kernel_size, sigma_color, sigma_space, border_type, color_distance_type)
+
+
+def adaptive_anisotropic_filter(x, g=None):
+    if g is None:
+        g = x
+    s, m = torch.std_mean(g, dim=(1, 2, 3), keepdim=True)
+    s = s + 1e-5
+    guidance = (g - m) / s
+    y = _bilateral_blur(x, guidance,
+                        kernel_size=(13, 13),
+                        sigma_color=3.0,
+                        sigma_space=3.0,
+                        border_type='reflect',
+                        color_distance_type='l1')
+    return y
+
+
+def joint_bilateral_blur(
+    input: Tensor,
+    guidance: Tensor,
+    kernel_size: tuple[int, int] | int,
+    sigma_color: float | Tensor,
+    sigma_space: tuple[float, float] | Tensor,
+    border_type: str = 'reflect',
+    color_distance_type: str = 'l1',
+) -> Tensor:
+    return _bilateral_blur(input, guidance, kernel_size, sigma_color, sigma_space, border_type, color_distance_type)
+
+
+class _BilateralBlur(torch.nn.Module):
+    def __init__(
+        self,
+        kernel_size: tuple[int, int] | int,
+        sigma_color: float | Tensor,
+        sigma_space: tuple[float, float] | Tensor,
+        border_type: str = 'reflect',
+        color_distance_type: str = "l1",
+    ) -> None:
+        super().__init__()
+        self.kernel_size = kernel_size
+        self.sigma_color = sigma_color
+        self.sigma_space = sigma_space
+        self.border_type = border_type
+        self.color_distance_type = color_distance_type
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}"
+            f"(kernel_size={self.kernel_size}, "
+            f"sigma_color={self.sigma_color}, "
+            f"sigma_space={self.sigma_space}, "
+            f"border_type={self.border_type}, "
+            f"color_distance_type={self.color_distance_type})"
+        )
+
+
+class BilateralBlur(_BilateralBlur):
+    def forward(self, input: Tensor) -> Tensor:
+        return bilateral_blur(
+            input, self.kernel_size, self.sigma_color, self.sigma_space, self.border_type, self.color_distance_type
+        )
+
+
+class JointBilateralBlur(_BilateralBlur):
+    def forward(self, input: Tensor, guidance: Tensor) -> Tensor:
+        return joint_bilateral_blur(
+            input,
+            guidance,
+            self.kernel_size,
+            self.sigma_color,
+            self.sigma_space,
+            self.border_type,
+            self.color_distance_type,
+        )

modules/async_worker.py

@@ -0,0 +1,1485 @@
+import threading
+
+from extras.inpaint_mask import generate_mask_from_image, SAMOptions
+from modules.patch import PatchSettings, patch_settings, patch_all
+import modules.config
+
+patch_all()
+
+
+class AsyncTask:
+    def __init__(self, args):
+        from modules.flags import Performance, MetadataScheme, ip_list, disabled
+        from modules.util import get_enabled_loras
+        from modules.config import default_max_lora_number
+        import args_manager
+
+        self.args = args.copy()
+        self.yields = []
+        self.results = []
+        self.last_stop = False
+        self.processing = False
+
+        self.performance_loras = []
+
+        if len(args) == 0:
+            return
+
+        args.reverse()
+        self.generate_image_grid = args.pop()
+        self.prompt = args.pop()
+        self.negative_prompt = args.pop()
+        self.style_selections = args.pop()
+
+        self.performance_selection = Performance(args.pop())
+        self.steps = self.performance_selection.steps()
+        self.original_steps = self.steps
+
+        self.aspect_ratios_selection = args.pop()
+        self.image_number = args.pop()
+        self.output_format = args.pop()
+        self.seed = int(args.pop())
+        self.read_wildcards_in_order = args.pop()
+        self.sharpness = args.pop()
+        self.cfg_scale = args.pop()
+        self.base_model_name = args.pop()
+        self.refiner_model_name = args.pop()
+        self.refiner_switch = args.pop()
+        self.loras = get_enabled_loras([(bool(args.pop()), str(args.pop()), float(args.pop())) for _ in
+                                        range(default_max_lora_number)])
+        self.input_image_checkbox = args.pop()
+        self.current_tab = args.pop()
+        self.uov_method = args.pop()
+        self.uov_input_image = args.pop()
+        self.outpaint_selections = args.pop()
+        self.inpaint_input_image = args.pop()
+        self.inpaint_additional_prompt = args.pop()
+        self.inpaint_mask_image_upload = args.pop()
+
+        self.disable_preview = args.pop()
+        self.disable_intermediate_results = args.pop()
+        self.disable_seed_increment = args.pop()
+        self.black_out_nsfw = args.pop()
+        self.adm_scaler_positive = args.pop()
+        self.adm_scaler_negative = args.pop()
+        self.adm_scaler_end = args.pop()
+        self.adaptive_cfg = args.pop()
+        self.clip_skip = args.pop()
+        self.sampler_name = args.pop()
+        self.scheduler_name = args.pop()
+        self.vae_name = args.pop()
+        self.overwrite_step = args.pop()
+        self.overwrite_switch = args.pop()
+        self.overwrite_width = args.pop()
+        self.overwrite_height = args.pop()
+        self.overwrite_vary_strength = args.pop()
+        self.overwrite_upscale_strength = args.pop()
+        self.mixing_image_prompt_and_vary_upscale = args.pop()
+        self.mixing_image_prompt_and_inpaint = args.pop()
+        self.debugging_cn_preprocessor = args.pop()
+        self.skipping_cn_preprocessor = args.pop()
+        self.canny_low_threshold = args.pop()
+        self.canny_high_threshold = args.pop()
+        self.refiner_swap_method = args.pop()
+        self.controlnet_softness = args.pop()
+        self.freeu_enabled = args.pop()
+        self.freeu_b1 = args.pop()
+        self.freeu_b2 = args.pop()
+        self.freeu_s1 = args.pop()
+        self.freeu_s2 = args.pop()
+        self.debugging_inpaint_preprocessor = args.pop()
+        self.inpaint_disable_initial_latent = args.pop()
+        self.inpaint_engine = args.pop()
+        self.inpaint_strength = args.pop()
+        self.inpaint_respective_field = args.pop()
+        self.inpaint_advanced_masking_checkbox = args.pop()
+        self.invert_mask_checkbox = args.pop()
+        self.inpaint_erode_or_dilate = args.pop()
+        self.save_final_enhanced_image_only = args.pop() if not args_manager.args.disable_image_log else False
+        self.save_metadata_to_images = args.pop() if not args_manager.args.disable_metadata else False
+        self.metadata_scheme = MetadataScheme(
+            args.pop()) if not args_manager.args.disable_metadata else MetadataScheme.FOOOCUS
+
+        self.cn_tasks = {x: [] for x in ip_list}
+        for _ in range(modules.config.default_controlnet_image_count):
+            cn_img = args.pop()
+            cn_stop = args.pop()
+            cn_weight = args.pop()
+            cn_type = args.pop()
+            if cn_img is not None:
+                self.cn_tasks[cn_type].append([cn_img, cn_stop, cn_weight])
+
+        self.debugging_dino = args.pop()
+        self.dino_erode_or_dilate = args.pop()
+        self.debugging_enhance_masks_checkbox = args.pop()
+
+        self.enhance_input_image = args.pop()
+        self.enhance_checkbox = args.pop()
+        self.enhance_uov_method = args.pop()
+        self.enhance_uov_processing_order = args.pop()
+        self.enhance_uov_prompt_type = args.pop()
+        self.enhance_ctrls = []
+        for _ in range(modules.config.default_enhance_tabs):
+            enhance_enabled = args.pop()
+            enhance_mask_dino_prompt_text = args.pop()
+            enhance_prompt = args.pop()
+            enhance_negative_prompt = args.pop()
+            enhance_mask_model = args.pop()
+            enhance_mask_cloth_category = args.pop()
+            enhance_mask_sam_model = args.pop()
+            enhance_mask_text_threshold = args.pop()
+            enhance_mask_box_threshold = args.pop()
+            enhance_mask_sam_max_detections = args.pop()
+            enhance_inpaint_disable_initial_latent = args.pop()
+            enhance_inpaint_engine = args.pop()
+            enhance_inpaint_strength = args.pop()
+            enhance_inpaint_respective_field = args.pop()
+            enhance_inpaint_erode_or_dilate = args.pop()
+            enhance_mask_invert = args.pop()
+            if enhance_enabled:
+                self.enhance_ctrls.append([
+                    enhance_mask_dino_prompt_text,
+                    enhance_prompt,
+                    enhance_negative_prompt,
+                    enhance_mask_model,
+                    enhance_mask_cloth_category,
+                    enhance_mask_sam_model,
+                    enhance_mask_text_threshold,
+                    enhance_mask_box_threshold,
+                    enhance_mask_sam_max_detections,
+                    enhance_inpaint_disable_initial_latent,
+                    enhance_inpaint_engine,
+                    enhance_inpaint_strength,
+                    enhance_inpaint_respective_field,
+                    enhance_inpaint_erode_or_dilate,
+                    enhance_mask_invert
+                ])
+        self.should_enhance = self.enhance_checkbox and (self.enhance_uov_method != disabled.casefold() or len(self.enhance_ctrls) > 0)
+        self.images_to_enhance_count = 0
+        self.enhance_stats = {}
+
+async_tasks = []
+
+
+class EarlyReturnException(BaseException):
+    pass
+
+
+def worker():
+    global async_tasks
+
+    import os
+    import traceback
+    import math
+    import numpy as np
+    import torch
+    import time
+    import shared
+    import random
+    import copy
+    import cv2
+    import modules.default_pipeline as pipeline
+    import modules.core as core
+    import modules.flags as flags
+    import modules.patch
+    import ldm_patched.modules.model_management
+    import extras.preprocessors as preprocessors
+    import modules.inpaint_worker as inpaint_worker
+    import modules.constants as constants
+    import extras.ip_adapter as ip_adapter
+    import extras.face_crop
+    import fooocus_version
+
+    from extras.censor import default_censor
+    from modules.sdxl_styles import apply_style, get_random_style, fooocus_expansion, apply_arrays, random_style_name
+    from modules.private_logger import log
+    from extras.expansion import safe_str
+    from modules.util import (remove_empty_str, HWC3, resize_image, get_image_shape_ceil, set_image_shape_ceil,
+                              get_shape_ceil, resample_image, erode_or_dilate, parse_lora_references_from_prompt,
+                              apply_wildcards)
+    from modules.upscaler import perform_upscale
+    from modules.flags import Performance
+    from modules.meta_parser import get_metadata_parser
+
+    pid = os.getpid()
+    print(f'Started worker with PID {pid}')
+
+    try:
+        async_gradio_app = shared.gradio_root
+        flag = f'''App started successful. Use the app with {str(async_gradio_app.local_url)} or {str(async_gradio_app.server_name)}:{str(async_gradio_app.server_port)}'''
+        if async_gradio_app.share:
+            flag += f''' or {async_gradio_app.share_url}'''
+        print(flag)
+    except Exception as e:
+        print(e)
+
+    def progressbar(async_task, number, text):
+        print(f'[Fooocus] {text}')
+        async_task.yields.append(['preview', (number, text, None)])
+
+    def yield_result(async_task, imgs, progressbar_index, black_out_nsfw, censor=True, do_not_show_finished_images=False):
+        if not isinstance(imgs, list):
+            imgs = [imgs]
+
+        if censor and (modules.config.default_black_out_nsfw or black_out_nsfw):
+            progressbar(async_task, progressbar_index, 'Checking for NSFW content ...')
+            imgs = default_censor(imgs)
+
+        async_task.results = async_task.results + imgs
+
+        if do_not_show_finished_images:
+            return
+
+        async_task.yields.append(['results', async_task.results])
+        return
+
+    def build_image_wall(async_task):
+        results = []
+
+        if len(async_task.results) < 2:
+            return
+
+        for img in async_task.results:
+            if isinstance(img, str) and os.path.exists(img):
+                img = cv2.imread(img)
+                img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+            if not isinstance(img, np.ndarray):
+                return
+            if img.ndim != 3:
+                return
+            results.append(img)
+
+        H, W, C = results[0].shape
+
+        for img in results:
+            Hn, Wn, Cn = img.shape
+            if H != Hn:
+                return
+            if W != Wn:
+                return
+            if C != Cn:
+                return
+
+        cols = float(len(results)) ** 0.5
+        cols = int(math.ceil(cols))
+        rows = float(len(results)) / float(cols)
+        rows = int(math.ceil(rows))
+
+        wall = np.zeros(shape=(H * rows, W * cols, C), dtype=np.uint8)
+
+        for y in range(rows):
+            for x in range(cols):
+                if y * cols + x < len(results):
+                    img = results[y * cols + x]
+                    wall[y * H:y * H + H, x * W:x * W + W, :] = img
+
+        # must use deep copy otherwise gradio is super laggy. Do not use list.append() .
+        async_task.results = async_task.results + [wall]
+        return
+
+    def process_task(all_steps, async_task, callback, controlnet_canny_path, controlnet_cpds_path, current_task_id,
+                     denoising_strength, final_scheduler_name, goals, initial_latent, steps, switch, positive_cond,
+                     negative_cond, task, loras, tiled, use_expansion, width, height, base_progress, preparation_steps,
+                     total_count, show_intermediate_results, persist_image=True):
+        if async_task.last_stop is not False:
+            ldm_patched.modules.model_management.interrupt_current_processing()
+        if 'cn' in goals:
+            for cn_flag, cn_path in [
+                (flags.cn_canny, controlnet_canny_path),
+                (flags.cn_cpds, controlnet_cpds_path)
+            ]:
+                for cn_img, cn_stop, cn_weight in async_task.cn_tasks[cn_flag]:
+                    positive_cond, negative_cond = core.apply_controlnet(
+                        positive_cond, negative_cond,
+                        pipeline.loaded_ControlNets[cn_path], cn_img, cn_weight, 0, cn_stop)
+        imgs = pipeline.process_diffusion(
+            positive_cond=positive_cond,
+            negative_cond=negative_cond,
+            steps=steps,
+            switch=switch,
+            width=width,
+            height=height,
+            image_seed=task['task_seed'],
+            callback=callback,
+            sampler_name=async_task.sampler_name,
+            scheduler_name=final_scheduler_name,
+            latent=initial_latent,
+            denoise=denoising_strength,
+            tiled=tiled,
+            cfg_scale=async_task.cfg_scale,
+            refiner_swap_method=async_task.refiner_swap_method,
+            disable_preview=async_task.disable_preview
+        )
+        del positive_cond, negative_cond  # Save memory
+        if inpaint_worker.current_task is not None:
+            imgs = [inpaint_worker.current_task.post_process(x) for x in imgs]
+        current_progress = int(base_progress + (100 - preparation_steps) / float(all_steps) * steps)
+        if modules.config.default_black_out_nsfw or async_task.black_out_nsfw:
+            progressbar(async_task, current_progress, 'Checking for NSFW content ...')
+            imgs = default_censor(imgs)
+        progressbar(async_task, current_progress, f'Saving image {current_task_id + 1}/{total_count} to system ...')
+        img_paths = save_and_log(async_task, height, imgs, task, use_expansion, width, loras, persist_image)
+        yield_result(async_task, img_paths, current_progress, async_task.black_out_nsfw, False,
+                     do_not_show_finished_images=not show_intermediate_results or async_task.disable_intermediate_results)
+
+        return imgs, img_paths, current_progress
+
+    def apply_patch_settings(async_task):
+        patch_settings[pid] = PatchSettings(
+            async_task.sharpness,
+            async_task.adm_scaler_end,
+            async_task.adm_scaler_positive,
+            async_task.adm_scaler_negative,
+            async_task.controlnet_softness,
+            async_task.adaptive_cfg
+        )
+
+    def save_and_log(async_task, height, imgs, task, use_expansion, width, loras, persist_image=True) -> list:
+        img_paths = []
+        for x in imgs:
+            d = [('Prompt', 'prompt', task['log_positive_prompt']),
+                 ('Negative Prompt', 'negative_prompt', task['log_negative_prompt']),
+                 ('Fooocus V2 Expansion', 'prompt_expansion', task['expansion']),
+                 ('Styles', 'styles',
+                  str(task['styles'] if not use_expansion else [fooocus_expansion] + task['styles'])),
+                 ('Performance', 'performance', async_task.performance_selection.value),
+                 ('Steps', 'steps', async_task.steps),
+                 ('Resolution', 'resolution', str((width, height))),
+                 ('Guidance Scale', 'guidance_scale', async_task.cfg_scale),
+                 ('Sharpness', 'sharpness', async_task.sharpness),
+                 ('ADM Guidance', 'adm_guidance', str((
+                     modules.patch.patch_settings[pid].positive_adm_scale,
+                     modules.patch.patch_settings[pid].negative_adm_scale,
+                     modules.patch.patch_settings[pid].adm_scaler_end))),
+                 ('Base Model', 'base_model', async_task.base_model_name),
+                 ('Refiner Model', 'refiner_model', async_task.refiner_model_name),
+                 ('Refiner Switch', 'refiner_switch', async_task.refiner_switch)]
+
+            if async_task.refiner_model_name != 'None':
+                if async_task.overwrite_switch > 0:
+                    d.append(('Overwrite Switch', 'overwrite_switch', async_task.overwrite_switch))
+                if async_task.refiner_swap_method != flags.refiner_swap_method:
+                    d.append(('Refiner Swap Method', 'refiner_swap_method', async_task.refiner_swap_method))
+            if modules.patch.patch_settings[pid].adaptive_cfg != modules.config.default_cfg_tsnr:
+                d.append(
+                    ('CFG Mimicking from TSNR', 'adaptive_cfg', modules.patch.patch_settings[pid].adaptive_cfg))
+
+            if async_task.clip_skip > 1:
+                d.append(('CLIP Skip', 'clip_skip', async_task.clip_skip))
+            d.append(('Sampler', 'sampler', async_task.sampler_name))
+            d.append(('Scheduler', 'scheduler', async_task.scheduler_name))
+            d.append(('VAE', 'vae', async_task.vae_name))
+            d.append(('Seed', 'seed', str(task['task_seed'])))
+
+            if async_task.freeu_enabled:
+                d.append(('FreeU', 'freeu',
+                          str((async_task.freeu_b1, async_task.freeu_b2, async_task.freeu_s1, async_task.freeu_s2))))
+
+            for li, (n, w) in enumerate(loras):
+                if n != 'None':
+                    d.append((f'LoRA {li + 1}', f'lora_combined_{li + 1}', f'{n} : {w}'))
+
+            metadata_parser = None
+            if async_task.save_metadata_to_images:
+                metadata_parser = modules.meta_parser.get_metadata_parser(async_task.metadata_scheme)
+                metadata_parser.set_data(task['log_positive_prompt'], task['positive'],
+                                         task['log_negative_prompt'], task['negative'],
+                                         async_task.steps, async_task.base_model_name, async_task.refiner_model_name,
+                                         loras, async_task.vae_name)
+            d.append(('Metadata Scheme', 'metadata_scheme',
+                      async_task.metadata_scheme.value if async_task.save_metadata_to_images else async_task.save_metadata_to_images))
+            d.append(('Version', 'version', 'Fooocus v' + fooocus_version.version))
+            img_paths.append(log(x, d, metadata_parser, async_task.output_format, task, persist_image))
+
+        return img_paths
+
+    def apply_control_nets(async_task, height, ip_adapter_face_path, ip_adapter_path, width, current_progress):
+        for task in async_task.cn_tasks[flags.cn_canny]:
+            cn_img, cn_stop, cn_weight = task
+            cn_img = resize_image(HWC3(cn_img), width=width, height=height)
+
+            if not async_task.skipping_cn_preprocessor:
+                cn_img = preprocessors.canny_pyramid(cn_img, async_task.canny_low_threshold,
+                                                     async_task.canny_high_threshold)
+
+            cn_img = HWC3(cn_img)
+            task[0] = core.numpy_to_pytorch(cn_img)
+            if async_task.debugging_cn_preprocessor:
+                yield_result(async_task, cn_img, current_progress, async_task.black_out_nsfw, do_not_show_finished_images=True)
+        for task in async_task.cn_tasks[flags.cn_cpds]:
+            cn_img, cn_stop, cn_weight = task
+            cn_img = resize_image(HWC3(cn_img), width=width, height=height)
+
+            if not async_task.skipping_cn_preprocessor:
+                cn_img = preprocessors.cpds(cn_img)
+
+            cn_img = HWC3(cn_img)
+            task[0] = core.numpy_to_pytorch(cn_img)
+            if async_task.debugging_cn_preprocessor:
+                yield_result(async_task, cn_img, current_progress, async_task.black_out_nsfw, do_not_show_finished_images=True)
+        for task in async_task.cn_tasks[flags.cn_ip]:
+            cn_img, cn_stop, cn_weight = task
+            cn_img = HWC3(cn_img)
+
+            # https://github.com/tencent-ailab/IP-Adapter/blob/d580c50a291566bbf9fc7ac0f760506607297e6d/README.md?plain=1#L75
+            cn_img = resize_image(cn_img, width=224, height=224, resize_mode=0)
+
+            task[0] = ip_adapter.preprocess(cn_img, ip_adapter_path=ip_adapter_path)
+            if async_task.debugging_cn_preprocessor:
+                yield_result(async_task, cn_img, current_progress, async_task.black_out_nsfw, do_not_show_finished_images=True)
+        for task in async_task.cn_tasks[flags.cn_ip_face]:
+            cn_img, cn_stop, cn_weight = task
+            cn_img = HWC3(cn_img)
+
+            if not async_task.skipping_cn_preprocessor:
+                cn_img = extras.face_crop.crop_image(cn_img)
+
+            # https://github.com/tencent-ailab/IP-Adapter/blob/d580c50a291566bbf9fc7ac0f760506607297e6d/README.md?plain=1#L75
+            cn_img = resize_image(cn_img, width=224, height=224, resize_mode=0)
+
+            task[0] = ip_adapter.preprocess(cn_img, ip_adapter_path=ip_adapter_face_path)
+            if async_task.debugging_cn_preprocessor:
+                yield_result(async_task, cn_img, current_progress, async_task.black_out_nsfw, do_not_show_finished_images=True)
+        all_ip_tasks = async_task.cn_tasks[flags.cn_ip] + async_task.cn_tasks[flags.cn_ip_face]
+        if len(all_ip_tasks) > 0:
+            pipeline.final_unet = ip_adapter.patch_model(pipeline.final_unet, all_ip_tasks)
+
+    def apply_vary(async_task, uov_method, denoising_strength, uov_input_image, switch, current_progress, advance_progress=False):
+        if 'subtle' in uov_method:
+            denoising_strength = 0.5
+        if 'strong' in uov_method:
+            denoising_strength = 0.85
+        if async_task.overwrite_vary_strength > 0:
+            denoising_strength = async_task.overwrite_vary_strength
+        shape_ceil = get_image_shape_ceil(uov_input_image)
+        if shape_ceil < 1024:
+            print(f'[Vary] Image is resized because it is too small.')
+            shape_ceil = 1024
+        elif shape_ceil > 2048:
+            print(f'[Vary] Image is resized because it is too big.')
+            shape_ceil = 2048
+        uov_input_image = set_image_shape_ceil(uov_input_image, shape_ceil)
+        initial_pixels = core.numpy_to_pytorch(uov_input_image)
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, current_progress, 'VAE encoding ...')
+        candidate_vae, _ = pipeline.get_candidate_vae(
+            steps=async_task.steps,
+            switch=switch,
+            denoise=denoising_strength,
+            refiner_swap_method=async_task.refiner_swap_method
+        )
+        initial_latent = core.encode_vae(vae=candidate_vae, pixels=initial_pixels)
+        B, C, H, W = initial_latent['samples'].shape
+        width = W * 8
+        height = H * 8
+        print(f'Final resolution is {str((width, height))}.')
+        return uov_input_image, denoising_strength, initial_latent, width, height, current_progress
+
+    def apply_inpaint(async_task, initial_latent, inpaint_head_model_path, inpaint_image,
+                      inpaint_mask, inpaint_parameterized, denoising_strength, inpaint_respective_field, switch,
+                      inpaint_disable_initial_latent, current_progress, skip_apply_outpaint=False,
+                      advance_progress=False):
+        if not skip_apply_outpaint:
+            inpaint_image, inpaint_mask = apply_outpaint(async_task, inpaint_image, inpaint_mask)
+
+        inpaint_worker.current_task = inpaint_worker.InpaintWorker(
+            image=inpaint_image,
+            mask=inpaint_mask,
+            use_fill=denoising_strength > 0.99,
+            k=inpaint_respective_field
+        )
+        if async_task.debugging_inpaint_preprocessor:
+            yield_result(async_task, inpaint_worker.current_task.visualize_mask_processing(), 100,
+                         async_task.black_out_nsfw, do_not_show_finished_images=True)
+            raise EarlyReturnException
+
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, current_progress, 'VAE Inpaint encoding ...')
+        inpaint_pixel_fill = core.numpy_to_pytorch(inpaint_worker.current_task.interested_fill)
+        inpaint_pixel_image = core.numpy_to_pytorch(inpaint_worker.current_task.interested_image)
+        inpaint_pixel_mask = core.numpy_to_pytorch(inpaint_worker.current_task.interested_mask)
+        candidate_vae, candidate_vae_swap = pipeline.get_candidate_vae(
+            steps=async_task.steps,
+            switch=switch,
+            denoise=denoising_strength,
+            refiner_swap_method=async_task.refiner_swap_method
+        )
+        latent_inpaint, latent_mask = core.encode_vae_inpaint(
+            mask=inpaint_pixel_mask,
+            vae=candidate_vae,
+            pixels=inpaint_pixel_image)
+        latent_swap = None
+        if candidate_vae_swap is not None:
+            if advance_progress:
+                current_progress += 1
+            progressbar(async_task, current_progress, 'VAE SD15 encoding ...')
+            latent_swap = core.encode_vae(
+                vae=candidate_vae_swap,
+                pixels=inpaint_pixel_fill)['samples']
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, current_progress, 'VAE encoding ...')
+        latent_fill = core.encode_vae(
+            vae=candidate_vae,
+            pixels=inpaint_pixel_fill)['samples']
+        inpaint_worker.current_task.load_latent(
+            latent_fill=latent_fill, latent_mask=latent_mask, latent_swap=latent_swap)
+        if inpaint_parameterized:
+            pipeline.final_unet = inpaint_worker.current_task.patch(
+                inpaint_head_model_path=inpaint_head_model_path,
+                inpaint_latent=latent_inpaint,
+                inpaint_latent_mask=latent_mask,
+                model=pipeline.final_unet
+            )
+        if not inpaint_disable_initial_latent:
+            initial_latent = {'samples': latent_fill}
+        B, C, H, W = latent_fill.shape
+        height, width = H * 8, W * 8
+        final_height, final_width = inpaint_worker.current_task.image.shape[:2]
+        print(f'Final resolution is {str((final_width, final_height))}, latent is {str((width, height))}.')
+
+        return denoising_strength, initial_latent, width, height, current_progress
+
+    def apply_outpaint(async_task, inpaint_image, inpaint_mask):
+        if len(async_task.outpaint_selections) > 0:
+            H, W, C = inpaint_image.shape
+            if 'top' in async_task.outpaint_selections:
+                inpaint_image = np.pad(inpaint_image, [[int(H * 0.3), 0], [0, 0], [0, 0]], mode='edge')
+                inpaint_mask = np.pad(inpaint_mask, [[int(H * 0.3), 0], [0, 0]], mode='constant',
+                                      constant_values=255)
+            if 'bottom' in async_task.outpaint_selections:
+                inpaint_image = np.pad(inpaint_image, [[0, int(H * 0.3)], [0, 0], [0, 0]], mode='edge')
+                inpaint_mask = np.pad(inpaint_mask, [[0, int(H * 0.3)], [0, 0]], mode='constant',
+                                      constant_values=255)
+
+            H, W, C = inpaint_image.shape
+            if 'left' in async_task.outpaint_selections:
+                inpaint_image = np.pad(inpaint_image, [[0, 0], [int(W * 0.3), 0], [0, 0]], mode='edge')
+                inpaint_mask = np.pad(inpaint_mask, [[0, 0], [int(W * 0.3), 0]], mode='constant',
+                                      constant_values=255)
+            if 'right' in async_task.outpaint_selections:
+                inpaint_image = np.pad(inpaint_image, [[0, 0], [0, int(W * 0.3)], [0, 0]], mode='edge')
+                inpaint_mask = np.pad(inpaint_mask, [[0, 0], [0, int(W * 0.3)]], mode='constant',
+                                      constant_values=255)
+
+            inpaint_image = np.ascontiguousarray(inpaint_image.copy())
+            inpaint_mask = np.ascontiguousarray(inpaint_mask.copy())
+            async_task.inpaint_strength = 1.0
+            async_task.inpaint_respective_field = 1.0
+        return inpaint_image, inpaint_mask
+
+    def apply_upscale(async_task, uov_input_image, uov_method, switch, current_progress, advance_progress=False):
+        H, W, C = uov_input_image.shape
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, current_progress, f'Upscaling image from {str((W, H))} ...')
+        uov_input_image = perform_upscale(uov_input_image)
+        print(f'Image upscaled.')
+        if '1.5x' in uov_method:
+            f = 1.5
+        elif '2x' in uov_method:
+            f = 2.0
+        else:
+            f = 1.0
+        shape_ceil = get_shape_ceil(H * f, W * f)
+        if shape_ceil < 1024:
+            print(f'[Upscale] Image is resized because it is too small.')
+            uov_input_image = set_image_shape_ceil(uov_input_image, 1024)
+            shape_ceil = 1024
+        else:
+            uov_input_image = resample_image(uov_input_image, width=W * f, height=H * f)
+        image_is_super_large = shape_ceil > 2800
+        if 'fast' in uov_method:
+            direct_return = True
+        elif image_is_super_large:
+            print('Image is too large. Directly returned the SR image. '
+                  'Usually directly return SR image at 4K resolution '
+                  'yields better results than SDXL diffusion.')
+            direct_return = True
+        else:
+            direct_return = False
+        if direct_return:
+            return direct_return, uov_input_image, None, None, None, None, None, current_progress
+
+        tiled = True
+        denoising_strength = 0.382
+        if async_task.overwrite_upscale_strength > 0:
+            denoising_strength = async_task.overwrite_upscale_strength
+        initial_pixels = core.numpy_to_pytorch(uov_input_image)
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, current_progress, 'VAE encoding ...')
+        candidate_vae, _ = pipeline.get_candidate_vae(
+            steps=async_task.steps,
+            switch=switch,
+            denoise=denoising_strength,
+            refiner_swap_method=async_task.refiner_swap_method
+        )
+        initial_latent = core.encode_vae(
+            vae=candidate_vae,
+            pixels=initial_pixels, tiled=True)
+        B, C, H, W = initial_latent['samples'].shape
+        width = W * 8
+        height = H * 8
+        print(f'Final resolution is {str((width, height))}.')
+        return direct_return, uov_input_image, denoising_strength, initial_latent, tiled, width, height, current_progress
+
+    def apply_overrides(async_task, steps, height, width):
+        if async_task.overwrite_step > 0:
+            steps = async_task.overwrite_step
+        switch = int(round(async_task.steps * async_task.refiner_switch))
+        if async_task.overwrite_switch > 0:
+            switch = async_task.overwrite_switch
+        if async_task.overwrite_width > 0:
+            width = async_task.overwrite_width
+        if async_task.overwrite_height > 0:
+            height = async_task.overwrite_height
+        return steps, switch, width, height
+
+    def process_prompt(async_task, prompt, negative_prompt, base_model_additional_loras, image_number, disable_seed_increment, use_expansion, use_style,
+                       use_synthetic_refiner, current_progress, advance_progress=False):
+        prompts = remove_empty_str([safe_str(p) for p in prompt.splitlines()], default='')
+        negative_prompts = remove_empty_str([safe_str(p) for p in negative_prompt.splitlines()], default='')
+        prompt = prompts[0]
+        negative_prompt = negative_prompts[0]
+        if prompt == '':
+            # disable expansion when empty since it is not meaningful and influences image prompt
+            use_expansion = False
+        extra_positive_prompts = prompts[1:] if len(prompts) > 1 else []
+        extra_negative_prompts = negative_prompts[1:] if len(negative_prompts) > 1 else []
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, current_progress, 'Loading models ...')
+        lora_filenames = modules.util.remove_performance_lora(modules.config.lora_filenames,
+                                                              async_task.performance_selection)
+        loras, prompt = parse_lora_references_from_prompt(prompt, async_task.loras,
+                                                          modules.config.default_max_lora_number,
+                                                          lora_filenames=lora_filenames)
+        loras += async_task.performance_loras
+        pipeline.refresh_everything(refiner_model_name=async_task.refiner_model_name,
+                                    base_model_name=async_task.base_model_name,
+                                    loras=loras, base_model_additional_loras=base_model_additional_loras,
+                                    use_synthetic_refiner=use_synthetic_refiner, vae_name=async_task.vae_name)
+        pipeline.set_clip_skip(async_task.clip_skip)
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, current_progress, 'Processing prompts ...')
+        tasks = []
+        for i in range(image_number):
+            if disable_seed_increment:
+                task_seed = async_task.seed % (constants.MAX_SEED + 1)
+            else:
+                task_seed = (async_task.seed + i) % (constants.MAX_SEED + 1)  # randint is inclusive, % is not
+
+            task_rng = random.Random(task_seed)  # may bind to inpaint noise in the future
+            task_prompt = apply_wildcards(prompt, task_rng, i, async_task.read_wildcards_in_order)
+            task_prompt = apply_arrays(task_prompt, i)
+            task_negative_prompt = apply_wildcards(negative_prompt, task_rng, i, async_task.read_wildcards_in_order)
+            task_extra_positive_prompts = [apply_wildcards(pmt, task_rng, i, async_task.read_wildcards_in_order) for pmt
+                                           in
+                                           extra_positive_prompts]
+            task_extra_negative_prompts = [apply_wildcards(pmt, task_rng, i, async_task.read_wildcards_in_order) for pmt
+                                           in
+                                           extra_negative_prompts]
+
+            positive_basic_workloads = []
+            negative_basic_workloads = []
+
+            task_styles = async_task.style_selections.copy()
+            if use_style:
+                placeholder_replaced = False
+
+                for j, s in enumerate(task_styles):
+                    if s == random_style_name:
+                        s = get_random_style(task_rng)
+                        task_styles[j] = s
+                    p, n, style_has_placeholder = apply_style(s, positive=task_prompt)
+                    if style_has_placeholder:
+                        placeholder_replaced = True
+                    positive_basic_workloads = positive_basic_workloads + p
+                    negative_basic_workloads = negative_basic_workloads + n
+
+                if not placeholder_replaced:
+                    positive_basic_workloads = [task_prompt] + positive_basic_workloads
+            else:
+                positive_basic_workloads.append(task_prompt)
+
+            negative_basic_workloads.append(task_negative_prompt)  # Always use independent workload for negative.
+
+            positive_basic_workloads = positive_basic_workloads + task_extra_positive_prompts
+            negative_basic_workloads = negative_basic_workloads + task_extra_negative_prompts
+
+            positive_basic_workloads = remove_empty_str(positive_basic_workloads, default=task_prompt)
+            negative_basic_workloads = remove_empty_str(negative_basic_workloads, default=task_negative_prompt)
+
+            tasks.append(dict(
+                task_seed=task_seed,
+                task_prompt=task_prompt,
+                task_negative_prompt=task_negative_prompt,
+                positive=positive_basic_workloads,
+                negative=negative_basic_workloads,
+                expansion='',
+                c=None,
+                uc=None,
+                positive_top_k=len(positive_basic_workloads),
+                negative_top_k=len(negative_basic_workloads),
+                log_positive_prompt='\n'.join([task_prompt] + task_extra_positive_prompts),
+                log_negative_prompt='\n'.join([task_negative_prompt] + task_extra_negative_prompts),
+                styles=task_styles
+            ))
+        if use_expansion:
+            if advance_progress:
+                current_progress += 1
+            for i, t in enumerate(tasks):
+
+                progressbar(async_task, current_progress, f'Preparing Fooocus text #{i + 1} ...')
+                expansion = pipeline.final_expansion(t['task_prompt'], t['task_seed'])
+                print(f'[Prompt Expansion] {expansion}')
+                t['expansion'] = expansion
+                t['positive'] = copy.deepcopy(t['positive']) + [expansion]  # Deep copy.
+        if advance_progress:
+            current_progress += 1
+        for i, t in enumerate(tasks):
+            progressbar(async_task, current_progress, f'Encoding positive #{i + 1} ...')
+            t['c'] = pipeline.clip_encode(texts=t['positive'], pool_top_k=t['positive_top_k'])
+        if advance_progress:
+            current_progress += 1
+        for i, t in enumerate(tasks):
+            if abs(float(async_task.cfg_scale) - 1.0) < 1e-4:
+                t['uc'] = pipeline.clone_cond(t['c'])
+            else:
+                progressbar(async_task, current_progress, f'Encoding negative #{i + 1} ...')
+                t['uc'] = pipeline.clip_encode(texts=t['negative'], pool_top_k=t['negative_top_k'])
+        return tasks, use_expansion, loras, current_progress
+
+    def apply_freeu(async_task):
+        print(f'FreeU is enabled!')
+        pipeline.final_unet = core.apply_freeu(
+            pipeline.final_unet,
+            async_task.freeu_b1,
+            async_task.freeu_b2,
+            async_task.freeu_s1,
+            async_task.freeu_s2
+        )
+
+    def patch_discrete(unet, scheduler_name):
+        return core.opModelSamplingDiscrete.patch(unet, scheduler_name, False)[0]
+
+    def patch_edm(unet, scheduler_name):
+        return core.opModelSamplingContinuousEDM.patch(unet, scheduler_name, 120.0, 0.002)[0]
+
+    def patch_samplers(async_task):
+        final_scheduler_name = async_task.scheduler_name
+
+        if async_task.scheduler_name in ['lcm', 'tcd']:
+            final_scheduler_name = 'sgm_uniform'
+            if pipeline.final_unet is not None:
+                pipeline.final_unet = patch_discrete(pipeline.final_unet, async_task.scheduler_name)
+            if pipeline.final_refiner_unet is not None:
+                pipeline.final_refiner_unet = patch_discrete(pipeline.final_refiner_unet, async_task.scheduler_name)
+
+        elif async_task.scheduler_name == 'edm_playground_v2.5':
+            final_scheduler_name = 'karras'
+            if pipeline.final_unet is not None:
+                pipeline.final_unet = patch_edm(pipeline.final_unet, async_task.scheduler_name)
+            if pipeline.final_refiner_unet is not None:
+                pipeline.final_refiner_unet = patch_edm(pipeline.final_refiner_unet, async_task.scheduler_name)
+
+        return final_scheduler_name
+
+    def set_hyper_sd_defaults(async_task, current_progress, advance_progress=False):
+        print('Enter Hyper-SD mode.')
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, current_progress, 'Downloading Hyper-SD components ...')
+        async_task.performance_loras += [(modules.config.downloading_sdxl_hyper_sd_lora(), 0.8)]
+        if async_task.refiner_model_name != 'None':
+            print(f'Refiner disabled in Hyper-SD mode.')
+        async_task.refiner_model_name = 'None'
+        async_task.sampler_name = 'dpmpp_sde_gpu'
+        async_task.scheduler_name = 'karras'
+        async_task.sharpness = 0.0
+        async_task.cfg_scale = 1.0
+        async_task.adaptive_cfg = 1.0
+        async_task.refiner_switch = 1.0
+        async_task.adm_scaler_positive = 1.0
+        async_task.adm_scaler_negative = 1.0
+        async_task.adm_scaler_end = 0.0
+        return current_progress
+
+    def set_lightning_defaults(async_task, current_progress, advance_progress=False):
+        print('Enter Lightning mode.')
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, 1, 'Downloading Lightning components ...')
+        async_task.performance_loras += [(modules.config.downloading_sdxl_lightning_lora(), 1.0)]
+        if async_task.refiner_model_name != 'None':
+            print(f'Refiner disabled in Lightning mode.')
+        async_task.refiner_model_name = 'None'
+        async_task.sampler_name = 'euler'
+        async_task.scheduler_name = 'sgm_uniform'
+        async_task.sharpness = 0.0
+        async_task.cfg_scale = 1.0
+        async_task.adaptive_cfg = 1.0
+        async_task.refiner_switch = 1.0
+        async_task.adm_scaler_positive = 1.0
+        async_task.adm_scaler_negative = 1.0
+        async_task.adm_scaler_end = 0.0
+        return current_progress
+
+    def set_lcm_defaults(async_task, current_progress, advance_progress=False):
+        print('Enter LCM mode.')
+        if advance_progress:
+            current_progress += 1
+        progressbar(async_task, 1, 'Downloading LCM components ...')
+        async_task.performance_loras += [(modules.config.downloading_sdxl_lcm_lora(), 1.0)]
+        if async_task.refiner_model_name != 'None':
+            print(f'Refiner disabled in LCM mode.')
+        async_task.refiner_model_name = 'None'
+        async_task.sampler_name = 'lcm'
+        async_task.scheduler_name = 'lcm'
+        async_task.sharpness = 0.0
+        async_task.cfg_scale = 1.0
+        async_task.adaptive_cfg = 1.0
+        async_task.refiner_switch = 1.0
+        async_task.adm_scaler_positive = 1.0
+        async_task.adm_scaler_negative = 1.0
+        async_task.adm_scaler_end = 0.0
+        return current_progress
+
+    def apply_image_input(async_task, base_model_additional_loras, clip_vision_path, controlnet_canny_path,
+                          controlnet_cpds_path, goals, inpaint_head_model_path, inpaint_image, inpaint_mask,
+                          inpaint_parameterized,  ip_adapter_face_path, ip_adapter_path, ip_negative_path,
+                          skip_prompt_processing, use_synthetic_refiner):
+        if (async_task.current_tab == 'uov' or (
+                async_task.current_tab == 'ip' and async_task.mixing_image_prompt_and_vary_upscale)) \
+                and async_task.uov_method != flags.disabled.casefold() and async_task.uov_input_image is not None:
+            async_task.uov_input_image, skip_prompt_processing, async_task.steps = prepare_upscale(
+                async_task, goals, async_task.uov_input_image, async_task.uov_method, async_task.performance_selection,
+                async_task.steps, 1, skip_prompt_processing=skip_prompt_processing)
+        if (async_task.current_tab == 'inpaint' or (
+                async_task.current_tab == 'ip' and async_task.mixing_image_prompt_and_inpaint)) \
+                and isinstance(async_task.inpaint_input_image, dict):
+            inpaint_image = async_task.inpaint_input_image['image']
+            inpaint_mask = async_task.inpaint_input_image['mask'][:, :, 0]
+
+            if async_task.inpaint_advanced_masking_checkbox:
+                if isinstance(async_task.inpaint_mask_image_upload, dict):
+                    if (isinstance(async_task.inpaint_mask_image_upload['image'], np.ndarray)
+                            and isinstance(async_task.inpaint_mask_image_upload['mask'], np.ndarray)
+                            and async_task.inpaint_mask_image_upload['image'].ndim == 3):
+                        async_task.inpaint_mask_image_upload = np.maximum(
+                            async_task.inpaint_mask_image_upload['image'],
+                            async_task.inpaint_mask_image_upload['mask'])
+                if isinstance(async_task.inpaint_mask_image_upload,
+                              np.ndarray) and async_task.inpaint_mask_image_upload.ndim == 3:
+                    H, W, C = inpaint_image.shape
+                    async_task.inpaint_mask_image_upload = resample_image(async_task.inpaint_mask_image_upload,
+                                                                          width=W, height=H)
+                    async_task.inpaint_mask_image_upload = np.mean(async_task.inpaint_mask_image_upload, axis=2)
+                    async_task.inpaint_mask_image_upload = (async_task.inpaint_mask_image_upload > 127).astype(
+                        np.uint8) * 255
+                    inpaint_mask = np.maximum(inpaint_mask, async_task.inpaint_mask_image_upload)
+
+            if int(async_task.inpaint_erode_or_dilate) != 0:
+                inpaint_mask = erode_or_dilate(inpaint_mask, async_task.inpaint_erode_or_dilate)
+
+            if async_task.invert_mask_checkbox:
+                inpaint_mask = 255 - inpaint_mask
+
+            inpaint_image = HWC3(inpaint_image)
+            if isinstance(inpaint_image, np.ndarray) and isinstance(inpaint_mask, np.ndarray) \
+                    and (np.any(inpaint_mask > 127) or len(async_task.outpaint_selections) > 0):
+                progressbar(async_task, 1, 'Downloading upscale models ...')
+                modules.config.downloading_upscale_model()
+                if inpaint_parameterized:
+                    progressbar(async_task, 1, 'Downloading inpainter ...')
+                    inpaint_head_model_path, inpaint_patch_model_path = modules.config.downloading_inpaint_models(
+                        async_task.inpaint_engine)
+                    base_model_additional_loras += [(inpaint_patch_model_path, 1.0)]
+                    print(f'[Inpaint] Current inpaint model is {inpaint_patch_model_path}')
+                    if async_task.refiner_model_name == 'None':
+                        use_synthetic_refiner = True
+                        async_task.refiner_switch = 0.8
+                else:
+                    inpaint_head_model_path, inpaint_patch_model_path = None, None
+                    print(f'[Inpaint] Parameterized inpaint is disabled.')
+                if async_task.inpaint_additional_prompt != '':
+                    if async_task.prompt == '':
+                        async_task.prompt = async_task.inpaint_additional_prompt
+                    else:
+                        async_task.prompt = async_task.inpaint_additional_prompt + '\n' + async_task.prompt
+                goals.append('inpaint')
+        if async_task.current_tab == 'ip' or \
+                async_task.mixing_image_prompt_and_vary_upscale or \
+                async_task.mixing_image_prompt_and_inpaint:
+            goals.append('cn')
+            progressbar(async_task, 1, 'Downloading control models ...')
+            if len(async_task.cn_tasks[flags.cn_canny]) > 0:
+                controlnet_canny_path = modules.config.downloading_controlnet_canny()
+            if len(async_task.cn_tasks[flags.cn_cpds]) > 0:
+                controlnet_cpds_path = modules.config.downloading_controlnet_cpds()
+            if len(async_task.cn_tasks[flags.cn_ip]) > 0:
+                clip_vision_path, ip_negative_path, ip_adapter_path = modules.config.downloading_ip_adapters('ip')
+            if len(async_task.cn_tasks[flags.cn_ip_face]) > 0:
+                clip_vision_path, ip_negative_path, ip_adapter_face_path = modules.config.downloading_ip_adapters(
+                    'face')
+        if async_task.current_tab == 'enhance' and async_task.enhance_input_image is not None:
+            goals.append('enhance')
+            skip_prompt_processing = True
+            async_task.enhance_input_image = HWC3(async_task.enhance_input_image)
+        return base_model_additional_loras, clip_vision_path, controlnet_canny_path, controlnet_cpds_path, inpaint_head_model_path, inpaint_image, inpaint_mask, ip_adapter_face_path, ip_adapter_path, ip_negative_path, skip_prompt_processing, use_synthetic_refiner
+
+    def prepare_upscale(async_task, goals, uov_input_image, uov_method, performance, steps, current_progress,
+                        advance_progress=False, skip_prompt_processing=False):
+        uov_input_image = HWC3(uov_input_image)
+        if 'vary' in uov_method:
+            goals.append('vary')
+        elif 'upscale' in uov_method:
+            goals.append('upscale')
+            if 'fast' in uov_method:
+                skip_prompt_processing = True
+                steps = 0
+            else:
+                steps = performance.steps_uov()
+
+            if advance_progress:
+                current_progress += 1
+            progressbar(async_task, current_progress, 'Downloading upscale models ...')
+            modules.config.downloading_upscale_model()
+        return uov_input_image, skip_prompt_processing, steps
+
+    def prepare_enhance_prompt(prompt: str, fallback_prompt: str):
+        if safe_str(prompt) == '' or len(remove_empty_str([safe_str(p) for p in prompt.splitlines()], default='')) == 0:
+            prompt = fallback_prompt
+
+        return prompt
+
+    def stop_processing(async_task, processing_start_time):
+        async_task.processing = False
+        processing_time = time.perf_counter() - processing_start_time
+        print(f'Processing time (total): {processing_time:.2f} seconds')
+
+    def process_enhance(all_steps, async_task, callback, controlnet_canny_path, controlnet_cpds_path,
+                        current_progress, current_task_id, denoising_strength, inpaint_disable_initial_latent,
+                        inpaint_engine, inpaint_respective_field, inpaint_strength,
+                        prompt, negative_prompt, final_scheduler_name, goals, height, img, mask,
+                        preparation_steps, steps, switch, tiled, total_count, use_expansion, use_style,
+                        use_synthetic_refiner, width, show_intermediate_results=True, persist_image=True):
+        base_model_additional_loras = []
+        inpaint_head_model_path = None
+        inpaint_parameterized = inpaint_engine != 'None'  # inpaint_engine = None, improve detail
+        initial_latent = None
+
+        prompt = prepare_enhance_prompt(prompt, async_task.prompt)
+        negative_prompt = prepare_enhance_prompt(negative_prompt, async_task.negative_prompt)
+
+        if 'vary' in goals:
+            img, denoising_strength, initial_latent, width, height, current_progress = apply_vary(
+                async_task, async_task.enhance_uov_method, denoising_strength, img, switch, current_progress)
+        if 'upscale' in goals:
+            direct_return, img, denoising_strength, initial_latent, tiled, width, height, current_progress = apply_upscale(
+                async_task, img, async_task.enhance_uov_method, switch, current_progress)
+            if direct_return:
+                d = [('Upscale (Fast)', 'upscale_fast', '2x')]
+                if modules.config.default_black_out_nsfw or async_task.black_out_nsfw:
+                    progressbar(async_task, current_progress, 'Checking for NSFW content ...')
+                    img = default_censor(img)
+                progressbar(async_task, current_progress, f'Saving image {current_task_id + 1}/{total_count} to system ...')
+                uov_image_path = log(img, d, output_format=async_task.output_format, persist_image=persist_image)
+                yield_result(async_task, uov_image_path, current_progress, async_task.black_out_nsfw, False,
+                             do_not_show_finished_images=not show_intermediate_results or async_task.disable_intermediate_results)
+                return current_progress, img, prompt, negative_prompt
+
+        if 'inpaint' in goals and inpaint_parameterized:
+            progressbar(async_task, current_progress, 'Downloading inpainter ...')
+            inpaint_head_model_path, inpaint_patch_model_path = modules.config.downloading_inpaint_models(
+                inpaint_engine)
+            if inpaint_patch_model_path not in base_model_additional_loras:
+                base_model_additional_loras += [(inpaint_patch_model_path, 1.0)]
+        progressbar(async_task, current_progress, 'Preparing enhance prompts ...')
+        # positive and negative conditioning aren't available here anymore, process prompt again
+        tasks_enhance, use_expansion, loras, current_progress = process_prompt(
+            async_task, prompt, negative_prompt, base_model_additional_loras, 1, True,
+            use_expansion, use_style, use_synthetic_refiner, current_progress)
+        task_enhance = tasks_enhance[0]
+        # TODO could support vary, upscale and CN in the future
+        # if 'cn' in goals:
+        #     apply_control_nets(async_task, height, ip_adapter_face_path, ip_adapter_path, width)
+        if async_task.freeu_enabled:
+            apply_freeu(async_task)
+        patch_samplers(async_task)
+        if 'inpaint' in goals:
+            denoising_strength, initial_latent, width, height, current_progress = apply_inpaint(
+                async_task, None, inpaint_head_model_path, img, mask,
+                inpaint_parameterized, inpaint_strength,
+                inpaint_respective_field, switch, inpaint_disable_initial_latent,
+                current_progress, True)
+        imgs, img_paths, current_progress = process_task(all_steps, async_task, callback, controlnet_canny_path,
+                                                         controlnet_cpds_path, current_task_id, denoising_strength,
+                                                         final_scheduler_name, goals, initial_latent, steps, switch,
+                                                         task_enhance['c'], task_enhance['uc'], task_enhance, loras,
+                                                         tiled, use_expansion, width, height, current_progress,
+                                                         preparation_steps, total_count, show_intermediate_results,
+                                                         persist_image)
+
+        del task_enhance['c'], task_enhance['uc']  # Save memory
+        return current_progress, imgs[0], prompt, negative_prompt
+
+    def enhance_upscale(all_steps, async_task, base_progress, callback, controlnet_canny_path, controlnet_cpds_path,
+                        current_task_id, denoising_strength, done_steps_inpainting, done_steps_upscaling, enhance_steps,
+                        prompt, negative_prompt, final_scheduler_name, height, img, preparation_steps, switch, tiled,
+                        total_count, use_expansion, use_style, use_synthetic_refiner, width, persist_image=True):
+        # reset inpaint worker to prevent tensor size issues and not mix upscale and inpainting
+        inpaint_worker.current_task = None
+
+        current_progress = int(base_progress + (100 - preparation_steps) / float(all_steps) * (done_steps_upscaling + done_steps_inpainting))
+        goals_enhance = []
+        img, skip_prompt_processing, steps = prepare_upscale(
+            async_task, goals_enhance, img, async_task.enhance_uov_method, async_task.performance_selection,
+            enhance_steps, current_progress)
+        steps, _, _, _ = apply_overrides(async_task, steps, height, width)
+        exception_result = ''
+        if len(goals_enhance) > 0:
+            try:
+                current_progress, img, prompt, negative_prompt = process_enhance(
+                    all_steps, async_task, callback, controlnet_canny_path,
+                    controlnet_cpds_path, current_progress, current_task_id, denoising_strength, False,
+                    'None', 0.0, 0.0, prompt, negative_prompt, final_scheduler_name,
+                    goals_enhance, height, img, None, preparation_steps, steps, switch, tiled, total_count,
+                    use_expansion, use_style, use_synthetic_refiner, width, persist_image=persist_image)
+
+            except ldm_patched.modules.model_management.InterruptProcessingException:
+                if async_task.last_stop == 'skip':
+                    print('User skipped')
+                    async_task.last_stop = False
+                    # also skip all enhance steps for this image, but add the steps to the progress bar
+                    if async_task.enhance_uov_processing_order == flags.enhancement_uov_before:
+                        done_steps_inpainting += len(async_task.enhance_ctrls) * enhance_steps
+                    exception_result = 'continue'
+                else:
+                    print('User stopped')
+                    exception_result = 'break'
+            finally:
+                done_steps_upscaling += steps
+        return current_task_id, done_steps_inpainting, done_steps_upscaling, img, exception_result
+
+    @torch.no_grad()
+    @torch.inference_mode()
+    def handler(async_task: AsyncTask):
+        preparation_start_time = time.perf_counter()
+        async_task.processing = True
+
+        async_task.outpaint_selections = [o.lower() for o in async_task.outpaint_selections]
+        base_model_additional_loras = []
+        async_task.uov_method = async_task.uov_method.casefold()
+        async_task.enhance_uov_method = async_task.enhance_uov_method.casefold()
+
+        if fooocus_expansion in async_task.style_selections:
+            use_expansion = True
+            async_task.style_selections.remove(fooocus_expansion)
+        else:
+            use_expansion = False
+
+        use_style = len(async_task.style_selections) > 0
+
+        if async_task.base_model_name == async_task.refiner_model_name:
+            print(f'Refiner disabled because base model and refiner are same.')
+            async_task.refiner_model_name = 'None'
+
+        current_progress = 0
+        if async_task.performance_selection == Performance.EXTREME_SPEED:
+            set_lcm_defaults(async_task, current_progress, advance_progress=True)
+        elif async_task.performance_selection == Performance.LIGHTNING:
+            set_lightning_defaults(async_task, current_progress, advance_progress=True)
+        elif async_task.performance_selection == Performance.HYPER_SD:
+            set_hyper_sd_defaults(async_task, current_progress, advance_progress=True)
+
+        print(f'[Parameters] Adaptive CFG = {async_task.adaptive_cfg}')
+        print(f'[Parameters] CLIP Skip = {async_task.clip_skip}')
+        print(f'[Parameters] Sharpness = {async_task.sharpness}')
+        print(f'[Parameters] ControlNet Softness = {async_task.controlnet_softness}')
+        print(f'[Parameters] ADM Scale = '
+              f'{async_task.adm_scaler_positive} : '
+              f'{async_task.adm_scaler_negative} : '
+              f'{async_task.adm_scaler_end}')
+        print(f'[Parameters] Seed = {async_task.seed}')
+
+        apply_patch_settings(async_task)
+
+        print(f'[Parameters] CFG = {async_task.cfg_scale}')
+
+        initial_latent = None
+        denoising_strength = 1.0
+        tiled = False
+
+        width, height = async_task.aspect_ratios_selection.replace('×', ' ').split(' ')[:2]
+        width, height = int(width), int(height)
+
+        skip_prompt_processing = False
+
+        inpaint_worker.current_task = None
+        inpaint_parameterized = async_task.inpaint_engine != 'None'
+        inpaint_image = None
+        inpaint_mask = None
+        inpaint_head_model_path = None
+
+        use_synthetic_refiner = False
+
+        controlnet_canny_path = None
+        controlnet_cpds_path = None
+        clip_vision_path, ip_negative_path, ip_adapter_path, ip_adapter_face_path = None, None, None, None
+
+        goals = []
+        tasks = []
+        current_progress = 1
+
+        if async_task.input_image_checkbox:
+            base_model_additional_loras, clip_vision_path, controlnet_canny_path, controlnet_cpds_path, inpaint_head_model_path, inpaint_image, inpaint_mask, ip_adapter_face_path, ip_adapter_path, ip_negative_path, skip_prompt_processing, use_synthetic_refiner = apply_image_input(
+                async_task, base_model_additional_loras, clip_vision_path, controlnet_canny_path, controlnet_cpds_path,
+                goals, inpaint_head_model_path, inpaint_image, inpaint_mask, inpaint_parameterized, ip_adapter_face_path,
+                ip_adapter_path, ip_negative_path, skip_prompt_processing, use_synthetic_refiner)
+
+        # Load or unload CNs
+        progressbar(async_task, current_progress, 'Loading control models ...')
+        pipeline.refresh_controlnets([controlnet_canny_path, controlnet_cpds_path])
+        ip_adapter.load_ip_adapter(clip_vision_path, ip_negative_path, ip_adapter_path)
+        ip_adapter.load_ip_adapter(clip_vision_path, ip_negative_path, ip_adapter_face_path)
+
+        async_task.steps, switch, width, height = apply_overrides(async_task, async_task.steps, height, width)
+
+        print(f'[Parameters] Sampler = {async_task.sampler_name} - {async_task.scheduler_name}')
+        print(f'[Parameters] Steps = {async_task.steps} - {switch}')
+
+        progressbar(async_task, current_progress, 'Initializing ...')
+
+        loras = async_task.loras
+        if not skip_prompt_processing:
+            tasks, use_expansion, loras, current_progress = process_prompt(async_task, async_task.prompt, async_task.negative_prompt,
+                                                         base_model_additional_loras, async_task.image_number,
+                                                         async_task.disable_seed_increment, use_expansion, use_style,
+                                                         use_synthetic_refiner, current_progress, advance_progress=True)
+
+        if len(goals) > 0:
+            current_progress += 1
+            progressbar(async_task, current_progress, 'Image processing ...')
+
+        should_enhance = async_task.enhance_checkbox and (async_task.enhance_uov_method != flags.disabled.casefold() or len(async_task.enhance_ctrls) > 0)
+
+        if 'vary' in goals:
+            async_task.uov_input_image, denoising_strength, initial_latent, width, height, current_progress = apply_vary(
+                async_task, async_task.uov_method, denoising_strength, async_task.uov_input_image, switch,
+                current_progress)
+
+        if 'upscale' in goals:
+            direct_return, async_task.uov_input_image, denoising_strength, initial_latent, tiled, width, height, current_progress = apply_upscale(
+                async_task, async_task.uov_input_image, async_task.uov_method, switch, current_progress,
+                advance_progress=True)
+            if direct_return:
+                d = [('Upscale (Fast)', 'upscale_fast', '2x')]
+                if modules.config.default_black_out_nsfw or async_task.black_out_nsfw:
+                    progressbar(async_task, 100, 'Checking for NSFW content ...')
+                    async_task.uov_input_image = default_censor(async_task.uov_input_image)
+                progressbar(async_task, 100, 'Saving image to system ...')
+                uov_input_image_path = log(async_task.uov_input_image, d, output_format=async_task.output_format)
+                yield_result(async_task, uov_input_image_path, 100, async_task.black_out_nsfw, False,
+                             do_not_show_finished_images=True)
+                return
+
+        if 'inpaint' in goals:
+            try:
+                denoising_strength, initial_latent, width, height, current_progress = apply_inpaint(async_task,
+                                                                                                    initial_latent,
+                                                                                                    inpaint_head_model_path,
+                                                                                                    inpaint_image,
+                                                                                                    inpaint_mask,
+                                                                                                    inpaint_parameterized,
+                                                                                                    async_task.inpaint_strength,
+                                                                                                    async_task.inpaint_respective_field,
+                                                                                                    switch,
+                                                                                                    async_task.inpaint_disable_initial_latent,
+                                                                                                    current_progress,
+                                                                                                    advance_progress=True)
+            except EarlyReturnException:
+                return
+
+        if 'cn' in goals:
+            apply_control_nets(async_task, height, ip_adapter_face_path, ip_adapter_path, width, current_progress)
+            if async_task.debugging_cn_preprocessor:
+                return
+
+        if async_task.freeu_enabled:
+            apply_freeu(async_task)
+
+        # async_task.steps can have value of uov steps here when upscale has been applied
+        steps, _, _, _ = apply_overrides(async_task, async_task.steps, height, width)
+
+        images_to_enhance = []
+        if 'enhance' in goals:
+            async_task.image_number = 1
+            images_to_enhance += [async_task.enhance_input_image]
+            height, width, _ = async_task.enhance_input_image.shape
+            # input image already provided, processing is skipped
+            steps = 0
+            yield_result(async_task, async_task.enhance_input_image, current_progress, async_task.black_out_nsfw, False,
+                         async_task.disable_intermediate_results)
+
+        all_steps = steps * async_task.image_number
+
+        if async_task.enhance_checkbox and async_task.enhance_uov_method != flags.disabled.casefold():
+            enhance_upscale_steps = async_task.performance_selection.steps()
+            if 'upscale' in async_task.enhance_uov_method:
+                if 'fast' in async_task.enhance_uov_method:
+                    enhance_upscale_steps = 0
+                else:
+                    enhance_upscale_steps = async_task.performance_selection.steps_uov()
+            enhance_upscale_steps, _, _, _ = apply_overrides(async_task, enhance_upscale_steps, height, width)
+            enhance_upscale_steps_total = async_task.image_number * enhance_upscale_steps
+            all_steps += enhance_upscale_steps_total
+
+        if async_task.enhance_checkbox and len(async_task.enhance_ctrls) != 0:
+            enhance_steps, _, _, _ = apply_overrides(async_task, async_task.original_steps, height, width)
+            all_steps += async_task.image_number * len(async_task.enhance_ctrls) * enhance_steps
+
+        all_steps = max(all_steps, 1)
+
+        print(f'[Parameters] Denoising Strength = {denoising_strength}')
+
+        if isinstance(initial_latent, dict) and 'samples' in initial_latent:
+            log_shape = initial_latent['samples'].shape
+        else:
+            log_shape = f'Image Space {(height, width)}'
+
+        print(f'[Parameters] Initial Latent shape: {log_shape}')
+
+        preparation_time = time.perf_counter() - preparation_start_time
+        print(f'Preparation time: {preparation_time:.2f} seconds')
+
+        final_scheduler_name = patch_samplers(async_task)
+        print(f'Using {final_scheduler_name} scheduler.')
+
+        async_task.yields.append(['preview', (current_progress, 'Moving model to GPU ...', None)])
+
+        processing_start_time = time.perf_counter()
+
+        preparation_steps = current_progress
+        total_count = async_task.image_number
+
+        def callback(step, x0, x, total_steps, y):
+            if step == 0:
+                async_task.callback_steps = 0
+            async_task.callback_steps += (100 - preparation_steps) / float(all_steps)
+            async_task.yields.append(['preview', (
+                int(current_progress + async_task.callback_steps),
+                f'Sampling step {step + 1}/{total_steps}, image {current_task_id + 1}/{total_count} ...', y)])
+
+        show_intermediate_results = len(tasks) > 1 or async_task.should_enhance
+        persist_image = not async_task.should_enhance or not async_task.save_final_enhanced_image_only
+
+        for current_task_id, task in enumerate(tasks):
+            progressbar(async_task, current_progress, f'Preparing task {current_task_id + 1}/{async_task.image_number} ...')
+            execution_start_time = time.perf_counter()
+
+            try:
+                imgs, img_paths, current_progress = process_task(all_steps, async_task, callback, controlnet_canny_path,
+                                                                 controlnet_cpds_path, current_task_id,
+                                                                 denoising_strength, final_scheduler_name, goals,
+                                                                 initial_latent, async_task.steps, switch, task['c'],
+                                                                 task['uc'], task, loras, tiled, use_expansion, width,
+                                                                 height, current_progress, preparation_steps,
+                                                                 async_task.image_number, show_intermediate_results,
+                                                                 persist_image)
+
+                current_progress = int(preparation_steps + (100 - preparation_steps) / float(all_steps) * async_task.steps * (current_task_id + 1))
+                images_to_enhance += imgs
+
+            except ldm_patched.modules.model_management.InterruptProcessingException:
+                if async_task.last_stop == 'skip':
+                    print('User skipped')
+                    async_task.last_stop = False
+                    continue
+                else:
+                    print('User stopped')
+                    break
+
+            del task['c'], task['uc']  # Save memory
+            execution_time = time.perf_counter() - execution_start_time
+            print(f'Generating and saving time: {execution_time:.2f} seconds')
+
+        if not async_task.should_enhance:
+            print(f'[Enhance] Skipping, preconditions aren\'t met')
+            stop_processing(async_task, processing_start_time)
+            return
+
+        progressbar(async_task, current_progress, 'Processing enhance ...')
+
+        active_enhance_tabs = len(async_task.enhance_ctrls)
+        should_process_enhance_uov = async_task.enhance_uov_method != flags.disabled.casefold()
+        enhance_uov_before = False
+        enhance_uov_after = False
+        if should_process_enhance_uov:
+            active_enhance_tabs += 1
+            enhance_uov_before = async_task.enhance_uov_processing_order == flags.enhancement_uov_before
+            enhance_uov_after = async_task.enhance_uov_processing_order == flags.enhancement_uov_after
+        total_count = len(images_to_enhance) * active_enhance_tabs
+        async_task.images_to_enhance_count = len(images_to_enhance)
+
+        base_progress = current_progress
+        current_task_id = -1
+        done_steps_upscaling = 0
+        done_steps_inpainting = 0
+        enhance_steps, _, _, _ = apply_overrides(async_task, async_task.original_steps, height, width)
+        exception_result = None
+        for index, img in enumerate(images_to_enhance):
+            async_task.enhance_stats[index] = 0
+            enhancement_image_start_time = time.perf_counter()
+
+            last_enhance_prompt = async_task.prompt
+            last_enhance_negative_prompt = async_task.negative_prompt
+
+            if enhance_uov_before:
+                current_task_id += 1
+                persist_image = not async_task.save_final_enhanced_image_only or active_enhance_tabs == 0
+                current_task_id, done_steps_inpainting, done_steps_upscaling, img, exception_result = enhance_upscale(
+                    all_steps, async_task, base_progress, callback, controlnet_canny_path, controlnet_cpds_path,
+                    current_task_id, denoising_strength, done_steps_inpainting, done_steps_upscaling, enhance_steps,
+                    async_task.prompt, async_task.negative_prompt, final_scheduler_name, height, img, preparation_steps,
+                    switch, tiled, total_count, use_expansion, use_style, use_synthetic_refiner, width, persist_image)
+                async_task.enhance_stats[index] += 1
+
+                if exception_result == 'continue':
+                    continue
+                elif exception_result == 'break':
+                    break
+
+            # inpaint for all other tabs
+            for enhance_mask_dino_prompt_text, enhance_prompt, enhance_negative_prompt, enhance_mask_model, enhance_mask_cloth_category, enhance_mask_sam_model, enhance_mask_text_threshold, enhance_mask_box_threshold, enhance_mask_sam_max_detections, enhance_inpaint_disable_initial_latent, enhance_inpaint_engine, enhance_inpaint_strength, enhance_inpaint_respective_field, enhance_inpaint_erode_or_dilate, enhance_mask_invert in async_task.enhance_ctrls:
+                current_task_id += 1
+                current_progress = int(base_progress + (100 - preparation_steps) / float(all_steps) * (done_steps_upscaling + done_steps_inpainting))
+                progressbar(async_task, current_progress, f'Preparing enhancement {current_task_id + 1}/{total_count} ...')
+                enhancement_task_start_time = time.perf_counter()
+                is_last_enhance_for_image = (current_task_id + 1) % active_enhance_tabs == 0 and not enhance_uov_after
+                persist_image = not async_task.save_final_enhanced_image_only or is_last_enhance_for_image
+
+                extras = {}
+                if enhance_mask_model == 'sam':
+                    print(f'[Enhance] Searching for "{enhance_mask_dino_prompt_text}"')
+                elif enhance_mask_model == 'u2net_cloth_seg':
+                    extras['cloth_category'] = enhance_mask_cloth_category
+
+                mask, dino_detection_count, sam_detection_count, sam_detection_on_mask_count = generate_mask_from_image(
+                    img, mask_model=enhance_mask_model, extras=extras, sam_options=SAMOptions(
+                        dino_prompt=enhance_mask_dino_prompt_text,
+                        dino_box_threshold=enhance_mask_box_threshold,
+                        dino_text_threshold=enhance_mask_text_threshold,
+                        dino_erode_or_dilate=async_task.dino_erode_or_dilate,
+                        dino_debug=async_task.debugging_dino,
+                        max_detections=enhance_mask_sam_max_detections,
+                        model_type=enhance_mask_sam_model,
+                    ))
+                if len(mask.shape) == 3:
+                    mask = mask[:, :, 0]
+
+                if int(enhance_inpaint_erode_or_dilate) != 0:
+                    mask = erode_or_dilate(mask, enhance_inpaint_erode_or_dilate)
+
+                if enhance_mask_invert:
+                    mask = 255 - mask
+
+                if async_task.debugging_enhance_masks_checkbox:
+                    async_task.yields.append(['preview', (current_progress, 'Loading ...', mask)])
+                    yield_result(async_task, mask, current_progress, async_task.black_out_nsfw, False,
+                                 async_task.disable_intermediate_results)
+                    async_task.enhance_stats[index] += 1
+
+                print(f'[Enhance] {dino_detection_count} boxes detected')
+                print(f'[Enhance] {sam_detection_count} segments detected in boxes')
+                print(f'[Enhance] {sam_detection_on_mask_count} segments applied to mask')
+
+                if enhance_mask_model == 'sam' and (dino_detection_count == 0 or not async_task.debugging_dino and sam_detection_on_mask_count == 0):
+                    print(f'[Enhance] No "{enhance_mask_dino_prompt_text}" detected, skipping')
+                    continue
+
+                goals_enhance = ['inpaint']
+
+                try:
+                    current_progress, img, enhance_prompt_processed, enhance_negative_prompt_processed = process_enhance(
+                        all_steps, async_task, callback, controlnet_canny_path, controlnet_cpds_path,
+                        current_progress, current_task_id, denoising_strength, enhance_inpaint_disable_initial_latent,
+                        enhance_inpaint_engine, enhance_inpaint_respective_field, enhance_inpaint_strength,
+                        enhance_prompt, enhance_negative_prompt, final_scheduler_name, goals_enhance, height, img, mask,
+                        preparation_steps, enhance_steps, switch, tiled, total_count, use_expansion, use_style,
+                        use_synthetic_refiner, width, persist_image=persist_image)
+                    async_task.enhance_stats[index] += 1
+
+                    if (should_process_enhance_uov and async_task.enhance_uov_processing_order == flags.enhancement_uov_after
+                            and async_task.enhance_uov_prompt_type == flags.enhancement_uov_prompt_type_last_filled):
+                        if enhance_prompt_processed != '':
+                            last_enhance_prompt = enhance_prompt_processed
+                        if enhance_negative_prompt_processed != '':
+                            last_enhance_negative_prompt = enhance_negative_prompt_processed
+
+                except ldm_patched.modules.model_management.InterruptProcessingException:
+                    if async_task.last_stop == 'skip':
+                        print('User skipped')
+                        async_task.last_stop = False
+                        continue
+                    else:
+                        print('User stopped')
+                        exception_result = 'break'
+                        break
+                finally:
+                    done_steps_inpainting += enhance_steps
+
+                enhancement_task_time = time.perf_counter() - enhancement_task_start_time
+                print(f'Enhancement time: {enhancement_task_time:.2f} seconds')
+
+            if exception_result == 'break':
+                break
+
+            if enhance_uov_after:
+                current_task_id += 1
+                # last step in enhance, always save
+                persist_image = True
+                current_task_id, done_steps_inpainting, done_steps_upscaling, img, exception_result = enhance_upscale(
+                    all_steps, async_task, base_progress, callback, controlnet_canny_path, controlnet_cpds_path,
+                    current_task_id, denoising_strength, done_steps_inpainting, done_steps_upscaling, enhance_steps,
+                    last_enhance_prompt, last_enhance_negative_prompt, final_scheduler_name, height, img,
+                    preparation_steps, switch, tiled, total_count, use_expansion, use_style, use_synthetic_refiner,
+                    width, persist_image)
+                async_task.enhance_stats[index] += 1
+                
+                if exception_result == 'continue':
+                    continue
+                elif exception_result == 'break':
+                    break
+
+            enhancement_image_time = time.perf_counter() - enhancement_image_start_time
+            print(f'Enhancement image time: {enhancement_image_time:.2f} seconds')
+
+        stop_processing(async_task, processing_start_time)
+        return
+
+    while True:
+        time.sleep(0.01)
+        if len(async_tasks) > 0:
+            task = async_tasks.pop(0)
+
+            try:
+                handler(task)
+                if task.generate_image_grid:
+                    build_image_wall(task)
+                task.yields.append(['finish', task.results])
+                pipeline.prepare_text_encoder(async_call=True)
+            except:
+                traceback.print_exc()
+                task.yields.append(['finish', task.results])
+            finally:
+                if pid in modules.patch.patch_settings:
+                    del modules.patch.patch_settings[pid]
+    pass
+
+
+threading.Thread(target=worker, daemon=True).start()

modules/auth.py

@@ -0,0 +1,41 @@
+import json
+import hashlib
+import modules.constants as constants
+
+from os.path import exists
+
+
+def auth_list_to_dict(auth_list):
+    auth_dict = {}
+    for auth_data in auth_list:
+        if 'user' in auth_data:
+            if 'hash' in auth_data:
+                auth_dict |= {auth_data['user']: auth_data['hash']}
+            elif 'pass' in auth_data:
+                auth_dict |= {auth_data['user']: hashlib.sha256(bytes(auth_data['pass'], encoding='utf-8')).hexdigest()}
+    return auth_dict
+
+
+def load_auth_data(filename=None):
+    auth_dict = None
+    if filename != None and exists(filename):
+        with open(filename, encoding='utf-8') as auth_file:
+            try:
+                auth_obj = json.load(auth_file)
+                if isinstance(auth_obj, list) and len(auth_obj) > 0:
+                    auth_dict = auth_list_to_dict(auth_obj)
+            except Exception as e:
+                print('load_auth_data, e: ' + str(e))
+    return auth_dict
+
+
+auth_dict = load_auth_data(constants.AUTH_FILENAME)
+
+auth_enabled = auth_dict != None
+
+
+def check_auth(user, password):
+    if user not in auth_dict:
+        return False
+    else:   
+        return hashlib.sha256(bytes(password, encoding='utf-8')).hexdigest() == auth_dict[user]

modules/config.py

@@ -0,0 +1,997 @@
+import os
+import json
+import math
+import numbers
+
+import args_manager
+import tempfile
+import modules.flags
+import modules.sdxl_styles
+
+from modules.model_loader import load_file_from_url
+from modules.extra_utils import makedirs_with_log, get_files_from_folder, try_eval_env_var
+from modules.flags import OutputFormat, Performance, MetadataScheme
+
+
+def get_config_path(key, default_value):
+    env = os.getenv(key)
+    if env is not None and isinstance(env, str):
+        print(f"Environment: {key} = {env}")
+        return env
+    else:
+        return os.path.abspath(default_value)
+
+wildcards_max_bfs_depth = 64
+config_path = get_config_path('config_path', "./config.txt")
+config_example_path = get_config_path('config_example_path', "config_modification_tutorial.txt")
+config_dict = {}
+always_save_keys = []
+visited_keys = []
+
+try:
+    with open(os.path.abspath(f'./presets/default.json'), "r", encoding="utf-8") as json_file:
+        config_dict.update(json.load(json_file))
+except Exception as e:
+    print(f'Load default preset failed.')
+    print(e)
+
+try:
+    if os.path.exists(config_path):
+        with open(config_path, "r", encoding="utf-8") as json_file:
+            config_dict.update(json.load(json_file))
+            always_save_keys = list(config_dict.keys())
+except Exception as e:
+    print(f'Failed to load config file "{config_path}" . The reason is: {str(e)}')
+    print('Please make sure that:')
+    print(f'1. The file "{config_path}" is a valid text file, and you have access to read it.')
+    print('2. Use "\\\\" instead of "\\" when describing paths.')
+    print('3. There is no "," before the last "}".')
+    print('4. All key/value formats are correct.')
+
+
+def try_load_deprecated_user_path_config():
+    global config_dict
+
+    if not os.path.exists('user_path_config.txt'):
+        return
+
+    try:
+        deprecated_config_dict = json.load(open('user_path_config.txt', "r", encoding="utf-8"))
+
+        def replace_config(old_key, new_key):
+            if old_key in deprecated_config_dict:
+                config_dict[new_key] = deprecated_config_dict[old_key]
+                del deprecated_config_dict[old_key]
+
+        replace_config('modelfile_path', 'path_checkpoints')
+        replace_config('lorafile_path', 'path_loras')
+        replace_config('embeddings_path', 'path_embeddings')
+        replace_config('vae_approx_path', 'path_vae_approx')
+        replace_config('upscale_models_path', 'path_upscale_models')
+        replace_config('inpaint_models_path', 'path_inpaint')
+        replace_config('controlnet_models_path', 'path_controlnet')
+        replace_config('clip_vision_models_path', 'path_clip_vision')
+        replace_config('fooocus_expansion_path', 'path_fooocus_expansion')
+        replace_config('temp_outputs_path', 'path_outputs')
+
+        if deprecated_config_dict.get("default_model", None) == 'juggernautXL_version6Rundiffusion.safetensors':
+            os.replace('user_path_config.txt', 'user_path_config-deprecated.txt')
+            print('Config updated successfully in silence. '
+                  'A backup of previous config is written to "user_path_config-deprecated.txt".')
+            return
+
+        if input("Newer models and configs are available. "
+                 "Download and update files? [Y/n]:") in ['n', 'N', 'No', 'no', 'NO']:
+            config_dict.update(deprecated_config_dict)
+            print('Loading using deprecated old models and deprecated old configs.')
+            return
+        else:
+            os.replace('user_path_config.txt', 'user_path_config-deprecated.txt')
+            print('Config updated successfully by user. '
+                  'A backup of previous config is written to "user_path_config-deprecated.txt".')
+            return
+    except Exception as e:
+        print('Processing deprecated config failed')
+        print(e)
+    return
+
+
+try_load_deprecated_user_path_config()
+
+def get_presets():
+    preset_folder = 'presets'
+    presets = ['initial']
+    if not os.path.exists(preset_folder):
+        print('No presets found.')
+        return presets
+
+    return presets + [f[:f.index(".json")] for f in os.listdir(preset_folder) if f.endswith('.json')]
+
+def update_presets():
+    global available_presets
+    available_presets = get_presets()
+
+def try_get_preset_content(preset):
+    if isinstance(preset, str):
+        preset_path = os.path.abspath(f'./presets/{preset}.json')
+        try:
+            if os.path.exists(preset_path):
+                with open(preset_path, "r", encoding="utf-8") as json_file:
+                    json_content = json.load(json_file)
+                    print(f'Loaded preset: {preset_path}')
+                    return json_content
+            else:
+                raise FileNotFoundError
+        except Exception as e:
+            print(f'Load preset [{preset_path}] failed')
+            print(e)
+    return {}
+
+available_presets = get_presets()
+preset = args_manager.args.preset
+config_dict.update(try_get_preset_content(preset))
+
+def get_path_output() -> str:
+    """
+    Checking output path argument and overriding default path.
+    """
+    global config_dict
+    path_output = get_dir_or_set_default('path_outputs', '../outputs/', make_directory=True)
+    if args_manager.args.output_path:
+        print(f'Overriding config value path_outputs with {args_manager.args.output_path}')
+        config_dict['path_outputs'] = path_output = args_manager.args.output_path
+    return path_output
+
+
+def get_dir_or_set_default(key, default_value, as_array=False, make_directory=False):
+    global config_dict, visited_keys, always_save_keys
+
+    if key not in visited_keys:
+        visited_keys.append(key)
+
+    if key not in always_save_keys:
+        always_save_keys.append(key)
+
+    v = os.getenv(key)
+    if v is not None:
+        print(f"Environment: {key} = {v}")
+        config_dict[key] = v
+    else:
+        v = config_dict.get(key, None)
+
+    if isinstance(v, str):
+        if make_directory:
+            makedirs_with_log(v)
+        if os.path.exists(v) and os.path.isdir(v):
+            return v if not as_array else [v]
+    elif isinstance(v, list):
+        if make_directory:
+            for d in v:
+                makedirs_with_log(d)
+        if all([os.path.exists(d) and os.path.isdir(d) for d in v]):
+            return v
+
+    if v is not None:
+        print(f'Failed to load config key: {json.dumps({key:v})} is invalid or does not exist; will use {json.dumps({key:default_value})} instead.')
+    if isinstance(default_value, list):
+        dp = []
+        for path in default_value:
+            abs_path = os.path.abspath(os.path.join(os.path.dirname(__file__), path))
+            dp.append(abs_path)
+            os.makedirs(abs_path, exist_ok=True)
+    else:
+        dp = os.path.abspath(os.path.join(os.path.dirname(__file__), default_value))
+        os.makedirs(dp, exist_ok=True)
+        if as_array:
+            dp = [dp]
+    config_dict[key] = dp
+    return dp
+
+
+paths_checkpoints = get_dir_or_set_default('path_checkpoints', ['../models/checkpoints/'], True)
+paths_loras = get_dir_or_set_default('path_loras', ['../models/loras/'], True)
+path_embeddings = get_dir_or_set_default('path_embeddings', '../models/embeddings/')
+path_vae_approx = get_dir_or_set_default('path_vae_approx', '../models/vae_approx/')
+path_vae = get_dir_or_set_default('path_vae', '../models/vae/')
+path_upscale_models = get_dir_or_set_default('path_upscale_models', '../models/upscale_models/')
+path_inpaint = get_dir_or_set_default('path_inpaint', '../models/inpaint/')
+path_controlnet = get_dir_or_set_default('path_controlnet', '../models/controlnet/')
+path_clip_vision = get_dir_or_set_default('path_clip_vision', '../models/clip_vision/')
+path_fooocus_expansion = get_dir_or_set_default('path_fooocus_expansion', '../models/prompt_expansion/fooocus_expansion')
+path_wildcards = get_dir_or_set_default('path_wildcards', '../wildcards/')
+path_safety_checker = get_dir_or_set_default('path_safety_checker', '../models/safety_checker/')
+path_sam = get_dir_or_set_default('path_sam', '../models/sam/')
+path_outputs = get_path_output()
+
+
+def get_config_item_or_set_default(key, default_value, validator, disable_empty_as_none=False, expected_type=None):
+    global config_dict, visited_keys
+
+    if key not in visited_keys:
+        visited_keys.append(key)
+    
+    v = os.getenv(key)
+    if v is not None:
+        v = try_eval_env_var(v, expected_type)
+        print(f"Environment: {key} = {v}")
+        config_dict[key] = v
+
+    if key not in config_dict:
+        config_dict[key] = default_value
+        return default_value
+
+    v = config_dict.get(key, None)
+    if not disable_empty_as_none:
+        if v is None or v == '':
+            v = 'None'
+    if validator(v):
+        return v
+    else:
+        if v is not None:
+            print(f'Failed to load config key: {json.dumps({key:v})} is invalid; will use {json.dumps({key:default_value})} instead.')
+        config_dict[key] = default_value
+        return default_value
+
+
+def init_temp_path(path: str | None, default_path: str) -> str:
+    if args_manager.args.temp_path:
+        path = args_manager.args.temp_path
+
+    if path != '' and path != default_path:
+        try:
+            if not os.path.isabs(path):
+                path = os.path.abspath(path)
+            os.makedirs(path, exist_ok=True)
+            print(f'Using temp path {path}')
+            return path
+        except Exception as e:
+            print(f'Could not create temp path {path}. Reason: {e}')
+            print(f'Using default temp path {default_path} instead.')
+
+    os.makedirs(default_path, exist_ok=True)
+    return default_path
+
+
+default_temp_path = os.path.join(tempfile.gettempdir(), 'fooocus')
+temp_path = init_temp_path(get_config_item_or_set_default(
+    key='temp_path',
+    default_value=default_temp_path,
+    validator=lambda x: isinstance(x, str),
+    expected_type=str
+), default_temp_path)
+temp_path_cleanup_on_launch = get_config_item_or_set_default(
+    key='temp_path_cleanup_on_launch',
+    default_value=True,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_base_model_name = default_model = get_config_item_or_set_default(
+    key='default_model',
+    default_value='model.safetensors',
+    validator=lambda x: isinstance(x, str),
+    expected_type=str
+)
+previous_default_models = get_config_item_or_set_default(
+    key='previous_default_models',
+    default_value=[],
+    validator=lambda x: isinstance(x, list) and all(isinstance(k, str) for k in x),
+    expected_type=list
+)
+default_refiner_model_name = default_refiner = get_config_item_or_set_default(
+    key='default_refiner',
+    default_value='None',
+    validator=lambda x: isinstance(x, str),
+    expected_type=str
+)
+default_refiner_switch = get_config_item_or_set_default(
+    key='default_refiner_switch',
+    default_value=0.8,
+    validator=lambda x: isinstance(x, numbers.Number) and 0 <= x <= 1,
+    expected_type=numbers.Number
+)
+default_loras_min_weight = get_config_item_or_set_default(
+    key='default_loras_min_weight',
+    default_value=-2,
+    validator=lambda x: isinstance(x, numbers.Number) and -10 <= x <= 10,
+    expected_type=numbers.Number
+)
+default_loras_max_weight = get_config_item_or_set_default(
+    key='default_loras_max_weight',
+    default_value=2,
+    validator=lambda x: isinstance(x, numbers.Number) and -10 <= x <= 10,
+    expected_type=numbers.Number
+)
+default_loras = get_config_item_or_set_default(
+    key='default_loras',
+    default_value=[
+        [
+            True,
+            "None",
+            1.0
+        ],
+        [
+            True,
+            "None",
+            1.0
+        ],
+        [
+            True,
+            "None",
+            1.0
+        ],
+        [
+            True,
+            "None",
+            1.0
+        ],
+        [
+            True,
+            "None",
+            1.0
+        ]
+    ],
+    validator=lambda x: isinstance(x, list) and all(
+        len(y) == 3 and isinstance(y[0], bool) and isinstance(y[1], str) and isinstance(y[2], numbers.Number)
+        or len(y) == 2 and isinstance(y[0], str) and isinstance(y[1], numbers.Number)
+        for y in x),
+    expected_type=list
+)
+default_loras = [(y[0], y[1], y[2]) if len(y) == 3 else (True, y[0], y[1]) for y in default_loras]
+default_max_lora_number = get_config_item_or_set_default(
+    key='default_max_lora_number',
+    default_value=len(default_loras) if isinstance(default_loras, list) and len(default_loras) > 0 else 5,
+    validator=lambda x: isinstance(x, int) and x >= 1,
+    expected_type=int
+)
+default_cfg_scale = get_config_item_or_set_default(
+    key='default_cfg_scale',
+    default_value=7.0,
+    validator=lambda x: isinstance(x, numbers.Number),
+    expected_type=numbers.Number
+)
+default_sample_sharpness = get_config_item_or_set_default(
+    key='default_sample_sharpness',
+    default_value=2.0,
+    validator=lambda x: isinstance(x, numbers.Number),
+    expected_type=numbers.Number
+)
+default_sampler = get_config_item_or_set_default(
+    key='default_sampler',
+    default_value='dpmpp_2m_sde_gpu',
+    validator=lambda x: x in modules.flags.sampler_list,
+    expected_type=str
+)
+default_scheduler = get_config_item_or_set_default(
+    key='default_scheduler',
+    default_value='karras',
+    validator=lambda x: x in modules.flags.scheduler_list,
+    expected_type=str
+)
+default_vae = get_config_item_or_set_default(
+    key='default_vae',
+    default_value=modules.flags.default_vae,
+    validator=lambda x: isinstance(x, str),
+    expected_type=str
+)
+default_styles = get_config_item_or_set_default(
+    key='default_styles',
+    default_value=[
+        "Fooocus V2",
+        "Fooocus Enhance",
+        "Fooocus Sharp"
+    ],
+    validator=lambda x: isinstance(x, list) and all(y in modules.sdxl_styles.legal_style_names for y in x),
+    expected_type=list
+)
+default_prompt_negative = get_config_item_or_set_default(
+    key='default_prompt_negative',
+    default_value='',
+    validator=lambda x: isinstance(x, str),
+    disable_empty_as_none=True,
+    expected_type=str
+)
+default_prompt = get_config_item_or_set_default(
+    key='default_prompt',
+    default_value='',
+    validator=lambda x: isinstance(x, str),
+    disable_empty_as_none=True,
+    expected_type=str
+)
+default_performance = get_config_item_or_set_default(
+    key='default_performance',
+    default_value=Performance.SPEED.value,
+    validator=lambda x: x in Performance.values(),
+    expected_type=str
+)
+default_image_prompt_checkbox = get_config_item_or_set_default(
+    key='default_image_prompt_checkbox',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_enhance_checkbox = get_config_item_or_set_default(
+    key='default_enhance_checkbox',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_advanced_checkbox = get_config_item_or_set_default(
+    key='default_advanced_checkbox',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_developer_debug_mode_checkbox = get_config_item_or_set_default(
+    key='default_developer_debug_mode_checkbox',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_image_prompt_advanced_checkbox = get_config_item_or_set_default(
+    key='default_image_prompt_advanced_checkbox',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_max_image_number = get_config_item_or_set_default(
+    key='default_max_image_number',
+    default_value=32,
+    validator=lambda x: isinstance(x, int) and x >= 1,
+    expected_type=int
+)
+default_output_format = get_config_item_or_set_default(
+    key='default_output_format',
+    default_value='png',
+    validator=lambda x: x in OutputFormat.list(),
+    expected_type=str
+)
+default_image_number = get_config_item_or_set_default(
+    key='default_image_number',
+    default_value=2,
+    validator=lambda x: isinstance(x, int) and 1 <= x <= default_max_image_number,
+    expected_type=int
+)
+checkpoint_downloads = get_config_item_or_set_default(
+    key='checkpoint_downloads',
+    default_value={},
+    validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()),
+    expected_type=dict
+)
+lora_downloads = get_config_item_or_set_default(
+    key='lora_downloads',
+    default_value={},
+    validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()),
+    expected_type=dict
+)
+embeddings_downloads = get_config_item_or_set_default(
+    key='embeddings_downloads',
+    default_value={},
+    validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()),
+    expected_type=dict
+)
+vae_downloads = get_config_item_or_set_default(
+    key='vae_downloads',
+    default_value={},
+    validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()),
+    expected_type=dict
+)
+available_aspect_ratios = get_config_item_or_set_default(
+    key='available_aspect_ratios',
+    default_value=modules.flags.sdxl_aspect_ratios,
+    validator=lambda x: isinstance(x, list) and all('*' in v for v in x) and len(x) > 1,
+    expected_type=list
+)
+default_aspect_ratio = get_config_item_or_set_default(
+    key='default_aspect_ratio',
+    default_value='1152*896' if '1152*896' in available_aspect_ratios else available_aspect_ratios[0],
+    validator=lambda x: x in available_aspect_ratios,
+    expected_type=str
+)
+default_inpaint_engine_version = get_config_item_or_set_default(
+    key='default_inpaint_engine_version',
+    default_value='v2.6',
+    validator=lambda x: x in modules.flags.inpaint_engine_versions,
+    expected_type=str
+)
+default_selected_image_input_tab_id = get_config_item_or_set_default(
+    key='default_selected_image_input_tab_id',
+    default_value=modules.flags.default_input_image_tab,
+    validator=lambda x: x in modules.flags.input_image_tab_ids,
+    expected_type=str
+)
+default_uov_method = get_config_item_or_set_default(
+    key='default_uov_method',
+    default_value=modules.flags.disabled,
+    validator=lambda x: x in modules.flags.uov_list,
+    expected_type=str
+)
+default_controlnet_image_count = get_config_item_or_set_default(
+    key='default_controlnet_image_count',
+    default_value=4,
+    validator=lambda x: isinstance(x, int) and x > 0,
+    expected_type=int
+)
+default_ip_images = {}
+default_ip_stop_ats = {}
+default_ip_weights = {}
+default_ip_types = {}
+
+for image_count in range(default_controlnet_image_count):
+    image_count += 1
+    default_ip_images[image_count] = get_config_item_or_set_default(
+        key=f'default_ip_image_{image_count}',
+        default_value='None',
+        validator=lambda x: x == 'None' or isinstance(x, str) and os.path.exists(x),
+        expected_type=str
+    )
+
+    if default_ip_images[image_count] == 'None':
+        default_ip_images[image_count] = None
+
+    default_ip_types[image_count] = get_config_item_or_set_default(
+        key=f'default_ip_type_{image_count}',
+        default_value=modules.flags.default_ip,
+        validator=lambda x: x in modules.flags.ip_list,
+        expected_type=str
+    )
+
+    default_end, default_weight = modules.flags.default_parameters[default_ip_types[image_count]]
+
+    default_ip_stop_ats[image_count] = get_config_item_or_set_default(
+        key=f'default_ip_stop_at_{image_count}',
+        default_value=default_end,
+        validator=lambda x: isinstance(x, float) and 0 <= x <= 1,
+        expected_type=float
+    )
+    default_ip_weights[image_count] = get_config_item_or_set_default(
+        key=f'default_ip_weight_{image_count}',
+        default_value=default_weight,
+        validator=lambda x: isinstance(x, float) and 0 <= x <= 2,
+        expected_type=float
+    )
+
+default_inpaint_advanced_masking_checkbox = get_config_item_or_set_default(
+    key='default_inpaint_advanced_masking_checkbox',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_inpaint_method = get_config_item_or_set_default(
+    key='default_inpaint_method',
+    default_value=modules.flags.inpaint_option_default,
+    validator=lambda x: x in modules.flags.inpaint_options,
+    expected_type=str
+)
+default_cfg_tsnr = get_config_item_or_set_default(
+    key='default_cfg_tsnr',
+    default_value=7.0,
+    validator=lambda x: isinstance(x, numbers.Number),
+    expected_type=numbers.Number
+)
+default_clip_skip = get_config_item_or_set_default(
+    key='default_clip_skip',
+    default_value=2,
+    validator=lambda x: isinstance(x, int) and 1 <= x <= modules.flags.clip_skip_max,
+    expected_type=int
+)
+default_overwrite_step = get_config_item_or_set_default(
+    key='default_overwrite_step',
+    default_value=-1,
+    validator=lambda x: isinstance(x, int),
+    expected_type=int
+)
+default_overwrite_switch = get_config_item_or_set_default(
+    key='default_overwrite_switch',
+    default_value=-1,
+    validator=lambda x: isinstance(x, int),
+    expected_type=int
+)
+default_overwrite_upscale = get_config_item_or_set_default(
+    key='default_overwrite_upscale',
+    default_value=-1,
+    validator=lambda x: isinstance(x, numbers.Number)
+)
+example_inpaint_prompts = get_config_item_or_set_default(
+    key='example_inpaint_prompts',
+    default_value=[
+        'highly detailed face', 'detailed girl face', 'detailed man face', 'detailed hand', 'beautiful eyes'
+    ],
+    validator=lambda x: isinstance(x, list) and all(isinstance(v, str) for v in x),
+    expected_type=list
+)
+example_enhance_detection_prompts = get_config_item_or_set_default(
+    key='example_enhance_detection_prompts',
+    default_value=[
+        'face', 'eye', 'mouth', 'hair', 'hand', 'body'
+    ],
+    validator=lambda x: isinstance(x, list) and all(isinstance(v, str) for v in x),
+    expected_type=list
+)
+default_enhance_tabs = get_config_item_or_set_default(
+    key='default_enhance_tabs',
+    default_value=3,
+    validator=lambda x: isinstance(x, int) and 1 <= x <= 5,
+    expected_type=int
+)
+default_enhance_uov_method = get_config_item_or_set_default(
+    key='default_enhance_uov_method',
+    default_value=modules.flags.disabled,
+    validator=lambda x: x in modules.flags.uov_list,
+    expected_type=int
+)
+default_enhance_uov_processing_order = get_config_item_or_set_default(
+    key='default_enhance_uov_processing_order',
+    default_value=modules.flags.enhancement_uov_before,
+    validator=lambda x: x in modules.flags.enhancement_uov_processing_order,
+    expected_type=int
+)
+default_enhance_uov_prompt_type = get_config_item_or_set_default(
+    key='default_enhance_uov_prompt_type',
+    default_value=modules.flags.enhancement_uov_prompt_type_original,
+    validator=lambda x: x in modules.flags.enhancement_uov_prompt_types,
+    expected_type=int
+)
+default_sam_max_detections = get_config_item_or_set_default(
+    key='default_sam_max_detections',
+    default_value=0,
+    validator=lambda x: isinstance(x, int) and 0 <= x <= 10,
+    expected_type=int
+)
+default_black_out_nsfw = get_config_item_or_set_default(
+    key='default_black_out_nsfw',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_save_only_final_enhanced_image = get_config_item_or_set_default(
+    key='default_save_only_final_enhanced_image',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_save_metadata_to_images = get_config_item_or_set_default(
+    key='default_save_metadata_to_images',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_metadata_scheme = get_config_item_or_set_default(
+    key='default_metadata_scheme',
+    default_value=MetadataScheme.FOOOCUS.value,
+    validator=lambda x: x in [y[1] for y in modules.flags.metadata_scheme if y[1] == x],
+    expected_type=str
+)
+metadata_created_by = get_config_item_or_set_default(
+    key='metadata_created_by',
+    default_value='',
+    validator=lambda x: isinstance(x, str),
+    expected_type=str
+)
+
+example_inpaint_prompts = [[x] for x in example_inpaint_prompts]
+example_enhance_detection_prompts = [[x] for x in example_enhance_detection_prompts]
+
+default_invert_mask_checkbox = get_config_item_or_set_default(
+    key='default_invert_mask_checkbox',
+    default_value=False,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+
+default_inpaint_mask_model = get_config_item_or_set_default(
+    key='default_inpaint_mask_model',
+    default_value='isnet-general-use',
+    validator=lambda x: x in modules.flags.inpaint_mask_models,
+    expected_type=str
+)
+
+default_enhance_inpaint_mask_model = get_config_item_or_set_default(
+    key='default_enhance_inpaint_mask_model',
+    default_value='sam',
+    validator=lambda x: x in modules.flags.inpaint_mask_models,
+    expected_type=str
+)
+
+default_inpaint_mask_cloth_category = get_config_item_or_set_default(
+    key='default_inpaint_mask_cloth_category',
+    default_value='full',
+    validator=lambda x: x in modules.flags.inpaint_mask_cloth_category,
+    expected_type=str
+)
+
+default_inpaint_mask_sam_model = get_config_item_or_set_default(
+    key='default_inpaint_mask_sam_model',
+    default_value='vit_b',
+    validator=lambda x: x in modules.flags.inpaint_mask_sam_model,
+    expected_type=str
+)
+
+default_describe_apply_prompts_checkbox = get_config_item_or_set_default(
+    key='default_describe_apply_prompts_checkbox',
+    default_value=True,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_describe_content_type = get_config_item_or_set_default(
+    key='default_describe_content_type',
+    default_value=[modules.flags.describe_type_photo],
+    validator=lambda x: all(k in modules.flags.describe_types for k in x),
+    expected_type=list
+)
+
+config_dict["default_loras"] = default_loras = default_loras[:default_max_lora_number] + [[True, 'None', 1.0] for _ in range(default_max_lora_number - len(default_loras))]
+
+# mapping config to meta parameter
+possible_preset_keys = {
+    "default_model": "base_model",
+    "default_refiner": "refiner_model",
+    "default_refiner_switch": "refiner_switch",
+    "previous_default_models": "previous_default_models",
+    "default_loras_min_weight": "default_loras_min_weight",
+    "default_loras_max_weight": "default_loras_max_weight",
+    "default_loras": "<processed>",
+    "default_cfg_scale": "guidance_scale",
+    "default_sample_sharpness": "sharpness",
+    "default_cfg_tsnr": "adaptive_cfg",
+    "default_clip_skip": "clip_skip",
+    "default_sampler": "sampler",
+    "default_scheduler": "scheduler",
+    "default_overwrite_step": "steps",
+    "default_overwrite_switch": "overwrite_switch",
+    "default_performance": "performance",
+    "default_image_number": "image_number",
+    "default_prompt": "prompt",
+    "default_prompt_negative": "negative_prompt",
+    "default_styles": "styles",
+    "default_aspect_ratio": "resolution",
+    "default_save_metadata_to_images": "default_save_metadata_to_images",
+    "checkpoint_downloads": "checkpoint_downloads",
+    "embeddings_downloads": "embeddings_downloads",
+    "lora_downloads": "lora_downloads",
+    "vae_downloads": "vae_downloads",
+    "default_vae": "vae",
+    # "default_inpaint_method": "inpaint_method", # disabled so inpaint mode doesn't refresh after every preset change
+    "default_inpaint_engine_version": "inpaint_engine_version",
+}
+
+REWRITE_PRESET = False
+
+if REWRITE_PRESET and isinstance(args_manager.args.preset, str):
+    save_path = 'presets/' + args_manager.args.preset + '.json'
+    with open(save_path, "w", encoding="utf-8") as json_file:
+        json.dump({k: config_dict[k] for k in possible_preset_keys}, json_file, indent=4)
+    print(f'Preset saved to {save_path}. Exiting ...')
+    exit(0)
+
+
+def add_ratio(x):
+    a, b = x.replace('*', ' ').split(' ')[:2]
+    a, b = int(a), int(b)
+    g = math.gcd(a, b)
+    return f'{a}×{b} <span style="color: grey;"> \U00002223 {a // g}:{b // g}</span>'
+
+
+default_aspect_ratio = add_ratio(default_aspect_ratio)
+available_aspect_ratios_labels = [add_ratio(x) for x in available_aspect_ratios]
+
+
+# Only write config in the first launch.
+if not os.path.exists(config_path):
+    with open(config_path, "w", encoding="utf-8") as json_file:
+        json.dump({k: config_dict[k] for k in always_save_keys}, json_file, indent=4)
+
+
+# Always write tutorials.
+with open(config_example_path, "w", encoding="utf-8") as json_file:
+    cpa = config_path.replace("\\", "\\\\")
+    json_file.write(f'You can modify your "{cpa}" using the below keys, formats, and examples.\n'
+                    f'Do not modify this file. Modifications in this file will not take effect.\n'
+                    f'This file is a tutorial and example. Please edit "{cpa}" to really change any settings.\n'
+                    + 'Remember to split the paths with "\\\\" rather than "\\", '
+                      'and there is no "," before the last "}". \n\n\n')
+    json.dump({k: config_dict[k] for k in visited_keys}, json_file, indent=4)
+
+model_filenames = []
+lora_filenames = []
+vae_filenames = []
+wildcard_filenames = []
+
+
+def get_model_filenames(folder_paths, extensions=None, name_filter=None):
+    if extensions is None:
+        extensions = ['.pth', '.ckpt', '.bin', '.safetensors', '.fooocus.patch']
+    files = []
+
+    if not isinstance(folder_paths, list):
+        folder_paths = [folder_paths]
+    for folder in folder_paths:
+        files += get_files_from_folder(folder, extensions, name_filter)
+
+    return files
+
+
+def update_files():
+    global model_filenames, lora_filenames, vae_filenames, wildcard_filenames, available_presets
+    model_filenames = get_model_filenames(paths_checkpoints)
+    lora_filenames = get_model_filenames(paths_loras)
+    vae_filenames = get_model_filenames(path_vae)
+    wildcard_filenames = get_files_from_folder(path_wildcards, ['.txt'])
+    available_presets = get_presets()
+    return
+
+
+def downloading_inpaint_models(v):
+    assert v in modules.flags.inpaint_engine_versions
+
+    load_file_from_url(
+        url='https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/fooocus_inpaint_head.pth',
+        model_dir=path_inpaint,
+        file_name='fooocus_inpaint_head.pth'
+    )
+    head_file = os.path.join(path_inpaint, 'fooocus_inpaint_head.pth')
+    patch_file = None
+
+    if v == 'v1':
+        load_file_from_url(
+            url='https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/inpaint.fooocus.patch',
+            model_dir=path_inpaint,
+            file_name='inpaint.fooocus.patch'
+        )
+        patch_file = os.path.join(path_inpaint, 'inpaint.fooocus.patch')
+
+    if v == 'v2.5':
+        load_file_from_url(
+            url='https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/inpaint_v25.fooocus.patch',
+            model_dir=path_inpaint,
+            file_name='inpaint_v25.fooocus.patch'
+        )
+        patch_file = os.path.join(path_inpaint, 'inpaint_v25.fooocus.patch')
+
+    if v == 'v2.6':
+        load_file_from_url(
+            url='https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/inpaint_v26.fooocus.patch',
+            model_dir=path_inpaint,
+            file_name='inpaint_v26.fooocus.patch'
+        )
+        patch_file = os.path.join(path_inpaint, 'inpaint_v26.fooocus.patch')
+
+    return head_file, patch_file
+
+
+def downloading_sdxl_lcm_lora():
+    load_file_from_url(
+        url='https://huggingface.co/lllyasviel/misc/resolve/main/sdxl_lcm_lora.safetensors',
+        model_dir=paths_loras[0],
+        file_name=modules.flags.PerformanceLoRA.EXTREME_SPEED.value
+    )
+    return modules.flags.PerformanceLoRA.EXTREME_SPEED.value
+
+
+def downloading_sdxl_lightning_lora():
+    load_file_from_url(
+        url='https://huggingface.co/mashb1t/misc/resolve/main/sdxl_lightning_4step_lora.safetensors',
+        model_dir=paths_loras[0],
+        file_name=modules.flags.PerformanceLoRA.LIGHTNING.value
+    )
+    return modules.flags.PerformanceLoRA.LIGHTNING.value
+
+
+def downloading_sdxl_hyper_sd_lora():
+    load_file_from_url(
+        url='https://huggingface.co/mashb1t/misc/resolve/main/sdxl_hyper_sd_4step_lora.safetensors',
+        model_dir=paths_loras[0],
+        file_name=modules.flags.PerformanceLoRA.HYPER_SD.value
+    )
+    return modules.flags.PerformanceLoRA.HYPER_SD.value
+
+
+def downloading_controlnet_canny():
+    load_file_from_url(
+        url='https://huggingface.co/lllyasviel/misc/resolve/main/control-lora-canny-rank128.safetensors',
+        model_dir=path_controlnet,
+        file_name='control-lora-canny-rank128.safetensors'
+    )
+    return os.path.join(path_controlnet, 'control-lora-canny-rank128.safetensors')
+
+
+def downloading_controlnet_cpds():
+    load_file_from_url(
+        url='https://huggingface.co/lllyasviel/misc/resolve/main/fooocus_xl_cpds_128.safetensors',
+        model_dir=path_controlnet,
+        file_name='fooocus_xl_cpds_128.safetensors'
+    )
+    return os.path.join(path_controlnet, 'fooocus_xl_cpds_128.safetensors')
+
+
+def downloading_ip_adapters(v):
+    assert v in ['ip', 'face']
+
+    results = []
+
+    load_file_from_url(
+        url='https://huggingface.co/lllyasviel/misc/resolve/main/clip_vision_vit_h.safetensors',
+        model_dir=path_clip_vision,
+        file_name='clip_vision_vit_h.safetensors'
+    )
+    results += [os.path.join(path_clip_vision, 'clip_vision_vit_h.safetensors')]
+
+    load_file_from_url(
+        url='https://huggingface.co/lllyasviel/misc/resolve/main/fooocus_ip_negative.safetensors',
+        model_dir=path_controlnet,
+        file_name='fooocus_ip_negative.safetensors'
+    )
+    results += [os.path.join(path_controlnet, 'fooocus_ip_negative.safetensors')]
+
+    if v == 'ip':
+        load_file_from_url(
+            url='https://huggingface.co/lllyasviel/misc/resolve/main/ip-adapter-plus_sdxl_vit-h.bin',
+            model_dir=path_controlnet,
+            file_name='ip-adapter-plus_sdxl_vit-h.bin'
+        )
+        results += [os.path.join(path_controlnet, 'ip-adapter-plus_sdxl_vit-h.bin')]
+
+    if v == 'face':
+        load_file_from_url(
+            url='https://huggingface.co/lllyasviel/misc/resolve/main/ip-adapter-plus-face_sdxl_vit-h.bin',
+            model_dir=path_controlnet,
+            file_name='ip-adapter-plus-face_sdxl_vit-h.bin'
+        )
+        results += [os.path.join(path_controlnet, 'ip-adapter-plus-face_sdxl_vit-h.bin')]
+
+    return results
+
+
+def downloading_upscale_model():
+    load_file_from_url(
+        url='https://huggingface.co/lllyasviel/misc/resolve/main/fooocus_upscaler_s409985e5.bin',
+        model_dir=path_upscale_models,
+        file_name='fooocus_upscaler_s409985e5.bin'
+    )
+    return os.path.join(path_upscale_models, 'fooocus_upscaler_s409985e5.bin')
+
+def downloading_safety_checker_model():
+    load_file_from_url(
+        url='https://huggingface.co/mashb1t/misc/resolve/main/stable-diffusion-safety-checker.bin',
+        model_dir=path_safety_checker,
+        file_name='stable-diffusion-safety-checker.bin'
+    )
+    return os.path.join(path_safety_checker, 'stable-diffusion-safety-checker.bin')
+
+
+def download_sam_model(sam_model: str) -> str:
+    match sam_model:
+        case 'vit_b':
+            return downloading_sam_vit_b()
+        case 'vit_l':
+            return downloading_sam_vit_l()
+        case 'vit_h':
+            return downloading_sam_vit_h()
+        case _:
+            raise ValueError(f"sam model {sam_model} does not exist.")
+
+
+def downloading_sam_vit_b():
+    load_file_from_url(
+        url='https://huggingface.co/mashb1t/misc/resolve/main/sam_vit_b_01ec64.pth',
+        model_dir=path_sam,
+        file_name='sam_vit_b_01ec64.pth'
+    )
+    return os.path.join(path_sam, 'sam_vit_b_01ec64.pth')
+
+
+def downloading_sam_vit_l():
+    load_file_from_url(
+        url='https://huggingface.co/mashb1t/misc/resolve/main/sam_vit_l_0b3195.pth',
+        model_dir=path_sam,
+        file_name='sam_vit_l_0b3195.pth'
+    )
+    return os.path.join(path_sam, 'sam_vit_l_0b3195.pth')
+
+
+def downloading_sam_vit_h():
+    load_file_from_url(
+        url='https://huggingface.co/mashb1t/misc/resolve/main/sam_vit_h_4b8939.pth',
+        model_dir=path_sam,
+        file_name='sam_vit_h_4b8939.pth'
+    )
+    return os.path.join(path_sam, 'sam_vit_h_4b8939.pth')

modules/constants.py

@@ -0,0 +1,5 @@
+# as in k-diffusion (sampling.py)
+MIN_SEED = 0
+MAX_SEED = 2**63 - 1
+
+AUTH_FILENAME = 'auth.json'

modules/core.py

@@ -0,0 +1,341 @@
+import os
+import einops
+import torch
+import numpy as np
+
+import ldm_patched.modules.model_management
+import ldm_patched.modules.model_detection
+import ldm_patched.modules.model_patcher
+import ldm_patched.modules.utils
+import ldm_patched.modules.controlnet
+import modules.sample_hijack
+import ldm_patched.modules.samplers
+import ldm_patched.modules.latent_formats
+
+from ldm_patched.modules.sd import load_checkpoint_guess_config
+from ldm_patched.contrib.external import VAEDecode, EmptyLatentImage, VAEEncode, VAEEncodeTiled, VAEDecodeTiled, \
+    ControlNetApplyAdvanced
+from ldm_patched.contrib.external_freelunch import FreeU_V2
+from ldm_patched.modules.sample import prepare_mask
+from modules.lora import match_lora
+from modules.util import get_file_from_folder_list
+from ldm_patched.modules.lora import model_lora_keys_unet, model_lora_keys_clip
+from modules.config import path_embeddings
+from ldm_patched.contrib.external_model_advanced import ModelSamplingDiscrete, ModelSamplingContinuousEDM
+
+opEmptyLatentImage = EmptyLatentImage()
+opVAEDecode = VAEDecode()
+opVAEEncode = VAEEncode()
+opVAEDecodeTiled = VAEDecodeTiled()
+opVAEEncodeTiled = VAEEncodeTiled()
+opControlNetApplyAdvanced = ControlNetApplyAdvanced()
+opFreeU = FreeU_V2()
+opModelSamplingDiscrete = ModelSamplingDiscrete()
+opModelSamplingContinuousEDM = ModelSamplingContinuousEDM()
+
+
+class StableDiffusionModel:
+    def __init__(self, unet=None, vae=None, clip=None, clip_vision=None, filename=None, vae_filename=None):
+        self.unet = unet
+        self.vae = vae
+        self.clip = clip
+        self.clip_vision = clip_vision
+        self.filename = filename
+        self.vae_filename = vae_filename
+        self.unet_with_lora = unet
+        self.clip_with_lora = clip
+        self.visited_loras = ''
+
+        self.lora_key_map_unet = {}
+        self.lora_key_map_clip = {}
+
+        if self.unet is not None:
+            self.lora_key_map_unet = model_lora_keys_unet(self.unet.model, self.lora_key_map_unet)
+            self.lora_key_map_unet.update({x: x for x in self.unet.model.state_dict().keys()})
+
+        if self.clip is not None:
+            self.lora_key_map_clip = model_lora_keys_clip(self.clip.cond_stage_model, self.lora_key_map_clip)
+            self.lora_key_map_clip.update({x: x for x in self.clip.cond_stage_model.state_dict().keys()})
+
+    @torch.no_grad()
+    @torch.inference_mode()
+    def refresh_loras(self, loras):
+        assert isinstance(loras, list)
+
+        if self.visited_loras == str(loras):
+            return
+
+        self.visited_loras = str(loras)
+
+        if self.unet is None:
+            return
+
+        print(f'Request to load LoRAs {str(loras)} for model [{self.filename}].')
+
+        loras_to_load = []
+
+        for filename, weight in loras:
+            if filename == 'None':
+                continue
+
+            if os.path.exists(filename):
+                lora_filename = filename
+            else:
+                lora_filename = get_file_from_folder_list(filename, modules.config.paths_loras)
+
+            if not os.path.exists(lora_filename):
+                print(f'Lora file not found: {lora_filename}')
+                continue
+
+            loras_to_load.append((lora_filename, weight))
+
+        self.unet_with_lora = self.unet.clone() if self.unet is not None else None
+        self.clip_with_lora = self.clip.clone() if self.clip is not None else None
+
+        for lora_filename, weight in loras_to_load:
+            lora_unmatch = ldm_patched.modules.utils.load_torch_file(lora_filename, safe_load=False)
+            lora_unet, lora_unmatch = match_lora(lora_unmatch, self.lora_key_map_unet)
+            lora_clip, lora_unmatch = match_lora(lora_unmatch, self.lora_key_map_clip)
+
+            if len(lora_unmatch) > 12:
+                # model mismatch
+                continue
+
+            if len(lora_unmatch) > 0:
+                print(f'Loaded LoRA [{lora_filename}] for model [{self.filename}] '
+                      f'with unmatched keys {list(lora_unmatch.keys())}')
+
+            if self.unet_with_lora is not None and len(lora_unet) > 0:
+                loaded_keys = self.unet_with_lora.add_patches(lora_unet, weight)
+                print(f'Loaded LoRA [{lora_filename}] for UNet [{self.filename}] '
+                      f'with {len(loaded_keys)} keys at weight {weight}.')
+                for item in lora_unet:
+                    if item not in loaded_keys:
+                        print("UNet LoRA key skipped: ", item)
+
+            if self.clip_with_lora is not None and len(lora_clip) > 0:
+                loaded_keys = self.clip_with_lora.add_patches(lora_clip, weight)
+                print(f'Loaded LoRA [{lora_filename}] for CLIP [{self.filename}] '
+                      f'with {len(loaded_keys)} keys at weight {weight}.')
+                for item in lora_clip:
+                    if item not in loaded_keys:
+                        print("CLIP LoRA key skipped: ", item)
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def apply_freeu(model, b1, b2, s1, s2):
+    return opFreeU.patch(model=model, b1=b1, b2=b2, s1=s1, s2=s2)[0]
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def load_controlnet(ckpt_filename):
+    return ldm_patched.modules.controlnet.load_controlnet(ckpt_filename)
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def apply_controlnet(positive, negative, control_net, image, strength, start_percent, end_percent):
+    return opControlNetApplyAdvanced.apply_controlnet(positive=positive, negative=negative, control_net=control_net,
+        image=image, strength=strength, start_percent=start_percent, end_percent=end_percent)
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def load_model(ckpt_filename, vae_filename=None):
+    unet, clip, vae, vae_filename, clip_vision = load_checkpoint_guess_config(ckpt_filename, embedding_directory=path_embeddings,
+                                                                vae_filename_param=vae_filename)
+    return StableDiffusionModel(unet=unet, clip=clip, vae=vae, clip_vision=clip_vision, filename=ckpt_filename, vae_filename=vae_filename)
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def generate_empty_latent(width=1024, height=1024, batch_size=1):
+    return opEmptyLatentImage.generate(width=width, height=height, batch_size=batch_size)[0]
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def decode_vae(vae, latent_image, tiled=False):
+    if tiled:
+        return opVAEDecodeTiled.decode(samples=latent_image, vae=vae, tile_size=512)[0]
+    else:
+        return opVAEDecode.decode(samples=latent_image, vae=vae)[0]
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def encode_vae(vae, pixels, tiled=False):
+    if tiled:
+        return opVAEEncodeTiled.encode(pixels=pixels, vae=vae, tile_size=512)[0]
+    else:
+        return opVAEEncode.encode(pixels=pixels, vae=vae)[0]
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def encode_vae_inpaint(vae, pixels, mask):
+    assert mask.ndim == 3 and pixels.ndim == 4
+    assert mask.shape[-1] == pixels.shape[-2]
+    assert mask.shape[-2] == pixels.shape[-3]
+
+    w = mask.round()[..., None]
+    pixels = pixels * (1 - w) + 0.5 * w
+
+    latent = vae.encode(pixels)
+    B, C, H, W = latent.shape
+
+    latent_mask = mask[:, None, :, :]
+    latent_mask = torch.nn.functional.interpolate(latent_mask, size=(H * 8, W * 8), mode="bilinear").round()
+    latent_mask = torch.nn.functional.max_pool2d(latent_mask, (8, 8)).round().to(latent)
+
+    return latent, latent_mask
+
+
+class VAEApprox(torch.nn.Module):
+    def __init__(self):
+        super(VAEApprox, self).__init__()
+        self.conv1 = torch.nn.Conv2d(4, 8, (7, 7))
+        self.conv2 = torch.nn.Conv2d(8, 16, (5, 5))
+        self.conv3 = torch.nn.Conv2d(16, 32, (3, 3))
+        self.conv4 = torch.nn.Conv2d(32, 64, (3, 3))
+        self.conv5 = torch.nn.Conv2d(64, 32, (3, 3))
+        self.conv6 = torch.nn.Conv2d(32, 16, (3, 3))
+        self.conv7 = torch.nn.Conv2d(16, 8, (3, 3))
+        self.conv8 = torch.nn.Conv2d(8, 3, (3, 3))
+        self.current_type = None
+
+    def forward(self, x):
+        extra = 11
+        x = torch.nn.functional.interpolate(x, (x.shape[2] * 2, x.shape[3] * 2))
+        x = torch.nn.functional.pad(x, (extra, extra, extra, extra))
+        for layer in [self.conv1, self.conv2, self.conv3, self.conv4, self.conv5, self.conv6, self.conv7, self.conv8]:
+            x = layer(x)
+            x = torch.nn.functional.leaky_relu(x, 0.1)
+        return x
+
+
+VAE_approx_models = {}
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def get_previewer(model):
+    global VAE_approx_models
+
+    from modules.config import path_vae_approx
+    is_sdxl = isinstance(model.model.latent_format, ldm_patched.modules.latent_formats.SDXL)
+    vae_approx_filename = os.path.join(path_vae_approx, 'xlvaeapp.pth' if is_sdxl else 'vaeapp_sd15.pth')
+
+    if vae_approx_filename in VAE_approx_models:
+        VAE_approx_model = VAE_approx_models[vae_approx_filename]
+    else:
+        sd = torch.load(vae_approx_filename, map_location='cpu', weights_only=True)
+        VAE_approx_model = VAEApprox()
+        VAE_approx_model.load_state_dict(sd)
+        del sd
+        VAE_approx_model.eval()
+
+        if ldm_patched.modules.model_management.should_use_fp16():
+            VAE_approx_model.half()
+            VAE_approx_model.current_type = torch.float16
+        else:
+            VAE_approx_model.float()
+            VAE_approx_model.current_type = torch.float32
+
+        VAE_approx_model.to(ldm_patched.modules.model_management.get_torch_device())
+        VAE_approx_models[vae_approx_filename] = VAE_approx_model
+
+    @torch.no_grad()
+    @torch.inference_mode()
+    def preview_function(x0, step, total_steps):
+        with torch.no_grad():
+            x_sample = x0.to(VAE_approx_model.current_type)
+            x_sample = VAE_approx_model(x_sample) * 127.5 + 127.5
+            x_sample = einops.rearrange(x_sample, 'b c h w -> b h w c')[0]
+            x_sample = x_sample.cpu().numpy().clip(0, 255).astype(np.uint8)
+            return x_sample
+
+    return preview_function
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sampler_name='dpmpp_2m_sde_gpu',
+             scheduler='karras', denoise=1.0, disable_noise=False, start_step=None, last_step=None,
+             force_full_denoise=False, callback_function=None, refiner=None, refiner_switch=-1,
+             previewer_start=None, previewer_end=None, sigmas=None, noise_mean=None, disable_preview=False):
+
+    if sigmas is not None:
+        sigmas = sigmas.clone().to(ldm_patched.modules.model_management.get_torch_device())
+
+    latent_image = latent["samples"]
+
+    if disable_noise:
+        noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu")
+    else:
+        batch_inds = latent["batch_index"] if "batch_index" in latent else None
+        noise = ldm_patched.modules.sample.prepare_noise(latent_image, seed, batch_inds)
+
+    if isinstance(noise_mean, torch.Tensor):
+        noise = noise + noise_mean - torch.mean(noise, dim=1, keepdim=True)
+
+    noise_mask = None
+    if "noise_mask" in latent:
+        noise_mask = latent["noise_mask"]
+
+    previewer = get_previewer(model)
+
+    if previewer_start is None:
+        previewer_start = 0
+
+    if previewer_end is None:
+        previewer_end = steps
+
+    def callback(step, x0, x, total_steps):
+        ldm_patched.modules.model_management.throw_exception_if_processing_interrupted()
+        y = None
+        if previewer is not None and not disable_preview:
+            y = previewer(x0, previewer_start + step, previewer_end)
+        if callback_function is not None:
+            callback_function(previewer_start + step, x0, x, previewer_end, y)
+
+    disable_pbar = False
+    modules.sample_hijack.current_refiner = refiner
+    modules.sample_hijack.refiner_switch_step = refiner_switch
+    ldm_patched.modules.samplers.sample = modules.sample_hijack.sample_hacked
+
+    try:
+        samples = ldm_patched.modules.sample.sample(model,
+                                                    noise, steps, cfg, sampler_name, scheduler,
+                                                    positive, negative, latent_image,
+                                                    denoise=denoise, disable_noise=disable_noise,
+                                                    start_step=start_step,
+                                                    last_step=last_step,
+                                                    force_full_denoise=force_full_denoise, noise_mask=noise_mask,
+                                                    callback=callback,
+                                                    disable_pbar=disable_pbar, seed=seed, sigmas=sigmas)
+
+        out = latent.copy()
+        out["samples"] = samples
+    finally:
+        modules.sample_hijack.current_refiner = None
+
+    return out
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def pytorch_to_numpy(x):
+    return [np.clip(255. * y.cpu().numpy(), 0, 255).astype(np.uint8) for y in x]
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def numpy_to_pytorch(x):
+    y = x.astype(np.float32) / 255.0
+    y = y[None]
+    y = np.ascontiguousarray(y.copy())
+    y = torch.from_numpy(y).float()
+    return y

modules/default_pipeline.py

@@ -0,0 +1,515 @@
+import modules.core as core
+import os
+import torch
+import modules.patch
+import modules.config
+import modules.flags
+import ldm_patched.modules.model_management
+import ldm_patched.modules.latent_formats
+import modules.inpaint_worker
+import extras.vae_interpose as vae_interpose
+from extras.expansion import FooocusExpansion
+
+from ldm_patched.modules.model_base import SDXL, SDXLRefiner
+from modules.sample_hijack import clip_separate
+from modules.util import get_file_from_folder_list, get_enabled_loras
+
+
+model_base = core.StableDiffusionModel()
+model_refiner = core.StableDiffusionModel()
+
+final_expansion = None
+final_unet = None
+final_clip = None
+final_vae = None
+final_refiner_unet = None
+final_refiner_vae = None
+
+loaded_ControlNets = {}
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def refresh_controlnets(model_paths):
+    global loaded_ControlNets
+    cache = {}
+    for p in model_paths:
+        if p is not None:
+            if p in loaded_ControlNets:
+                cache[p] = loaded_ControlNets[p]
+            else:
+                cache[p] = core.load_controlnet(p)
+    loaded_ControlNets = cache
+    return
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def assert_model_integrity():
+    error_message = None
+
+    if not isinstance(model_base.unet_with_lora.model, SDXL):
+        error_message = 'You have selected base model other than SDXL. This is not supported yet.'
+
+    if error_message is not None:
+        raise NotImplementedError(error_message)
+
+    return True
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def refresh_base_model(name, vae_name=None):
+    global model_base
+
+    filename = get_file_from_folder_list(name, modules.config.paths_checkpoints)
+
+    vae_filename = None
+    if vae_name is not None and vae_name != modules.flags.default_vae:
+        vae_filename = get_file_from_folder_list(vae_name, modules.config.path_vae)
+
+    if model_base.filename == filename and model_base.vae_filename == vae_filename:
+        return
+
+    model_base = core.load_model(filename, vae_filename)
+    print(f'Base model loaded: {model_base.filename}')
+    print(f'VAE loaded: {model_base.vae_filename}')
+    return
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def refresh_refiner_model(name):
+    global model_refiner
+
+    filename = get_file_from_folder_list(name, modules.config.paths_checkpoints)
+
+    if model_refiner.filename == filename:
+        return
+
+    model_refiner = core.StableDiffusionModel()
+
+    if name == 'None':
+        print(f'Refiner unloaded.')
+        return
+
+    model_refiner = core.load_model(filename)
+    print(f'Refiner model loaded: {model_refiner.filename}')
+
+    if isinstance(model_refiner.unet.model, SDXL):
+        model_refiner.clip = None
+        model_refiner.vae = None
+    elif isinstance(model_refiner.unet.model, SDXLRefiner):
+        model_refiner.clip = None
+        model_refiner.vae = None
+    else:
+        model_refiner.clip = None
+
+    return
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def synthesize_refiner_model():
+    global model_base, model_refiner
+
+    print('Synthetic Refiner Activated')
+    model_refiner = core.StableDiffusionModel(
+        unet=model_base.unet,
+        vae=model_base.vae,
+        clip=model_base.clip,
+        clip_vision=model_base.clip_vision,
+        filename=model_base.filename
+    )
+    model_refiner.vae = None
+    model_refiner.clip = None
+    model_refiner.clip_vision = None
+
+    return
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def refresh_loras(loras, base_model_additional_loras=None):
+    global model_base, model_refiner
+
+    if not isinstance(base_model_additional_loras, list):
+        base_model_additional_loras = []
+
+    model_base.refresh_loras(loras + base_model_additional_loras)
+    model_refiner.refresh_loras(loras)
+
+    return
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def clip_encode_single(clip, text, verbose=False):
+    cached = clip.fcs_cond_cache.get(text, None)
+    if cached is not None:
+        if verbose:
+            print(f'[CLIP Cached] {text}')
+        return cached
+    tokens = clip.tokenize(text)
+    result = clip.encode_from_tokens(tokens, return_pooled=True)
+    clip.fcs_cond_cache[text] = result
+    if verbose:
+        print(f'[CLIP Encoded] {text}')
+    return result
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def clone_cond(conds):
+    results = []
+
+    for c, p in conds:
+        p = p["pooled_output"]
+
+        if isinstance(c, torch.Tensor):
+            c = c.clone()
+
+        if isinstance(p, torch.Tensor):
+            p = p.clone()
+
+        results.append([c, {"pooled_output": p}])
+
+    return results
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def clip_encode(texts, pool_top_k=1):
+    global final_clip
+
+    if final_clip is None:
+        return None
+    if not isinstance(texts, list):
+        return None
+    if len(texts) == 0:
+        return None
+
+    cond_list = []
+    pooled_acc = 0
+
+    for i, text in enumerate(texts):
+        cond, pooled = clip_encode_single(final_clip, text)
+        cond_list.append(cond)
+        if i < pool_top_k:
+            pooled_acc += pooled
+
+    return [[torch.cat(cond_list, dim=1), {"pooled_output": pooled_acc}]]
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def set_clip_skip(clip_skip: int):
+    global final_clip
+
+    if final_clip is None:
+        return
+
+    final_clip.clip_layer(-abs(clip_skip))
+    return
+
+@torch.no_grad()
+@torch.inference_mode()
+def clear_all_caches():
+    final_clip.fcs_cond_cache = {}
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def prepare_text_encoder(async_call=True):
+    if async_call:
+        # TODO: make sure that this is always called in an async way so that users cannot feel it.
+        pass
+    assert_model_integrity()
+    ldm_patched.modules.model_management.load_models_gpu([final_clip.patcher, final_expansion.patcher])
+    return
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def refresh_everything(refiner_model_name, base_model_name, loras,
+                       base_model_additional_loras=None, use_synthetic_refiner=False, vae_name=None):
+    global final_unet, final_clip, final_vae, final_refiner_unet, final_refiner_vae, final_expansion
+
+    final_unet = None
+    final_clip = None
+    final_vae = None
+    final_refiner_unet = None
+    final_refiner_vae = None
+
+    if use_synthetic_refiner and refiner_model_name == 'None':
+        print('Synthetic Refiner Activated')
+        refresh_base_model(base_model_name, vae_name)
+        synthesize_refiner_model()
+    else:
+        refresh_refiner_model(refiner_model_name)
+        refresh_base_model(base_model_name, vae_name)
+
+    refresh_loras(loras, base_model_additional_loras=base_model_additional_loras)
+    assert_model_integrity()
+
+    final_unet = model_base.unet_with_lora
+    final_clip = model_base.clip_with_lora
+    final_vae = model_base.vae
+
+    final_refiner_unet = model_refiner.unet_with_lora
+    final_refiner_vae = model_refiner.vae
+
+    if final_expansion is None:
+        final_expansion = FooocusExpansion()
+
+    prepare_text_encoder(async_call=True)
+    clear_all_caches()
+    return
+
+
+refresh_everything(
+    refiner_model_name=modules.config.default_refiner_model_name,
+    base_model_name=modules.config.default_base_model_name,
+    loras=get_enabled_loras(modules.config.default_loras),
+    vae_name=modules.config.default_vae,
+)
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def vae_parse(latent):
+    if final_refiner_vae is None:
+        return latent
+
+    result = vae_interpose.parse(latent["samples"])
+    return {'samples': result}
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def calculate_sigmas_all(sampler, model, scheduler, steps):
+    from ldm_patched.modules.samplers import calculate_sigmas_scheduler
+
+    discard_penultimate_sigma = False
+    if sampler in ['dpm_2', 'dpm_2_ancestral']:
+        steps += 1
+        discard_penultimate_sigma = True
+
+    sigmas = calculate_sigmas_scheduler(model, scheduler, steps)
+
+    if discard_penultimate_sigma:
+        sigmas = torch.cat([sigmas[:-2], sigmas[-1:]])
+    return sigmas
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def calculate_sigmas(sampler, model, scheduler, steps, denoise):
+    if denoise is None or denoise > 0.9999:
+        sigmas = calculate_sigmas_all(sampler, model, scheduler, steps)
+    else:
+        new_steps = int(steps / denoise)
+        sigmas = calculate_sigmas_all(sampler, model, scheduler, new_steps)
+        sigmas = sigmas[-(steps + 1):]
+    return sigmas
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def get_candidate_vae(steps, switch, denoise=1.0, refiner_swap_method='joint'):
+    assert refiner_swap_method in ['joint', 'separate', 'vae']
+
+    if final_refiner_vae is not None and final_refiner_unet is not None:
+        if denoise > 0.9:
+            return final_vae, final_refiner_vae
+        else:
+            if denoise > (float(steps - switch) / float(steps)) ** 0.834:  # karras 0.834
+                return final_vae, None
+            else:
+                return final_refiner_vae, None
+
+    return final_vae, final_refiner_vae
+
+
+@torch.no_grad()
+@torch.inference_mode()
+def process_diffusion(positive_cond, negative_cond, steps, switch, width, height, image_seed, callback, sampler_name, scheduler_name, latent=None, denoise=1.0, tiled=False, cfg_scale=7.0, refiner_swap_method='joint', disable_preview=False):
+    target_unet, target_vae, target_refiner_unet, target_refiner_vae, target_clip \
+        = final_unet, final_vae, final_refiner_unet, final_refiner_vae, final_clip
+
+    assert refiner_swap_method in ['joint', 'separate', 'vae']
+
+    if final_refiner_vae is not None and final_refiner_unet is not None:
+        # Refiner Use Different VAE (then it is SD15)
+        if denoise > 0.9:
+            refiner_swap_method = 'vae'
+        else:
+            refiner_swap_method = 'joint'
+            if denoise > (float(steps - switch) / float(steps)) ** 0.834:  # karras 0.834
+                target_unet, target_vae, target_refiner_unet, target_refiner_vae \
+                    = final_unet, final_vae, None, None
+                print(f'[Sampler] only use Base because of partial denoise.')
+            else:
+                positive_cond = clip_separate(positive_cond, target_model=final_refiner_unet.model, target_clip=final_clip)
+                negative_cond = clip_separate(negative_cond, target_model=final_refiner_unet.model, target_clip=final_clip)
+                target_unet, target_vae, target_refiner_unet, target_refiner_vae \
+                    = final_refiner_unet, final_refiner_vae, None, None
+                print(f'[Sampler] only use Refiner because of partial denoise.')
+
+    print(f'[Sampler] refiner_swap_method = {refiner_swap_method}')
+
+    if latent is None:
+        initial_latent = core.generate_empty_latent(width=width, height=height, batch_size=1)
+    else:
+        initial_latent = latent
+
+    minmax_sigmas = calculate_sigmas(sampler=sampler_name, scheduler=scheduler_name, model=final_unet.model, steps=steps, denoise=denoise)
+    sigma_min, sigma_max = minmax_sigmas[minmax_sigmas > 0].min(), minmax_sigmas.max()
+    sigma_min = float(sigma_min.cpu().numpy())
+    sigma_max = float(sigma_max.cpu().numpy())
+    print(f'[Sampler] sigma_min = {sigma_min}, sigma_max = {sigma_max}')
+
+    modules.patch.BrownianTreeNoiseSamplerPatched.global_init(
+        initial_latent['samples'].to(ldm_patched.modules.model_management.get_torch_device()),
+        sigma_min, sigma_max, seed=image_seed, cpu=False)
+
+    decoded_latent = None
+
+    if refiner_swap_method == 'joint':
+        sampled_latent = core.ksampler(
+            model=target_unet,
+            refiner=target_refiner_unet,
+            positive=positive_cond,
+            negative=negative_cond,
+            latent=initial_latent,
+            steps=steps, start_step=0, last_step=steps, disable_noise=False, force_full_denoise=True,
+            seed=image_seed,
+            denoise=denoise,
+            callback_function=callback,
+            cfg=cfg_scale,
+            sampler_name=sampler_name,
+            scheduler=scheduler_name,
+            refiner_switch=switch,
+            previewer_start=0,
+            previewer_end=steps,
+            disable_preview=disable_preview
+        )
+        decoded_latent = core.decode_vae(vae=target_vae, latent_image=sampled_latent, tiled=tiled)
+
+    if refiner_swap_method == 'separate':
+        sampled_latent = core.ksampler(
+            model=target_unet,
+            positive=positive_cond,
+            negative=negative_cond,
+            latent=initial_latent,
+            steps=steps, start_step=0, last_step=switch, disable_noise=False, force_full_denoise=False,
+            seed=image_seed,
+            denoise=denoise,
+            callback_function=callback,
+            cfg=cfg_scale,
+            sampler_name=sampler_name,
+            scheduler=scheduler_name,
+            previewer_start=0,
+            previewer_end=steps,
+            disable_preview=disable_preview
+        )
+        print('Refiner swapped by changing ksampler. Noise preserved.')
+
+        target_model = target_refiner_unet
+        if target_model is None:
+            target_model = target_unet
+            print('Use base model to refine itself - this may because of developer mode.')
+
+        sampled_latent = core.ksampler(
+            model=target_model,
+            positive=clip_separate(positive_cond, target_model=target_model.model, target_clip=target_clip),
+            negative=clip_separate(negative_cond, target_model=target_model.model, target_clip=target_clip),
+            latent=sampled_latent,
+            steps=steps, start_step=switch, last_step=steps, disable_noise=True, force_full_denoise=True,
+            seed=image_seed,
+            denoise=denoise,
+            callback_function=callback,
+            cfg=cfg_scale,
+            sampler_name=sampler_name,
+            scheduler=scheduler_name,
+            previewer_start=switch,
+            previewer_end=steps,
+            disable_preview=disable_preview
+        )
+
+        target_model = target_refiner_vae
+        if target_model is None:
+            target_model = target_vae
+        decoded_latent = core.decode_vae(vae=target_model, latent_image=sampled_latent, tiled=tiled)
+
+    if refiner_swap_method == 'vae':
+        modules.patch.patch_settings[os.getpid()].eps_record = 'vae'
+
+        if modules.inpaint_worker.current_task is not None:
+            modules.inpaint_worker.current_task.unswap()
+
+        sampled_latent = core.ksampler(
+            model=target_unet,
+            positive=positive_cond,
+            negative=negative_cond,
+            latent=initial_latent,
+            steps=steps, start_step=0, last_step=switch, disable_noise=False, force_full_denoise=True,
+            seed=image_seed,
+            denoise=denoise,
+            callback_function=callback,
+            cfg=cfg_scale,
+            sampler_name=sampler_name,
+            scheduler=scheduler_name,
+            previewer_start=0,
+            previewer_end=steps,
+            disable_preview=disable_preview
+        )
+        print('Fooocus VAE-based swap.')
+
+        target_model = target_refiner_unet
+        if target_model is None:
+            target_model = target_unet
+            print('Use base model to refine itself - this may because of developer mode.')
+
+        sampled_latent = vae_parse(sampled_latent)
+
+        k_sigmas = 1.4
+        sigmas = calculate_sigmas(sampler=sampler_name,
+                                  scheduler=scheduler_name,
+                                  model=target_model.model,
+                                  steps=steps,
+                                  denoise=denoise)[switch:] * k_sigmas
+        len_sigmas = len(sigmas) - 1
+
+        noise_mean = torch.mean(modules.patch.patch_settings[os.getpid()].eps_record, dim=1, keepdim=True)
+
+        if modules.inpaint_worker.current_task is not None:
+            modules.inpaint_worker.current_task.swap()
+
+        sampled_latent = core.ksampler(
+            model=target_model,
+            positive=clip_separate(positive_cond, target_model=target_model.model, target_clip=target_clip),
+            negative=clip_separate(negative_cond, target_model=target_model.model, target_clip=target_clip),
+            latent=sampled_latent,
+            steps=len_sigmas, start_step=0, last_step=len_sigmas, disable_noise=False, force_full_denoise=True,
+            seed=image_seed+1,
+            denoise=denoise,
+            callback_function=callback,
+            cfg=cfg_scale,
+            sampler_name=sampler_name,
+            scheduler=scheduler_name,
+            previewer_start=switch,
+            previewer_end=steps,
+            sigmas=sigmas,
+            noise_mean=noise_mean,
+            disable_preview=disable_preview
+        )
+
+        target_model = target_refiner_vae
+        if target_model is None:
+            target_model = target_vae
+        decoded_latent = core.decode_vae(vae=target_model, latent_image=sampled_latent, tiled=tiled)
+
+    images = core.pytorch_to_numpy(decoded_latent)
+    modules.patch.patch_settings[os.getpid()].eps_record = None
+    return images

modules/extra_utils.py

@@ -0,0 +1,41 @@
+import os
+from ast import literal_eval
+
+
+def makedirs_with_log(path):
+    try:
+        os.makedirs(path, exist_ok=True)
+    except OSError as error:
+        print(f'Directory {path} could not be created, reason: {error}')
+
+
+def get_files_from_folder(folder_path, extensions=None, name_filter=None):
+    if not os.path.isdir(folder_path):
+        raise ValueError("Folder path is not a valid directory.")
+
+    filenames = []
+
+    for root, _, files in os.walk(folder_path, topdown=False):
+        relative_path = os.path.relpath(root, folder_path)
+        if relative_path == ".":
+            relative_path = ""
+        for filename in sorted(files, key=lambda s: s.casefold()):
+            _, file_extension = os.path.splitext(filename)
+            if (extensions is None or file_extension.lower() in extensions) and (name_filter is None or name_filter in _):
+                path = os.path.join(relative_path, filename)
+                filenames.append(path)
+
+    return filenames
+
+
+def try_eval_env_var(value: str, expected_type=None):
+    try:
+        value_eval = value
+        if expected_type is bool:
+            value_eval = value.title()
+        value_eval = literal_eval(value_eval)
+        if expected_type is not None and not isinstance(value_eval, expected_type):
+            return value
+        return value_eval
+    except:
+        return value

modules/flags.py

@@ -0,0 +1,191 @@
+from enum import IntEnum, Enum
+
+disabled = 'Disabled'
+enabled = 'Enabled'
+subtle_variation = 'Vary (Subtle)'
+strong_variation = 'Vary (Strong)'
+upscale_15 = 'Upscale (1.5x)'
+upscale_2 = 'Upscale (2x)'
+upscale_fast = 'Upscale (Fast 2x)'
+
+uov_list = [disabled, subtle_variation, strong_variation, upscale_15, upscale_2, upscale_fast]
+
+enhancement_uov_before = "Before First Enhancement"
+enhancement_uov_after = "After Last Enhancement"
+enhancement_uov_processing_order = [enhancement_uov_before, enhancement_uov_after]
+
+enhancement_uov_prompt_type_original = 'Original Prompts'
+enhancement_uov_prompt_type_last_filled = 'Last Filled Enhancement Prompts'
+enhancement_uov_prompt_types = [enhancement_uov_prompt_type_original, enhancement_uov_prompt_type_last_filled]
+
+CIVITAI_NO_KARRAS = ["euler", "euler_ancestral", "heun", "dpm_fast", "dpm_adaptive", "ddim", "uni_pc"]
+
+# fooocus: a1111 (Civitai)
+KSAMPLER = {
+    "euler": "Euler",
+    "euler_ancestral": "Euler a",
+    "heun": "Heun",
+    "heunpp2": "",
+    "dpm_2": "DPM2",
+    "dpm_2_ancestral": "DPM2 a",
+    "lms": "LMS",
+    "dpm_fast": "DPM fast",
+    "dpm_adaptive": "DPM adaptive",
+    "dpmpp_2s_ancestral": "DPM++ 2S a",
+    "dpmpp_sde": "DPM++ SDE",
+    "dpmpp_sde_gpu": "DPM++ SDE",
+    "dpmpp_2m": "DPM++ 2M",
+    "dpmpp_2m_sde": "DPM++ 2M SDE",
+    "dpmpp_2m_sde_gpu": "DPM++ 2M SDE",
+    "dpmpp_3m_sde": "",
+    "dpmpp_3m_sde_gpu": "",
+    "ddpm": "",
+    "lcm": "LCM",
+    "tcd": "TCD",
+    "restart": "Restart"
+}
+
+SAMPLER_EXTRA = {
+    "ddim": "DDIM",
+    "uni_pc": "UniPC",
+    "uni_pc_bh2": ""
+}
+
+SAMPLERS = KSAMPLER | SAMPLER_EXTRA
+
+KSAMPLER_NAMES = list(KSAMPLER.keys())
+
+SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo", "align_your_steps", "tcd", "edm_playground_v2.5"]
+SAMPLER_NAMES = KSAMPLER_NAMES + list(SAMPLER_EXTRA.keys())
+
+sampler_list = SAMPLER_NAMES
+scheduler_list = SCHEDULER_NAMES
+
+clip_skip_max = 12
+
+default_vae = 'Default (model)'
+
+refiner_swap_method = 'joint'
+
+default_input_image_tab = 'uov_tab'
+input_image_tab_ids = ['uov_tab', 'ip_tab', 'inpaint_tab', 'describe_tab', 'enhance_tab', 'metadata_tab']
+
+cn_ip = "ImagePrompt"
+cn_ip_face = "FaceSwap"
+cn_canny = "PyraCanny"
+cn_cpds = "CPDS"
+
+ip_list = [cn_ip, cn_canny, cn_cpds, cn_ip_face]
+default_ip = cn_ip
+
+default_parameters = {
+    cn_ip: (0.5, 0.6), cn_ip_face: (0.9, 0.75), cn_canny: (0.5, 1.0), cn_cpds: (0.5, 1.0)
+}  # stop, weight
+
+output_formats = ['png', 'jpeg', 'webp']
+
+inpaint_mask_models = ['u2net', 'u2netp', 'u2net_human_seg', 'u2net_cloth_seg', 'silueta', 'isnet-general-use', 'isnet-anime', 'sam']
+inpaint_mask_cloth_category = ['full', 'upper', 'lower']
+inpaint_mask_sam_model = ['vit_b', 'vit_l', 'vit_h']
+
+inpaint_engine_versions = ['None', 'v1', 'v2.5', 'v2.6']
+inpaint_option_default = 'Inpaint or Outpaint (default)'
+inpaint_option_detail = 'Improve Detail (face, hand, eyes, etc.)'
+inpaint_option_modify = 'Modify Content (add objects, change background, etc.)'
+inpaint_options = [inpaint_option_default, inpaint_option_detail, inpaint_option_modify]
+
+describe_type_photo = 'Photograph'
+describe_type_anime = 'Art/Anime'
+describe_types = [describe_type_photo, describe_type_anime]
+
+sdxl_aspect_ratios = [
+    '704*1408', '704*1344', '768*1344', '768*1280', '832*1216', '832*1152',
+    '896*1152', '896*1088', '960*1088', '960*1024', '1024*1024', '1024*960',
+    '1088*960', '1088*896', '1152*896', '1152*832', '1216*832', '1280*768',
+    '1344*768', '1344*704', '1408*704', '1472*704', '1536*640', '1600*640',
+    '1664*576', '1728*576'
+]
+
+
+class MetadataScheme(Enum):
+    FOOOCUS = 'fooocus'
+    A1111 = 'a1111'
+
+
+metadata_scheme = [
+    (f'{MetadataScheme.FOOOCUS.value} (json)', MetadataScheme.FOOOCUS.value),
+    (f'{MetadataScheme.A1111.value} (plain text)', MetadataScheme.A1111.value),
+]
+
+
+class OutputFormat(Enum):
+    PNG = 'png'
+    JPEG = 'jpeg'
+    WEBP = 'webp'
+
+    @classmethod
+    def list(cls) -> list:
+        return list(map(lambda c: c.value, cls))
+
+
+class PerformanceLoRA(Enum):
+    QUALITY = None
+    SPEED = None
+    EXTREME_SPEED = 'sdxl_lcm_lora.safetensors'
+    LIGHTNING = 'sdxl_lightning_4step_lora.safetensors'
+    HYPER_SD = 'sdxl_hyper_sd_4step_lora.safetensors'
+
+
+class Steps(IntEnum):
+    QUALITY = 60
+    SPEED = 30
+    EXTREME_SPEED = 8
+    LIGHTNING = 4
+    HYPER_SD = 4
+
+    @classmethod
+    def keys(cls) -> list:
+        return list(map(lambda c: c, Steps.__members__))
+
+
+class StepsUOV(IntEnum):
+    QUALITY = 36
+    SPEED = 18
+    EXTREME_SPEED = 8
+    LIGHTNING = 4
+    HYPER_SD = 4
+
+
+class Performance(Enum):
+    QUALITY = 'Quality'
+    SPEED = 'Speed'
+    EXTREME_SPEED = 'Extreme Speed'
+    LIGHTNING = 'Lightning'
+    HYPER_SD = 'Hyper-SD'
+
+    @classmethod
+    def list(cls) -> list:
+        return list(map(lambda c: (c.name, c.value), cls))
+
+    @classmethod
+    def values(cls) -> list:
+        return list(map(lambda c: c.value, cls))
+
+    @classmethod
+    def by_steps(cls, steps: int | str):
+        return cls[Steps(int(steps)).name]
+
+    @classmethod
+    def has_restricted_features(cls, x) -> bool:
+        if isinstance(x, Performance):
+            x = x.value
+        return x in [cls.EXTREME_SPEED.value, cls.LIGHTNING.value, cls.HYPER_SD.value]
+
+    def steps(self) -> int | None:
+        return Steps[self.name].value if self.name in Steps.__members__ else None
+
+    def steps_uov(self) -> int | None:
+        return StepsUOV[self.name].value if self.name in StepsUOV.__members__ else None
+
+    def lora_filename(self) -> str | None:
+        return PerformanceLoRA[self.name].value if self.name in PerformanceLoRA.__members__ else None