Buckets:
import os
import time
from typing import List, Optional, Literal, Tuple, Union, TYPE_CHECKING, Dict
import random
import torch
import torchaudio
from toolkit.audio.album_artwork import add_album_artwork
from toolkit.prompt_utils import PromptEmbeds
from torchao.quantization.quant_primitives import _DTYPE_TO_BIT_WIDTH
ImgExt = Literal['jpg', 'png', 'webp']
SaveFormat = Literal['safetensors', 'diffusers']
if TYPE_CHECKING:
from toolkit.guidance import GuidanceType
from toolkit.logging_aitk import EmptyLogger
else:
EmptyLogger = None
class SaveConfig:
def __init__(self, **kwargs):
self.save_every: int = kwargs.get('save_every', 1000)
self.dtype: str = kwargs.get('dtype', 'float16')
self.max_step_saves_to_keep: int = kwargs.get('max_step_saves_to_keep', 5)
self.save_format: SaveFormat = kwargs.get('save_format', 'safetensors')
if self.save_format not in ['safetensors', 'diffusers']:
raise ValueError(f"save_format must be safetensors or diffusers, got {self.save_format}")
self.push_to_hub: bool = kwargs.get("push_to_hub", False)
self.hf_repo_id: Optional[str] = kwargs.get("hf_repo_id", None)
self.hf_private: Optional[str] = kwargs.get("hf_private", False)
class LoggingConfig:
def __init__(self, **kwargs):
self.log_every: int = kwargs.get('log_every', 100)
self.verbose: bool = kwargs.get('verbose', False)
self.use_wandb: bool = kwargs.get('use_wandb', False)
self.use_ui_logger: bool = kwargs.get('use_ui_logger', False)
self.project_name: str = kwargs.get('project_name', 'ai-toolkit')
self.run_name: str = kwargs.get('run_name', None)
class SampleItem:
def __init__(
self,
sample_config: 'SampleConfig',
**kwargs
):
# prompt should always be in the kwargs
self.prompt = kwargs.get('prompt', None)
self.width: int = kwargs.get('width', sample_config.width)
self.height: int = kwargs.get('height', sample_config.height)
self.neg: str = kwargs.get('neg', sample_config.neg)
self.seed: Optional[int] = kwargs.get('seed', None) # if none, default to autogen seed
self.guidance_scale: float = kwargs.get('guidance_scale', sample_config.guidance_scale)
self.sample_steps: int = kwargs.get('sample_steps', sample_config.sample_steps)
self.fps: int = kwargs.get('fps', sample_config.fps)
self.num_frames: int = kwargs.get('num_frames', sample_config.num_frames)
self.ctrl_img: Optional[str] = kwargs.get('ctrl_img', None)
self.ctrl_idx: int = kwargs.get('ctrl_idx', 0)
# for multi control image models
self.ctrl_img_1: Optional[str] = kwargs.get('ctrl_img_1', self.ctrl_img)
self.ctrl_img_2: Optional[str] = kwargs.get('ctrl_img_2', None)
self.ctrl_img_3: Optional[str] = kwargs.get('ctrl_img_3', None)
self.network_multiplier: float = kwargs.get('network_multiplier', sample_config.network_multiplier)
# convert to a number if it is a string
if isinstance(self.network_multiplier, str):
try:
self.network_multiplier = float(self.network_multiplier)
except:
print(f"Invalid network_multiplier {self.network_multiplier}, defaulting to 1.0")
self.network_multiplier = 1.0
# only for models that support it, (qwen image edit 2509 for now)
self.do_cfg_norm: bool = kwargs.get('do_cfg_norm', False)
class SampleConfig:
def __init__(self, **kwargs):
self.sampler: str = kwargs.get('sampler', 'ddpm')
self.sample_every: int = kwargs.get('sample_every', 100)
self.sample_start_step: int = kwargs.get('sample_start_step', 0)
self.width: int = kwargs.get('width', 512)
self.height: int = kwargs.get('height', 512)
self.neg = kwargs.get('neg', False)
self.seed = kwargs.get('seed', 0)
self.walk_seed = kwargs.get('walk_seed', False)
self.guidance_scale = kwargs.get('guidance_scale', 7)
self.sample_steps = kwargs.get('sample_steps', 20)
self.network_multiplier = kwargs.get('network_multiplier', 1)
self.guidance_rescale = kwargs.get('guidance_rescale', 0.0)
self.ext: ImgExt = kwargs.get('format', 'jpg')
self.adapter_conditioning_scale = kwargs.get('adapter_conditioning_scale', 1.0)
self.refiner_start_at = kwargs.get('refiner_start_at',
0.5) # step to start using refiner on sample if it exists
self.extra_values = kwargs.get('extra_values', [])
self.num_frames = kwargs.get('num_frames', 1)
self.fps: int = kwargs.get('fps', 16)
if self.num_frames > 1 and self.ext not in ['webp']:
print("Changing sample extention to animated webp")
self.ext = 'webp'
prompts: list[str] = kwargs.get('prompts', [])
self.samples: Optional[List[SampleItem]] = None
# use the legacy prompts if it is passed that way to get samples object
default_samples_kwargs = [
{"prompt": x} for x in prompts
]
raw_samples = kwargs.get('samples', default_samples_kwargs)
self.samples = [SampleItem(self, **item) for item in raw_samples]
# only for models that support it, (qwen image edit 2509 for now)
self.do_cfg_norm: bool = kwargs.get('do_cfg_norm', False)
@property
def prompts(self):
# for backwards compatibility as this is checked for length frequently
return [sample.prompt for sample in self.samples if sample.prompt is not None]
class LormModuleSettingsConfig:
def __init__(self, **kwargs):
self.contains: str = kwargs.get('contains', '4nt$3')
self.extract_mode: str = kwargs.get('extract_mode', 'ratio')
# min num parameters to attach to
self.parameter_threshold: int = kwargs.get('parameter_threshold', 0)
self.extract_mode_param: dict = kwargs.get('extract_mode_param', 0.25)
class LoRMConfig:
def __init__(self, **kwargs):
self.extract_mode: str = kwargs.get('extract_mode', 'ratio')
self.do_conv: bool = kwargs.get('do_conv', False)
self.extract_mode_param: dict = kwargs.get('extract_mode_param', 0.25)
self.parameter_threshold: int = kwargs.get('parameter_threshold', 0)
module_settings = kwargs.get('module_settings', [])
default_module_settings = {
'extract_mode': self.extract_mode,
'extract_mode_param': self.extract_mode_param,
'parameter_threshold': self.parameter_threshold,
}
module_settings = [{**default_module_settings, **module_setting, } for module_setting in module_settings]
self.module_settings: List[LormModuleSettingsConfig] = [LormModuleSettingsConfig(**module_setting) for
module_setting in module_settings]
def get_config_for_module(self, block_name):
for setting in self.module_settings:
contain_pieces = setting.contains.split('|')
if all(contain_piece in block_name for contain_piece in contain_pieces):
return setting
# try replacing the . with _
contain_pieces = setting.contains.replace('.', '_').split('|')
if all(contain_piece in block_name for contain_piece in contain_pieces):
return setting
# do default
return LormModuleSettingsConfig(**{
'extract_mode': self.extract_mode,
'extract_mode_param': self.extract_mode_param,
'parameter_threshold': self.parameter_threshold,
})
NetworkType = Literal['lora', 'locon', 'lorm', 'lokr']
class NetworkConfig:
def __init__(self, **kwargs):
self.type: NetworkType = kwargs.get('type', 'lora')
rank = kwargs.get('rank', None)
linear = kwargs.get('linear', None)
if rank is not None:
self.rank: int = rank # rank for backward compatibility
self.linear: int = rank
elif linear is not None:
self.rank: int = linear
self.linear: int = linear
else:
self.rank: int = 4
self.linear: int = 4
self.conv: int = kwargs.get('conv', None)
self.alpha: float = kwargs.get('alpha', 1.0)
self.linear_alpha: float = kwargs.get('linear_alpha', self.alpha)
self.conv_alpha: float = kwargs.get('conv_alpha', self.conv)
self.dropout: Union[float, None] = kwargs.get('dropout', None)
self.network_kwargs: dict = kwargs.get('network_kwargs', {})
self.lorm_config: Union[LoRMConfig, None] = None
lorm = kwargs.get('lorm', None)
if lorm is not None:
self.lorm_config: LoRMConfig = LoRMConfig(**lorm)
if self.type == 'lorm':
# set linear to arbitrary values so it makes them
self.linear = 4
self.rank = 4
if self.lorm_config.do_conv:
self.conv = 4
self.transformer_only = kwargs.get('transformer_only', True)
self.lokr_full_rank = kwargs.get('lokr_full_rank', True)
if self.lokr_full_rank and self.type.lower() == 'lokr':
self.linear = 9999999999
self.linear_alpha = 9999999999
self.conv = 9999999999
self.conv_alpha = 9999999999
# -1 automatically finds the largest factor
self.lokr_factor = kwargs.get('lokr_factor', -1)
# Use the old lokr format
self.old_lokr_format = kwargs.get('old_lokr_format', False)
# for multi stage models
self.split_multistage_loras = kwargs.get('split_multistage_loras', True)
# ramtorch, doesn't work yet
self.layer_offloading = kwargs.get('layer_offloading', False)
# start from a pretrained lora
self.pretrained_lora_path = kwargs.get('pretrained_lora_path', None)
# will create diffirential full weight modules for layers not conv/linear
# only useful in very special cases.
self.all_layers = kwargs.get('all_layers', False)
AdapterTypes = Literal['t2i', 'ip', 'ip+', 'clip', 'ilora', 'photo_maker', 'control_net', 'control_lora', 'i2v']
CLIPLayer = Literal['penultimate_hidden_states', 'image_embeds', 'last_hidden_state']
class AdapterConfig:
def __init__(self, **kwargs):
self.type: AdapterTypes = kwargs.get('type', 't2i') # t2i, ip, clip, control_net, i2v
self.in_channels: int = kwargs.get('in_channels', 3)
self.channels: List[int] = kwargs.get('channels', [320, 640, 1280, 1280])
self.num_res_blocks: int = kwargs.get('num_res_blocks', 2)
self.downscale_factor: int = kwargs.get('downscale_factor', 8)
self.adapter_type: str = kwargs.get('adapter_type', 'full_adapter')
self.image_dir: str = kwargs.get('image_dir', None)
self.test_img_path: List[str] = kwargs.get('test_img_path', None)
if self.test_img_path is not None:
if isinstance(self.test_img_path, str):
self.test_img_path = self.test_img_path.split(',')
self.test_img_path = [p.strip() for p in self.test_img_path]
self.test_img_path = [p for p in self.test_img_path if p != '']
self.train: str = kwargs.get('train', False)
self.image_encoder_path: str = kwargs.get('image_encoder_path', None)
self.name_or_path = kwargs.get('name_or_path', None)
num_tokens = kwargs.get('num_tokens', None)
if num_tokens is None and self.type.startswith('ip'):
if self.type == 'ip+':
num_tokens = 16
num_tokens = 16
elif self.type == 'ip':
num_tokens = 4
self.num_tokens: int = num_tokens
self.train_image_encoder: bool = kwargs.get('train_image_encoder', False)
self.train_only_image_encoder: bool = kwargs.get('train_only_image_encoder', False)
if self.train_only_image_encoder:
self.train_image_encoder = True
self.train_only_image_encoder_positional_embedding: bool = kwargs.get(
'train_only_image_encoder_positional_embedding', False)
self.image_encoder_arch: str = kwargs.get('image_encoder_arch', 'clip') # clip vit vit_hybrid, safe
self.safe_reducer_channels: int = kwargs.get('safe_reducer_channels', 512)
self.safe_channels: int = kwargs.get('safe_channels', 2048)
self.safe_tokens: int = kwargs.get('safe_tokens', 8)
self.quad_image: bool = kwargs.get('quad_image', False)
# clip vision
self.trigger = kwargs.get('trigger', 'tri993r')
self.trigger_class_name = kwargs.get('trigger_class_name', None)
self.class_names = kwargs.get('class_names', [])
self.clip_layer: CLIPLayer = kwargs.get('clip_layer', None)
if self.clip_layer is None:
if self.type.startswith('ip+'):
self.clip_layer = 'penultimate_hidden_states'
else:
self.clip_layer = 'last_hidden_state'
# text encoder
self.text_encoder_path: str = kwargs.get('text_encoder_path', None)
self.text_encoder_arch: str = kwargs.get('text_encoder_arch', 'clip') # clip t5
self.train_scaler: bool = kwargs.get('train_scaler', False)
self.scaler_lr: Optional[float] = kwargs.get('scaler_lr', None)
# trains with a scaler to easy channel bias but merges it in on save
self.merge_scaler: bool = kwargs.get('merge_scaler', False)
# for ilora
self.head_dim: int = kwargs.get('head_dim', 1024)
self.num_heads: int = kwargs.get('num_heads', 1)
self.ilora_down: bool = kwargs.get('ilora_down', True)
self.ilora_mid: bool = kwargs.get('ilora_mid', True)
self.ilora_up: bool = kwargs.get('ilora_up', True)
self.pixtral_max_image_size: int = kwargs.get('pixtral_max_image_size', 512)
self.pixtral_random_image_size: int = kwargs.get('pixtral_random_image_size', False)
self.flux_only_double: bool = kwargs.get('flux_only_double', False)
# train and use a conv layer to pool the embedding
self.conv_pooling: bool = kwargs.get('conv_pooling', False)
self.conv_pooling_stacks: int = kwargs.get('conv_pooling_stacks', 1)
self.sparse_autoencoder_dim: Optional[int] = kwargs.get('sparse_autoencoder_dim', None)
# for llm adapter
self.num_cloned_blocks: int = kwargs.get('num_cloned_blocks', 0)
self.quantize_llm: bool = kwargs.get('quantize_llm', False)
# for control lora only
lora_config: dict = kwargs.get('lora_config', None)
if lora_config is not None:
self.lora_config: NetworkConfig = NetworkConfig(**lora_config)
else:
self.lora_config = None
self.num_control_images: int = kwargs.get('num_control_images', 1)
# decimal for how often the control is dropped out and replaced with noise 1.0 is 100%
self.control_image_dropout: float = kwargs.get('control_image_dropout', 0.0)
self.has_inpainting_input: bool = kwargs.get('has_inpainting_input', False)
self.invert_inpaint_mask_chance: float = kwargs.get('invert_inpaint_mask_chance', 0.0)
# for subpixel adapter
self.subpixel_downscale_factor: int = kwargs.get('subpixel_downscale_factor', 8)
# for i2v adapter
# append the masked start frame. During pretraining we will only do the vision encoder
self.i2v_do_start_frame: bool = kwargs.get('i2v_do_start_frame', False)
class ValidationItem:
def __init__(self, **kwargs):
self.image_path: str = kwargs.get('image_path', '')
self.prompt: str = kwargs.get('prompt', '')
class ValidationConfig:
def __init__(self, **kwargs):
self.validation_items: List[ValidationItem] = [
item if isinstance(item, ValidationItem) else ValidationItem(**item)
for item in kwargs.get('validation_items', [])
]
self.resolution: int = kwargs.get('resolution', 512)
self.validate_every_n_steps: int = kwargs.get('validate_every_n_steps', 10)
self.validation_sigmas: List[float] = kwargs.get('validation_sigmas', [1.0, 0.75, 0.5, 0.25])
class EmbeddingConfig:
def __init__(self, **kwargs):
self.trigger = kwargs.get('trigger', 'custom_embedding')
self.tokens = kwargs.get('tokens', 4)
self.init_words = kwargs.get('init_words', '*')
self.save_format = kwargs.get('save_format', 'safetensors')
self.trigger_class_name = kwargs.get('trigger_class_name', None) # used for inverted masked prior
class DecoratorConfig:
def __init__(self, **kwargs):
self.num_tokens: str = kwargs.get('num_tokens', 4)
ContentOrStyleType = Literal['balanced', 'style', 'content']
LossTarget = Literal['noise', 'source', 'unaugmented', 'differential_noise']
class TrainConfig:
def __init__(self, **kwargs):
self.noise_scheduler = kwargs.get('noise_scheduler', 'ddpm')
self.content_or_style: ContentOrStyleType = kwargs.get('content_or_style', 'balanced')
self.content_or_style_reg: ContentOrStyleType = kwargs.get('content_or_style', 'balanced')
self.steps: int = kwargs.get('steps', 1000)
self.lr = kwargs.get('lr', 1e-6)
self.unet_lr = kwargs.get('unet_lr', self.lr)
self.text_encoder_lr = kwargs.get('text_encoder_lr', self.lr)
self.refiner_lr = kwargs.get('refiner_lr', self.lr)
self.embedding_lr = kwargs.get('embedding_lr', self.lr)
self.adapter_lr = kwargs.get('adapter_lr', self.lr)
self.optimizer = kwargs.get('optimizer', 'adamw')
self.optimizer_params = kwargs.get('optimizer_params', {})
self.lr_scheduler = kwargs.get('lr_scheduler', 'constant')
self.lr_scheduler_params = kwargs.get('lr_scheduler_params', {})
self.min_denoising_steps: int = kwargs.get('min_denoising_steps', 0)
self.max_denoising_steps: int = kwargs.get('max_denoising_steps', 999)
self.batch_size: int = kwargs.get('batch_size', 1)
self.orig_batch_size: int = self.batch_size
self.dtype: str = kwargs.get('dtype', 'fp32')
self.xformers = kwargs.get('xformers', False)
self.sdp = kwargs.get('sdp', False)
# see https://huggingface.co/docs/diffusers/main/optimization/attention_backends#available-backends for options
self.attention_backend: str = kwargs.get('attention_backend', 'native') # native, flash, _flash_3_hub, _flash_3,
self.train_unet = kwargs.get('train_unet', True)
self.train_text_encoder = kwargs.get('train_text_encoder', False)
self.train_refiner = kwargs.get('train_refiner', True)
self.train_turbo = kwargs.get('train_turbo', False)
self.show_turbo_outputs = kwargs.get('show_turbo_outputs', False)
self.min_snr_gamma = kwargs.get('min_snr_gamma', None)
self.snr_gamma = kwargs.get('snr_gamma', None)
# trains a gamma, offset, and scale to adjust loss to adapt to timestep differentials
# this should balance the learning rate across all timesteps over time
self.learnable_snr_gos = kwargs.get('learnable_snr_gos', False)
self.noise_offset = kwargs.get('noise_offset', 0.0)
self.skip_first_sample = kwargs.get('skip_first_sample', False)
self.force_first_sample = kwargs.get('force_first_sample', False)
self.gradient_checkpointing = kwargs.get('gradient_checkpointing', True)
self.weight_jitter = kwargs.get('weight_jitter', 0.0)
self.merge_network_on_save = kwargs.get('merge_network_on_save', False)
self.merge_network_on_save_strength = kwargs.get('merge_network_on_save_strength', 1.0)
self.max_grad_norm = kwargs.get('max_grad_norm', 1.0)
self.start_step = kwargs.get('start_step', None)
self.free_u = kwargs.get('free_u', False)
self.adapter_assist_name_or_path: Optional[str] = kwargs.get('adapter_assist_name_or_path', None)
self.adapter_assist_type: Optional[str] = kwargs.get('adapter_assist_type', 't2i') # t2i, control_net
self.noise_multiplier = kwargs.get('noise_multiplier', 1.0)
self.target_noise_multiplier = kwargs.get('target_noise_multiplier', 1.0)
self.random_noise_multiplier = kwargs.get('random_noise_multiplier', 0.0)
self.do_signal_correction_noise = kwargs.get('do_signal_correction_noise', False)
# batch noise correction adds other images in the batch as noise to correct away from other images
self.do_batch_noise_correction = kwargs.get('do_batch_noise_correction', False)
self.batch_noise_correction_scale = kwargs.get('batch_noise_correction_scale', 0.1)
self.do_signal_amplification = kwargs.get('do_signal_amplification', False)
self.signal_amplification_strength = kwargs.get('signal_amplification_strength', 0.5)
self.signal_correction_noise_scale = kwargs.get('signal_correction_noise_scale', 1.0)
self.random_noise_shift = kwargs.get('random_noise_shift', 0.0)
self.img_multiplier = kwargs.get('img_multiplier', 1.0)
self.noisy_latent_multiplier = kwargs.get('noisy_latent_multiplier', 1.0)
self.negative_prompt = kwargs.get('negative_prompt', None)
self.max_negative_prompts = kwargs.get('max_negative_prompts', 1)
# multiplier applied to loos on regularization images
self.reg_weight = kwargs.get('reg_weight', 1.0)
self.num_train_timesteps = kwargs.get('num_train_timesteps', 1000)
# automatically adapte the vae scaling based on the image norm
self.adaptive_scaling_factor = kwargs.get('adaptive_scaling_factor', False)
# dropout that happens before encoding. It functions independently per text encoder
self.prompt_dropout_prob = kwargs.get('prompt_dropout_prob', 0.0)
# match the norm of the noise before computing loss. This will help the model maintain its
# current understandin of the brightness of images.
self.match_noise_norm = kwargs.get('match_noise_norm', False)
# set to -1 to accumulate gradients for entire epoch
# warning, only do this with a small dataset or you will run out of memory
# This is legacy but left in for backwards compatibility
self.gradient_accumulation_steps = kwargs.get('gradient_accumulation_steps', 1)
# this will do proper gradient accumulation where you will not see a step until the end of the accumulation
# the method above will show a step every accumulation
self.gradient_accumulation = kwargs.get('gradient_accumulation', 1)
if self.gradient_accumulation > 1:
if self.gradient_accumulation_steps != 1:
raise ValueError("gradient_accumulation and gradient_accumulation_steps are mutually exclusive")
# short long captions will double your batch size. This only works when a dataset is
# prepared with a json caption file that has both short and long captions in it. It will
# Double up every image and run it through with both short and long captions. The idea
# is that the network will learn how to generate good images with both short and long captions
self.short_and_long_captions = kwargs.get('short_and_long_captions', False)
# if above is NOT true, this will make it so the long caption foes to te2 and the short caption goes to te1 for sdxl only
self.short_and_long_captions_encoder_split = kwargs.get('short_and_long_captions_encoder_split', False)
# basically gradient accumulation but we run just 1 item through the network
# and accumulate gradients. This can be used as basic gradient accumulation but is very helpful
# for training tricks that increase batch size but need a single gradient step
self.single_item_batching = kwargs.get('single_item_batching', False)
match_adapter_assist = kwargs.get('match_adapter_assist', False)
self.match_adapter_chance = kwargs.get('match_adapter_chance', 0.0)
self.loss_target: LossTarget = kwargs.get('loss_target',
'noise') # noise, source, unaugmented, differential_noise
# When a mask is passed in a dataset, and this is true,
# we will predict noise without a the LoRa network and use the prediction as a target for
# unmasked reign. It is unmasked regularization basically
self.inverted_mask_prior = kwargs.get('inverted_mask_prior', False)
self.inverted_mask_prior_multiplier = kwargs.get('inverted_mask_prior_multiplier', 0.5)
# DOP will will run the same image and prompt through the network without the trigger word blank and use it as a target
self.diff_output_preservation = kwargs.get('diff_output_preservation', False)
self.diff_output_preservation_multiplier = kwargs.get('diff_output_preservation_multiplier', 1.0)
# If the trigger word is in the prompt, we will use this class name to replace it eg. "sks woman" -> "woman"
self.diff_output_preservation_class = kwargs.get('diff_output_preservation_class', '')
# blank prompt preservation will preserve the model's knowledge of a blank prompt
self.blank_prompt_preservation = kwargs.get('blank_prompt_preservation', False)
self.blank_prompt_preservation_multiplier = kwargs.get('blank_prompt_preservation_multiplier', 1.0)
# legacy
if match_adapter_assist and self.match_adapter_chance == 0.0:
self.match_adapter_chance = 1.0
# standardize inputs to the meand std of the model knowledge
self.standardize_images = kwargs.get('standardize_images', False)
# if self.train_turbo and not self.noise_scheduler.startswith("euler"):
# raise ValueError(f"train_turbo is only supported with euler and wuler_a noise schedulers")
self.dynamic_noise_offset = kwargs.get('dynamic_noise_offset', False)
self.do_cfg = kwargs.get('do_cfg', False)
self.do_random_cfg = kwargs.get('do_random_cfg', False)
self.cfg_scale = kwargs.get('cfg_scale', 1.0)
self.max_cfg_scale = kwargs.get('max_cfg_scale', self.cfg_scale)
self.cfg_rescale = kwargs.get('cfg_rescale', None)
if self.cfg_rescale is None:
self.cfg_rescale = self.cfg_scale
# applies the inverse of the prediction mean and std to the target to correct
# for norm drift
self.correct_pred_norm = kwargs.get('correct_pred_norm', False)
self.correct_pred_norm_multiplier = kwargs.get('correct_pred_norm_multiplier', 1.0)
self.loss_type = kwargs.get('loss_type', 'mse') # mse, mae, wavelet, pixelspace, mean_flow, pseudo_huber
# do the loss on a timestep to 0 prediction
self.t0_loss_target = kwargs.get('t0_loss_target', False)
self.t0_velocity_equiv_weight = kwargs.get('t0_velocity_equiv_weight', False)
# do additional fft loss
self.do_fft_loss = kwargs.get('do_fft_loss', False)
self.do_fft_velocity_equiv_weight = kwargs.get('do_fft_velocity_equiv_weight', False)
# scale the prediction by this. Increase for more detail, decrease for less
self.pred_scaler = kwargs.get('pred_scaler', 1.0)
# repeats the prompt a few times to saturate the encoder
self.prompt_saturation_chance = kwargs.get('prompt_saturation_chance', 0.0)
# applies negative loss on the prior to encourage network to diverge from it
self.do_prior_divergence = kwargs.get('do_prior_divergence', False)
ema_config: Union[Dict, None] = kwargs.get('ema_config', None)
# if it is set explicitly to false, leave it false.
if ema_config is not None and ema_config.get('use_ema', False):
ema_config['use_ema'] = True
print(f"Using EMA")
else:
ema_config = {'use_ema': False}
self.ema_config: EMAConfig = EMAConfig(**ema_config)
# adds an additional loss to the network to encourage it output a normalized standard deviation
self.target_norm_std = kwargs.get('target_norm_std', None)
self.target_norm_std_value = kwargs.get('target_norm_std_value', 1.0)
self.timestep_type = kwargs.get('timestep_type', 'sigmoid') # sigmoid, linear, lognorm_blend, next_sample, weighted, one_step
self.first_timestep_chance = kwargs.get('first_timestep_chance', 0.0)
self.next_sample_timesteps = kwargs.get('next_sample_timesteps', 8)
self.linear_timesteps = kwargs.get('linear_timesteps', False)
self.linear_timesteps2 = kwargs.get('linear_timesteps2', False)
self.disable_sampling = kwargs.get('disable_sampling', False)
# will cache a blank prompt or the trigger word, and unload the text encoder to cpu
# will make training faster and use less vram
self.unload_text_encoder = kwargs.get('unload_text_encoder', False)
# will toggle all datasets to cache text embeddings
self.cache_text_embeddings: bool = kwargs.get('cache_text_embeddings', False)
# for swapping which parameters are trained during training
self.do_paramiter_swapping = kwargs.get('do_paramiter_swapping', False)
# 0.1 is 10% of the parameters active at a time lower is less vram, higher is more
self.paramiter_swapping_factor = kwargs.get('paramiter_swapping_factor', 0.1)
# bypass the guidance embedding for training. For open flux with guidance embedding
self.bypass_guidance_embedding = kwargs.get('bypass_guidance_embedding', False)
# diffusion feature extractor
self.latent_feature_extractor_path = kwargs.get('latent_feature_extractor_path', None)
self.latent_feature_loss_weight = kwargs.get('latent_feature_loss_weight', 1.0)
# we use this in the code, but it really needs to be called latent_feature_extractor as that makes more sense with new architecture
self.diffusion_feature_extractor_path = kwargs.get('diffusion_feature_extractor_path', self.latent_feature_extractor_path)
self.diffusion_feature_extractor_weight = kwargs.get('diffusion_feature_extractor_weight', self.latent_feature_loss_weight)
# optimal noise pairing
self.optimal_noise_pairing_samples = kwargs.get('optimal_noise_pairing_samples', 1)
# forces same noise for the same image at a given size.
self.force_consistent_noise = kwargs.get('force_consistent_noise', False)
self.blended_blur_noise = kwargs.get('blended_blur_noise', False)
# contrastive loss
self.do_guidance_loss = kwargs.get('do_guidance_loss', False)
self.guidance_loss_target: Union[int, List[int, int]] = kwargs.get('guidance_loss_target', 3.0)
self.do_guidance_loss_cfg_zero: bool = kwargs.get('do_guidance_loss_cfg_zero', False)
# 'constant' uses guidance_loss_target as is. 'sigma' decays the target
# toward 1.0 as sigma falls (effective = 1 + (target - 1) * sigma) so the
# extrapolation never amplifies the unpredictable fresh-noise term at low
# sigma. Needed for guidance-distilled models with no guidance embedding.
self.guidance_loss_schedule: str = kwargs.get('guidance_loss_schedule', 'constant')
self.unconditional_prompt: str = kwargs.get('unconditional_prompt', '')
if isinstance(self.guidance_loss_target, tuple):
self.guidance_loss_target = list(self.guidance_loss_target)
self.do_differential_guidance = kwargs.get('do_differential_guidance', False)
self.differential_guidance_scale = kwargs.get('differential_guidance_scale', 3.0)
# for multi stage models, how often to switch the boundary
self.switch_boundary_every: int = kwargs.get('switch_boundary_every', 1)
# stabilizes empty prompts to be zeroed predictions
self.do_blank_stabilization = kwargs.get('do_blank_stabilization', False)
self.audio_loss_multiplier = kwargs.get("audio_loss_multiplier", 1.0)
# will throw detailed error when it goes over
self.max_loss_debug: bool = kwargs.get("max_loss_debug", False)
# will clip the loss to this amount to prevent wild outliers
self.max_loss: Optional[float] = kwargs.get("max_loss", None)
self.validation_config: Optional[ValidationConfig] = None
validation = kwargs.get('validation_config', None)
if validation is not None:
self.validation_config: ValidationConfig = ValidationConfig(**validation)
ModelArch = Literal['sd1', 'sd2', 'sd3', 'sdxl', 'pixart', 'pixart_sigma', 'auraflow', 'flux', 'flex1', 'flex2', 'lumina2', 'vega', 'ssd', 'wan21', 'anima']
class ModelConfig:
def __init__(self, **kwargs):
self.name_or_path: str = kwargs.get('name_or_path', None)
# name or path is updated on fine tuning. Keep a copy of the original
self.name_or_path_original: str = self.name_or_path
self.is_v2: bool = kwargs.get('is_v2', False)
self.is_xl: bool = kwargs.get('is_xl', False)
self.is_pixart: bool = kwargs.get('is_pixart', False)
self.is_pixart_sigma: bool = kwargs.get('is_pixart_sigma', False)
self.is_auraflow: bool = kwargs.get('is_auraflow', False)
self.is_v3: bool = kwargs.get('is_v3', False)
self.is_flux: bool = kwargs.get('is_flux', False)
self.is_lumina2: bool = kwargs.get('is_lumina2', False)
if self.is_pixart_sigma:
self.is_pixart = True
self.use_flux_cfg = kwargs.get('use_flux_cfg', False)
self.is_ssd: bool = kwargs.get('is_ssd', False)
self.is_vega: bool = kwargs.get('is_vega', False)
self.is_v_pred: bool = kwargs.get('is_v_pred', False)
self.dtype: str = kwargs.get('dtype', 'float16')
self.vae_path = kwargs.get('vae_path', None)
self.refiner_name_or_path = kwargs.get('refiner_name_or_path', None)
self._original_refiner_name_or_path = self.refiner_name_or_path
self.refiner_start_at = kwargs.get('refiner_start_at', 0.5)
self.lora_path = kwargs.get('lora_path', None)
# mainly for decompression loras for distilled models
self.assistant_lora_path = kwargs.get('assistant_lora_path', None)
self.inference_lora_path = kwargs.get('inference_lora_path', None)
# a lora that stays inactive except during the unconditional (negative)
# CFG pass -- used to learn the unconditional branch without a second model
self.unconditional_lora_path = kwargs.get('unconditional_lora_path', None)
self.latent_space_version = kwargs.get('latent_space_version', None)
# only for SDXL models for now
self.use_text_encoder_1: bool = kwargs.get('use_text_encoder_1', True)
self.use_text_encoder_2: bool = kwargs.get('use_text_encoder_2', True)
self.experimental_xl: bool = kwargs.get('experimental_xl', False)
if self.name_or_path is None:
raise ValueError('name_or_path must be specified')
if self.is_ssd:
# sed sdxl as true since it is mostly the same architecture
self.is_xl = True
if self.is_vega:
self.is_xl = True
# for text encoder quant. Only works with pixart currently
self.text_encoder_bits = kwargs.get('text_encoder_bits', 16) # 16, 8, 4
self.unet_path = kwargs.get("unet_path", None)
self.unet_sample_size = kwargs.get("unet_sample_size", None)
self.vae_device = kwargs.get("vae_device", None)
self.vae_dtype = kwargs.get("vae_dtype", self.dtype)
self.te_device = kwargs.get("te_device", None)
self.te_dtype = kwargs.get("te_dtype", self.dtype)
# only for flux for now
self.quantize = kwargs.get("quantize", False)
self.quantize_te = kwargs.get("quantize_te", self.quantize)
self.qtype = kwargs.get("qtype", "qfloat8")
self.qtype_te = kwargs.get("qtype_te", "qfloat8")
self.low_vram = kwargs.get("low_vram", False)
self.attn_masking = kwargs.get("attn_masking", False)
if self.attn_masking and not self.is_flux:
raise ValueError("attn_masking is only supported with flux models currently")
# for targeting a specific layers
self.ignore_if_contains: Optional[List[str]] = kwargs.get("ignore_if_contains", None)
self.only_if_contains: Optional[List[str]] = kwargs.get("only_if_contains", None)
self.quantize_kwargs = kwargs.get("quantize_kwargs", {})
# splits the model over the available gpus WIP
self.split_model_over_gpus = kwargs.get("split_model_over_gpus", False)
if self.split_model_over_gpus and not self.is_flux:
raise ValueError("split_model_over_gpus is only supported with flux models currently")
self.split_model_other_module_param_count_scale = kwargs.get("split_model_other_module_param_count_scale", 0.3)
self.te_name_or_path = kwargs.get("te_name_or_path", None)
self.arch: ModelArch = kwargs.get("arch", None)
# auto memory management, only for some models
self.auto_memory = kwargs.get("auto_memory", False)
# auto memory is deprecated, use layer offloading instead
if self.auto_memory:
print("auto_memory is deprecated, use layer_offloading instead")
self.layer_offloading = kwargs.get("layer_offloading", self.auto_memory )
if self.layer_offloading and self.qtype == "qfloat8":
self.qtype = "float8"
if self.layer_offloading and self.qtype_te == "qfloat8":
self.qtype_te = "float8"
# MPS has no fp8 dtype, so qfloat8 has to become an 8 bit integer format.
# convrot8, not torchao int8: measured on an M3 against bf16, convrot8
# trains at 0.79x and holds 1.04 GB of resident weight where torchao int8
# trains at 0.52x and holds 1.21 GB, and convrot8 quantizes in 19ms
# against 2.8s. See scripts/test_quantizations.py --device mps.
if torch.backends.mps.is_available() and self.qtype == "qfloat8":
self.qtype = "convrot8"
if torch.backends.mps.is_available() and self.qtype_te == "qfloat8":
self.qtype_te = "convrot8"
# 0 is off and 1.0 is 100% of the layers
self.layer_offloading_transformer_percent = kwargs.get("layer_offloading_transformer_percent", 1.0)
self.layer_offloading_text_encoder_percent = kwargs.get("layer_offloading_text_encoder_percent", 1.0)
# can be used to load the extras like text encoder or vae from here
# only setup for some models but will prevent having to download the te for
# 20 different model variants
self.extras_name_or_path = kwargs.get("extras_name_or_path", self.name_or_path)
# path to an accuracy recovery adapter, either local or remote
self.accuracy_recovery_adapter = kwargs.get("accuracy_recovery_adapter", None)
# parse ARA from qtype
if self.qtype is not None and "|" in self.qtype:
self.qtype, self.accuracy_recovery_adapter = self.qtype.split('|')
# compile the model with torch compile
self.compile = kwargs.get("compile", False)
if self.compile and self.quantize:
print("Quantized model detected - allowing torch.compile (experimental)")
self.block_compile = kwargs.get("block_compile", False)
self.compile_mode = kwargs.get("compile_mode", "default")
self.compile_fullgraph = kwargs.get("compile_fullgraph", False)
self.compile_dynamic = kwargs.get("compile_dynamic", True)
self.cache_size_limit = kwargs.get("cache_size_limit", None)
# kwargs to pass to the model
self.model_kwargs = kwargs.get("model_kwargs", {})
# model paths for models that support it
self.model_paths = kwargs.get("model_paths", {})
self.in_context = kwargs.get("in_context", False)
# allow frontend to pass arch with a color like arch:tag
# but remove the tag
if self.arch is not None:
if ':' in self.arch:
self.arch = self.arch.split(':')[0]
if self.arch == "flex1":
self.arch = "flux"
# handle migrating to new model arch
if self.arch is not None:
# reverse the arch to the old style
if self.arch == 'sd2':
self.is_v2 = True
elif self.arch == 'sd3':
self.is_v3 = True
elif self.arch == 'sdxl':
self.is_xl = True
elif self.arch == 'pixart':
self.is_pixart = True
elif self.arch == 'pixart_sigma':
self.is_pixart_sigma = True
elif self.arch == 'auraflow':
self.is_auraflow = True
elif self.arch == 'flux':
self.is_flux = True
elif self.arch == 'lumina2':
self.is_lumina2 = True
elif self.arch == 'vega':
self.is_vega = True
elif self.arch == 'ssd':
self.is_ssd = True
else:
pass
if self.arch is None:
if kwargs.get('is_v2', False):
self.arch = 'sd2'
elif kwargs.get('is_v3', False):
self.arch = 'sd3'
elif kwargs.get('is_xl', False):
self.arch = 'sdxl'
elif kwargs.get('is_pixart', False):
self.arch = 'pixart'
elif kwargs.get('is_pixart_sigma', False):
self.arch = 'pixart_sigma'
elif kwargs.get('is_auraflow', False):
self.arch = 'auraflow'
elif kwargs.get('is_flux', False):
self.arch = 'flux'
elif kwargs.get('is_lumina2', False):
self.arch = 'lumina2'
elif kwargs.get('is_vega', False):
self.arch = 'vega'
elif kwargs.get('is_ssd', False):
self.arch = 'ssd'
else:
self.arch = 'sd1'
class EMAConfig:
def __init__(self, **kwargs):
self.use_ema: bool = kwargs.get('use_ema', False)
self.ema_decay: float = kwargs.get('ema_decay', 0.999)
# feeds back the decay difference into the parameter
self.use_feedback: bool = kwargs.get('use_feedback', False)
# per-step fraction of (shadow - param) pulled back into the param; keep well below 1 - ema_decay
self.feedback_rate: float = kwargs.get('feedback_rate', 0.001)
# every update, the params are multiplied by this amount
# only use for things without a bias like lora
# similar to a decay in an optimizer but the opposite
self.param_multiplier: float = kwargs.get('param_multiplier', 1.0)
class ReferenceDatasetConfig:
def __init__(self, **kwargs):
# can pass with a side by side pait or a folder with pos and neg folder
self.pair_folder: str = kwargs.get('pair_folder', None)
self.pos_folder: str = kwargs.get('pos_folder', None)
self.neg_folder: str = kwargs.get('neg_folder', None)
self.network_weight: float = float(kwargs.get('network_weight', 1.0))
self.pos_weight: float = float(kwargs.get('pos_weight', self.network_weight))
self.neg_weight: float = float(kwargs.get('neg_weight', self.network_weight))
# make sure they are all absolute values no negatives
self.pos_weight = abs(self.pos_weight)
self.neg_weight = abs(self.neg_weight)
self.target_class: str = kwargs.get('target_class', '')
self.size: int = kwargs.get('size', 512)
class SliderTargetConfig:
def __init__(self, **kwargs):
self.target_class: str = kwargs.get('target_class', '')
self.positive: str = kwargs.get('positive', '')
self.negative: str = kwargs.get('negative', '')
self.multiplier: float = kwargs.get('multiplier', 1.0)
self.weight: float = kwargs.get('weight', 1.0)
self.shuffle: bool = kwargs.get('shuffle', False)
class GuidanceConfig:
def __init__(self, **kwargs):
self.target_class: str = kwargs.get('target_class', '')
self.guidance_scale: float = kwargs.get('guidance_scale', 1.0)
self.positive_prompt: str = kwargs.get('positive_prompt', '')
self.negative_prompt: str = kwargs.get('negative_prompt', '')
class SliderConfigAnchors:
def __init__(self, **kwargs):
self.prompt = kwargs.get('prompt', '')
self.neg_prompt = kwargs.get('neg_prompt', '')
self.multiplier = kwargs.get('multiplier', 1.0)
class SliderConfig:
def __init__(self, **kwargs):
targets = kwargs.get('targets', [])
anchors = kwargs.get('anchors', [])
anchors = [SliderConfigAnchors(**anchor) for anchor in anchors]
self.anchors: List[SliderConfigAnchors] = anchors
self.resolutions: List[List[int]] = kwargs.get('resolutions', [[512, 512]])
self.prompt_file: str = kwargs.get('prompt_file', None)
self.prompt_tensors: str = kwargs.get('prompt_tensors', None)
self.batch_full_slide: bool = kwargs.get('batch_full_slide', True)
self.use_adapter: bool = kwargs.get('use_adapter', None) # depth
self.adapter_img_dir = kwargs.get('adapter_img_dir', None)
self.low_ram = kwargs.get('low_ram', False)
# expand targets if shuffling
from toolkit.prompt_utils import get_slider_target_permutations
self.targets: List[SliderTargetConfig] = []
targets = [SliderTargetConfig(**target) for target in targets]
# do permutations if shuffle is true
print(f"Building slider targets")
for target in targets:
if target.shuffle:
target_permutations = get_slider_target_permutations(target, max_permutations=8)
self.targets = self.targets + target_permutations
else:
self.targets.append(target)
print(f"Built {len(self.targets)} slider targets (with permutations)")
ControlTypes = Literal['depth', 'line', 'pose', 'inpaint', 'mask', 'sapiens2_mask']
class DatasetConfig:
"""
Dataset config for sd-datasets
"""
def __init__(self, **kwargs):
self.batch_size: Union[int, None] = kwargs.get('batch_size', None)
self.type = kwargs.get('type', 'image') # sd, slider, reference
# will be legacy
self.folder_path: str = kwargs.get('folder_path', None)
# can be json or folder path
self.dataset_path: str = kwargs.get('dataset_path', None)
self.default_caption: str = kwargs.get('default_caption', None)
# trigger word for just this dataset
self.trigger_word: str = kwargs.get('trigger_word', None)
# set automatically from the train config when diff output preservation is enabled.
# the dataset trigger word is replaced with the class in the caption for DOP embeddings
self.diff_output_preservation: bool = kwargs.get('diff_output_preservation', False)
self.diff_output_preservation_class: str = kwargs.get('diff_output_preservation_class', '')
random_triggers = kwargs.get('random_triggers', [])
# if they are a string, load them from a file
if isinstance(random_triggers, str) and os.path.exists(random_triggers):
with open(random_triggers, 'r') as f:
random_triggers = f.read().splitlines()
# remove empty lines
random_triggers = [line for line in random_triggers if line.strip() != '']
self.random_triggers: List[str] = random_triggers
self.random_triggers_max: int = kwargs.get('random_triggers_max', 1)
self.caption_ext: str = kwargs.get('caption_ext', '.txt')
# if caption_ext doesnt start with a dot, add it
if self.caption_ext and not self.caption_ext.startswith('.'):
self.caption_ext = '.' + self.caption_ext
self.random_scale: bool = kwargs.get('random_scale', False)
self.random_crop: bool = kwargs.get('random_crop', False)
self.resolution: int = kwargs.get('resolution', 512)
self.scale: float = kwargs.get('scale', 1.0)
self.buckets: bool = kwargs.get('buckets', True)
self.bucket_tolerance: int = kwargs.get('bucket_tolerance', 64)
self.is_reg: bool = kwargs.get('is_reg', False)
self.prior_reg: bool = kwargs.get('prior_reg', False)
self.network_weight: float = float(kwargs.get('network_weight', 1.0))
self.token_dropout_rate: float = float(kwargs.get('token_dropout_rate', 0.0))
self.shuffle_tokens: bool = kwargs.get('shuffle_tokens', False)
self.caption_dropout_rate: float = float(kwargs.get('caption_dropout_rate', 0.0))
self.keep_tokens: int = kwargs.get('keep_tokens', 0) # #of first tokens to always keep unless caption dropped
self.flip_x: bool = kwargs.get('flip_x', False)
self.flip_y: bool = kwargs.get('flip_y', False)
self.augments: List[str] = kwargs.get('augments', [])
self.control_path: Union[str,List[str]] = kwargs.get('control_path', None) # depth maps, etc
# pull a random control image from the same folder as the image. Useful for folder grouped pairs.
self.control_from_same_folder: bool = kwargs.get('control_from_same_folder', False)
self.num_controls_from_same_folder: int = kwargs.get('num_controls_from_same_folder', 1)
if self.control_path == '':
self.control_path = None
# handle multi control inputs from the ui. It is just easier to handle it here for a cleaner ui experience
control_path_1 = kwargs.get('control_path_1', None)
control_path_2 = kwargs.get('control_path_2', None)
control_path_3 = kwargs.get('control_path_3', None)
if any([control_path_1, control_path_2, control_path_3]):
control_paths = []
if control_path_1:
control_paths.append(control_path_1)
if control_path_2:
control_paths.append(control_path_2)
if control_path_3:
control_paths.append(control_path_3)
self.control_path = control_paths
# color for transparent reigon of control images with transparency
self.control_transparent_color: List[int] = kwargs.get('control_transparent_color', [0, 0, 0])
# inpaint images should be webp/png images with alpha channel. The alpha 0 (invisible) section will
# be the part conditioned to be inpainted. The alpha 1 (visible) section will be the part that is ignored
self.inpaint_path: Union[str,List[str]] = kwargs.get('inpaint_path', None)
# instead of cropping ot match image, it will serve the full size control image (clip images ie for ip adapters)
self.full_size_control_images: bool = kwargs.get('full_size_control_images', True)
self.alpha_mask: bool = kwargs.get('alpha_mask', False) # if true, will use alpha channel as mask
self.mask_path: str = kwargs.get('mask_path',
None) # focus mask (black and white. White has higher loss than black)
self.unconditional_path: str = kwargs.get('unconditional_path',
None) # path where matching unconditional images are located
self.invert_mask: bool = kwargs.get('invert_mask', False) # invert mask
self.mask_min_value: float = kwargs.get('mask_min_value', 0.0) # min value for . 0 - 1
self.poi: Union[str, None] = kwargs.get('poi', None)
if self.poi is not None:
raise ValueError("poi is deprecated and is no longer supported")
self.use_short_captions: bool = kwargs.get('use_short_captions', False) # if true, will use 'caption_short' from json
self.num_repeats: int = kwargs.get('num_repeats', 1) # number of times to repeat dataset
# cache latents will store them in memory
self.cache_latents: bool = kwargs.get('cache_latents', False)
# cache latents to disk will store them on disk. If both are true, it will save to disk, but keep in memory
self.cache_latents_to_disk: bool = kwargs.get('cache_latents_to_disk', False)
# cache tensors to disk. Useful for saving video files tensors to the disk so we have the clean pixelspace versions of video and audio
self.cache_tensors_to_disk: bool = kwargs.get('cache_tensors_to_disk', False)
self.cache_clip_vision_to_disk: bool = kwargs.get('cache_clip_vision_to_disk', False)
self.cache_text_embeddings: bool = kwargs.get('cache_text_embeddings', False)
self.load_image_when_caching_latents: bool = kwargs.get('load_image_when_caching_latents', False)
self.standardize_images: bool = kwargs.get('standardize_images', False)
# https://albumentations.ai/docs/api_reference/augmentations/transforms
# augmentations are returned as a separate image and cannot currently be cached
self.augmentations: List[dict] = kwargs.get('augmentations', None)
self.shuffle_augmentations: bool = kwargs.get('shuffle_augmentations', False)
has_augmentations = self.augmentations is not None and len(self.augmentations) > 0
if (len(self.augments) > 0 or has_augmentations) and (self.cache_latents or self.cache_latents_to_disk):
print(f"WARNING: Augments are not supported with caching latents. Setting cache_latents to False")
self.cache_latents = False
self.cache_latents_to_disk = False
# legacy compatability
legacy_caption_type = kwargs.get('caption_type', None)
if legacy_caption_type:
self.caption_ext = legacy_caption_type
self.caption_type = self.caption_ext
self.guidance_type: GuidanceType = kwargs.get('guidance_type', 'targeted')
# ip adapter / reference dataset
self.clip_image_path: str = kwargs.get('clip_image_path', None) # depth maps, etc
# get the clip image randomly from the same folder as the image. Useful for folder grouped pairs.
self.clip_image_from_same_folder: bool = kwargs.get('clip_image_from_same_folder', False)
self.clip_image_augmentations: List[dict] = kwargs.get('clip_image_augmentations', None)
self.clip_image_shuffle_augmentations: bool = kwargs.get('clip_image_shuffle_augmentations', False)
self.replacements: List[str] = kwargs.get('replacements', [])
self.loss_multiplier: float = kwargs.get('loss_multiplier', 1.0)
self.num_workers: int = kwargs.get('num_workers', 2)
self.prefetch_factor: int = kwargs.get('prefetch_factor', 2)
# Pin DataLoader output tensors in page-locked RAM for faster CPU->GPU
# transfer. Off by default because page-locked RAM cannot be relocated
# by NVIDIA's Windows driver shared-memory VRAM-overflow fallback,
# which can cause severe PCIe thrashing for users at the VRAM ceiling.
# Opt in if you have stable VRAM headroom and want the transfer speedup.
self.pin_memory: bool = kwargs.get('pin_memory', False)
# threads used to prep (decode/resize) items ahead of the VAE while caching latents
self.cache_latents_num_workers: int = kwargs.get('cache_latents_num_workers', min(6, os.cpu_count() or 1))
self.extra_values: List[float] = kwargs.get('extra_values', [])
self.square_crop: bool = kwargs.get('square_crop', False)
# apply same augmentations to control images. Usually want this true unless special case
self.replay_transforms: bool = kwargs.get('replay_transforms', True)
# for video
# if num_frames is greater than 1, the dataloader will look for video files.
# num_frames will be the number of frames in the training batch. If num_frames is 1, it will look for images
self.num_frames: int = kwargs.get('num_frames', 1)
# if true, will shrink video to our frames. For instance, if we have a video with 100 frames and num_frames is 10,
# we would pull frame 0, 10, 20, 30, 40, 50, 60, 70, 80, 90 so they are evenly spaced
self.shrink_video_to_frames: bool = kwargs.get('shrink_video_to_frames', True)
# fps is only used if shrink_video_to_frames is false. This will attempt to pull the num_frames at the given fps
# it will select a random start frame and pull the frames at the given fps
# this could have various issues with shorter videos and videos with variable fps
# I recommend trimming your videos to the desired length and using shrink_video_to_frames(default)
self.fps: int = kwargs.get('fps', 24)
# auto_frame_count pull as many frames as in the video at given fps
# Important, make sure fps for dataset is set correctly.
# this wont work with bucketing for now until I can handle this before bucketing.
self.auto_frame_count: bool = kwargs.get('auto_frame_count', False)
# old behavior shrank the video to fit the temporal spacing of the model. Which fits the whole video, but
# can lead to fast motion/chipmunking. This will prevent the video from shrinking to fit, and instead, trim
# the tail of the video. Usually only a few frames.
self.trim_auto_frame_count_tail: bool = kwargs.get('trim_auto_frame_count_tail', True)
# debug the frame count and frame selection. You dont need this. It is for debugging.
self.debug: bool = kwargs.get('debug', False)
# automatic controls
self.controls: List[ControlTypes] = kwargs.get('controls', [])
if isinstance(self.controls, str):
self.controls = [self.controls]
# remove empty strings
self.controls = [control for control in self.controls if control.strip() != '']
# if true, will use a fask method to get image sizes. This can result in errors. Do not use unless you know what you are doing
self.fast_image_size: bool = kwargs.get('fast_image_size', False)
self.do_i2v: bool = kwargs.get('do_i2v', False) # do image to video on models that are both t2i and i2v capable
self.do_audio: bool = kwargs.get('do_audio', False) # load audio from video files for models that support it
self.audio_preserve_pitch: bool = kwargs.get('audio_preserve_pitch', False) # preserve pitch when stretching audio to fit num_frames
self.audio_normalize: bool = kwargs.get('audio_normalize', False) # normalize audio volume levels when loading
def preprocess_dataset_raw_config(raw_config: List[dict]) -> List[dict]:
"""
This just splits up the datasets by resolutions so you dont have to do it manually
:param raw_config:
:return:
"""
# split up datasets by resolutions
new_config = []
for dataset in raw_config:
resolution = dataset.get('resolution', 512)
if isinstance(resolution, list):
resolution_list = resolution
else:
resolution_list = [resolution]
for res in resolution_list:
dataset_copy = dataset.copy()
dataset_copy['resolution'] = res
new_config.append(dataset_copy)
return new_config
class GenerateImageConfig:
def __init__(
self,
prompt: str = '',
prompt_2: Optional[str] = None,
width: int = 512,
height: int = 512,
num_inference_steps: int = 50,
guidance_scale: float = 7.5,
negative_prompt: str = '',
negative_prompt_2: Optional[str] = None,
seed: int = -1,
network_multiplier: float = 1.0,
guidance_rescale: float = 0.0,
# the tag [time] will be replaced with milliseconds since epoch
output_path: str = None, # full image path
output_folder: str = None, # folder to save image in if output_path is not specified
output_ext: str = ImgExt, # extension to save image as if output_path is not specified
output_tail: str = '', # tail to add to output filename
add_prompt_file: bool = False, # add a prompt file with generated image
adapter_image_path: str = None, # path to adapter image
adapter_conditioning_scale: float = 1.0, # scale for adapter conditioning
latents: Union[torch.Tensor | None] = None, # input latent to start with,
extra_kwargs: dict = None, # extra data to save with prompt file
refiner_start_at: float = 0.5, # start at this percentage of a step. 0.0 to 1.0 . 1.0 is the end
extra_values: List[float] = None, # extra values to save with prompt file
logger: Optional[EmptyLogger] = None,
ctrl_img: Optional[str] = None, # control image for controlnet
ctrl_img_1: Optional[str] = None, # first control image for multi control model
ctrl_img_2: Optional[str] = None, # second control image for multi control model
ctrl_img_3: Optional[str] = None, # third control image for multi control model
num_frames: int = 1,
fps: int = 15,
ctrl_idx: int = 0,
do_cfg_norm: bool = False,
):
self.width: int = width
self.height: int = height
self.num_inference_steps: int = num_inference_steps
self.guidance_scale: float = guidance_scale
self.guidance_rescale: float = guidance_rescale
self.prompt: str = prompt
self.prompt_2: str = prompt_2
self.negative_prompt: str = negative_prompt
self.negative_prompt_2: str = negative_prompt_2
self.latents: Union[torch.Tensor | None] = latents
self.output_path: str = output_path
self.seed: int = seed
if self.seed == -1:
# generate random one
self.seed = random.randint(0, 2 ** 32 - 1)
self.network_multiplier: float = network_multiplier
self.output_folder: str = output_folder
self.output_ext: str = output_ext
self.add_prompt_file: bool = add_prompt_file
self.output_tail: str = output_tail
self.gen_time: int = int(time.time() * 1000)
self.adapter_image_path: str = adapter_image_path
self.adapter_conditioning_scale: float = adapter_conditioning_scale
self.extra_kwargs = extra_kwargs if extra_kwargs is not None else {}
self.refiner_start_at = refiner_start_at
self.extra_values = extra_values if extra_values is not None else []
self.num_frames = num_frames
self.fps = fps
self.ctrl_img = ctrl_img
self.ctrl_idx = ctrl_idx
if ctrl_img_1 is None and ctrl_img is not None:
ctrl_img_1 = ctrl_img
self.ctrl_img_1 = ctrl_img_1
self.ctrl_img_2 = ctrl_img_2
self.ctrl_img_3 = ctrl_img_3
# prompt string will override any settings above
self._process_prompt_string()
# handle dual text encoder prompts if nothing passed
if negative_prompt_2 is None:
self.negative_prompt_2 = negative_prompt
if prompt_2 is None:
self.prompt_2 = self.prompt
# parse prompt paths
if self.output_path is None and self.output_folder is None:
raise ValueError('output_path or output_folder must be specified')
elif self.output_path is not None:
self.output_folder = os.path.dirname(self.output_path)
self.output_ext = os.path.splitext(self.output_path)[1][1:]
self.output_filename_no_ext = os.path.splitext(os.path.basename(self.output_path))[0]
else:
self.output_filename_no_ext = '[time]_[count]'
if len(self.output_tail) > 0:
self.output_filename_no_ext += '_' + self.output_tail
self.output_path = os.path.join(self.output_folder, self.output_filename_no_ext + '.' + self.output_ext)
# adjust height
self.height = max(64, self.height - self.height % 8) # round to divisible by 8
self.width = max(64, self.width - self.width % 8) # round to divisible by 8
self.logger = logger
self.do_cfg_norm: bool = do_cfg_norm
def set_gen_time(self, gen_time: int = None):
if gen_time is not None:
self.gen_time = gen_time
else:
self.gen_time = int(time.time() * 1000)
def _get_path_no_ext(self, count: int = 0, max_count=0):
# zero pad count
count_str = str(count).zfill(len(str(max_count)))
# replace [time] with gen time
filename = self.output_filename_no_ext.replace('[time]', str(self.gen_time))
# replace [count] with count
filename = filename.replace('[count]', count_str)
return filename
def get_image_path(self, count: int = 0, max_count=0):
filename = self._get_path_no_ext(count, max_count)
ext = self.output_ext
# if it does not start with a dot add one
if ext[0] != '.':
ext = '.' + ext
filename += ext
# join with folder
return os.path.join(self.output_folder, filename)
def get_prompt_path(self, count: int = 0, max_count=0):
filename = self._get_path_no_ext(count, max_count)
filename += '.txt'
# join with folder
return os.path.join(self.output_folder, filename)
def save_image_atomic(self, image, count: int = 0, max_count=0):
# write into a hidden tmp subfolder, then atomically move into place so
# watchers (UI/CDN) never see and cache a partially written file. Wraps
# self.save_image so it also covers models that replace that function.
real_folder = self.output_folder
tmp_folder = os.path.join(real_folder, '.tmp')
os.makedirs(tmp_folder, exist_ok=True)
self.output_folder = tmp_folder
try:
self.save_image(image, count, max_count)
finally:
self.output_folder = real_folder
files = os.listdir(tmp_folder)
# thumbs move into place first so they already exist when the media
# file appears in the samples folder
thumbs_folder = os.path.join(real_folder, '.thumbs')
for file in files:
tmp_thumb = os.path.join(tmp_folder, file + '.thumb')
try:
if self._generate_thumbnail(os.path.join(tmp_folder, file), tmp_thumb):
os.makedirs(thumbs_folder, exist_ok=True)
os.replace(tmp_thumb, os.path.join(thumbs_folder, file + '.jpg'))
except Exception as e:
print(f"Failed to generate thumbnail for {file}: {e}")
for file in files:
os.replace(os.path.join(tmp_folder, file), os.path.join(real_folder, file))
def _generate_thumbnail(self, media_path, thumb_path):
# 300x300 center-cropped 90% jpg. Returns True if one was written.
from PIL import Image as PILImage
ext = os.path.splitext(media_path)[1].lower()
img = None
if ext in ['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp']:
img = PILImage.open(media_path) # animated formats open on the first frame
elif ext == '.mp4':
import cv2
cap = cv2.VideoCapture(media_path)
ok, frame = cap.read()
cap.release()
if ok:
img = PILImage.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
if img is None:
return False
img = img.convert('RGB')
w, h = img.size
side = min(w, h)
left = (w - side) // 2
top = (h - side) // 2
img = img.crop((left, top, left + side, top + side)).resize((300, 300), PILImage.LANCZOS)
img.save(thumb_path, format='JPEG', quality=90)
return True
def save_image(self, image, count: int = 0, max_count=0):
# make parent dirs
os.makedirs(self.output_folder, exist_ok=True)
self.set_gen_time()
if isinstance(image, list):
# video
if self.num_frames == 1:
raise ValueError(f"Expected 1 img but got a list {len(image)}")
if self.num_frames > 1 and self.output_ext not in ['webp']:
self.output_ext = 'webp'
if self.output_ext == 'webp':
# save as animated webp
duration = 1000 // self.fps # Convert fps to milliseconds per frame
image[0].save(
self.get_image_path(count, max_count),
format='WEBP',
append_images=image[1:],
save_all=True,
duration=duration, # Duration per frame in milliseconds
loop=0, # 0 means loop forever
quality=80 # Quality setting (0-100)
)
else:
raise ValueError(f"Unsupported video format {self.output_ext}")
elif self.output_ext in ['wav', 'mp3', 'flac', 'ogg']:
# save audio file
audio_path = self.get_image_path(count, max_count)
torchaudio.save(
audio_path,
image[0].to('cpu'),
sample_rate=48000,
format=None,
backend=None
)
if self.output_ext == 'mp3':
add_album_artwork(audio_path)
else:
# TODO save image gen header info for A1111 and us, our seeds probably wont match
image.save(self.get_image_path(count, max_count))
# do prompt file
if self.add_prompt_file:
self.save_prompt_file(count, max_count)
def save_prompt_file(self, count: int = 0, max_count=0):
# save prompt file
with open(self.get_prompt_path(count, max_count), 'w') as f:
prompt = self.prompt
if self.prompt_2 is not None:
prompt += ' --p2 ' + self.prompt_2
if self.negative_prompt is not None:
prompt += ' --n ' + self.negative_prompt
if self.negative_prompt_2 is not None:
prompt += ' --n2 ' + self.negative_prompt_2
prompt += ' --w ' + str(self.width)
prompt += ' --h ' + str(self.height)
prompt += ' --seed ' + str(self.seed)
prompt += ' --cfg ' + str(self.guidance_scale)
prompt += ' --steps ' + str(self.num_inference_steps)
prompt += ' --m ' + str(self.network_multiplier)
prompt += ' --gr ' + str(self.guidance_rescale)
# get gen info
try:
f.write(self.prompt)
except Exception as e:
print(f"Error writing prompt file. Prompt contains non-unicode characters. {e}")
def _process_prompt_string(self):
# we will try to support all sd-scripts where we can
# FROM SD-SCRIPTS
# --n Treat everything until the next option as a negative prompt.
# --w Specify the width of the generated image.
# --h Specify the height of the generated image.
# --d Specify the seed for the generated image.
# --l Specify the CFG scale for the generated image.
# --s Specify the number of steps during generation.
# OURS and some QOL additions
# --m Specify the network multiplier for the generated image.
# --p2 Prompt for the second text encoder (SDXL only)
# --n2 Negative prompt for the second text encoder (SDXL only)
# --gr Specify the guidance rescale for the generated image (SDXL only)
# --seed Specify the seed for the generated image same as --d
# --cfg Specify the CFG scale for the generated image same as --l
# --steps Specify the number of steps during generation same as --s
# --network_multiplier Specify the network multiplier for the generated image same as --m
# process prompt string and update values if it has some
if self.prompt is not None and len(self.prompt) > 0:
# process prompt string
prompt = self.prompt
prompt = prompt.strip()
p_split = prompt.split('--')
self.prompt = p_split[0].strip()
if len(p_split) > 1:
for split in p_split[1:]:
# allows multi char flags
flag = split.split(' ')[0].strip()
content = split[len(flag):].strip()
if flag == 'p2':
self.prompt_2 = content
elif flag == 'n':
self.negative_prompt = content
elif flag == 'n2':
self.negative_prompt_2 = content
elif flag == 'w':
self.width = int(content)
elif flag == 'h':
self.height = int(content)
elif flag == 'd':
self.seed = int(content)
elif flag == 'seed':
self.seed = int(content)
elif flag == 'l':
self.guidance_scale = float(content)
elif flag == 'cfg':
self.guidance_scale = float(content)
elif flag == 's':
self.num_inference_steps = int(content)
elif flag == 'steps':
self.num_inference_steps = int(content)
elif flag == 'm':
self.network_multiplier = float(content)
elif flag == 'network_multiplier':
self.network_multiplier = float(content)
elif flag == 'gr':
self.guidance_rescale = float(content)
elif flag == 'a':
self.adapter_conditioning_scale = float(content)
elif flag == 'ref':
self.refiner_start_at = float(content)
elif flag == 'ev':
# split by comma
self.extra_values = [float(val) for val in content.split(',')]
elif flag == 'extra_values':
# split by comma
self.extra_values = [float(val) for val in content.split(',')]
elif flag == 'frames':
self.num_frames = int(content)
elif flag == 'num_frames':
self.num_frames = int(content)
elif flag == 'fps':
self.fps = int(content)
elif flag == 'ctrl_img':
self.ctrl_img = content
elif flag == 'ctrl_idx':
self.ctrl_idx = int(content)
def post_process_embeddings(
self,
conditional_prompt_embeds: PromptEmbeds,
unconditional_prompt_embeds: Optional[PromptEmbeds] = None,
):
# this is called after prompt embeds are encoded. We can override them in the future here
pass
def log_image(self, image, count: int = 0, max_count=0):
if self.logger is None:
return
self.logger.log_image(image, count, self.prompt)
def validate_configs(
train_config: TrainConfig,
model_config: ModelConfig,
save_config: SaveConfig,
dataset_configs: List[DatasetConfig]
):
if model_config.is_flux:
if save_config.save_format != 'diffusers':
# make it diffusers
save_config.save_format = 'diffusers'
if model_config.use_flux_cfg:
# bypass the embedding
train_config.bypass_guidance_embedding = True
if train_config.bypass_guidance_embedding and train_config.do_guidance_loss:
raise ValueError("Cannot bypass guidance embedding and do guidance loss at the same time. "
"Please set bypass_guidance_embedding to False or do_guidance_loss to False.")
if model_config.accuracy_recovery_adapter is not None:
if model_config.assistant_lora_path is not None:
raise ValueError("Cannot use accuracy recovery adapter and assistant lora at the same time. "
"Please set one of them to None.")
# see if any datasets are caching text embeddings
is_caching_text_embeddings = any(dataset.cache_text_embeddings for dataset in dataset_configs)
if is_caching_text_embeddings:
# make sure they are all cached
for dataset in dataset_configs:
if not dataset.cache_text_embeddings:
raise ValueError("All datasets must have cache_text_embeddings set to True when caching text embeddings is enabled.")
# qwen image edit cannot cache text embeddings
if model_config.arch in ['qwen_image_edit', 'boogu_image_edit']:
if train_config.unload_text_encoder:
raise ValueError(f"Cannot cache unload text encoder with {model_config.arch} model. Control images are encoded with text embeddings. You can cache the text embeddings though")
if train_config.diff_output_preservation and train_config.blank_prompt_preservation:
raise ValueError("Cannot use both differential output preservation and blank prompt preservation at the same time. Please set one of them to False.")
import os
import time
from typing import List, Optional, Literal, Tuple, Union, TYPE_CHECKING, Dict
import random
import torch
import torchaudio
from toolkit.audio.album_artwork import add_album_artwork
from toolkit.prompt_utils import PromptEmbeds
from torchao.quantization.quant_primitives import _DTYPE_TO_BIT_WIDTH
ImgExt = Literal['jpg', 'png', 'webp']
SaveFormat = Literal['safetensors', 'diffusers']
if TYPE_CHECKING:
from toolkit.guidance import GuidanceType
from toolkit.logging_aitk import EmptyLogger
else:
EmptyLogger = None
class SaveConfig:
def __init__(self, **kwargs):
self.save_every: int = kwargs.get('save_every', 1000)
self.dtype: str = kwargs.get('dtype', 'float16')
self.max_step_saves_to_keep: int = kwargs.get('max_step_saves_to_keep', 5)
self.save_format: SaveFormat = kwargs.get('save_format', 'safetensors')
if self.save_format not in ['safetensors', 'diffusers']:
raise ValueError(f"save_format must be safetensors or diffusers, got {self.save_format}")
self.push_to_hub: bool = kwargs.get("push_to_hub", False)
self.hf_repo_id: Optional[str] = kwargs.get("hf_repo_id", None)
self.hf_private: Optional[str] = kwargs.get("hf_private", False)
class LoggingConfig:
def __init__(self, **kwargs):
self.log_every: int = kwargs.get('log_every', 100)
self.verbose: bool = kwargs.get('verbose', False)
self.use_wandb: bool = kwargs.get('use_wandb', False)
self.use_ui_logger: bool = kwargs.get('use_ui_logger', False)
self.project_name: str = kwargs.get('project_name', 'ai-toolkit')
self.run_name: str = kwargs.get('run_name', None)
class SampleItem:
def __init__(
self,
sample_config: 'SampleConfig',
**kwargs
):
# prompt should always be in the kwargs
self.prompt = kwargs.get('prompt', None)
self.width: int = kwargs.get('width', sample_config.width)
self.height: int = kwargs.get('height', sample_config.height)
self.neg: str = kwargs.get('neg', sample_config.neg)
self.seed: Optional[int] = kwargs.get('seed', None) # if none, default to autogen seed
self.guidance_scale: float = kwargs.get('guidance_scale', sample_config.guidance_scale)
self.sample_steps: int = kwargs.get('sample_steps', sample_config.sample_steps)
self.fps: int = kwargs.get('fps', sample_config.fps)
self.num_frames: int = kwargs.get('num_frames', sample_config.num_frames)
self.ctrl_img: Optional[str] = kwargs.get('ctrl_img', None)
self.ctrl_idx: int = kwargs.get('ctrl_idx', 0)
# for multi control image models
self.ctrl_img_1: Optional[str] = kwargs.get('ctrl_img_1', self.ctrl_img)
self.ctrl_img_2: Optional[str] = kwargs.get('ctrl_img_2', None)
self.ctrl_img_3: Optional[str] = kwargs.get('ctrl_img_3', None)
self.network_multiplier: float = kwargs.get('network_multiplier', sample_config.network_multiplier)
# convert to a number if it is a string
if isinstance(self.network_multiplier, str):
try:
self.network_multiplier = float(self.network_multiplier)
except:
print(f"Invalid network_multiplier {self.network_multiplier}, defaulting to 1.0")
self.network_multiplier = 1.0
# only for models that support it, (qwen image edit 2509 for now)
self.do_cfg_norm: bool = kwargs.get('do_cfg_norm', False)
class SampleConfig:
def __init__(self, **kwargs):
self.sampler: str = kwargs.get('sampler', 'ddpm')
self.sample_every: int = kwargs.get('sample_every', 100)
self.sample_start_step: int = kwargs.get('sample_start_step', 0)
self.width: int = kwargs.get('width', 512)
self.height: int = kwargs.get('height', 512)
self.neg = kwargs.get('neg', False)
self.seed = kwargs.get('seed', 0)
self.walk_seed = kwargs.get('walk_seed', False)
self.guidance_scale = kwargs.get('guidance_scale', 7)
self.sample_steps = kwargs.get('sample_steps', 20)
self.network_multiplier = kwargs.get('network_multiplier', 1)
self.guidance_rescale = kwargs.get('guidance_rescale', 0.0)
self.ext: ImgExt = kwargs.get('format', 'jpg')
self.adapter_conditioning_scale = kwargs.get('adapter_conditioning_scale', 1.0)
self.refiner_start_at = kwargs.get('refiner_start_at',
0.5) # step to start using refiner on sample if it exists
self.extra_values = kwargs.get('extra_values', [])
self.num_frames = kwargs.get('num_frames', 1)
self.fps: int = kwargs.get('fps', 16)
if self.num_frames > 1 and self.ext not in ['webp']:
Xet Storage Details
- Size:
- 78.5 kB
- Xet hash:
- 7ed48574f02c40644b7f3cbff89bce9196a31d8b33189bc5805f8fcc80ffd7f7
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.