diff --git a/qwen2509_object_removal_512/.job_config.json b/qwen2509_object_removal_512/.job_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d53b58c141708b568612d506bffbab9ce53d7811 --- /dev/null +++ b/qwen2509_object_removal_512/.job_config.json @@ -0,0 +1,144 @@ +{ + "job": "extension", + "config": { + "name": "qwen2509_object_removal_512", + "process": [ + { + "type": "diffusion_trainer", + "training_folder": "/mnt/d/Github/ai-toolkit/output/", + "sqlite_db_path": "/mnt/d/Github/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 16, + "linear_alpha": 16, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 250, + "max_step_saves_to_keep": 4, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/target", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "Remove Object", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "do_i2v": true, + "flip_x": false, + "flip_y": false, + "control_path_1": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/with_object", + "control_path_2": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mask", + "control_path_3": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mae_output" + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 13582, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 1, + "loss_type": "mse" + }, + "model": { + "name_or_path": "Qwen/Qwen-Image-Edit-2509", + "quantize": true, + "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_edit_2509_torchao_uint3.safetensors", + "quantize_te": true, + "qtype_te": "uint4", + "arch": "qwen_image_edit_plus", + "low_vram": true, + "model_kwargs": { + "match_target_res": true + }, + "layer_offloading": true, + "layer_offloading_text_encoder_percent": 0, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/8184cef5-9303-484b-bc7c-fd3d2e865e24.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/74de8410-0630-41b7-bf80-747bdc03df50.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/4b3a523f-8bd2-4db2-b734-ebe0b629f17a.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/66a6d4bc-7c71-40f3-a8b7-55719a0d0596.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/3f7e34c1-54d0-4095-b4ed-ec7f8f3ed475.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/17521be8-a7fe-4240-9340-da1657874313.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/dba76474-f415-4e8b-a410-caa17c1ea158.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/47b8f626-147d-41da-a8c0-6c21382ae206.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/7dfc0f6d-6c7f-4f09-b855-248f6f39614a.jpg" + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 25, + "num_frames": 1, + "fps": 1 + } + } + ] + }, + "meta": { + "name": "[name]", + "version": "1.0" + } +} \ No newline at end of file diff --git a/qwen2509_object_removal_512/config.yaml b/qwen2509_object_removal_512/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ef3c34cef85dad710ca9437c8bfce971c89c9a4 --- /dev/null +++ b/qwen2509_object_removal_512/config.yaml @@ -0,0 +1,117 @@ +job: extension +config: + name: qwen2509_object_removal_512 + process: + - type: diffusion_trainer + training_folder: /mnt/d/Github/ai-toolkit/output/ + sqlite_db_path: /mnt/d/Github/ai-toolkit/aitk_db.db + device: cuda + trigger_word: null + performance_log_every: 10 + network: + type: lora + linear: 16 + linear_alpha: 16 + conv: 16 + conv_alpha: 16 + lokr_full_rank: true + lokr_factor: -1 + network_kwargs: + ignore_if_contains: [] + save: + dtype: bf16 + save_every: 250 + max_step_saves_to_keep: 4 + save_format: diffusers + push_to_hub: false + datasets: + - folder_path: /mnt/d/Github/ai-toolkit/datasets/qwen_or/train/target + mask_path: null + mask_min_value: 0.1 + default_caption: Remove Object + caption_ext: txt + caption_dropout_rate: 0.05 + cache_latents_to_disk: true + is_reg: false + network_weight: 1 + resolution: + - 512 + controls: [] + shrink_video_to_frames: true + num_frames: 1 + do_i2v: true + flip_x: false + flip_y: false + control_path_1: /mnt/d/Github/ai-toolkit/datasets/qwen_or/train/with_object + control_path_2: /mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mask + control_path_3: /mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mae_output + train: + batch_size: 1 + bypass_guidance_embedding: false + steps: 13582 + gradient_accumulation: 1 + train_unet: true + train_text_encoder: false + gradient_checkpointing: true + noise_scheduler: flowmatch + optimizer: adamw8bit + timestep_type: weighted + content_or_style: balanced + optimizer_params: + weight_decay: 0.0001 + unload_text_encoder: false + cache_text_embeddings: true + lr: 0.0001 + ema_config: + use_ema: false + ema_decay: 0.99 + skip_first_sample: false + force_first_sample: false + disable_sampling: false + dtype: bf16 + diff_output_preservation: false + diff_output_preservation_multiplier: 1 + diff_output_preservation_class: person + switch_boundary_every: 1 + loss_type: mse + model: + name_or_path: Qwen/Qwen-Image-Edit-2509 + quantize: true + qtype: uint3|ostris/accuracy_recovery_adapters/qwen_image_edit_2509_torchao_uint3.safetensors + quantize_te: true + qtype_te: uint4 + arch: qwen_image_edit_plus + low_vram: true + model_kwargs: + match_target_res: true + layer_offloading: true + layer_offloading_text_encoder_percent: 0 + layer_offloading_transformer_percent: 1 + sample: + sampler: flowmatch + sample_every: 250 + width: 512 + height: 512 + samples: + - prompt: Remove Object + ctrl_img_1: /mnt/d/Github/ai-toolkit/data/images/8184cef5-9303-484b-bc7c-fd3d2e865e24.jpg + ctrl_img_2: /mnt/d/Github/ai-toolkit/data/images/74de8410-0630-41b7-bf80-747bdc03df50.jpg + ctrl_img_3: /mnt/d/Github/ai-toolkit/data/images/4b3a523f-8bd2-4db2-b734-ebe0b629f17a.jpg + - prompt: Remove Object + ctrl_img_1: /mnt/d/Github/ai-toolkit/data/images/66a6d4bc-7c71-40f3-a8b7-55719a0d0596.jpg + ctrl_img_2: /mnt/d/Github/ai-toolkit/data/images/3f7e34c1-54d0-4095-b4ed-ec7f8f3ed475.jpg + ctrl_img_3: /mnt/d/Github/ai-toolkit/data/images/17521be8-a7fe-4240-9340-da1657874313.jpg + - prompt: Remove Object + ctrl_img_1: /mnt/d/Github/ai-toolkit/data/images/dba76474-f415-4e8b-a410-caa17c1ea158.jpg + ctrl_img_2: /mnt/d/Github/ai-toolkit/data/images/47b8f626-147d-41da-a8c0-6c21382ae206.jpg + ctrl_img_3: /mnt/d/Github/ai-toolkit/data/images/7dfc0f6d-6c7f-4f09-b855-248f6f39614a.jpg + neg: '' + seed: 42 + walk_seed: true + guidance_scale: 4 + sample_steps: 25 + num_frames: 1 + fps: 1 +meta: + name: qwen2509_object_removal_512 + version: '1.0' diff --git a/qwen2509_object_removal_512/log.txt b/qwen2509_object_removal_512/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..93a592e58c105c1ec4a8fa5e733bd3728fcbeaf7 --- /dev/null +++ b/qwen2509_object_removal_512/log.txt @@ -0,0 +1,1069 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/mnt/d/Github/ai-toolkit/output/", + "sqlite_db_path": "/mnt/d/Github/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 16, + "linear_alpha": 16, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 250, + "max_step_saves_to_keep": 4, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/target", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "Remove Object", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "do_i2v": true, + "flip_x": false, + "flip_y": false, + "control_path_1": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/with_object", + "control_path_2": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mask", + "control_path_3": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mae_output" + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 13582, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 1, + "loss_type": "mse" + }, + "model": { + "name_or_path": "Qwen/Qwen-Image-Edit-2509", + "quantize": true, + "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_edit_2509_torchao_uint3.safetensors", + "quantize_te": true, + "qtype_te": "uint4", + "arch": "qwen_image_edit_plus", + "low_vram": true, + "model_kwargs": { + "match_target_res": true + }, + "layer_offloading": true, + "layer_offloading_text_encoder_percent": 0, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/8184cef5-9303-484b-bc7c-fd3d2e865e24.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/74de8410-0630-41b7-bf80-747bdc03df50.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/4b3a523f-8bd2-4db2-b734-ebe0b629f17a.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/66a6d4bc-7c71-40f3-a8b7-55719a0d0596.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/3f7e34c1-54d0-4095-b4ed-ec7f8f3ed475.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/17521be8-a7fe-4240-9340-da1657874313.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/dba76474-f415-4e8b-a410-caa17c1ea158.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/47b8f626-147d-41da-a8c0-6c21382ae206.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/7dfc0f6d-6c7f-4f09-b855-248f6f39614a.jpg" + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 25, + "num_frames": 1, + "fps": 1 + } +} +Using SQLite database at /mnt/d/Github/ai-toolkit/aitk_db.db +Job ID: "ab192e4e-2fbc-4fe0-8067-c4de78a005b2" + +############################################# +# Running job: qwen2509_object_removal_512 +############################################# + + +Running 1 process +Loading Qwen Image model +Loading transformer + Loading checkpoint shards: 0%| | 0/5 [00:00 + File "/mnt/d/Github/ai-toolkit/run.py", line 120, in + main()main() + + File "/mnt/d/Github/ai-toolkit/run.py", line 108, in main + File "/mnt/d/Github/ai-toolkit/run.py", line 108, in main + raise eraise e + + File "/mnt/d/Github/ai-toolkit/run.py", line 96, in main + File "/mnt/d/Github/ai-toolkit/run.py", line 96, in main + job.run()job.run() + + File "/mnt/d/Github/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + File "/mnt/d/Github/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run()process.run() + + File "/mnt/d/Github/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1995, in run + File "/mnt/d/Github/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1995, in run + self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd)self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd) + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + File "/mnt/d/Github/ai-toolkit/toolkit/data_loader.py", line 626, in get_dataloader_from_datasets + File "/mnt/d/Github/ai-toolkit/toolkit/data_loader.py", line 626, in get_dataloader_from_datasets + dataset = AiToolkitDataset(config, batch_size=batch_size, sd=sd)dataset = AiToolkitDataset(config, batch_size=batch_size, sd=sd) + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + File "/mnt/d/Github/ai-toolkit/toolkit/data_loader.py", line 431, in __init__ + File "/mnt/d/Github/ai-toolkit/toolkit/data_loader.py", line 431, in __init__ + with open(self.dataset_path, 'r') as f:with open(self.dataset_path, 'r') as f: + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +FileNotFoundErrorFileNotFoundError: : [Errno 2] No such file or directory: 'D:\\Github\\ai-toolkit\\datasets\\qwen_or\\train/mae_output'[Errno 2] No such file or directory: 'D:\\Github\\ai-toolkit\\datasets\\qwen_or\\train/mae_output' + diff --git a/qwen2509_object_removal_512/logs/1_log.txt b/qwen2509_object_removal_512/logs/1_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3f374548d5592c6e1d81e9f45af86c42db964b9b --- /dev/null +++ b/qwen2509_object_removal_512/logs/1_log.txt @@ -0,0 +1,378 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "D:\\Github\\ai-toolkit\\output", + "sqlite_db_path": "/mnt/d/Github/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 16, + "linear_alpha": 16, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 250, + "max_step_saves_to_keep": 4, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/target", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "Remove Object", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "do_i2v": true, + "flip_x": false, + "flip_y": false, + "control_path_1": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/with_object", + "control_path_2": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mask", + "control_path_3": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mae_output" + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 13582, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 1, + "loss_type": "mse" + }, + "model": { + "name_or_path": "Qwen/Qwen-Image-Edit-2509", + "quantize": true, + "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_edit_2509_torchao_uint3.safetensors", + "quantize_te": true, + "qtype_te": "uint4", + "arch": "qwen_image_edit_plus", + "low_vram": true, + "model_kwargs": { + "match_target_res": true + }, + "layer_offloading": true, + "layer_offloading_text_encoder_percent": 0, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/8184cef5-9303-484b-bc7c-fd3d2e865e24.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/74de8410-0630-41b7-bf80-747bdc03df50.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/4b3a523f-8bd2-4db2-b734-ebe0b629f17a.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/66a6d4bc-7c71-40f3-a8b7-55719a0d0596.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/3f7e34c1-54d0-4095-b4ed-ec7f8f3ed475.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/17521be8-a7fe-4240-9340-da1657874313.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/dba76474-f415-4e8b-a410-caa17c1ea158.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/47b8f626-147d-41da-a8c0-6c21382ae206.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/7dfc0f6d-6c7f-4f09-b855-248f6f39614a.jpg" + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 25, + "num_frames": 1, + "fps": 1 + } +} +Using SQLite database at /mnt/d/Github/ai-toolkit/aitk_db.db +Job ID: "ab192e4e-2fbc-4fe0-8067-c4de78a005b2" + +############################################# +# Running job: qwen2509_object_removal_512 +############################################# + + +Running 1 process +Loading Qwen Image model +Loading transformer + Loading checkpoint shards: 0%| | 0/5 [00:00 + File "/mnt/d/Github/ai-toolkit/run.py", line 120, in + main()main() + + File "/mnt/d/Github/ai-toolkit/run.py", line 108, in main + File "/mnt/d/Github/ai-toolkit/run.py", line 108, in main + raise eraise e + + File "/mnt/d/Github/ai-toolkit/run.py", line 96, in main + File "/mnt/d/Github/ai-toolkit/run.py", line 96, in main + job.run()job.run() + + File "/mnt/d/Github/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + File "/mnt/d/Github/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run()process.run() + + File "/mnt/d/Github/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2324, in run + File "/mnt/d/Github/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2324, in run + self.end_step_hook()self.end_step_hook() + + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook + self.maybe_stop()self.maybe_stop() + + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop + raise Exception("Job stopped")raise Exception("Job stopped") + +ExceptionException: : Job stoppedJob stopped + + qwen2509_object_removal_512: 7%|7 | 975/13582 [3:37:40<46:54:41, 13.40s/it, lr: 1.0e-04 loss: 1.081e-02] qwen2509_object_removal_512: 7%|7 | 975/13582 [3:37:40<46:54:41, 13.40s/it, lr: 1.0e-04 loss: 1.081e-02] + diff --git a/qwen2509_object_removal_512/logs/2_log.txt b/qwen2509_object_removal_512/logs/2_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..77487ee267eced54a128b6e7fa380348cc6c0437 --- /dev/null +++ b/qwen2509_object_removal_512/logs/2_log.txt @@ -0,0 +1,181 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "D:\\Github\\ai-toolkit\\output", + "sqlite_db_path": "/mnt/d/Github/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 16, + "linear_alpha": 16, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 250, + "max_step_saves_to_keep": 4, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/target", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "Remove Object", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "do_i2v": true, + "flip_x": false, + "flip_y": false, + "control_path_1": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/with_object", + "control_path_2": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mask", + "control_path_3": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mae_output" + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 13582, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 1, + "loss_type": "mse" + }, + "model": { + "name_or_path": "Qwen/Qwen-Image-Edit-2509", + "quantize": true, + "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_edit_2509_torchao_uint3.safetensors", + "quantize_te": true, + "qtype_te": "uint4", + "arch": "qwen_image_edit_plus", + "low_vram": true, + "model_kwargs": { + "match_target_res": true + }, + "layer_offloading": true, + "layer_offloading_text_encoder_percent": 0, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/8184cef5-9303-484b-bc7c-fd3d2e865e24.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/74de8410-0630-41b7-bf80-747bdc03df50.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/4b3a523f-8bd2-4db2-b734-ebe0b629f17a.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/66a6d4bc-7c71-40f3-a8b7-55719a0d0596.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/3f7e34c1-54d0-4095-b4ed-ec7f8f3ed475.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/17521be8-a7fe-4240-9340-da1657874313.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/dba76474-f415-4e8b-a410-caa17c1ea158.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/47b8f626-147d-41da-a8c0-6c21382ae206.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/7dfc0f6d-6c7f-4f09-b855-248f6f39614a.jpg" + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 25, + "num_frames": 1, + "fps": 1 + } +} +Using SQLite database at /mnt/d/Github/ai-toolkit/aitk_db.db +Job ID: "ab192e4e-2fbc-4fe0-8067-c4de78a005b2" + +############################################# +# Running job: qwen2509_object_removal_512 +############################################# + + +Running 1 process +Error running job: Job stopped + +======================================== +Result: + - 0 completed jobs + - 1 failure +======================================== +Traceback (most recent call last): +Traceback (most recent call last): + File "/mnt/d/Github/ai-toolkit/run.py", line 120, in + File "/mnt/d/Github/ai-toolkit/run.py", line 120, in + main()main() + + File "/mnt/d/Github/ai-toolkit/run.py", line 108, in main + File "/mnt/d/Github/ai-toolkit/run.py", line 108, in main + raise eraise e + + File "/mnt/d/Github/ai-toolkit/run.py", line 96, in main + File "/mnt/d/Github/ai-toolkit/run.py", line 96, in main + job.run()job.run() + + File "/mnt/d/Github/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + File "/mnt/d/Github/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run()process.run() + + File "/mnt/d/Github/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1513, in run + File "/mnt/d/Github/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1513, in run + self.hook_before_model_load()self.hook_before_model_load() + + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 269, in hook_before_model_load + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 269, in hook_before_model_load + self.maybe_stop()self.maybe_stop() + + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop + raise Exception("Job stopped")raise Exception("Job stopped") + +ExceptionException: : Job stoppedJob stopped + diff --git a/qwen2509_object_removal_512/logs/3_log.txt b/qwen2509_object_removal_512/logs/3_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..e16efc3213069d224a876a305c1a1346a34e22b2 --- /dev/null +++ b/qwen2509_object_removal_512/logs/3_log.txt @@ -0,0 +1,761 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "D:\\Github\\ai-toolkit\\output", + "sqlite_db_path": "/mnt/d/Github/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 16, + "linear_alpha": 16, + "conv": 16, + "conv_alpha": 16,ten + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 250, + "max_step_saves_to_keep": 4, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/target", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "Remove Object", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "do_i2v": true, + "flip_x": false, + "flip_y": false, + "control_path_1": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/with_object", + "control_path_2": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mask", + "control_path_3": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mae_output" + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 13582, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 1, + "loss_type": "mse" + }, + "model": { + "name_or_path": "Qwen/Qwen-Image-Edit-2509", + "quantize": true, + "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_edit_2509_torchao_uint3.safetensors", + "quantize_te": true, + "qtype_te": "uint4", + "arch": "qwen_image_edit_plus", + "low_vram": true, + "model_kwargs": { + "match_target_res": true + }, + "layer_offloading": true, + "layer_offloading_text_encoder_percent": 0, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/8184cef5-9303-484b-bc7c-fd3d2e865e24.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/74de8410-0630-41b7-bf80-747bdc03df50.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/4b3a523f-8bd2-4db2-b734-ebe0b629f17a.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/66a6d4bc-7c71-40f3-a8b7-55719a0d0596.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/3f7e34c1-54d0-4095-b4ed-ec7f8f3ed475.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/17521be8-a7fe-4240-9340-da1657874313.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/dba76474-f415-4e8b-a410-caa17c1ea158.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/47b8f626-147d-41da-a8c0-6c21382ae206.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/7dfc0f6d-6c7f-4f09-b855-248f6f39614a.jpg" + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 25, + "num_frames": 1, + "fps": 1 + } +} +Using SQLite database at /mnt/d/Github/ai-toolkit/aitk_db.db +Job ID: "ab192e4e-2fbc-4fe0-8067-c4de78a005b2" + +############################################# +# Running job: qwen2509_object_removal_512 +############################################# + + +Running 1 process +Loading Qwen Image model +Loading transformer + +Loading checkpoint shards: 0%| | 0/5 [00:00 + File "/mnt/d/Github/ai-toolkit/run.py", line 120, in + main()main() + + File "/mnt/d/Github/ai-toolkit/run.py", line 108, in main + File "/mnt/d/Github/ai-toolkit/run.py", line 108, in main + raise eraise e + + File "/mnt/d/Github/ai-toolkit/run.py", line 96, in main + File "/mnt/d/Github/ai-toolkit/run.py", line 96, in main + job.run()job.run() + + File "/mnt/d/Github/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + File "/mnt/d/Github/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run()process.run() + + File "/mnt/d/Github/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1992, in run + File "/mnt/d/Github/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1992, in run + self.before_dataset_load()self.before_dataset_load() + + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 275, in before_dataset_load + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 275, in before_dataset_load + self.maybe_stop()self.maybe_stop() + + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop + File "/mnt/d/Github/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop + raise Exception("Job stopped")raise Exception("Job stopped") + +ExceptionException: : Job stoppedJob stopped + diff --git a/qwen2509_object_removal_512/logs/4_log.txt b/qwen2509_object_removal_512/logs/4_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..553163bb377bd85ab1bb323e9e899b38126013a5 --- /dev/null +++ b/qwen2509_object_removal_512/logs/4_log.txt @@ -0,0 +1,289 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/mnt/d/Github/ai-toolkit/output/", + "sqlite_db_path": "/mnt/d/Github/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 16, + "linear_alpha": 16, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 250, + "max_step_saves_to_keep": 4, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mae_output", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "Remove Object", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "do_i2v": true, + "flip_x": false, + "flip_y": false, + "control_path_1": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/with_object", + "control_path_2": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mask", + "control_path_3": "/mnt/d/Github/ai-toolkit/datasets/qwen_or/train/mae_output" + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 13582, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 1, + "loss_type": "mse" + }, + "model": { + "name_or_path": "Qwen/Qwen-Image-Edit-2509", + "quantize": true, + "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_edit_2509_torchao_uint3.safetensors", + "quantize_te": true, + "qtype_te": "uint4", + "arch": "qwen_image_edit_plus", + "low_vram": true, + "model_kwargs": { + "match_target_res": true + }, + "layer_offloading": true, + "layer_offloading_text_encoder_percent": 0, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/8184cef5-9303-484b-bc7c-fd3d2e865e24.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/74de8410-0630-41b7-bf80-747bdc03df50.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/4b3a523f-8bd2-4db2-b734-ebe0b629f17a.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/66a6d4bc-7c71-40f3-a8b7-55719a0d0596.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/3f7e34c1-54d0-4095-b4ed-ec7f8f3ed475.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/17521be8-a7fe-4240-9340-da1657874313.jpg" + }, + { + "prompt": "Remove Object", + "ctrl_img_1": "/mnt/d/Github/ai-toolkit/data/images/dba76474-f415-4e8b-a410-caa17c1ea158.jpg", + "ctrl_img_2": "/mnt/d/Github/ai-toolkit/data/images/47b8f626-147d-41da-a8c0-6c21382ae206.jpg", + "ctrl_img_3": "/mnt/d/Github/ai-toolkit/data/images/7dfc0f6d-6c7f-4f09-b855-248f6f39614a.jpg" + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 25, + "num_frames": 1, + "fps": 1 + } +} +Using SQLite database at /mnt/d/Github/ai-toolkit/aitk_db.db +Job ID: "ab192e4e-2fbc-4fe0-8067-c4de78a005b2" + +############################################# +# Running job: qwen2509_object_removal_512 +############################################# + + +Running 1 process +Loading Qwen Image model +Loading transformer + Loading checkpoint shards: 0%| | 0/5 [00:00