Upload logs/0_log.txt with huggingface_hub

logs/0_log.txt

@@ -0,0 +1,223 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/teamspace/studios/this_studio/ai-toolkit/output",
+    "sqlite_db_path": "/teamspace/studios/this_studio/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": "xdipszit1",
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 32,
+        "linear_alpha": 32,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 250,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/teamspace/studios/this_studio/ai-toolkit/datasets/anty",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 4000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "weighted",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse"
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "Tongyi-MAI/Z-Image-Turbo",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "zimage:turbo",
+        "low_vram": true,
+        "model_kwargs": {},
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1,
+        "assistant_lora_path": "ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v2.safetensors"
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [
+            {
+                "prompt": "\"A stunning Indian woman with lustrous black hair cascading over her shoulders kneels gracefully on a sleek gray tiled bathroom floor, her full nude form captured in perfect, sensual detail. Her body is sculpted with flawless, taut curves \u2014 her breasts are perfectly round and full, with deep brownish areolas and erect nipples drawing the eye, while her toned abdomen and hips flow seamlessly into a lush, curvaceous ass that glistens with natural sheen. Her legs are bent at the knees, one foot tucked beneath her, the other resting lightly on the floor, creating a poised, almost reverent posture as she gazes directly into the camera with an intense, knowing expression. Her skin is smooth and radiant, catching the soft, diffused light that highlights every contour, from the delicate arch of her collarbone to the subtle definition of her glutes.  The composition is intimate and uncluttered, with the background reduced to a minimalist gray tile wall and a pristine white toilet just behind her, grounding the scene in a modern, clean bathroom setting. The monochromatic backdrop ensures all attention remains on her flawless anatomy \u2014 the subtle curve of her spine, the gentle slope of her shoulders, the fullness of her thighs, and the delicate arch of her back as she leans slightly forward. Her long, dark hair frames her face like a natural halo, contrasting beautifully with her warm, sun-kissed skin tone, while her eyes hold a quiet, magnetic allure that invites the viewer into her world. The lighting is soft yet precise, casting gentle shadows that accentuate her curves without overpowering her natural beauty.  This portrait is a celebration of raw, unfiltered sensuality \u2014 her pose is both vulnerable and commanding, her nudity not merely exposed but intentional, a testament to her confidence and grace. The simplicity of the environment amplifies the power of her form, making her appear both ethereal and grounded, like a goddess emerging from the mundane. Every detail \u2014 from the slight tension in her shoulders to the relaxed curve of her hips \u2014 speaks of a woman who owns her body, her presence, and her moment. The image is a masterful blend of eroticism and elegance, capturing not just her physical beauty but the quiet strength and allure that radiates from within.\""
+            },
+            {
+                "prompt": "\"A stunning Indian woman with lustrous black hair tied in a high, messy bun, her body glistening with natural oils and sweat, is captured in a moment of intense, sensual intimacy. Her full, perfectly rounded breasts with deep brownish-tinted areolas and erect nipples press against her back as she arches her spine, her hips thrusting forward in a powerful, rhythmic motion. Her thighs are spread wide, revealing the glistening, wet folds of her vagina and pussy as she rides the thick, dark brown shaft of her partner\u2019s erect penis, her fingers gripping his shoulders for balance. Her face is turned over her shoulder, eyes locked with the viewer, lips parted in a breathy moan, her expression a mix of ecstasy and dominance.  The scene is set against a minimalist, warm-toned beige background that accentuates her flawless, sun-kissed skin and the rich, earthy tones of her dark hair and the deep brown of her partner\u2019s skin. The environment is simple yet intimate \u2014 a wooden floor, a few potted plants with trailing vines, and a large blue exercise ball in the background \u2014 all serving to frame the raw, unfiltered passion of the moment. Her body is sculpted with perfect, athletic curves, her ass cheeks taut and rounded, her legs strong and toned, and her entire form radiates a powerful, sensual energy that is both commanding and inviting.  The lighting is soft and natural, casting gentle shadows that highlight the contours of her body and the texture of her skin, while the camera angle, positioned low and close, captures the full, unfiltered drama of the encounter. Her partner\u2019s muscular legs and arms are visible beneath her, his body a solid, grounding presence as she moves with fluid, controlled power. The image is a celebration of raw, unapologetic desire \u2014 a portrait of a woman in complete command of her body and her pleasure, her every movement a testament to her strength, beauty, and erotic power.\""
+            },
+            {
+                "prompt": "A first-person view of a Indian 21y woman sucking a man's penis. She's kneeling in front of him, drooling over his cock and take the penis in her mouth. Dramatic lighting and great detail. Her face has innocent expressions. She is looking at the man's body while she is sucking the man's penis."
+            },
+            {
+                "prompt": "\"A stunning Indian woman with lustrous black hair styled in a thick, cascading braid down her back is captured from behind, seated provocatively on a dark gray couch. Her full, rounded buttocks are the focal point, glistening with natural skin tones and subtle freckles, as she wears only a tight pink tank top and matching pink socks. Her posture is relaxed yet inviting, with her arms resting casually on the couch cushions, emphasizing the curve of her spine and the smooth expanse of her back. The setting is minimalist, with a plain, soft beige background that draws all attention to her flawless, curvaceous form.  In an intimate, sensual moment, a pair of hands\u2014belonging to an unseen partner\u2014gracefully caresses her lower back and gently presses against her cheeks, while one finger delicately traces the crevice between her cheeks, teasingly exploring the hidden depths of her anatomy. Her nude form is rendered with hyper-realistic detail: her perfectly round, deep brownish-tinted breasts are subtly hinted at beneath the snug fabric of her top, her nipples are visible and erect, and her vaginal opening is exposed and glistening with natural moisture, framed by the lush folds of her pussy. Her ass is impossibly plump and firm, with every contour and dimple rendered in exquisite detail, inviting the viewer\u2019s gaze to linger on the sheer perfection of her curves.  The scene is bathed in soft, natural light that enhances the smoothness of her skin and the rich, dark hue of her hair, creating a warm, sensual atmosphere. The portrait-style composition focuses entirely on her body, capturing the raw, unfiltered beauty of her form in a moment of quiet, erotic intimacy. The simple, uncluttered background and the tight framing amplify the emotional and physical connection between the woman and her partner, making this image a celebration of sensuality, confidence, and the unapologetic allure of the female form.\""
+            },
+            {
+                "prompt": "\"A full-body portrait of a stunning Indian woman with long, flowing black hair cascading down her back, standing nude and poised in a serene, softly lit bedroom. Her body is sculpted with perfect, natural curves \u2014 full, round breasts with dark brownish areolas and nipples, a smooth midriff leading to a gently curved waist, and a deeply toned, naturally darkened pubic area that accentuates her feminine form. Her legs are long and slender, ending in a firm, rounded ass that subtly tenses as she stands with her arms relaxed at her sides, fingers slightly curled, exuding quiet confidence and sensual grace. Her gaze is direct and unflinching, meeting the viewer\u2019s eyes with a calm, alluring intensity that draws you in.  The environment is minimalist and intimate, set against a soft, neutral beige background that enhances the natural tones of her skin and hair. Behind her, sheer white curtains filter in diffused daylight, casting gentle shadows across the room, while a simple bed with rumpled white linens and a potted green plant in the corner add subtle domestic warmth without distracting from the central focus. The lighting is soft and even, highlighting the smooth texture of her skin, the subtle contours of her anatomy, and the natural sheen of her hair \u2014 all rendered with photorealistic precision to emphasize her flawless, voluptuous form.  This is a portrait of raw, unfiltered beauty \u2014 a celebration of the female body in its most natural state. Her posture is relaxed yet commanding, her expression serene and self-assured, embodying both innocence and sensuality. The composition is vertically framed, drawing attention to her full, symmetrical silhouette, while the muted, monochromatic backdrop ensures every curve, every detail of her anatomy \u2014 from the delicate arch of her collarbone to the subtle definition of her hips \u2014 is rendered with exquisite clarity. She stands as a living sculpture, a vision of erotic elegance, inviting the viewer to appreciate the harmony of her form and the quiet power of her presence.\""
+            }
+        ],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 1,
+        "sample_steps": 8,
+        "num_frames": 1,
+        "fps": 1
+    }
+}
+Using SQLite database at /teamspace/studios/this_studio/ai-toolkit/aitk_db.db
+Job ID: "769b5a2a-3423-4c87-979a-b1981216279e"
+
+#############################################
+# Running job: my_first_lora_v4
+#############################################
+
+
+Running  1 process
+Loading ZImage model
+Loading transformer
+
+
+Loading assistant LoRA
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 276 modules.
+enable LoRA for U-Net
+Merging in assistant LoRA
+Missing keys: []
+Quantizing Transformer
+ - quantizing 30 transformer blocks
+
  0%|          | 0/30 [00:00<?, ?it/s]
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|3         | 1/30 [00:00<00:04,  6.70it/s]
  3%|3         | 1/30 [00:00<00:04,  6.70it/s]
  7%|6         | 2/30 [00:00<00:03,  7.86it/s]
  7%|6         | 2/30 [00:00<00:03,  7.86it/s]
 13%|#3        | 4/30 [00:00<00:02,  9.33it/s]
 13%|#3        | 4/30 [00:00<00:02,  9.33it/s]
 17%|#6        | 5/30 [00:00<00:02,  9.23it/s]
 17%|#6        | 5/30 [00:00<00:02,  9.23it/s]
 20%|##        | 6/30 [00:00<00:02,  9.16it/s]
 20%|##        | 6/30 [00:00<00:02,  9.16it/s]
 23%|##3       | 7/30 [00:00<00:02,  9.11it/s]
 23%|##3       | 7/30 [00:00<00:02,  9.11it/s]
 27%|##6       | 8/30 [00:00<00:02,  9.08it/s]
 27%|##6       | 8/30 [00:00<00:02,  9.08it/s]
 33%|###3      | 10/30 [00:01<00:01, 10.06it/s]
 33%|###3      | 10/30 [00:01<00:01, 10.06it/s]
 37%|###6      | 11/30 [00:01<00:01,  9.77it/s]
 37%|###6      | 11/30 [00:01<00:01,  9.77it/s]
 40%|####      | 12/30 [00:01<00:01,  9.50it/s]
 40%|####      | 12/30 [00:01<00:01,  9.50it/s]
 43%|####3     | 13/30 [00:01<00:01,  9.35it/s]
 43%|####3     | 13/30 [00:01<00:01,  9.35it/s]
 47%|####6     | 14/30 [00:01<00:01,  9.22it/s]
 47%|####6     | 14/30 [00:01<00:01,  9.22it/s]
 50%|#####     | 15/30 [00:01<00:01,  9.14it/s]
 50%|#####     | 15/30 [00:01<00:01,  9.14it/s]
 53%|#####3    | 16/30 [00:01<00:01,  8.84it/s]
 53%|#####3    | 16/30 [00:01<00:01,  8.84it/s]
 60%|######    | 18/30 [00:01<00:01,  9.02it/s]
 60%|######    | 18/30 [00:01<00:01,  9.02it/s]
 67%|######6   | 20/30 [00:02<00:01,  9.76it/s]
 67%|######6   | 20/30 [00:02<00:01,  9.76it/s]
 73%|#######3  | 22/30 [00:02<00:00,  9.94it/s]
 73%|#######3  | 22/30 [00:02<00:00,  9.94it/s]
 80%|########  | 24/30 [00:02<00:00, 10.07it/s]
 80%|########  | 24/30 [00:02<00:00, 10.07it/s]
 83%|########3 | 25/30 [00:02<00:00,  9.82it/s]
 83%|########3 | 25/30 [00:02<00:00,  9.82it/s]
 90%|######### | 27/30 [00:02<00:00,  9.77it/s]
 90%|######### | 27/30 [00:02<00:00,  9.77it/s]
 93%|#########3| 28/30 [00:02<00:00,  9.60it/s]
 93%|#########3| 28/30 [00:02<00:00,  9.60it/s]
 97%|#########6| 29/30 [00:03<00:00,  9.69it/s]
 97%|#########6| 29/30 [00:03<00:00,  9.69it/s]
+
+ - quantizing extras
+Moving transformer to CPU
+Text Encoder
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing Text Encoder
+Loading VAE
+Making pipe
+Preparing Model
+Model Loaded
+create LoRA network. base dim (rank): 32, alpha: 32
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 240 modules.
+enable LoRA for U-Net
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/anty
+  -  Preprocessing image dimensions
+
  0%|          | 0/26 [00:00<?, ?it/s]
  0%|          | 0/26 [00:00<?, ?it/s]
  8%|7         | 2/26 [00:00<00:01, 16.49it/s]
  8%|7         | 2/26 [00:00<00:01, 16.49it/s]
 15%|#5        | 4/26 [00:00<00:01, 16.66it/s]
 15%|#5        | 4/26 [00:00<00:01, 16.66it/s]
 23%|##3       | 6/26 [00:00<00:01, 17.61it/s]
 23%|##3       | 6/26 [00:00<00:01, 17.61it/s]
 31%|###       | 8/26 [00:00<00:01, 17.60it/s]
 31%|###       | 8/26 [00:00<00:01, 17.60it/s]
 38%|###8      | 10/26 [00:00<00:00, 17.63it/s]
 38%|###8      | 10/26 [00:00<00:00, 17.63it/s]
 46%|####6     | 12/26 [00:00<00:00, 17.58it/s]
 46%|####6     | 12/26 [00:00<00:00, 17.58it/s]
 54%|#####3    | 14/26 [00:00<00:00, 17.67it/s]
 54%|#####3    | 14/26 [00:00<00:00, 17.67it/s]
 62%|######1   | 16/26 [00:00<00:00, 17.32it/s]
 62%|######1   | 16/26 [00:00<00:00, 17.32it/s]
 69%|######9   | 18/26 [00:01<00:00, 17.56it/s]
 69%|######9   | 18/26 [00:01<00:00, 17.56it/s]
 77%|#######6  | 20/26 [00:01<00:00, 17.96it/s]
 77%|#######6  | 20/26 [00:01<00:00, 17.96it/s]
 85%|########4 | 22/26 [00:01<00:00, 18.09it/s]
 85%|########4 | 22/26 [00:01<00:00, 18.09it/s]
 92%|#########2| 24/26 [00:01<00:00, 17.95it/s]
 92%|#########2| 24/26 [00:01<00:00, 17.95it/s]
+
+  -  Found 26 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/anty:
+768x1344: 26 files
+1 buckets made
+Generating baseline samples before training
+Unloading assistant lora
+
                                                                
                                                                
+Error running job: Job stopped
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2039, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2039, in run
+        self.sample(self.step_num)self.sample(self.step_num)
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 307, in sample
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 307, in sample
+        self.maybe_stop()self.maybe_stop()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+