Upload logs/0_log.txt with huggingface_hub

logs/0_log.txt

@@ -0,0 +1,252 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/teamspace/studios/this_studio/ai-toolkit/output",
+    "sqlite_db_path": "/teamspace/studios/this_studio/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": "xdipszit1",
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 32,
+        "linear_alpha": 32,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 250,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/teamspace/studios/this_studio/ai-toolkit/datasets/dips",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512,
+                1024,
+                768
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 4000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "weighted",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse"
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "Tongyi-MAI/Z-Image-Turbo",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "zimage:turbo",
+        "low_vram": true,
+        "model_kwargs": {},
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1,
+        "assistant_lora_path": "ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v2.safetensors"
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [
+            {
+                "prompt": "\"A stunning Indian woman with lustrous black hair cascading over her shoulders kneels gracefully on a sleek gray tiled bathroom floor, her full nude form captured in perfect, sensual detail. Her body is sculpted with flawless, taut curves \u2014 her breasts are perfectly round and full, with deep brownish areolas and erect nipples drawing the eye, while her toned abdomen and hips flow seamlessly into a lush, curvaceous ass that glistens with natural sheen. Her legs are bent at the knees, one foot tucked beneath her, the other resting lightly on the floor, creating a poised, almost reverent posture as she gazes directly into the camera with an intense, knowing expression. Her skin is smooth and radiant, catching the soft, diffused light that highlights every contour, from the delicate arch of her collarbone to the subtle definition of her glutes.  The composition is intimate and uncluttered, with the background reduced to a minimalist gray tile wall and a pristine white toilet just behind her, grounding the scene in a modern, clean bathroom setting. The monochromatic backdrop ensures all attention remains on her flawless anatomy \u2014 the subtle curve of her spine, the gentle slope of her shoulders, the fullness of her thighs, and the delicate arch of her back as she leans slightly forward. Her long, dark hair frames her face like a natural halo, contrasting beautifully with her warm, sun-kissed skin tone, while her eyes hold a quiet, magnetic allure that invites the viewer into her world. The lighting is soft yet precise, casting gentle shadows that accentuate her curves without overpowering her natural beauty.  This portrait is a celebration of raw, unfiltered sensuality \u2014 her pose is both vulnerable and commanding, her nudity not merely exposed but intentional, a testament to her confidence and grace. The simplicity of the environment amplifies the power of her form, making her appear both ethereal and grounded, like a goddess emerging from the mundane. Every detail \u2014 from the slight tension in her shoulders to the relaxed curve of her hips \u2014 speaks of a woman who owns her body, her presence, and her moment. The image is a masterful blend of eroticism and elegance, capturing not just her physical beauty but the quiet strength and allure that radiates from within.\""
+            },
+            {
+                "prompt": "\"A stunning Indian woman with lustrous black hair cascading in soft waves down her back, kneels gracefully in a full-body nude pose, her voluptuous curves accentuated by the elegant curve of her spine and the gentle arch of her hips. Her perfectly round, deep brownish-tinted breasts swell with natural fullness, their firm, dark nipples standing out against her smooth, sun-kissed skin. Her gaze is locked with intense, sultry eyes as she leans forward, her lips parted to take in the thick, throbbing cock of her partner, her tongue flicking out to trace the tip with a teasing, erotic motion. Her body is poised in a moment of pure, uninhibited desire, her hands resting lightly on her thighs as she submits to the pleasure of the moment.  Her anatomy is rendered with exquisite detail \u2014 the soft, glistening curve of her pussy, the delicate folds of her inner thighs, and the rounded, firm swell of her ass, all highlighted by the warm, golden lighting that bathes her skin. She wears only a pair of black lace panties that cling to her hips, framing the lushness of her lower body and drawing attention to the natural beauty of her form. Her neck is adorned with a delicate gold necklace, its pendant catching the light as she tilts her head back, her expression a mix of ecstasy and surrender. The scene is intimate and sensual, capturing the raw, unfiltered passion of the moment.  The background is a simple, elegant setting \u2014 a single, rich red rose bouquet in a vase stands behind her, its vibrant color contrasting with the deep, luxurious black of the leather couch and the ornate gold-patterned wall behind. A chilled bottle of champagne and two flutes sit on a nearby table, hinting at a decadent, romantic evening. The entire scene is framed in a portrait-style composition, with the focus entirely on the woman\u2019s body and her erotic engagement with her partner, creating a powerful, visually arresting image that celebrates the beauty and sensuality of the human form.\""
+            },
+            {
+                "prompt": "\"A stunning Indian woman with lustrous black hair cascading down her back stands in a full-body nude pose, her voluptuous figure commanding attention. Her perfectly round, deep brownish tits are prominently displayed, with erect, dark nipples drawing the eye. Her wide, expressive eyes and open mouth suggest a moment of ecstatic surprise or pleasure, as she holds a pint of Guinness aloft in one hand. Her body is sculpted with flawless, taut skin, and her full, curvaceous hips and thighs are spread wide in a provocative stance, revealing her glistening, wet pussy and the thick, erect cock penetrating her from below. Her ass is perfectly rounded and firm, accentuating her powerful, athletic build.  The scene is set against a simple, monochromatic background that isolates her form, drawing all focus to her erotic anatomy. The lighting is soft yet dramatic, highlighting the natural contours of her body and the sheen of sweat and arousal on her skin. Her pose is both powerful and vulnerable \u2014 she stands tall, yet her legs are spread wide in a classic, sensual stance that invites the viewer into the moment. Her expression is one of uninhibited passion, as if she\u2019s caught mid-pleasure, her mouth slightly parted in a gasp or moan.  This portrait captures the raw, unfiltered essence of eroticism \u2014 the perfect blend of strength and sensuality. Her Indian heritage is subtly hinted at through her features, while her black hair adds a striking contrast to her fair skin. The Guinness pint in her hand adds a touch of casual, everyday life to the scene, grounding the moment in a relatable context. The image is a celebration of female beauty, power, and desire \u2014 a full-body, nude depiction that leaves no detail untouched, from the delicate curve of her collarbone to the powerful swell of her hips and the glistening wetness of her pussy.\""
+            },
+            {
+                "prompt": "\"A stunning Indian woman with lustrous black hair styled in a thick, cascading braid down her back is captured from behind, seated provocatively on a dark gray couch. Her full, rounded buttocks are the focal point, glistening with natural skin tones and subtle freckles, as she wears only a tight pink tank top and matching pink socks. Her posture is relaxed yet inviting, with her arms resting casually on the couch cushions, emphasizing the curve of her spine and the smooth expanse of her back. The setting is minimalist, with a plain, soft beige background that draws all attention to her flawless, curvaceous form.  In an intimate, sensual moment, a pair of hands\u2014belonging to an unseen partner\u2014gracefully caresses her lower back and gently presses against her cheeks, while one finger delicately traces the crevice between her cheeks, teasingly exploring the hidden depths of her anatomy. Her nude form is rendered with hyper-realistic detail: her perfectly round, deep brownish-tinted breasts are subtly hinted at beneath the snug fabric of her top, her nipples are visible and erect, and her vaginal opening is exposed and glistening with natural moisture, framed by the lush folds of her pussy. Her ass is impossibly plump and firm, with every contour and dimple rendered in exquisite detail, inviting the viewer\u2019s gaze to linger on the sheer perfection of her curves.  The scene is bathed in soft, natural light that enhances the smoothness of her skin and the rich, dark hue of her hair, creating a warm, sensual atmosphere. The portrait-style composition focuses entirely on her body, capturing the raw, unfiltered beauty of her form in a moment of quiet, erotic intimacy. The simple, uncluttered background and the tight framing amplify the emotional and physical connection between the woman and her partner, making this image a celebration of sensuality, confidence, and the unapologetic allure of the female form.\""
+            },
+            {
+                "prompt": "A close-up portrait of a young Indian woman smiling gently at the camera. She has long, straight black hair and warm brown skin, with well-defined eyebrows and expressive eyes enhanced by soft pink and smoky eye makeup. She has subtle eyeliner, mascara, and bold red lipstick. A small nose ring adorns her left nostril, and she wears a delicate silver necklace with a pendant. She is nude and visible deep cleavage with perfect round big breasts with brownish nipples and full, round breasts with dark brownish areolas and nipples, a smooth midriff leading to a gently curved waist, and a deeply toned, naturally darkened pubic area that accentuates her feminine form. The background is an indoor setting with soft, warm lighting. The composition is natural and candid, with a focus on her face, smooth skin texture, and realistic details. No bindi on the forehead."
+            }
+        ],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 1,
+        "sample_steps": 8,
+        "num_frames": 1,
+        "fps": 1
+    }
+}
+Using SQLite database at /teamspace/studios/this_studio/ai-toolkit/aitk_db.db
+Job ID: "959ea2b9-d727-4106-9608-239c9754f9fa"
+
+#############################################
+# Running job: my_first_lora_v3
+#############################################
+
+
+Running  1 process
+Loading ZImage model
+Loading transformer
+
+
+Loading assistant LoRA
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 276 modules.
+enable LoRA for U-Net
+Merging in assistant LoRA
+Missing keys: []
+Quantizing Transformer
+ - quantizing 30 transformer blocks
+
  0%|          | 0/30 [00:00<?, ?it/s]
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|3         | 1/30 [00:00<00:04,  6.58it/s]
  3%|3         | 1/30 [00:00<00:04,  6.58it/s]
  7%|6         | 2/30 [00:00<00:03,  7.66it/s]
  7%|6         | 2/30 [00:00<00:03,  7.66it/s]
 10%|#         | 3/30 [00:00<00:03,  7.95it/s]
 10%|#         | 3/30 [00:00<00:03,  7.95it/s]
 17%|#6        | 5/30 [00:00<00:02,  9.04it/s]
 17%|#6        | 5/30 [00:00<00:02,  9.04it/s]
 20%|##        | 6/30 [00:00<00:02,  8.88it/s]
 20%|##        | 6/30 [00:00<00:02,  8.88it/s]
 27%|##6       | 8/30 [00:00<00:02,  9.05it/s]
 27%|##6       | 8/30 [00:00<00:02,  9.05it/s]
 33%|###3      | 10/30 [00:01<00:02,  9.16it/s]
 33%|###3      | 10/30 [00:01<00:02,  9.16it/s]
 37%|###6      | 11/30 [00:01<00:02,  9.29it/s]
 37%|###6      | 11/30 [00:01<00:02,  9.29it/s]
 40%|####      | 12/30 [00:01<00:01,  9.11it/s]
 40%|####      | 12/30 [00:01<00:01,  9.11it/s]
 43%|####3     | 13/30 [00:01<00:01,  8.99it/s]
 43%|####3     | 13/30 [00:01<00:01,  8.99it/s]
 47%|####6     | 14/30 [00:01<00:01,  8.87it/s]
 47%|####6     | 14/30 [00:01<00:01,  8.87it/s]
 53%|#####3    | 16/30 [00:01<00:01,  9.02it/s]
 53%|#####3    | 16/30 [00:01<00:01,  9.02it/s]
 60%|######    | 18/30 [00:01<00:01,  9.47it/s]
 60%|######    | 18/30 [00:01<00:01,  9.47it/s]
 63%|######3   | 19/30 [00:02<00:01,  9.29it/s]
 63%|######3   | 19/30 [00:02<00:01,  9.29it/s]
 70%|#######   | 21/30 [00:02<00:00,  9.69it/s]
 70%|#######   | 21/30 [00:02<00:00,  9.69it/s]
 77%|#######6  | 23/30 [00:02<00:00,  9.93it/s]
 77%|#######6  | 23/30 [00:02<00:00,  9.93it/s]
 83%|########3 | 25/30 [00:02<00:00,  9.81it/s]
 83%|########3 | 25/30 [00:02<00:00,  9.81it/s]
 87%|########6 | 26/30 [00:02<00:00,  9.61it/s]
 87%|########6 | 26/30 [00:02<00:00,  9.61it/s]
 90%|######### | 27/30 [00:02<00:00,  9.43it/s]
 90%|######### | 27/30 [00:02<00:00,  9.43it/s]
 93%|#########3| 28/30 [00:03<00:00,  9.30it/s]
 93%|#########3| 28/30 [00:03<00:00,  9.30it/s]
 97%|#########6| 29/30 [00:03<00:00,  9.25it/s]
 97%|#########6| 29/30 [00:03<00:00,  9.25it/s]
+
+ - quantizing extras
+Moving transformer to CPU
+Text Encoder
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing Text Encoder
+Loading VAE
+Making pipe
+Preparing Model
+Model Loaded
+create LoRA network. base dim (rank): 32, alpha: 32
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 240 modules.
+enable LoRA for U-Net
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/dips
+  -  Preprocessing image dimensions
+
  0%|          | 0/17 [00:00<?, ?it/s]
  0%|          | 0/17 [00:00<?, ?it/s]
 18%|#7        | 3/17 [00:00<00:00, 20.42it/s]
 18%|#7        | 3/17 [00:00<00:00, 20.42it/s]
 41%|####1     | 7/17 [00:00<00:00, 22.54it/s]
 41%|####1     | 7/17 [00:00<00:00, 22.54it/s]
 59%|#####8    | 10/17 [00:00<00:00, 17.76it/s]
 59%|#####8    | 10/17 [00:00<00:00, 17.76it/s]
 71%|#######   | 12/17 [00:00<00:00, 17.42it/s]
 71%|#######   | 12/17 [00:00<00:00, 17.42it/s]
 82%|########2 | 14/17 [00:00<00:00, 16.58it/s]
 82%|########2 | 14/17 [00:00<00:00, 16.58it/s]
 94%|#########4| 16/17 [00:00<00:00, 16.41it/s]
 94%|#########4| 16/17 [00:00<00:00, 16.41it/s]
+
+  -  Found 17 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/dips:
+416x576: 5 files
+320x768: 4 files
+416x608: 3 files
+384x672: 5 files
+4 buckets made
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/dips
+  -  Preprocessing image dimensions
+
  0%|          | 0/17 [00:00<?, ?it/s]
  0%|          | 0/17 [00:00<?, ?it/s]
+
+  -  Found 17 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/dips:
+832x1152: 5 files
+640x1536: 4 files
+832x1216: 3 files
+768x1344: 5 files
+4 buckets made
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/dips
+  -  Preprocessing image dimensions
+
  0%|          | 0/17 [00:00<?, ?it/s]
  0%|          | 0/17 [00:00<?, ?it/s]
+
+  -  Found 17 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/dips:
+672x864: 5 files
+512x1088: 4 files
+608x896: 3 files
+576x992: 5 files
+4 buckets made
+Generating baseline samples before training
+Unloading assistant lora
+
                                                                
                                                                
+
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+