Upload folder using huggingface_hub

.job_config.json

@@ -0,0 +1,122 @@
+{
+  "job": "extension",
+  "config": {
+    "name": "anw0ll",
+    "process": [
+      {
+        "type": "diffusion_trainer",
+        "training_folder": "/workspace/ai-toolkit/output",
+        "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+        "device": "cuda",
+        "trigger_word": null,
+        "performance_log_every": 10,
+        "network": {
+          "type": "lora",
+          "linear": 16,
+          "linear_alpha": 16,
+          "conv": 16,
+          "conv_alpha": 16,
+          "lokr_full_rank": true,
+          "lokr_factor": -1,
+          "network_kwargs": {
+            "ignore_if_contains": []
+          }
+        },
+        "save": {
+          "dtype": "bf16",
+          "save_every": 250,
+          "max_step_saves_to_keep": 8,
+          "save_format": "diffusers",
+          "push_to_hub": false
+        },
+        "datasets": [
+          {
+            "folder_path": "/workspace/ai-toolkit/datasets/anw0ll",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+              512,
+              768,
+              1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "do_i2v": true,
+            "flip_x": false,
+            "flip_y": false
+          }
+        ],
+        "train": {
+          "batch_size": 1,
+          "bypass_guidance_embedding": false,
+          "steps": 3000,
+          "gradient_accumulation": 1,
+          "train_unet": true,
+          "train_text_encoder": false,
+          "gradient_checkpointing": true,
+          "noise_scheduler": "flowmatch",
+          "optimizer": "adamw8bit",
+          "timestep_type": "sigmoid",
+          "content_or_style": "balanced",
+          "optimizer_params": {
+            "weight_decay": 0.0001
+          },
+          "unload_text_encoder": false,
+          "cache_text_embeddings": true,
+          "lr": 0.0002,
+          "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+          },
+          "skip_first_sample": false,
+          "force_first_sample": false,
+          "disable_sampling": false,
+          "dtype": "bf16",
+          "diff_output_preservation": false,
+          "diff_output_preservation_multiplier": 1,
+          "diff_output_preservation_class": "person",
+          "switch_boundary_every": 1,
+          "loss_type": "mse"
+        },
+        "model": {
+          "name_or_path": "Qwen/Qwen-Image",
+          "quantize": true,
+          "qtype": "uint6",
+          "quantize_te": true,
+          "qtype_te": "qfloat8",
+          "arch": "qwen_image",
+          "low_vram": true,
+          "model_kwargs": {},
+          "layer_offloading": false,
+          "layer_offloading_text_encoder_percent": 1,
+          "layer_offloading_transformer_percent": 1
+        },
+        "sample": {
+          "sampler": "flowmatch",
+          "sample_every": 250,
+          "width": 1024,
+          "height": 1024,
+          "samples": [],
+          "neg": "",
+          "seed": 42,
+          "walk_seed": true,
+          "guidance_scale": 4,
+          "sample_steps": 25,
+          "num_frames": 1,
+          "fps": 1
+        }
+      }
+    ]
+  },
+  "meta": {
+    "name": "[name]",
+    "version": "1.0"
+  }
+}

anw0ll.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b602bb6dd95f679946b1e0c0f978e5f45c3ca78b438e531dc590ea69499b221
+size 295146160

anw0ll_000002000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d22562e241b243e80ea6c9b88aeb08f6654ae9cfbe562715ce097da41e5d6f8
+size 295146160

anw0ll_000002250.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ec61190a0372eb3439200f522d9c649b47a33ac10c7d335b875504ff01b8a5d
+size 295146160

anw0ll_000002500.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03e0f62ec55e18082d7d578b83205d79508e69cdf3ab973cc72e0c03c1ca36c3
+size 295146160

anw0ll_000002750.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa9a0f94de14a28dafb3771f3d74ec4e5dae36b1899e6cddabd79ac33bfa2905
+size 295146160

config.yaml

@@ -0,0 +1,103 @@
+job: extension
+config:
+  name: anw0ll
+  process:
+  - type: diffusion_trainer
+    training_folder: /workspace/ai-toolkit/output
+    sqlite_db_path: /workspace/ai-toolkit/aitk_db.db
+    device: cuda
+    trigger_word: null
+    performance_log_every: 10
+    network:
+      type: lora
+      linear: 16
+      linear_alpha: 16
+      conv: 16
+      conv_alpha: 16
+      lokr_full_rank: true
+      lokr_factor: -1
+      network_kwargs:
+        ignore_if_contains: []
+    save:
+      dtype: bf16
+      save_every: 250
+      max_step_saves_to_keep: 8
+      save_format: diffusers
+      push_to_hub: false
+    datasets:
+    - folder_path: /workspace/ai-toolkit/datasets/anw0ll
+      mask_path: null
+      mask_min_value: 0.1
+      default_caption: ''
+      caption_ext: txt
+      caption_dropout_rate: 0.05
+      cache_latents_to_disk: false
+      is_reg: false
+      network_weight: 1
+      resolution:
+      - 512
+      - 768
+      - 1024
+      controls: []
+      shrink_video_to_frames: true
+      num_frames: 1
+      do_i2v: true
+      flip_x: false
+      flip_y: false
+    train:
+      batch_size: 1
+      bypass_guidance_embedding: false
+      steps: 3000
+      gradient_accumulation: 1
+      train_unet: true
+      train_text_encoder: false
+      gradient_checkpointing: true
+      noise_scheduler: flowmatch
+      optimizer: adamw8bit
+      timestep_type: sigmoid
+      content_or_style: balanced
+      optimizer_params:
+        weight_decay: 0.0001
+      unload_text_encoder: false
+      cache_text_embeddings: true
+      lr: 0.0002
+      ema_config:
+        use_ema: false
+        ema_decay: 0.99
+      skip_first_sample: false
+      force_first_sample: false
+      disable_sampling: false
+      dtype: bf16
+      diff_output_preservation: false
+      diff_output_preservation_multiplier: 1
+      diff_output_preservation_class: person
+      switch_boundary_every: 1
+      loss_type: mse
+    model:
+      name_or_path: Qwen/Qwen-Image
+      quantize: true
+      qtype: uint6
+      quantize_te: true
+      qtype_te: qfloat8
+      arch: qwen_image
+      low_vram: true
+      model_kwargs: {}
+      layer_offloading: false
+      layer_offloading_text_encoder_percent: 1
+      layer_offloading_transformer_percent: 1
+    sample:
+      sampler: flowmatch
+      sample_every: 250
+      width: 1024
+      height: 1024
+      samples: []
+      neg: ''
+      seed: 42
+      walk_seed: true
+      guidance_scale: 4
+      sample_steps: 25
+      num_frames: 1
+      fps: 1
+meta:
+  name: anw0ll
+  version: '1.0'

logs/0_log.txt

@@ -0,0 +1,206 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/workspace/ai-toolkit/output",
+    "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": null,
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 16,
+        "linear_alpha": 16,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 250,
+        "max_step_saves_to_keep": 8,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/workspace/ai-toolkit/datasets/anw0ll",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512,
+                768,
+                1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "do_i2v": true,
+            "flip_x": false,
+            "flip_y": false
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 3000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "sigmoid",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": true,
+        "lr": 0.0002,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse"
+    },
+    "model": {
+        "name_or_path": "Qwen/Qwen-Image",
+        "quantize": true,
+        "qtype": "uint6",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "qwen_image",
+        "low_vram": true,
+        "model_kwargs": {},
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 4,
+        "sample_steps": 25,
+        "num_frames": 1,
+        "fps": 1
+    }
+}
+Using SQLite database at /workspace/ai-toolkit/aitk_db.db
+Job ID: "c0204165-6ad4-405b-954b-58c03d00ba04"
+
+#############################################
+# Running job: anw0ll
+#############################################
+
+
+Running  1 process
+Loading Qwen Image model
+Loading transformer
+
+
+Quantizing Transformer
+ - quantizing 60 transformer blocks
+
  0%|          | 0/60 [00:00<?, ?it/s]
  0%|          | 0/60 [00:00<?, ?it/s]
  2%|1         | 1/60 [00:00<00:48,  1.22it/s]
  2%|1         | 1/60 [00:00<00:48,  1.22it/s]
  3%|3         | 2/60 [00:01<00:34,  1.70it/s]
  3%|3         | 2/60 [00:01<00:34,  1.70it/s]
  5%|5         | 3/60 [00:01<00:29,  1.96it/s]
  5%|5         | 3/60 [00:01<00:29,  1.96it/s]
  7%|6         | 4/60 [00:02<00:26,  2.15it/s]
  7%|6         | 4/60 [00:02<00:26,  2.15it/s]
  8%|8         | 5/60 [00:02<00:22,  2.47it/s]
  8%|8         | 5/60 [00:02<00:22,  2.47it/s]
 10%|#         | 6/60 [00:02<00:19,  2.73it/s]
 10%|#         | 6/60 [00:02<00:19,  2.73it/s]
 12%|#1        | 7/60 [00:02<00:18,  2.92it/s]
 12%|#1        | 7/60 [00:02<00:18,  2.92it/s]
 13%|#3        | 8/60 [00:03<00:17,  3.05it/s]
 13%|#3        | 8/60 [00:03<00:17,  3.05it/s]
 15%|#5        | 9/60 [00:03<00:16,  3.15it/s]
 15%|#5        | 9/60 [00:03<00:16,  3.15it/s]
 17%|#6        | 10/60 [00:03<00:15,  3.24it/s]
 17%|#6        | 10/60 [00:03<00:15,  3.24it/s]
 18%|#8        | 11/60 [00:04<00:14,  3.30it/s]
 18%|#8        | 11/60 [00:04<00:14,  3.30it/s]
 20%|##        | 12/60 [00:04<00:15,  3.00it/s]
 20%|##        | 12/60 [00:04<00:15,  3.00it/s]
 22%|##1       | 13/60 [00:04<00:16,  2.87it/s]
 22%|##1       | 13/60 [00:04<00:16,  2.87it/s]
 23%|##3       | 14/60 [00:05<00:16,  2.82it/s]
 23%|##3       | 14/60 [00:05<00:16,  2.82it/s]
 25%|##5       | 15/60 [00:06<00:21,  2.08it/s]
 25%|##5       | 15/60 [00:06<00:21,  2.08it/s]
 27%|##6       | 16/60 [00:06<00:19,  2.22it/s]
 27%|##6       | 16/60 [00:06<00:19,  2.22it/s]
 28%|##8       | 17/60 [00:06<00:18,  2.30it/s]
 28%|##8       | 17/60 [00:06<00:18,  2.30it/s]
 30%|###       | 18/60 [00:07<00:17,  2.34it/s]
 30%|###       | 18/60 [00:07<00:17,  2.34it/s]
 32%|###1      | 19/60 [00:07<00:17,  2.38it/s]
 32%|###1      | 19/60 [00:07<00:17,  2.38it/s]
 33%|###3      | 20/60 [00:08<00:16,  2.40it/s]
 33%|###3      | 20/60 [00:08<00:16,  2.40it/s]
 35%|###5      | 21/60 [00:08<00:16,  2.43it/s]
 35%|###5      | 21/60 [00:08<00:16,  2.43it/s]
 37%|###6      | 22/60 [00:09<00:20,  1.87it/s]
 37%|###6      | 22/60 [00:09<00:20,  1.87it/s]
 38%|###8      | 23/60 [00:09<00:18,  1.96it/s]
 38%|###8      | 23/60 [00:09<00:18,  1.96it/s]
 40%|####      | 24/60 [00:10<00:16,  2.12it/s]
 40%|####      | 24/60 [00:10<00:16,  2.12it/s]
 42%|####1     | 25/60 [00:10<00:15,  2.26it/s]
 42%|####1     | 25/60 [00:10<00:15,  2.26it/s]
 43%|####3     | 26/60 [00:10<00:14,  2.35it/s]
 43%|####3     | 26/60 [00:10<00:14,  2.35it/s]
 45%|####5     | 27/60 [00:11<00:13,  2.39it/s]
 45%|####5     | 27/60 [00:11<00:13,  2.39it/s]
 47%|####6     | 28/60 [00:11<00:13,  2.38it/s]
 47%|####6     | 28/60 [00:11<00:13,  2.38it/s]
 48%|####8     | 29/60 [00:12<00:13,  2.36it/s]
 48%|####8     | 29/60 [00:12<00:13,  2.36it/s]
 50%|#####     | 30/60 [00:12<00:15,  1.97it/s]
 50%|#####     | 30/60 [00:12<00:15,  1.97it/s]
 52%|#####1    | 31/60 [00:13<00:13,  2.22it/s]
 52%|#####1    | 31/60 [00:13<00:13,  2.22it/s]
 53%|#####3    | 32/60 [00:13<00:11,  2.41it/s]
 53%|#####3    | 32/60 [00:13<00:11,  2.41it/s]
 55%|#####5    | 33/60 [00:13<00:10,  2.58it/s]
 55%|#####5    | 33/60 [00:13<00:10,  2.58it/s]
 57%|#####6    | 34/60 [00:14<00:09,  2.69it/s]
 57%|#####6    | 34/60 [00:14<00:09,  2.69it/s]
 58%|#####8    | 35/60 [00:14<00:08,  2.84it/s]
 58%|#####8    | 35/60 [00:14<00:08,  2.84it/s]
 60%|######    | 36/60 [00:14<00:08,  2.73it/s]
 60%|######    | 36/60 [00:14<00:08,  2.73it/s]
 62%|######1   | 37/60 [00:15<00:11,  2.01it/s]
 62%|######1   | 37/60 [00:15<00:11,  2.01it/s]
 63%|######3   | 38/60 [00:16<00:10,  2.10it/s]
 63%|######3   | 38/60 [00:16<00:10,  2.10it/s]
 65%|######5   | 39/60 [00:16<00:09,  2.14it/s]
 65%|######5   | 39/60 [00:16<00:09,  2.14it/s]
 67%|######6   | 40/60 [00:16<00:09,  2.21it/s]
 67%|######6   | 40/60 [00:16<00:09,  2.21it/s]
 68%|######8   | 41/60 [00:17<00:08,  2.30it/s]
 68%|######8   | 41/60 [00:17<00:08,  2.30it/s]
 70%|#######   | 42/60 [00:17<00:07,  2.33it/s]
 70%|#######   | 42/60 [00:17<00:07,  2.33it/s]
 72%|#######1  | 43/60 [00:18<00:07,  2.38it/s]
 72%|#######1  | 43/60 [00:18<00:07,  2.38it/s]
 73%|#######3  | 44/60 [00:18<00:08,  1.85it/s]
 73%|#######3  | 44/60 [00:18<00:08,  1.85it/s]
 75%|#######5  | 45/60 [00:19<00:07,  2.00it/s]
 75%|#######5  | 45/60 [00:19<00:07,  2.00it/s]
 77%|#######6  | 46/60 [00:19<00:06,  2.08it/s]
 77%|#######6  | 46/60 [00:19<00:06,  2.08it/s]
 78%|#######8  | 47/60 [00:20<00:06,  2.15it/s]
 78%|#######8  | 47/60 [00:20<00:06,  2.15it/s]
 80%|########  | 48/60 [00:20<00:05,  2.24it/s]
 80%|########  | 48/60 [00:20<00:05,  2.24it/s]
 82%|########1 | 49/60 [00:21<00:04,  2.32it/s]
 82%|########1 | 49/60 [00:21<00:04,  2.32it/s]
 83%|########3 | 50/60 [00:21<00:04,  2.37it/s]
 83%|########3 | 50/60 [00:21<00:04,  2.37it/s]
 85%|########5 | 51/60 [00:22<00:04,  1.84it/s]
 85%|########5 | 51/60 [00:22<00:04,  1.84it/s]
 87%|########6 | 52/60 [00:22<00:04,  1.96it/s]
 87%|########6 | 52/60 [00:22<00:04,  1.96it/s]
 88%|########8 | 53/60 [00:23<00:03,  2.08it/s]
 88%|########8 | 53/60 [00:23<00:03,  2.08it/s]
 90%|######### | 54/60 [00:23<00:02,  2.31it/s]
 90%|######### | 54/60 [00:23<00:02,  2.31it/s]
 92%|#########1| 55/60 [00:23<00:01,  2.53it/s]
 92%|#########1| 55/60 [00:23<00:01,  2.53it/s]
 93%|#########3| 56/60 [00:24<00:01,  2.71it/s]
 93%|#########3| 56/60 [00:24<00:01,  2.71it/s]
 95%|#########5| 57/60 [00:24<00:01,  2.85it/s]
 95%|#########5| 57/60 [00:24<00:01,  2.85it/s]
 97%|#########6| 58/60 [00:24<00:00,  2.95it/s]
 97%|#########6| 58/60 [00:24<00:00,  2.95it/s]
 98%|#########8| 59/60 [00:25<00:00,  2.38it/s]
 98%|#########8| 59/60 [00:25<00:00,  2.38it/s]
+
+ - quantizing extras
+Moving transformer to CPU
+Text Encoder
+
+
+Error running job: CUDA out of memory. Tried to allocate 130.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 21.69 MiB is free. Process 21085 has 29.54 GiB memory in use. Including non-PyTorch memory, this process has 1.78 GiB memory in use. Of the allocated memory 1.20 GiB is allocated by PyTorch, and 17.99 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1564, in run
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1564, in run
+        self.sd.load_model()self.sd.load_model()
+
+  File "/workspace/ai-toolkit/extensions_built_in/diffusion_models/qwen_image/qwen_image.py", line 161, in load_model
+  File "/workspace/ai-toolkit/extensions_built_in/diffusion_models/qwen_image/qwen_image.py", line 161, in load_model
+        text_encoder.to(self.device_torch, dtype=dtype)text_encoder.to(self.device_torch, dtype=dtype)
+
+  File "/venv/main/lib/python3.12/site-packages/transformers/modeling_utils.py", line 3851, in to
+  File "/venv/main/lib/python3.12/site-packages/transformers/modeling_utils.py", line 3851, in to
+        return super().to(*args, **kwargs)return super().to(*args, **kwargs)
+
+                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1355, in to
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1355, in to
+        return self._apply(convert)return self._apply(convert)
+
+                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 915, in _apply
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 915, in _apply
+        module._apply(fn)module._apply(fn)
+
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 915, in _apply
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 915, in _apply
+        module._apply(fn)module._apply(fn)
+
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 915, in _apply
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 915, in _apply
+        module._apply(fn)module._apply(fn)
+
+  [Previous line repeated 3 more times]
+  [Previous line repeated 3 more times]
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 942, in _apply
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 942, in _apply
+        param_applied = fn(param)param_applied = fn(param)
+
+                                        ^^^^^^^^^^^^^^^^^^
+
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1341, in convert
+  File "/venv/main/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1341, in convert
+        return t.to(return t.to(
+
+                      ^^^^^^^^^^
+
+torchtorch..OutOfMemoryErrorOutOfMemoryError: : CUDA out of memory. Tried to allocate 130.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 21.69 MiB is free. Process 21085 has 29.54 GiB memory in use. Including non-PyTorch memory, this process has 1.78 GiB memory in use. Of the allocated memory 1.20 GiB is allocated by PyTorch, and 17.99 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)CUDA out of memory. Tried to allocate 130.00 MiB. GPU 0 has a total capacity of 31.36 GiB of which 21.69 MiB is free. Process 21085 has 29.54 GiB memory in use. Including non-PyTorch memory, this process has 1.78 GiB memory in use. Of the allocated memory 1.20 GiB is allocated by PyTorch, and 17.99 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7833cea23a1ef1384912a33c17d4ccd01a9d1d29b4677d30d5b98feae47c933e
+size 300280909

pid.txt

@@ -0,0 +1 @@
+4288