exp004b_anima_relay/relay_s0: ship-on-completion from Colab

exp004b_anima_relay/relay_s0/aleph_relay.canonical.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4899f8fb1f18783995a382377a355ef5059cd4b10f9172f139bfd5d8634dfad
+size 28592600

exp004b_anima_relay/relay_s0/aleph_relay.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d496472145654b85f33f4446913606d9b05e910e4f97805a104815fab5b140e4
+size 28654279

exp004b_anima_relay/relay_s0/aleph_relay.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dddbd42092aac30d6308b5d8b0ae2e2f152f5e80cdd59d85998d445e9d8ad0c
+size 28581800

exp004b_anima_relay/relay_s0/e004b_relay_s0.toml

@@ -0,0 +1,48 @@
+# e004b_relay_s0 — generated by anima_closeout.ipynb
+output_dir = '/content/anima_campaign/runs/e004b_relay_s0'
+dataset = '/content/anima_campaign/tomls/dataset_train.toml'
+seed = 0
+bf16_master_weights = true   # the exp004 gate-freeze fix
+epochs = 1000
+max_steps = 10000
+micro_batch_size_per_gpu = 8
+pipeline_stages = 1
+gradient_accumulation_steps = 1
+gradient_clipping = 0
+save_every_n_steps = 2500
+checkpoint_every_n_minutes = 30
+activation_checkpointing = false   # 96GB rider
+save_dtype = 'bfloat16'
+caching_batch_size = 8
+eval_datasets = [{name = 'heldout', config = '/content/anima_campaign/tomls/dataset_eval.toml'}]
+eval_every_n_steps = 1000
+eval_before_first_step = true
+eval_micro_batch_size_per_gpu = 8
+eval_gradient_accumulation_steps = 1
+
+[model]
+type = 'anima'
+transformer_path = '/content/anima_campaign/models/anima/split_files/diffusion_models/anima-base-v1.0.safetensors'
+llm_path = '/content/anima_campaign/models/anima/split_files/text_encoders/qwen_3_06b_base.safetensors'
+vae_path = '/content/anima_campaign/models/anima/split_files/vae/qwen_image_vae.safetensors'
+dtype = 'bfloat16'
+timestep_sample_method = 'logit_normal'
+aleph_relay = true
+aleph_relay_mode = 'relay'
+aleph_relay_rank = 16
+aleph_relay_every = 1
+aleph_relay_lr = 1e-3
+self_attn_lr = 0
+cross_attn_lr = 0
+mlp_lr = 0
+mod_lr = 0
+llm_adapter_lr = 0   # NON-NEGOTIABLE (anima-trainer law)
+
+[optimizer]
+type = 'adam'   # pure Adam — never adamw on aleph paths
+lr = 0
+betas = [0.9, 0.99]
+weight_decay = 0.0
+
+[monitoring]
+enable_wandb = false

exp004b_anima_relay/relay_s0/eval_curve.json

@@ -0,0 +1,462 @@
+{
+ "heldout/loss_quantile_0.10": [
+  [
+   0,
+   0.16606006026268005
+  ],
+  [
+   1000,
+   0.16268260776996613
+  ],
+  [
+   2000,
+   0.16177482903003693
+  ],
+  [
+   3000,
+   0.16111871600151062
+  ],
+  [
+   4000,
+   0.16071383655071259
+  ],
+  [
+   5000,
+   0.16035397350788116
+  ],
+  [
+   6000,
+   0.16004572808742523
+  ],
+  [
+   7000,
+   0.15984301269054413
+  ],
+  [
+   8000,
+   0.1596382111310959
+  ],
+  [
+   9000,
+   0.15941977500915527
+  ],
+  [
+   10000,
+   0.15913474559783936
+  ]
+ ],
+ "heldout/loss_quantile_0.20": [
+  [
+   0,
+   0.1285930871963501
+  ],
+  [
+   1000,
+   0.12510550022125244
+  ],
+  [
+   2000,
+   0.12423108518123627
+  ],
+  [
+   3000,
+   0.12358711659908295
+  ],
+  [
+   4000,
+   0.12315299361944199
+  ],
+  [
+   5000,
+   0.12276514619588852
+  ],
+  [
+   6000,
+   0.12242186814546585
+  ],
+  [
+   7000,
+   0.12220000475645065
+  ],
+  [
+   8000,
+   0.12197216600179672
+  ],
+  [
+   9000,
+   0.1217254102230072
+  ],
+  [
+   10000,
+   0.12145788222551346
+  ]
+ ],
+ "heldout/loss_quantile_0.30": [
+  [
+   0,
+   0.11038035154342651
+  ],
+  [
+   1000,
+   0.1067766398191452
+  ],
+  [
+   2000,
+   0.10591248422861099
+  ],
+  [
+   3000,
+   0.10524100810289383
+  ],
+  [
+   4000,
+   0.1047937199473381
+  ],
+  [
+   5000,
+   0.10441001504659653
+  ],
+  [
+   6000,
+   0.10404980927705765
+  ],
+  [
+   7000,
+   0.103817880153656
+  ],
+  [
+   8000,
+   0.10356983542442322
+  ],
+  [
+   9000,
+   0.10334514826536179
+  ],
+  [
+   10000,
+   0.10300949960947037
+  ]
+ ],
+ "heldout/loss_quantile_0.40": [
+  [
+   0,
+   0.09999153017997742
+  ],
+  [
+   1000,
+   0.09621915966272354
+  ],
+  [
+   2000,
+   0.09536629170179367
+  ],
+  [
+   3000,
+   0.09464913606643677
+  ],
+  [
+   4000,
+   0.09420210868120193
+  ],
+  [
+   5000,
+   0.09378992021083832
+  ],
+  [
+   6000,
+   0.09337989240884781
+  ],
+  [
+   7000,
+   0.09313254803419113
+  ],
+  [
+   8000,
+   0.09281160682439804
+  ],
+  [
+   9000,
+   0.09257414191961288
+  ],
+  [
+   10000,
+   0.09220588207244873
+  ]
+ ],
+ "heldout/loss_quantile_0.50": [
+  [
+   0,
+   0.09324966371059418
+  ],
+  [
+   1000,
+   0.08925876021385193
+  ],
+  [
+   2000,
+   0.08836200088262558
+  ],
+  [
+   3000,
+   0.08764582127332687
+  ],
+  [
+   4000,
+   0.08714890480041504
+  ],
+  [
+   5000,
+   0.0866379663348198
+  ],
+  [
+   6000,
+   0.08620858937501907
+  ],
+  [
+   7000,
+   0.08590159565210342
+  ],
+  [
+   8000,
+   0.0855848640203476
+  ],
+  [
+   9000,
+   0.08530429005622864
+  ],
+  [
+   10000,
+   0.08496716618537903
+  ]
+ ],
+ "heldout/loss_quantile_0.60": [
+  [
+   0,
+   0.08952882140874863
+  ],
+  [
+   1000,
+   0.0852133110165596
+  ],
+  [
+   2000,
+   0.08437198400497437
+  ],
+  [
+   3000,
+   0.08363045006990433
+  ],
+  [
+   4000,
+   0.08312531560659409
+  ],
+  [
+   5000,
+   0.08261635899543762
+  ],
+  [
+   6000,
+   0.08211397379636765
+  ],
+  [
+   7000,
+   0.08179517835378647
+  ],
+  [
+   8000,
+   0.08145128190517426
+  ],
+  [
+   9000,
+   0.08119434118270874
+  ],
+  [
+   10000,
+   0.08075518161058426
+  ]
+ ],
+ "heldout/loss_quantile_0.70": [
+  [
+   0,
+   0.08823437243700027
+  ],
+  [
+   1000,
+   0.08339894562959671
+  ],
+  [
+   2000,
+   0.08241903781890869
+  ],
+  [
+   3000,
+   0.08161813765764236
+  ],
+  [
+   4000,
+   0.08110307157039642
+  ],
+  [
+   5000,
+   0.0805315151810646
+  ],
+  [
+   6000,
+   0.08004327863454819
+  ],
+  [
+   7000,
+   0.07970591634511948
+  ],
+  [
+   8000,
+   0.07931864261627197
+  ],
+  [
+   9000,
+   0.0790276750922203
+  ],
+  [
+   10000,
+   0.07857183367013931
+  ]
+ ],
+ "heldout/loss_quantile_0.80": [
+  [
+   0,
+   0.08956252038478851
+  ],
+  [
+   1000,
+   0.08382070064544678
+  ],
+  [
+   2000,
+   0.08284620940685272
+  ],
+  [
+   3000,
+   0.0819382295012474
+  ],
+  [
+   4000,
+   0.08138864487409592
+  ],
+  [
+   5000,
+   0.08086816221475601
+  ],
+  [
+   6000,
+   0.08032213151454926
+  ],
+  [
+   7000,
+   0.0799277275800705
+  ],
+  [
+   8000,
+   0.07944628596305847
+  ],
+  [
+   9000,
+   0.07916602492332458
+  ],
+  [
+   10000,
+   0.07869710773229599
+  ]
+ ],
+ "heldout/loss_quantile_0.90": [
+  [
+   0,
+   0.0974397286772728
+  ],
+  [
+   1000,
+   0.09015677124261856
+  ],
+  [
+   2000,
+   0.08905632793903351
+  ],
+  [
+   3000,
+   0.08808011561632156
+  ],
+  [
+   4000,
+   0.08745108544826508
+  ],
+  [
+   5000,
+   0.0867486223578453
+  ],
+  [
+   6000,
+   0.0861261636018753
+  ],
+  [
+   7000,
+   0.08558878302574158
+  ],
+  [
+   8000,
+   0.08499374985694885
+  ],
+  [
+   9000,
+   0.08471350371837616
+  ],
+  [
+   10000,
+   0.0839509591460228
+  ]
+ ],
+ "heldout/loss": [
+  [
+   0,
+   0.10700446367263794
+  ],
+  [
+   1000,
+   0.10251470655202866
+  ],
+  [
+   2000,
+   0.10159336030483246
+  ],
+  [
+   3000,
+   0.10083430260419846
+  ],
+  [
+   4000,
+   0.10034218430519104
+  ],
+  [
+   5000,
+   0.09985796362161636
+  ],
+  [
+   6000,
+   0.09941238164901733
+  ],
+  [
+   7000,
+   0.09910140186548233
+  ],
+  [
+   8000,
+   0.09875407069921494
+  ],
+  [
+   9000,
+   0.0984966978430748
+  ],
+  [
+   10000,
+   0.09808336198329926
+  ]
+ ]
+}

exp004b_anima_relay/relay_s0/log_tail.txt

@@ -0,0 +1,400 @@
+ 31%|███       | 175/567 [00:32<01:12,  5.44it/s]
+ 31%|███       | 176/567 [00:32<01:11,  5.45it/s]
+ 31%|███       | 177/567 [00:32<01:11,  5.43it/s]
+ 31%|███▏      | 178/567 [00:32<01:11,  5.44it/s]
+ 32%|███▏      | 179/567 [00:33<01:11,  5.45it/s]
+ 32%|███▏      | 180/567 [00:33<01:11,  5.42it/s]
+ 32%|███▏      | 181/567 [00:33<01:11,  5.40it/s]
+ 32%|███▏      | 182/567 [00:33<01:11,  5.39it/s]
+ 32%|███▏      | 183/567 [00:33<01:10,  5.42it/s]
+ 32%|███▏      | 184/567 [00:34<01:10,  5.44it/s]
+ 33%|███▎      | 185/567 [00:34<01:10,  5.42it/s]
+ 33%|███▎      | 186/567 [00:34<01:10,  5.44it/s]
+ 33%|███▎      | 187/567 [00:34<01:09,  5.45it/s]
+ 33%|███▎      | 188/567 [00:34<01:09,  5.43it/s]
+ 33%|███▎      | 189/567 [00:34<01:08,  5.52it/s]
+ 34%|███▎      | 190/567 [00:35<01:09,  5.43it/s]
+ 34%|███▎      | 191/567 [00:35<01:09,  5.41it/s]
+ 34%|███▍      | 192/567 [00:35<01:09,  5.43it/s]
+ 34%|███▍      | 193/567 [00:35<01:08,  5.44it/s]
+ 34%|███▍      | 194/567 [00:35<01:08,  5.41it/s]
+ 34%|███▍      | 195/567 [00:36<01:08,  5.43it/s]
+ 35%|███▍      | 196/567 [00:36<01:08,  5.41it/s]
+ 35%|███▍      | 197/567 [00:36<01:08,  5.43it/s]
+ 35%|███▍      | 198/567 [00:36<01:07,  5.44it/s]
+ 35%|███▌      | 199/567 [00:36<01:07,  5.42it/s]
+ 35%|███▌      | 200/567 [00:36<01:07,  5.43it/s]
+ 35%|███▌      | 201/567 [00:37<01:07,  5.40it/s]
+ 36%|███▌      | 202/567 [00:37<01:07,  5.39it/s]
+ 36%|███▌      | 203/567 [00:37<01:07,  5.38it/s]
+ 36%|███▌      | 204/567 [00:37<01:07,  5.41it/s]
+ 36%|███▌      | 205/567 [00:37<01:06,  5.43it/s]
+ 36%|███▋      | 206/567 [00:38<01:06,  5.44it/s]
+ 37%|███▋      | 207/567 [00:38<01:06,  5.45it/s]
+ 37%|███▋      | 208/567 [00:38<01:05,  5.46it/s]
+ 37%|███▋      | 209/567 [00:38<01:05,  5.46it/s]
+ 37%|███▋      | 210/567 [00:38<01:05,  5.43it/s]
+ 37%|███▋      | 211/567 [00:39<01:05,  5.41it/s]
+ 37%|███▋      | 212/567 [00:39<01:05,  5.43it/s]
+ 38%|███▊      | 213/567 [00:39<01:05,  5.44it/s]
+ 38%|███▊      | 214/567 [00:39<01:04,  5.45it/s]
+ 38%|███▊      | 215/567 [00:39<01:04,  5.46it/s]
+ 38%|███▊      | 216/567 [00:39<01:04,  5.46it/s]
+ 38%|███▊      | 217/567 [00:40<01:04,  5.43it/s]
+ 38%|███▊      | 218/567 [00:40<01:04,  5.45it/s]
+ 39%|███▊      | 219/567 [00:40<01:03,  5.46it/s]
+ 39%|███▉      | 220/567 [00:40<01:03,  5.47it/s]
+ 39%|███▉      | 221/567 [00:40<01:03,  5.43it/s]
+ 39%|███▉      | 222/567 [00:41<01:03,  5.44it/s]
+ 39%|███▉      | 223/567 [00:41<01:03,  5.45it/s]
+ 40%|███▉      | 224/567 [00:41<01:02,  5.46it/s]
+ 40%|███▉      | 225/567 [00:41<01:02,  5.43it/s]
+ 40%|███▉      | 226/567 [00:41<01:02,  5.44it/s]
+ 40%|████      | 227/567 [00:41<01:02,  5.45it/s]
+ 40%|████      | 228/567 [00:42<01:02,  5.42it/s]
+ 40%|████      | 229/567 [00:42<01:02,  5.41it/s]
+ 41%|████      | 230/567 [00:42<01:02,  5.43it/s]
+ 41%|████      | 231/567 [00:42<01:02,  5.40it/s]
+ 41%|████      | 232/567 [00:42<01:01,  5.43it/s]
+ 41%|████      | 233/567 [00:43<01:01,  5.40it/s]
+ 41%|████▏     | 234/567 [00:43<01:01,  5.43it/s]
+ 41%|████▏     | 235/567 [00:43<01:01,  5.44it/s]
+ 42%|████▏     | 236/567 [00:43<01:00,  5.45it/s]
+ 42%|████▏     | 237/567 [00:43<01:00,  5.45it/s]
+ 42%|████▏     | 238/567 [00:43<01:00,  5.42it/s]
+ 42%|████▏     | 239/567 [00:44<01:00,  5.43it/s]
+ 42%|████▏     | 240/567 [00:44<01:00,  5.41it/s]
+ 43%|████▎     | 241/567 [00:44<01:00,  5.43it/s]
+ 43%|████▎     | 242/567 [00:44<00:59,  5.44it/s]
+ 43%|████▎     | 243/567 [00:44<00:59,  5.41it/s]
+ 43%|████▎     | 244/567 [00:45<00:59,  5.39it/s]
+ 43%|████▎     | 245/567 [00:45<00:59,  5.38it/s]
+ 43%|████▎     | 246/567 [00:45<00:59,  5.41it/s]
+ 44%|████▎     | 247/567 [00:45<00:58,  5.43it/s]
+ 44%|████▎     | 248/567 [00:45<00:59,  5.41it/s]
+ 44%|████▍     | 249/567 [00:45<00:58,  5.42it/s]
+ 44%|████▍     | 250/567 [00:46<00:58,  5.44it/s]
+ 44%|████▍     | 251/567 [00:46<00:58,  5.41it/s]
+ 44%|████▍     | 252/567 [00:46<00:57,  5.51it/s]
+ 45%|████▍     | 253/567 [00:46<00:57,  5.43it/s]
+ 45%|████▍     | 254/567 [00:46<00:57,  5.40it/s]
+ 45%|████▍     | 255/567 [00:47<00:57,  5.43it/s]
+ 45%|████▌     | 256/567 [00:47<00:57,  5.45it/s]
+ 45%|████▌     | 257/567 [00:47<00:57,  5.42it/s]
+ 46%|████▌     | 258/567 [00:47<00:56,  5.42it/s]
+ 46%|████▌     | 259/567 [00:47<00:57,  5.39it/s]
+ 46%|████▌     | 260/567 [00:48<00:56,  5.41it/s]
+ 46%|████▌     | 261/567 [00:48<00:56,  5.42it/s]
+ 46%|████▌     | 262/567 [00:48<00:56,  5.39it/s]
+ 46%|████▋     | 263/567 [00:48<00:56,  5.41it/s]
+ 47%|████▋     | 264/567 [00:48<00:56,  5.38it/s]
+ 47%|████▋     | 265/567 [00:48<00:56,  5.37it/s]
+ 47%|████▋     | 266/567 [00:49<00:56,  5.36it/s]
+ 47%|████▋     | 267/567 [00:49<00:55,  5.38it/s]
+ 47%|████▋     | 268/567 [00:49<00:55,  5.40it/s]
+ 47%|████▋     | 269/567 [00:49<00:55,  5.41it/s]
+ 48%|████▊     | 270/567 [00:49<00:54,  5.42it/s]
+ 48%|████▊     | 271/567 [00:50<00:54,  5.43it/s]
+ 48%|████▊     | 272/567 [00:50<00:54,  5.43it/s]
+ 48%|████▊     | 273/567 [00:50<00:54,  5.39it/s]
+ 48%|████▊     | 274/567 [00:50<00:54,  5.37it/s]
+ 49%|████▊     | 275/567 [00:50<00:54,  5.40it/s]
+ 49%|████▊     | 276/567 [00:50<00:53,  5.41it/s]
+ 49%|████▉     | 277/567 [00:51<00:53,  5.42it/s]
+ 49%|████▉     | 278/567 [00:51<00:53,  5.43it/s]
+ 49%|████▉     | 279/567 [00:51<00:53,  5.43it/s]
+ 49%|████▉     | 280/567 [00:51<00:53,  5.40it/s]
+ 50%|████▉     | 281/567 [00:51<00:52,  5.41it/s]
+ 50%|████▉     | 282/567 [00:52<00:52,  5.42it/s]
+ 50%|████▉     | 283/567 [00:52<00:52,  5.43it/s]
+ 50%|█████     | 284/567 [00:52<00:52,  5.40it/s]
+ 50%|█████     | 285/567 [00:52<00:52,  5.41it/s]
+ 50%|█████     | 286/567 [00:52<00:51,  5.42it/s]
+ 51%|█████     | 287/567 [00:53<00:51,  5.43it/s]
+ 51%|█████     | 288/567 [00:53<00:51,  5.40it/s]
+ 51%|█████     | 289/567 [00:53<00:51,  5.41it/s]
+ 51%|█████     | 290/567 [00:53<00:51,  5.42it/s]
+ 51%|█████▏    | 291/567 [00:53<00:51,  5.39it/s]
+ 51%|█████▏    | 292/567 [00:53<00:51,  5.37it/s]
+ 52%|█████▏    | 293/567 [00:54<00:50,  5.39it/s]
+ 52%|█████▏    | 294/567 [00:54<00:50,  5.37it/s]
+ 52%|█████▏    | 295/567 [00:54<00:50,  5.39it/s]
+ 52%|█████▏    | 296/567 [00:54<00:50,  5.37it/s]
+ 52%|█████▏    | 297/567 [00:54<00:50,  5.39it/s]
+ 53%|█████▎    | 298/567 [00:55<00:49,  5.41it/s]
+ 53%|█████▎    | 299/567 [00:55<00:49,  5.42it/s]
+ 53%|█████▎    | 300/567 [00:55<00:49,  5.43it/s]
+ 53%|█████▎    | 301/567 [00:55<00:49,  5.40it/s]
+ 53%|█████▎    | 302/567 [00:55<00:49,  5.41it/s]
+ 53%|█████▎    | 303/567 [00:55<00:49,  5.38it/s]
+ 54%|█████▎    | 304/567 [00:56<00:48,  5.40it/s]
+ 54%|█████▍    | 305/567 [00:56<00:48,  5.41it/s]
+ 54%|█████▍    | 306/567 [00:56<00:48,  5.39it/s]
+ 54%|█████▍    | 307/567 [00:56<00:48,  5.37it/s]
+ 54%|█████▍    | 308/567 [00:56<00:48,  5.36it/s]
+ 54%|█████▍    | 309/567 [00:57<00:47,  5.38it/s]
+ 55%|█████▍    | 310/567 [00:57<00:47,  5.40it/s]
+ 55%|█████▍    | 311/567 [00:57<00:47,  5.38it/s]
+ 55%|█████▌    | 312/567 [00:57<00:47,  5.40it/s]
+ 55%|█████▌    | 313/567 [00:57<00:46,  5.41it/s]
+ 55%|█████▌    | 314/567 [00:58<00:46,  5.39it/s]
+ 56%|█████▌    | 315/567 [00:58<00:45,  5.49it/s]
+ 56%|█████▌    | 316/567 [00:58<00:46,  5.39it/s]
+ 56%|█████▌    | 317/567 [00:58<00:46,  5.36it/s]
+ 56%|█████▌    | 318/567 [00:58<00:46,  5.39it/s]
+ 56%|█████▋    | 319/567 [00:58<00:45,  5.40it/s]
+ 56%|█████▋    | 320/567 [00:59<00:45,  5.38it/s]
+ 57%|█████▋    | 321/567 [00:59<00:45,  5.40it/s]
+ 57%|█████▋    | 322/567 [00:59<00:45,  5.38it/s]
+ 57%|█████▋    | 323/567 [00:59<00:45,  5.40it/s]
+ 57%|█████▋    | 324/567 [00:59<00:44,  5.41it/s]
+ 57%|█████▋    | 325/567 [01:00<00:44,  5.38it/s]
+ 57%|█████▋    | 326/567 [01:00<00:44,  5.40it/s]
+ 58%|█████▊    | 327/567 [01:00<00:44,  5.37it/s]
+ 58%|█████▊    | 328/567 [01:00<00:44,  5.36it/s]
+ 58%|█████▊    | 329/567 [01:00<00:44,  5.35it/s]
+ 58%|█████▊    | 330/567 [01:00<00:44,  5.38it/s]
+ 58%|█████▊    | 331/567 [01:01<00:43,  5.40it/s]
+ 59%|█████▊    | 332/567 [01:01<00:43,  5.41it/s]
+ 59%|█████▊    | 333/567 [01:01<00:43,  5.42it/s]
+ 59%|█████▉    | 334/567 [01:01<00:42,  5.43it/s]
+ 59%|█████▉    | 335/567 [01:01<00:42,  5.44it/s]
+ 59%|█████▉    | 336/567 [01:02<00:42,  5.40it/s]
+ 59%|█████▉    | 337/567 [01:02<00:42,  5.38it/s]
+ 60%|█████▉    | 338/567 [01:02<00:42,  5.40it/s]
+ 60%|█████▉    | 339/567 [01:02<00:42,  5.41it/s]
+ 60%|█████▉    | 340/567 [01:02<00:41,  5.42it/s]
+ 60%|██████    | 341/567 [01:03<00:41,  5.43it/s]
+ 60%|██████    | 342/567 [01:03<00:41,  5.43it/s]
+ 60%|██████    | 343/567 [01:03<00:41,  5.40it/s]
+ 61%|██████    | 344/567 [01:03<00:41,  5.41it/s]
+ 61%|██████    | 345/567 [01:03<00:40,  5.42it/s]
+ 61%|██████    | 346/567 [01:03<00:40,  5.43it/s]
+ 61%|██████    | 347/567 [01:04<00:40,  5.40it/s]
+ 61%|██████▏   | 348/567 [01:04<00:40,  5.41it/s]
+ 62%|██████▏   | 349/567 [01:04<00:40,  5.42it/s]
+ 62%|██████▏   | 350/567 [01:04<00:39,  5.43it/s]
+ 62%|██████▏   | 351/567 [01:04<00:39,  5.40it/s]
+ 62%|██████▏   | 352/567 [01:05<00:39,  5.41it/s]
+ 62%|██████▏   | 353/567 [01:05<00:39,  5.42it/s]
+ 62%|██████▏   | 354/567 [01:05<00:39,  5.39it/s]
+ 63%|██████▎   | 355/567 [01:05<00:39,  5.37it/s]
+ 63%|██████▎   | 356/567 [01:05<00:39,  5.39it/s]
+ 63%|██████▎   | 357/567 [01:05<00:39,  5.38it/s]
+ 63%|██████▎   | 358/567 [01:06<00:38,  5.40it/s]
+ 63%|██████▎   | 359/567 [01:06<00:38,  5.37it/s]
+ 63%|██████▎   | 360/567 [01:06<00:38,  5.39it/s]
+ 64%|██████▎   | 361/567 [01:06<00:38,  5.41it/s]
+ 64%|██████▍   | 362/567 [01:06<00:37,  5.42it/s]
+ 64%|██████▍   | 363/567 [01:07<00:37,  5.43it/s]
+ 64%|██████▍   | 364/567 [01:07<00:37,  5.40it/s]
+ 64%|██████▍   | 365/567 [01:07<00:37,  5.41it/s]
+ 65%|██████▍   | 366/567 [01:07<00:37,  5.38it/s]
+ 65%|██████▍   | 367/567 [01:07<00:37,  5.40it/s]
+ 65%|██████▍   | 368/567 [01:08<00:36,  5.41it/s]
+ 65%|██████▌   | 369/567 [01:08<00:36,  5.38it/s]
+ 65%|██████▌   | 370/567 [01:08<00:36,  5.36it/s]
+ 65%|██████▌   | 371/567 [01:08<00:36,  5.36it/s]
+ 66%|██████▌   | 372/567 [01:08<00:36,  5.38it/s]
+ 66%|██████▌   | 373/567 [01:08<00:35,  5.40it/s]
+ 66%|██████▌   | 374/567 [01:09<00:35,  5.38it/s]
+ 66%|██████▌   | 375/567 [01:09<00:35,  5.40it/s]
+ 66%|██████▋   | 376/567 [01:09<00:35,  5.42it/s]
+ 66%|██████▋   | 377/567 [01:09<00:35,  5.39it/s]
+ 67%|██████▋   | 378/567 [01:09<00:34,  5.49it/s]
+ 67%|██████▋   | 379/567 [01:10<00:34,  5.40it/s]
+ 67%|██████▋   | 380/567 [01:10<00:34,  5.38it/s]
+ 67%|██████▋   | 381/567 [01:10<00:34,  5.40it/s]
+ 67%|██████▋   | 382/567 [01:10<00:34,  5.41it/s]
+ 68%|██████▊   | 383/567 [01:10<00:34,  5.38it/s]
+ 68%|██████▊   | 384/567 [01:10<00:33,  5.40it/s]
+ 68%|██████▊   | 385/567 [01:11<00:33,  5.38it/s]
+ 68%|██████▊   | 386/567 [01:11<00:33,  5.40it/s]
+ 68%|██████▊   | 387/567 [01:11<00:33,  5.41it/s]
+ 68%|██████▊   | 388/567 [01:11<00:33,  5.39it/s]
+ 69%|██████▊   | 389/567 [01:11<00:32,  5.41it/s]
+ 69%|██████▉   | 390/567 [01:12<00:32,  5.38it/s]
+ 69%|██████▉   | 391/567 [01:12<00:32,  5.37it/s]
+ 69%|██████▉   | 392/567 [01:12<00:32,  5.36it/s]
+ 69%|██████▉   | 393/567 [01:12<00:32,  5.39it/s]
+ 69%|██████▉   | 394/567 [01:12<00:32,  5.40it/s]
+ 70%|██████▉   | 395/567 [01:13<00:31,  5.42it/s]
+ 70%|██████▉   | 396/567 [01:13<00:31,  5.43it/s]
+ 70%|███████   | 397/567 [01:13<00:31,  5.44it/s]
+ 70%|███████   | 398/567 [01:13<00:31,  5.44it/s]
+ 70%|███████   | 399/567 [01:13<00:31,  5.40it/s]
+ 71%|███████   | 400/567 [01:13<00:31,  5.38it/s]
+ 71%|███████   | 401/567 [01:14<00:30,  5.40it/s]
+ 71%|███████   | 402/567 [01:14<00:30,  5.42it/s]
+ 71%|███████   | 403/567 [01:14<00:30,  5.43it/s]
+ 71%|███████▏  | 404/567 [01:14<00:30,  5.43it/s]
+ 71%|███████▏  | 405/567 [01:14<00:29,  5.44it/s]
+ 72%|███████▏  | 406/567 [01:15<00:29,  5.41it/s]
+ 72%|███████▏  | 407/567 [01:15<00:29,  5.41it/s]
+ 72%|███████▏  | 408/567 [01:15<00:29,  5.43it/s]
+ 72%|███████▏  | 409/567 [01:15<00:29,  5.43it/s]
+ 72%|███████▏  | 410/567 [01:15<00:29,  5.40it/s]
+ 72%|███████▏  | 411/567 [01:15<00:28,  5.41it/s]
+ 73%|███████▎  | 412/567 [01:16<00:28,  5.43it/s]
+ 73%|███████▎  | 413/567 [01:16<00:28,  5.43it/s]
+ 73%|███████▎  | 414/567 [01:16<00:28,  5.40it/s]
+ 73%|███████▎  | 415/567 [01:16<00:28,  5.41it/s]
+ 73%|███████▎  | 416/567 [01:16<00:27,  5.42it/s]
+ 74%|███████▎  | 417/567 [01:17<00:27,  5.39it/s]
+ 74%|███████▎  | 418/567 [01:17<00:27,  5.37it/s]
+ 74%|███████▍  | 419/567 [01:17<00:27,  5.40it/s]
+ 74%|███████▍  | 420/567 [01:17<00:27,  5.38it/s]
+ 74%|████��██▍  | 421/567 [01:17<00:27,  5.39it/s]
+ 74%|███████▍  | 422/567 [01:18<00:26,  5.38it/s]
+ 75%|███████▍  | 423/567 [01:18<00:26,  5.40it/s]
+ 75%|███████▍  | 424/567 [01:18<00:26,  5.41it/s]
+ 75%|███████▍  | 425/567 [01:18<00:26,  5.42it/s]
+ 75%|███████▌  | 426/567 [01:18<00:25,  5.43it/s]
+ 75%|███████▌  | 427/567 [01:18<00:25,  5.39it/s]
+ 75%|███████▌  | 428/567 [01:19<00:25,  5.41it/s]
+ 76%|███████▌  | 429/567 [01:19<00:25,  5.39it/s]
+ 76%|███████▌  | 430/567 [01:19<00:25,  5.40it/s]
+ 76%|███████▌  | 431/567 [01:19<00:25,  5.41it/s]
+ 76%|███████▌  | 432/567 [01:19<00:25,  5.39it/s]
+ 76%|███████▋  | 433/567 [01:20<00:24,  5.36it/s]
+ 77%|███████▋  | 434/567 [01:20<00:24,  5.35it/s]
+ 77%|███████▋  | 435/567 [01:20<00:24,  5.38it/s]
+ 77%|███████▋  | 436/567 [01:20<00:24,  5.40it/s]
+ 77%|███████▋  | 437/567 [01:20<00:24,  5.38it/s]
+ 77%|███████▋  | 438/567 [01:20<00:23,  5.40it/s]
+ 77%|███████▋  | 439/567 [01:21<00:23,  5.41it/s]
+ 78%|███████▊  | 440/567 [01:21<00:23,  5.39it/s]
+ 78%|███████▊  | 441/567 [01:21<00:22,  5.49it/s]
+ 78%|███████▊  | 442/567 [01:21<00:23,  5.39it/s]
+ 78%|███████▊  | 443/567 [01:21<00:23,  5.37it/s]
+ 78%|███████▊  | 444/567 [01:22<00:22,  5.39it/s]
+ 78%|███████▊  | 445/567 [01:22<00:22,  5.41it/s]
+ 79%|███████▊  | 446/567 [01:22<00:22,  5.38it/s]
+ 79%|███████▉  | 447/567 [01:22<00:22,  5.40it/s]
+ 79%|███████▉  | 448/567 [01:22<00:22,  5.38it/s]
+ 79%|███████▉  | 449/567 [01:23<00:21,  5.40it/s]
+ 79%|███████▉  | 450/567 [01:23<00:21,  5.42it/s]
+ 80%|███████▉  | 451/567 [01:23<00:21,  5.39it/s]
+ 80%|███████▉  | 452/567 [01:23<00:21,  5.41it/s]
+ 80%|███████▉  | 453/567 [01:23<00:21,  5.38it/s]
+ 80%|████████  | 454/567 [01:23<00:21,  5.37it/s]
+ 80%|████████  | 455/567 [01:24<00:20,  5.36it/s]
+ 80%|████████  | 456/567 [01:24<00:20,  5.39it/s]
+ 81%|████████  | 457/567 [01:24<00:20,  5.41it/s]
+ 81%|████████  | 458/567 [01:24<00:20,  5.41it/s]
+ 81%|████████  | 459/567 [01:24<00:19,  5.42it/s]
+ 81%|████████  | 460/567 [01:25<00:19,  5.43it/s]
+ 81%|████████▏ | 461/567 [01:25<00:19,  5.44it/s]
+ 81%|████████▏ | 462/567 [01:25<00:19,  5.40it/s]
+ 82%|████████▏ | 463/567 [01:25<00:19,  5.38it/s]
+ 82%|████████▏ | 464/567 [01:25<00:19,  5.40it/s]
+ 82%|████████▏ | 465/567 [01:25<00:18,  5.42it/s]
+ 82%|████████▏ | 466/567 [01:26<00:18,  5.43it/s]
+ 82%|████████▏ | 467/567 [01:26<00:18,  5.44it/s]
+ 83%|████████▎ | 468/567 [01:26<00:18,  5.44it/s]
+ 83%|████████▎ | 469/567 [01:26<00:18,  5.41it/s]
+ 83%|████████▎ | 470/567 [01:26<00:17,  5.42it/s]
+ 83%|████████▎ | 471/567 [01:27<00:17,  5.43it/s]
+ 83%|████████▎ | 472/567 [01:27<00:17,  5.44it/s]
+ 83%|████████▎ | 473/567 [01:27<00:17,  5.41it/s]
+ 84%|████████▎ | 474/567 [01:27<00:17,  5.42it/s]
+ 84%|████████▍ | 475/567 [01:27<00:16,  5.43it/s]
+ 84%|████████▍ | 476/567 [01:28<00:16,  5.44it/s]
+ 84%|████████▍ | 477/567 [01:28<00:16,  5.41it/s]
+ 84%|████████▍ | 478/567 [01:28<00:16,  5.42it/s]
+ 84%|████████▍ | 479/567 [01:28<00:16,  5.43it/s]
+ 85%|████████▍ | 480/567 [01:28<00:16,  5.40it/s]
+ 85%|████████▍ | 481/567 [01:28<00:15,  5.38it/s]
+ 85%|████████▌ | 482/567 [01:29<00:15,  5.40it/s]
+ 85%|████████▌ | 483/567 [01:29<00:15,  5.38it/s]
+ 85%|████████▌ | 484/567 [01:29<00:15,  5.41it/s]
+ 86%|████████▌ | 485/567 [01:29<00:15,  5.39it/s]
+ 86%|████████▌ | 486/567 [01:29<00:15,  5.40it/s]
+ 86%|████████▌ | 487/567 [01:30<00:14,  5.42it/s]
+ 86%|████████▌ | 488/567 [01:30<00:14,  5.43it/s]
+ 86%|████████▌ | 489/567 [01:30<00:14,  5.44it/s]
+ 86%|████████▋ | 490/567 [01:30<00:14,  5.41it/s]
+ 87%|████████▋ | 491/567 [01:30<00:14,  5.42it/s]
+ 87%|████████▋ | 492/567 [01:30<00:13,  5.39it/s]
+ 87%|████████▋ | 493/567 [01:31<00:13,  5.41it/s]
+ 87%|████████▋ | 494/567 [01:31<00:13,  5.41it/s]
+ 87%|████████▋ | 495/567 [01:31<00:13,  5.38it/s]
+ 87%|████████▋ | 496/567 [01:31<00:13,  5.36it/s]
+ 88%|████████▊ | 497/567 [01:31<00:13,  5.36it/s]
+ 88%|████████▊ | 498/567 [01:32<00:12,  5.38it/s]
+ 88%|████████▊ | 499/567 [01:32<00:12,  5.40it/s]
+ 88%|████████▊ | 500/567 [01:32<00:12,  5.38it/s]
+ 88%|████████▊ | 501/567 [01:32<00:12,  5.40it/s]
+ 89%|████████▊ | 502/567 [01:32<00:12,  5.41it/s]
+ 89%|████████▊ | 503/567 [01:33<00:11,  5.38it/s]
+ 89%|████████▉ | 504/567 [01:33<00:11,  5.49it/s]
+ 89%|████████▉ | 505/567 [01:33<00:11,  5.38it/s]
+ 89%|████████▉ | 506/567 [01:33<00:11,  5.36it/s]
+ 89%|████████▉ | 507/567 [01:33<00:11,  5.39it/s]
+ 90%|████████▉ | 508/567 [01:33<00:10,  5.40it/s]
+ 90%|████████▉ | 509/567 [01:34<00:10,  5.38it/s]
+ 90%|████████▉ | 510/567 [01:34<00:10,  5.40it/s]
+ 90%|█████████ | 511/567 [01:34<00:10,  5.39it/s]
+ 90%|█████████ | 512/567 [01:34<00:10,  5.41it/s]
+ 90%|█████████ | 513/567 [01:34<00:09,  5.42it/s]
+ 91%|█████████ | 514/567 [01:35<00:09,  5.40it/s]
+ 91%|█████████ | 515/567 [01:35<00:09,  5.41it/s]
+ 91%|█████████ | 516/567 [01:35<00:09,  5.39it/s]
+ 91%|█████████ | 517/567 [01:35<00:09,  5.37it/s]
+ 91%|█████████▏| 518/567 [01:35<00:09,  5.36it/s]
+ 92%|█████████▏| 519/567 [01:35<00:08,  5.39it/s]
+ 92%|█████████▏| 520/567 [01:36<00:08,  5.41it/s]
+ 92%|█████████▏| 521/567 [01:36<00:08,  5.42it/s]
+ 92%|█████████▏| 522/567 [01:36<00:08,  5.43it/s]
+ 92%|█████████▏| 523/567 [01:36<00:08,  5.44it/s]
+ 92%|█████████▏| 524/567 [01:36<00:07,  5.45it/s]
+ 93%|█████████▎| 525/567 [01:37<00:07,  5.41it/s]
+ 93%|█████████▎| 526/567 [01:37<00:07,  5.38it/s]
+ 93%|█████████▎| 527/567 [01:37<00:07,  5.40it/s]
+ 93%|█████████▎| 528/567 [01:37<00:07,  5.42it/s]
+ 93%|█████████▎| 529/567 [01:37<00:06,  5.43it/s]
+ 93%|█████████▎| 530/567 [01:38<00:06,  5.44it/s]
+ 94%|█████████▎| 531/567 [01:38<00:06,  5.44it/s]
+ 94%|█████████▍| 532/567 [01:38<00:06,  5.41it/s]
+ 94%|█████████▍| 533/567 [01:38<00:06,  5.43it/s]
+ 94%|█████████▍| 534/567 [01:38<00:06,  5.43it/s]
+ 94%|█████████▍| 535/567 [01:38<00:05,  5.44it/s]
+ 95%|█████████▍| 536/567 [01:39<00:05,  5.41it/s]
+ 95%|█████████▍| 537/567 [01:39<00:05,  5.42it/s]
+ 95%|█████████▍| 538/567 [01:39<00:05,  5.43it/s]
+ 95%|█████████▌| 539/567 [01:39<00:05,  5.44it/s]
+ 95%|█████████▌| 540/567 [01:39<00:04,  5.41it/s]
+ 95%|█████████▌| 541/567 [01:40<00:04,  5.42it/s]
+ 96%|█████████▌| 542/567 [01:40<00:04,  5.43it/s]
+ 96%|█████████▌| 543/567 [01:40<00:04,  5.40it/s]
+ 96%|█████████▌| 544/567 [01:40<00:04,  5.39it/s]
+ 96%|█████████▌| 545/567 [01:40<00:04,  5.40it/s]
+ 96%|█████████▋| 546/567 [01:40<00:03,  5.38it/s]
+ 96%|█████████▋| 547/567 [01:41<00:03,  5.40it/s]
+ 97%|█████████▋| 548/567 [01:41<00:03,  5.38it/s]
+ 97%|█████████▋| 549/567 [01:41<00:03,  5.40it/s]
+ 97%|█████████▋| 550/567 [01:41<00:03,  5.41it/s]
+ 97%|█████████▋| 551/567 [01:41<00:02,  5.42it/s]
+ 97%|█████████▋| 552/567 [01:42<00:02,  5.43it/s]
+ 98%|█████████▊| 553/567 [01:42<00:02,  5.40it/s]
+ 98%|█████████▊| 554/567 [01:42<00:02,  5.41it/s]
+ 98%|█████████▊| 555/567 [01:42<00:02,  5.39it/s]
+ 98%|█████████▊| 556/567 [01:42<00:02,  5.41it/s]
+ 98%|█████████▊| 557/567 [01:42<00:01,  5.42it/s]
+ 98%|█████████▊| 558/567 [01:43<00:01,  5.39it/s]
+ 99%|█████████▊| 559/567 [01:43<00:01,  5.36it/s]
+ 99%|█████████▉| 560/567 [01:43<00:01,  5.35it/s]
+ 99%|█████████▉| 561/567 [01:43<00:01,  5.38it/s]
+ 99%|█████████▉| 562/567 [01:43<00:00,  5.40it/s]
+ 99%|█████████▉| 563/567 [01:44<00:00,  5.38it/s]
+ 99%|█████████▉| 564/567 [01:44<00:00,  5.40it/s]
+100%|█████████▉| 565/567 [01:44<00:00,  5.42it/s]
+100%|█████████▉| 566/567 [01:44<00:00,  5.39it/s]
+100%|██████████| 567/567 [01:44<00:00,  5.49it/s]
+100%|██████████| 567/567 [01:44<00:00,  5.41it/s]
+Saving model to directory step10000
+[2026-07-22 06:10:48,714] [INFO] [logging.py:123:log_dist] [Rank 0] [Torch] Checkpoint global_step10000 is about to be saved!
+[2026-07-22 06:10:48,747] [INFO] [logging.py:123:log_dist] [Rank 0] Saving model checkpoint: /content/anima_campaign/runs/e004b_relay_s0/20260722_04-29-49/global_step10000/mp_rank_00_model_states.pt
+TRAINING COMPLETE!
+[rank0]:[W722 06:10:50.292716876 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[2026-07-22 06:10:56,487] [INFO] [launch.py:367:main] Process 53072 exits successfully.