Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +286 -0
- pretrain/config.json +237 -0
- pretrain/events.out.tfevents.1776907229.047f74d76241.302462.0 +3 -0
- pretrain/events.out.tfevents.1776907336.047f74d76241.304586.0 +3 -0
- pretrain/events.out.tfevents.1776909527.047f74d76241.309272.0 +3 -0
- pretrain/events.out.tfevents.1776910803.047f74d76241.313106.0 +3 -0
- pretrain/events.out.tfevents.1776911725.047f74d76241.316403.0 +3 -0
- pretrain/events.out.tfevents.1776911987.047f74d76241.317785.0 +3 -0
- pretrain/events.out.tfevents.1776912176.047f74d76241.319180.0 +3 -0
- pretrain/gradient_steps_skipped_0030.txt +1 -0
- pretrain/model_0030.pth +3 -0
- pretrain/optimizer_0030.pth +3 -0
- pretrain/scaler_0030.pth +3 -0
- pretrain/scheduler_0030.pth +3 -0
- pretrain/wandb/debug-internal.log +0 -0
- pretrain/wandb/debug.log +24 -0
- pretrain/wandb/run-20260423_012244-al9a0z6q/files/output.log +135 -0
- pretrain/wandb/run-20260423_012244-al9a0z6q/files/requirements.txt +309 -0
- pretrain/wandb/run-20260423_012244-al9a0z6q/files/wandb-metadata.json +103 -0
- pretrain/wandb/run-20260423_012244-al9a0z6q/logs/debug-core.log +8 -0
- pretrain/wandb/run-20260423_012244-al9a0z6q/logs/debug-internal.log +27 -0
- pretrain/wandb/run-20260423_012244-al9a0z6q/logs/debug.log +19 -0
- pretrain/wandb/run-20260423_012244-al9a0z6q/run-al9a0z6q.wandb +0 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/files/config.yaml +530 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/files/output.log +204 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/files/requirements.txt +309 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/files/wandb-metadata.json +103 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/files/wandb-summary.json +1 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/logs/debug-core.log +15 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/logs/debug-internal.log +26 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/logs/debug.log +21 -0
- pretrain/wandb/run-20260423_015848-rau44hjb/run-rau44hjb.wandb +0 -0
- pretrain/wandb/run-20260423_022003-8rqq7dwf/files/output.log +45 -0
- pretrain/wandb/run-20260423_022003-8rqq7dwf/files/requirements.txt +309 -0
- pretrain/wandb/run-20260423_022003-8rqq7dwf/files/wandb-metadata.json +103 -0
- pretrain/wandb/run-20260423_022003-8rqq7dwf/logs/debug-core.log +8 -0
- pretrain/wandb/run-20260423_022003-8rqq7dwf/logs/debug-internal.log +19 -0
- pretrain/wandb/run-20260423_022003-8rqq7dwf/logs/debug.log +19 -0
- pretrain/wandb/run-20260423_022003-8rqq7dwf/run-8rqq7dwf.wandb +0 -0
- pretrain/wandb/run-20260423_023526-gi92or6f/files/output.log +7 -0
- pretrain/wandb/run-20260423_023526-gi92or6f/files/requirements.txt +309 -0
- pretrain/wandb/run-20260423_023526-gi92or6f/files/wandb-metadata.json +103 -0
- pretrain/wandb/run-20260423_023526-gi92or6f/logs/debug-core.log +8 -0
- pretrain/wandb/run-20260423_023526-gi92or6f/logs/debug-internal.log +13 -0
- pretrain/wandb/run-20260423_023526-gi92or6f/logs/debug.log +19 -0
- pretrain/wandb/run-20260423_023526-gi92or6f/run-gi92or6f.wandb +0 -0
- pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/train_viz/feature_maps_0_4e1f0beb5dc171629abe.png +3 -0
- pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/train_viz/feature_maps_0_a4bfb42de805ac3cdd19.png +3 -0
- pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/viz/_train_gt_0_3d3ff6913bfd3113bd14.png +3 -0
- pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/viz/_train_gt_0_96988caced64988329d4.png +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,289 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/train_viz/feature_maps_0_4e1f0beb5dc171629abe.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/train_viz/feature_maps_0_a4bfb42de805ac3cdd19.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/viz/_train_gt_0_3d3ff6913bfd3113bd14.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/viz/_train_gt_0_96988caced64988329d4.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/viz/train_pred_0_4932e082ac8e5c8cdcb3.png filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/viz/train_pred_0_b28fbe1baf8466c648e8.png filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_0_31ba490ac252da95bdf9.png filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_0_5adb30cf2398632ac2f0.png filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_102219_bd307721a6020d8e4e0a.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_102219_cb3510b796ecdab457e4.png filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_107189_4085170ae65bf6b71a6b.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_110739_2374cdefc30239676395.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_110739_b3231aabed4d6a883fab.png filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_115709_3fa394090a62c6ff54d1.png filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_119259_136f900be9ac320e4a75.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_119259_f49a13af8d46ba7d8ed3.png filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_124229_078d1f3118d46fc631e6.png filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_127779_1810971c371cfbbb5b8f.png filter=lfs diff=lfs merge=lfs -text
|
| 54 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_127779_df0eccad7fccfc77601e.png filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_132749_d332547ae5e856266046.png filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_13469_a37757f1106534739b1e.png filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_136299_86bff141a45a99ab4de9.png filter=lfs diff=lfs merge=lfs -text
|
| 58 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_136299_ad22719ca6f17c4fe388.png filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_141269_efcf717c1db34978c806.png filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_144819_a0daf2b793e0761788c0.png filter=lfs diff=lfs merge=lfs -text
|
| 61 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_144819_d75e44c065db8dc731ae.png filter=lfs diff=lfs merge=lfs -text
|
| 62 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_149789_a0d5beafae1d1736a81b.png filter=lfs diff=lfs merge=lfs -text
|
| 63 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_153339_11cde66c1cea9956bc7f.png filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_153339_f72cfa116c13f9e443cc.png filter=lfs diff=lfs merge=lfs -text
|
| 65 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_158309_fee39a88b97d0d1872bc.png filter=lfs diff=lfs merge=lfs -text
|
| 66 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_161859_2e5cab9f64a2524abdbc.png filter=lfs diff=lfs merge=lfs -text
|
| 67 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_161859_4aeb3e26bd19df393f5c.png filter=lfs diff=lfs merge=lfs -text
|
| 68 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_166829_48d85969588a60f27002.png filter=lfs diff=lfs merge=lfs -text
|
| 69 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_17019_49276b1dd3625629ec6c.png filter=lfs diff=lfs merge=lfs -text
|
| 70 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_17019_b7d9d5ff79c6bce20f21.png filter=lfs diff=lfs merge=lfs -text
|
| 71 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_170379_1b8072f53b1481c0b7dd.png filter=lfs diff=lfs merge=lfs -text
|
| 72 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_170379_9c10435b8f4feefbfe8b.png filter=lfs diff=lfs merge=lfs -text
|
| 73 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_175349_17de9173ce705787fa87.png filter=lfs diff=lfs merge=lfs -text
|
| 74 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_178899_5f361d4e3184af8b0512.png filter=lfs diff=lfs merge=lfs -text
|
| 75 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_178899_ad278b1b40e9402c1886.png filter=lfs diff=lfs merge=lfs -text
|
| 76 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_183869_ea79e800335e30fe3a23.png filter=lfs diff=lfs merge=lfs -text
|
| 77 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_187419_1c7dbaf5661799a67429.png filter=lfs diff=lfs merge=lfs -text
|
| 78 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_187419_3d02bd1436a62e559d7b.png filter=lfs diff=lfs merge=lfs -text
|
| 79 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_192389_4451a2a2fdc36595e810.png filter=lfs diff=lfs merge=lfs -text
|
| 80 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_195939_0d7f825690fcadc710e2.png filter=lfs diff=lfs merge=lfs -text
|
| 81 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_195939_4ae0618f37605f482d2c.png filter=lfs diff=lfs merge=lfs -text
|
| 82 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_200909_4f09663bf02a1d5fb387.png filter=lfs diff=lfs merge=lfs -text
|
| 83 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_204459_4ccc91758988d7076d95.png filter=lfs diff=lfs merge=lfs -text
|
| 84 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_204459_ca7d9ad9c9808650bb8b.png filter=lfs diff=lfs merge=lfs -text
|
| 85 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_209429_8ccd47b64464f277a697.png filter=lfs diff=lfs merge=lfs -text
|
| 86 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_212979_5062df8008675f0aeebc.png filter=lfs diff=lfs merge=lfs -text
|
| 87 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_212979_eb62dc1e8c66f7e426e0.png filter=lfs diff=lfs merge=lfs -text
|
| 88 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_217949_ddcb7397d30a76c9b418.png filter=lfs diff=lfs merge=lfs -text
|
| 89 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_21989_180175dd24e51c663494.png filter=lfs diff=lfs merge=lfs -text
|
| 90 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_221499_53ebb3df56958052e9a1.png filter=lfs diff=lfs merge=lfs -text
|
| 91 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_221499_98fb36ffb174f2ec57bf.png filter=lfs diff=lfs merge=lfs -text
|
| 92 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_226469_5be3752f39d050bf9e05.png filter=lfs diff=lfs merge=lfs -text
|
| 93 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_230019_a517b2d285233ad682cd.png filter=lfs diff=lfs merge=lfs -text
|
| 94 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_230019_f360adf4d01f071cf517.png filter=lfs diff=lfs merge=lfs -text
|
| 95 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_234989_23323b0e4f9cafc5bd3a.png filter=lfs diff=lfs merge=lfs -text
|
| 96 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_238539_641453bc0112ee566e3e.png filter=lfs diff=lfs merge=lfs -text
|
| 97 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_238539_a79a3aa498f0f721bc43.png filter=lfs diff=lfs merge=lfs -text
|
| 98 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_243509_8cff333a34223c3be142.png filter=lfs diff=lfs merge=lfs -text
|
| 99 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_247059_091d72468881897d5432.png filter=lfs diff=lfs merge=lfs -text
|
| 100 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_247059_60c9dae3af78915d2b4a.png filter=lfs diff=lfs merge=lfs -text
|
| 101 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_252029_f4b05035ac8bbb2496c3.png filter=lfs diff=lfs merge=lfs -text
|
| 102 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_25539_4fb182ddc9979ab4b563.png filter=lfs diff=lfs merge=lfs -text
|
| 103 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_25539_dd4175f8ac29a6ff05be.png filter=lfs diff=lfs merge=lfs -text
|
| 104 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_255579_36c02eab622d09cff56e.png filter=lfs diff=lfs merge=lfs -text
|
| 105 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_255579_88d7e0ce9f56e5e81cd0.png filter=lfs diff=lfs merge=lfs -text
|
| 106 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_260549_c17e7617d8d8947b746e.png filter=lfs diff=lfs merge=lfs -text
|
| 107 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_30509_129ba2d8fbe4f206f261.png filter=lfs diff=lfs merge=lfs -text
|
| 108 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_34059_6f4bb13f8227b03b881c.png filter=lfs diff=lfs merge=lfs -text
|
| 109 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_34059_85263f925323a48c6fe3.png filter=lfs diff=lfs merge=lfs -text
|
| 110 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_39029_12a776c372f76fc8e431.png filter=lfs diff=lfs merge=lfs -text
|
| 111 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_42579_1aff59cb6cde39cdcd7a.png filter=lfs diff=lfs merge=lfs -text
|
| 112 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_42579_589036cc7298b3d529ea.png filter=lfs diff=lfs merge=lfs -text
|
| 113 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_47549_3354a11415329b077113.png filter=lfs diff=lfs merge=lfs -text
|
| 114 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_4949_3dcbf2d7d49d30f49f06.png filter=lfs diff=lfs merge=lfs -text
|
| 115 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_51099_01f8a05610331725e4a9.png filter=lfs diff=lfs merge=lfs -text
|
| 116 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_51099_7fe035dee865bef67267.png filter=lfs diff=lfs merge=lfs -text
|
| 117 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_56069_807205a88c98514a1a01.png filter=lfs diff=lfs merge=lfs -text
|
| 118 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_59619_70d55c97837fffa3ee63.png filter=lfs diff=lfs merge=lfs -text
|
| 119 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_59619_f2811acebb535f896fb0.png filter=lfs diff=lfs merge=lfs -text
|
| 120 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_64589_6f26e8de15a28c5e8036.png filter=lfs diff=lfs merge=lfs -text
|
| 121 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_68139_1fd893a321281dc9a079.png filter=lfs diff=lfs merge=lfs -text
|
| 122 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_68139_b0d0f9a2536593c6e6d1.png filter=lfs diff=lfs merge=lfs -text
|
| 123 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_73109_e94f39b992e640f98a37.png filter=lfs diff=lfs merge=lfs -text
|
| 124 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_76659_4ea2b08647fce2264d17.png filter=lfs diff=lfs merge=lfs -text
|
| 125 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_76659_a5088187b29f026c28aa.png filter=lfs diff=lfs merge=lfs -text
|
| 126 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_81629_f0f51612b771fa019030.png filter=lfs diff=lfs merge=lfs -text
|
| 127 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_8499_9c027f1e7f8fbe5df13c.png filter=lfs diff=lfs merge=lfs -text
|
| 128 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_8499_d6923d1e85609908d06e.png filter=lfs diff=lfs merge=lfs -text
|
| 129 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_85179_07165309fb1a82ad6dc6.png filter=lfs diff=lfs merge=lfs -text
|
| 130 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_85179_5456b889b12db03b4a33.png filter=lfs diff=lfs merge=lfs -text
|
| 131 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_90149_0a818994edb02a32239e.png filter=lfs diff=lfs merge=lfs -text
|
| 132 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_93699_2d70203626a03b8ca82e.png filter=lfs diff=lfs merge=lfs -text
|
| 133 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_93699_9974c6825b5c476e58b5.png filter=lfs diff=lfs merge=lfs -text
|
| 134 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/train_viz/feature_maps_98669_8f59ba3a4c2072a81cea.png filter=lfs diff=lfs merge=lfs -text
|
| 135 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_0_6d70233d19312b289963.png filter=lfs diff=lfs merge=lfs -text
|
| 136 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_0_96988caced64988329d4.png filter=lfs diff=lfs merge=lfs -text
|
| 137 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_102219_0c7dd92731cc9ddaa6a7.png filter=lfs diff=lfs merge=lfs -text
|
| 138 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_102219_588e53ed69a803d76978.png filter=lfs diff=lfs merge=lfs -text
|
| 139 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_107189_04a68d5b99fb6adf7865.png filter=lfs diff=lfs merge=lfs -text
|
| 140 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_110739_175012dcf0ebf6240e29.png filter=lfs diff=lfs merge=lfs -text
|
| 141 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_110739_7acdff7500ea00e98316.png filter=lfs diff=lfs merge=lfs -text
|
| 142 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_115709_04a68d5b99fb6adf7865.png filter=lfs diff=lfs merge=lfs -text
|
| 143 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_119259_1fc525e14272ff4d0f13.png filter=lfs diff=lfs merge=lfs -text
|
| 144 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_119259_588e53ed69a803d76978.png filter=lfs diff=lfs merge=lfs -text
|
| 145 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_124229_e48e04f1340d4dca1e2f.png filter=lfs diff=lfs merge=lfs -text
|
| 146 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_127779_22ecc220e0237a278587.png filter=lfs diff=lfs merge=lfs -text
|
| 147 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_127779_95582d72aa226408e4a4.png filter=lfs diff=lfs merge=lfs -text
|
| 148 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_132749_81df01d60bbb8e9c9973.png filter=lfs diff=lfs merge=lfs -text
|
| 149 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_13469_eeeb2f53fd286596ed54.png filter=lfs diff=lfs merge=lfs -text
|
| 150 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_136299_38a5fc261da5af145d4e.png filter=lfs diff=lfs merge=lfs -text
|
| 151 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_136299_572fd15fbee4e09a7dd6.png filter=lfs diff=lfs merge=lfs -text
|
| 152 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_141269_b2f83d0b0686d1a0f7d5.png filter=lfs diff=lfs merge=lfs -text
|
| 153 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_144819_922f43a16d9e093ee0a0.png filter=lfs diff=lfs merge=lfs -text
|
| 154 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_144819_c218c2bf102b1ab216d8.png filter=lfs diff=lfs merge=lfs -text
|
| 155 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_149789_9c06ece8a64e630e83e8.png filter=lfs diff=lfs merge=lfs -text
|
| 156 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_153339_588e53ed69a803d76978.png filter=lfs diff=lfs merge=lfs -text
|
| 157 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_153339_cb0d8e275477577e43e6.png filter=lfs diff=lfs merge=lfs -text
|
| 158 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_158309_ac73294c9f246dd45cf5.png filter=lfs diff=lfs merge=lfs -text
|
| 159 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_161859_588e53ed69a803d76978.png filter=lfs diff=lfs merge=lfs -text
|
| 160 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_161859_afdda8ff707dca9441f7.png filter=lfs diff=lfs merge=lfs -text
|
| 161 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_166829_112e66f64af89939cb2f.png filter=lfs diff=lfs merge=lfs -text
|
| 162 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_17019_15404e9fcf646e460823.png filter=lfs diff=lfs merge=lfs -text
|
| 163 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_17019_fc3e0f2ecb7123d554d5.png filter=lfs diff=lfs merge=lfs -text
|
| 164 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_170379_bcdd48a9fe295334494b.png filter=lfs diff=lfs merge=lfs -text
|
| 165 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_170379_fb72cd76bfee0ea328f9.png filter=lfs diff=lfs merge=lfs -text
|
| 166 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_175349_06b4644c677cc3a68d12.png filter=lfs diff=lfs merge=lfs -text
|
| 167 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_178899_152d2975b65ed3658361.png filter=lfs diff=lfs merge=lfs -text
|
| 168 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_178899_cff3b7470628fe6cdd29.png filter=lfs diff=lfs merge=lfs -text
|
| 169 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_183869_04a68d5b99fb6adf7865.png filter=lfs diff=lfs merge=lfs -text
|
| 170 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_187419_0e4ce06d9f43ef5e21d6.png filter=lfs diff=lfs merge=lfs -text
|
| 171 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_187419_38a5fc261da5af145d4e.png filter=lfs diff=lfs merge=lfs -text
|
| 172 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_192389_73c9c6048a9f3acf148a.png filter=lfs diff=lfs merge=lfs -text
|
| 173 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_195939_1487bc3e46a8d0025255.png filter=lfs diff=lfs merge=lfs -text
|
| 174 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_195939_5d2ec4de58a7f15fca6c.png filter=lfs diff=lfs merge=lfs -text
|
| 175 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_200909_21ba7dfd8e8fcbb1ece4.png filter=lfs diff=lfs merge=lfs -text
|
| 176 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_204459_25d77da73438f8555368.png filter=lfs diff=lfs merge=lfs -text
|
| 177 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_204459_588e53ed69a803d76978.png filter=lfs diff=lfs merge=lfs -text
|
| 178 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_209429_d1c20a37cf792119de2d.png filter=lfs diff=lfs merge=lfs -text
|
| 179 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_212979_38a5fc261da5af145d4e.png filter=lfs diff=lfs merge=lfs -text
|
| 180 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_212979_f6835fff68175be7df36.png filter=lfs diff=lfs merge=lfs -text
|
| 181 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_217949_04a68d5b99fb6adf7865.png filter=lfs diff=lfs merge=lfs -text
|
| 182 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_21989_ec7efaf006f32e26149d.png filter=lfs diff=lfs merge=lfs -text
|
| 183 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_221499_38a5fc261da5af145d4e.png filter=lfs diff=lfs merge=lfs -text
|
| 184 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_221499_be74328ecf2d5b962bd9.png filter=lfs diff=lfs merge=lfs -text
|
| 185 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_226469_04a68d5b99fb6adf7865.png filter=lfs diff=lfs merge=lfs -text
|
| 186 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_230019_9a9602e77479ab77d71c.png filter=lfs diff=lfs merge=lfs -text
|
| 187 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_230019_a5448780a7fb57809733.png filter=lfs diff=lfs merge=lfs -text
|
| 188 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_234989_448924b64297b63a8426.png filter=lfs diff=lfs merge=lfs -text
|
| 189 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_238539_12e3a623f24946eddb3f.png filter=lfs diff=lfs merge=lfs -text
|
| 190 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_238539_43d0cdb2f7ec2a723b0b.png filter=lfs diff=lfs merge=lfs -text
|
| 191 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_243509_0449eb9b73c626d6311c.png filter=lfs diff=lfs merge=lfs -text
|
| 192 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_247059_38a5fc261da5af145d4e.png filter=lfs diff=lfs merge=lfs -text
|
| 193 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_247059_be74328ecf2d5b962bd9.png filter=lfs diff=lfs merge=lfs -text
|
| 194 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_252029_fee60f50de9178d78c24.png filter=lfs diff=lfs merge=lfs -text
|
| 195 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_25539_8c4bf884c2bb96a14d8c.png filter=lfs diff=lfs merge=lfs -text
|
| 196 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_25539_d36e9e5625f7d70a299a.png filter=lfs diff=lfs merge=lfs -text
|
| 197 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_255579_57f99c2854f97314d8b5.png filter=lfs diff=lfs merge=lfs -text
|
| 198 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_255579_8561a31044a621828546.png filter=lfs diff=lfs merge=lfs -text
|
| 199 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_260549_9a59a776c2bb39744918.png filter=lfs diff=lfs merge=lfs -text
|
| 200 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_30509_d10ccf6f7966b283d144.png filter=lfs diff=lfs merge=lfs -text
|
| 201 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_34059_482d877e3a931567a8a9.png filter=lfs diff=lfs merge=lfs -text
|
| 202 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_34059_588e53ed69a803d76978.png filter=lfs diff=lfs merge=lfs -text
|
| 203 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_39029_04a68d5b99fb6adf7865.png filter=lfs diff=lfs merge=lfs -text
|
| 204 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_42579_588e53ed69a803d76978.png filter=lfs diff=lfs merge=lfs -text
|
| 205 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_42579_7291980d4f8ee0a4ed75.png filter=lfs diff=lfs merge=lfs -text
|
| 206 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_47549_af29ac95a24bb6ae7c96.png filter=lfs diff=lfs merge=lfs -text
|
| 207 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_4949_a722af70d2b07db7913a.png filter=lfs diff=lfs merge=lfs -text
|
| 208 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_51099_50fbba48c91d8d6e664b.png filter=lfs diff=lfs merge=lfs -text
|
| 209 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_51099_d4142c0f1a6466b6a451.png filter=lfs diff=lfs merge=lfs -text
|
| 210 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_56069_654d2de6c19328d1f286.png filter=lfs diff=lfs merge=lfs -text
|
| 211 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_59619_60e5d3a7105242559e08.png filter=lfs diff=lfs merge=lfs -text
|
| 212 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_59619_afdda8ff707dca9441f7.png filter=lfs diff=lfs merge=lfs -text
|
| 213 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_64589_b8aee0a81b3b259da7bb.png filter=lfs diff=lfs merge=lfs -text
|
| 214 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_68139_588e53ed69a803d76978.png filter=lfs diff=lfs merge=lfs -text
|
| 215 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_68139_9e0e0874316e80c3e8d9.png filter=lfs diff=lfs merge=lfs -text
|
| 216 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_73109_04a68d5b99fb6adf7865.png filter=lfs diff=lfs merge=lfs -text
|
| 217 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_76659_6db3e2aa941cb8bd16d2.png filter=lfs diff=lfs merge=lfs -text
|
| 218 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_76659_853e2e5c93048f5bde25.png filter=lfs diff=lfs merge=lfs -text
|
| 219 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_81629_7f9feec362d1d6c97a8f.png filter=lfs diff=lfs merge=lfs -text
|
| 220 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_8499_38a5fc261da5af145d4e.png filter=lfs diff=lfs merge=lfs -text
|
| 221 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_8499_be74328ecf2d5b962bd9.png filter=lfs diff=lfs merge=lfs -text
|
| 222 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_85179_35d89e54e9a1b3acf07e.png filter=lfs diff=lfs merge=lfs -text
|
| 223 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_85179_92e5175e8ef20a440529.png filter=lfs diff=lfs merge=lfs -text
|
| 224 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_90149_ef75d104b4e6fc2b0b2b.png filter=lfs diff=lfs merge=lfs -text
|
| 225 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_93699_38a5fc261da5af145d4e.png filter=lfs diff=lfs merge=lfs -text
|
| 226 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_93699_df2d5cf52c45b01df3dc.png filter=lfs diff=lfs merge=lfs -text
|
| 227 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/_train_gt_98669_185929b554b48197607c.png filter=lfs diff=lfs merge=lfs -text
|
| 228 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_0_bbcbb0717d9649dea274.png filter=lfs diff=lfs merge=lfs -text
|
| 229 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_0_bbd54b778399c306b445.png filter=lfs diff=lfs merge=lfs -text
|
| 230 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_102219_14a8541563093678e902.png filter=lfs diff=lfs merge=lfs -text
|
| 231 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_102219_96cd000138baf56b01d6.png filter=lfs diff=lfs merge=lfs -text
|
| 232 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_107189_c393ca73d75028d8da76.png filter=lfs diff=lfs merge=lfs -text
|
| 233 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_110739_936132343ac3a96be644.png filter=lfs diff=lfs merge=lfs -text
|
| 234 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_110739_ae6cb264388022f80d01.png filter=lfs diff=lfs merge=lfs -text
|
| 235 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_115709_b5ddb0df36e6ed5e0c4b.png filter=lfs diff=lfs merge=lfs -text
|
| 236 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_119259_83dabc779013a34b0e7c.png filter=lfs diff=lfs merge=lfs -text
|
| 237 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_119259_dfb3abcc8d0892ce3c0e.png filter=lfs diff=lfs merge=lfs -text
|
| 238 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_124229_f697187aa42f9c1bc004.png filter=lfs diff=lfs merge=lfs -text
|
| 239 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_127779_aa3dd85c28c986a07fb1.png filter=lfs diff=lfs merge=lfs -text
|
| 240 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_127779_ce6daa40cdfe4c43daee.png filter=lfs diff=lfs merge=lfs -text
|
| 241 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_132749_51e34a7f8a0638d52020.png filter=lfs diff=lfs merge=lfs -text
|
| 242 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_13469_1f346502f8b017de5c61.png filter=lfs diff=lfs merge=lfs -text
|
| 243 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_136299_3a89b57f0a0fc2d7cdc5.png filter=lfs diff=lfs merge=lfs -text
|
| 244 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_136299_68fb6e0af47899863923.png filter=lfs diff=lfs merge=lfs -text
|
| 245 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_141269_6760fbc78d190261b195.png filter=lfs diff=lfs merge=lfs -text
|
| 246 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_144819_2fd62379ad0630a4f7c6.png filter=lfs diff=lfs merge=lfs -text
|
| 247 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_144819_4f38605eb80a114af6cb.png filter=lfs diff=lfs merge=lfs -text
|
| 248 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_149789_6444af0fa55472b31bd1.png filter=lfs diff=lfs merge=lfs -text
|
| 249 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_153339_6b7b29dbbbdb9c11258e.png filter=lfs diff=lfs merge=lfs -text
|
| 250 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_153339_e64a50dbfdd68041dbee.png filter=lfs diff=lfs merge=lfs -text
|
| 251 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_158309_e8a17dcf1c92aec2f64b.png filter=lfs diff=lfs merge=lfs -text
|
| 252 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_161859_be48c598e2a306645b0e.png filter=lfs diff=lfs merge=lfs -text
|
| 253 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_161859_f66e941c1b9513c42289.png filter=lfs diff=lfs merge=lfs -text
|
| 254 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_166829_cdc2c35d970cab8481d1.png filter=lfs diff=lfs merge=lfs -text
|
| 255 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_17019_519cd452e194fa135f21.png filter=lfs diff=lfs merge=lfs -text
|
| 256 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_17019_9d5e5670691c8d6bc5fb.png filter=lfs diff=lfs merge=lfs -text
|
| 257 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_170379_65008e9f95cd7ce377e8.png filter=lfs diff=lfs merge=lfs -text
|
| 258 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_170379_67a4128d4e85d0058247.png filter=lfs diff=lfs merge=lfs -text
|
| 259 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_175349_52465d685e7e15469522.png filter=lfs diff=lfs merge=lfs -text
|
| 260 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_178899_1c3f646a8aae8ecb9eef.png filter=lfs diff=lfs merge=lfs -text
|
| 261 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_178899_47b600bdbed196aef093.png filter=lfs diff=lfs merge=lfs -text
|
| 262 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_183869_c4a6fe9abb24c47c76b8.png filter=lfs diff=lfs merge=lfs -text
|
| 263 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_187419_9690a2a6c343fc1578d4.png filter=lfs diff=lfs merge=lfs -text
|
| 264 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_187419_c10683a5a62490529c90.png filter=lfs diff=lfs merge=lfs -text
|
| 265 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_192389_bd39e6b295704759fabb.png filter=lfs diff=lfs merge=lfs -text
|
| 266 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_195939_51895d571e9fbdf95d5f.png filter=lfs diff=lfs merge=lfs -text
|
| 267 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_195939_81362f1401adc966c972.png filter=lfs diff=lfs merge=lfs -text
|
| 268 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_200909_9c340caf2065d8b28a71.png filter=lfs diff=lfs merge=lfs -text
|
| 269 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_204459_d95ff93cb188d789f379.png filter=lfs diff=lfs merge=lfs -text
|
| 270 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_204459_fc77566df2e7c71fd590.png filter=lfs diff=lfs merge=lfs -text
|
| 271 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_209429_7658e2184fc20bbaab70.png filter=lfs diff=lfs merge=lfs -text
|
| 272 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_212979_36d73efdb05024a831b0.png filter=lfs diff=lfs merge=lfs -text
|
| 273 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_212979_d37c4a308682593b7679.png filter=lfs diff=lfs merge=lfs -text
|
| 274 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_217949_9bd516e585bbcdff6f7a.png filter=lfs diff=lfs merge=lfs -text
|
| 275 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_21989_b154bf93fa20eea0c0ef.png filter=lfs diff=lfs merge=lfs -text
|
| 276 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_221499_94ccd8891ffc7f15c842.png filter=lfs diff=lfs merge=lfs -text
|
| 277 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_221499_a00f51f2b65c59d5067b.png filter=lfs diff=lfs merge=lfs -text
|
| 278 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_226469_7ffecb614a4c3fac5112.png filter=lfs diff=lfs merge=lfs -text
|
| 279 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_230019_7ca6d8178534968e1a13.png filter=lfs diff=lfs merge=lfs -text
|
| 280 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_230019_f89a3f30d22d897a38f0.png filter=lfs diff=lfs merge=lfs -text
|
| 281 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_234989_97a7121c80c0d42a202a.png filter=lfs diff=lfs merge=lfs -text
|
| 282 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_238539_decefaa109110d09428c.png filter=lfs diff=lfs merge=lfs -text
|
| 283 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_238539_ed9fa1b05ae4c0af6624.png filter=lfs diff=lfs merge=lfs -text
|
| 284 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_243509_b596922208b0b61259a2.png filter=lfs diff=lfs merge=lfs -text
|
| 285 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_247059_1eeb3c386fdb736b76ab.png filter=lfs diff=lfs merge=lfs -text
|
| 286 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_247059_ad3d141e08c8889338d7.png filter=lfs diff=lfs merge=lfs -text
|
| 287 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_252029_db3975c0f7cc9ae84fdd.png filter=lfs diff=lfs merge=lfs -text
|
| 288 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_25539_3ec7424a01cccb40129a.png filter=lfs diff=lfs merge=lfs -text
|
| 289 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_25539_a53c8a4c4cb3b6ea145b.png filter=lfs diff=lfs merge=lfs -text
|
| 290 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_255579_89c2cc3fe152fe4788c7.png filter=lfs diff=lfs merge=lfs -text
|
| 291 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_255579_cdac32716635003dc430.png filter=lfs diff=lfs merge=lfs -text
|
| 292 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_260549_7b3d83206f31309b83ba.png filter=lfs diff=lfs merge=lfs -text
|
| 293 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_30509_a6ea7a3e2ef9f61fdda6.png filter=lfs diff=lfs merge=lfs -text
|
| 294 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_34059_004fb12b1c5520cbf81d.png filter=lfs diff=lfs merge=lfs -text
|
| 295 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_34059_c93dc1850893c9f450f6.png filter=lfs diff=lfs merge=lfs -text
|
| 296 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_39029_e815c2ed59c4f42a334e.png filter=lfs diff=lfs merge=lfs -text
|
| 297 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_42579_4885e120b108afe05296.png filter=lfs diff=lfs merge=lfs -text
|
| 298 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_42579_dc5dead468f52506d717.png filter=lfs diff=lfs merge=lfs -text
|
| 299 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_47549_306abfe64704febdc648.png filter=lfs diff=lfs merge=lfs -text
|
| 300 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_4949_f43d00f6140fb6a198d6.png filter=lfs diff=lfs merge=lfs -text
|
| 301 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_51099_ac5713510000e6a8f4ff.png filter=lfs diff=lfs merge=lfs -text
|
| 302 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_51099_addd803584dafbfc7505.png filter=lfs diff=lfs merge=lfs -text
|
| 303 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_56069_120ebbc7d05dd4ee388d.png filter=lfs diff=lfs merge=lfs -text
|
| 304 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_59619_5faa4f8c52e847668ceb.png filter=lfs diff=lfs merge=lfs -text
|
| 305 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_59619_dba1282c66d9623125fc.png filter=lfs diff=lfs merge=lfs -text
|
| 306 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_64589_19ec449e1c8575a47bb7.png filter=lfs diff=lfs merge=lfs -text
|
| 307 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_68139_2f543781806f3e35c4e2.png filter=lfs diff=lfs merge=lfs -text
|
| 308 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_68139_e63c2357bd0da97125c9.png filter=lfs diff=lfs merge=lfs -text
|
| 309 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_73109_2e0d28b83fdc7686afa9.png filter=lfs diff=lfs merge=lfs -text
|
| 310 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_76659_2e853c2572d81d1ef3bd.png filter=lfs diff=lfs merge=lfs -text
|
| 311 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_76659_65f1880abef6697aebe4.png filter=lfs diff=lfs merge=lfs -text
|
| 312 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_81629_30f77c7e6d095d92d17d.png filter=lfs diff=lfs merge=lfs -text
|
| 313 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_8499_36b9ec7a9870823f80fc.png filter=lfs diff=lfs merge=lfs -text
|
| 314 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_8499_7cc29a8f4210582bd2bd.png filter=lfs diff=lfs merge=lfs -text
|
| 315 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_85179_66a93390a3d8edd82168.png filter=lfs diff=lfs merge=lfs -text
|
| 316 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_85179_e5dc904959a78c0ea0e2.png filter=lfs diff=lfs merge=lfs -text
|
| 317 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_90149_2fc7a5f71756e9784382.png filter=lfs diff=lfs merge=lfs -text
|
| 318 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_93699_1e88a08760921fbc7dab.png filter=lfs diff=lfs merge=lfs -text
|
| 319 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_93699_1e9a06b6860bfbcdbee2.png filter=lfs diff=lfs merge=lfs -text
|
| 320 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/files/media/images/viz/train_pred_98669_8312c67e94e905510aba.png filter=lfs diff=lfs merge=lfs -text
|
| 321 |
+
pretrain/wandb/run-20260423_024256-2u07ixdc/run-2u07ixdc.wandb filter=lfs diff=lfs merge=lfs -text
|
pretrain/config.json
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"target_dataset": 1,
|
| 3 |
+
"num_available_cameras": 3,
|
| 4 |
+
"used_cameras": [
|
| 5 |
+
true,
|
| 6 |
+
true,
|
| 7 |
+
true
|
| 8 |
+
],
|
| 9 |
+
"num_used_cameras": 3,
|
| 10 |
+
"pixels_per_meter": 4.0,
|
| 11 |
+
"min_x_meter": -32,
|
| 12 |
+
"max_x_meter": 64,
|
| 13 |
+
"min_y_meter": -40,
|
| 14 |
+
"max_y_meter": 40,
|
| 15 |
+
"lidar_width_pixel": 384,
|
| 16 |
+
"lidar_height_pixel": 320,
|
| 17 |
+
"lidar_width_meter": 96,
|
| 18 |
+
"lidar_height_meter": 80,
|
| 19 |
+
"visualize_dataset": false,
|
| 20 |
+
"visualize_failed_scenarios": false,
|
| 21 |
+
"load_bev_3rd_person_images": false,
|
| 22 |
+
"seed": 0,
|
| 23 |
+
"wandb_id": null,
|
| 24 |
+
"wandb_resume": "never",
|
| 25 |
+
"wandb_project_name": "lead_pretrain",
|
| 26 |
+
"description": "An example experiment description.",
|
| 27 |
+
"visualize_training": true,
|
| 28 |
+
"id": "Experiment 1",
|
| 29 |
+
"load_file": null,
|
| 30 |
+
"continue_failed_training": false,
|
| 31 |
+
"epoch_checkpoints_keep": [],
|
| 32 |
+
"use_training_session_cache": true,
|
| 33 |
+
"use_persistent_cache": true,
|
| 34 |
+
"force_rebuild_data_cache": false,
|
| 35 |
+
"carla_cache_path": [
|
| 36 |
+
"1152",
|
| 37 |
+
"384",
|
| 38 |
+
"-32",
|
| 39 |
+
"64",
|
| 40 |
+
"-40",
|
| 41 |
+
"40",
|
| 42 |
+
"True",
|
| 43 |
+
"True",
|
| 44 |
+
"True",
|
| 45 |
+
"True",
|
| 46 |
+
"False",
|
| 47 |
+
"10"
|
| 48 |
+
],
|
| 49 |
+
"training_session_cache_path": "/tmp/divyanshu",
|
| 50 |
+
"carla_root": "data/carla_leaderboard2",
|
| 51 |
+
"carla_data": "data/carla_leaderboard2/data",
|
| 52 |
+
"logdir": "outputs/local_training/pretrain",
|
| 53 |
+
"training_png_compression_level": 6,
|
| 54 |
+
"vehicle_min_num_lidar_points": 1,
|
| 55 |
+
"vehicle_min_num_visible_pixels": 1,
|
| 56 |
+
"pedestrian_min_num_lidar_points": 5,
|
| 57 |
+
"pedestrian_min_num_visible_pixels": 15,
|
| 58 |
+
"parking_vehicle_min_num_lidar_points": 3,
|
| 59 |
+
"parking_vehicle_min_num_visible_pixels": 5,
|
| 60 |
+
"grad_scaler_init_scale": 1024,
|
| 61 |
+
"grad_scaler_growth_factor": 2,
|
| 62 |
+
"grad_scaler_backoff_factor": 0.5,
|
| 63 |
+
"grad_scaler_growth_interval": 256,
|
| 64 |
+
"grad_scaler_max_grad_scale": 65536,
|
| 65 |
+
"sync_batchnorm": false,
|
| 66 |
+
"epochs": 31,
|
| 67 |
+
"batch_size": 128,
|
| 68 |
+
"use_mixed_precision_training": true,
|
| 69 |
+
"need_grad_scaler": false,
|
| 70 |
+
"use_zero_redundancy": false,
|
| 71 |
+
"save_model_checkpoint": true,
|
| 72 |
+
"is_pretraining": true,
|
| 73 |
+
"prefetch_factor": 16,
|
| 74 |
+
"compile": true,
|
| 75 |
+
"channel_last": true,
|
| 76 |
+
"lr": 0.0003,
|
| 77 |
+
"skip_first": 1,
|
| 78 |
+
"skip_last": 1,
|
| 79 |
+
"radar_detection": true,
|
| 80 |
+
"use_radar_detection": true,
|
| 81 |
+
"num_radar_points_per_sensor": 75,
|
| 82 |
+
"num_radar_queries": 20,
|
| 83 |
+
"radar_hidden_dim_tokenizer": 1024,
|
| 84 |
+
"radar_token_dim": 256,
|
| 85 |
+
"radar_tf_dim_ff": 1024,
|
| 86 |
+
"radar_dropout": 0.1,
|
| 87 |
+
"radar_num_heads": 8,
|
| 88 |
+
"radar_num_layers": 4,
|
| 89 |
+
"radar_hidden_dim_decoder": 1024,
|
| 90 |
+
"radar_classification_loss_weight": 1.0,
|
| 91 |
+
"radar_regression_loss_weight": 5.0,
|
| 92 |
+
"num_radar_sensors": 4,
|
| 93 |
+
"force_rebuild_bucket": false,
|
| 94 |
+
"randomize_route_order": false,
|
| 95 |
+
"hold_out_town13_routes": false,
|
| 96 |
+
"bucket_collection_path": "data/carla_leaderboard2/buckets",
|
| 97 |
+
"use_sensor_perburtation": true,
|
| 98 |
+
"use_sensor_perburtation_prob": 0.5,
|
| 99 |
+
"use_color_aug": true,
|
| 100 |
+
"use_color_aug_prob": 0.2,
|
| 101 |
+
"weight_decay": 0.01,
|
| 102 |
+
"use_cosine_annealing_with_restarts": true,
|
| 103 |
+
"use_depth": true,
|
| 104 |
+
"training_used_lidar_steps": 10,
|
| 105 |
+
"min_z": -4,
|
| 106 |
+
"max_z": 4,
|
| 107 |
+
"hist_max_per_pixel": 5,
|
| 108 |
+
"lidar_vert_anchors": 10,
|
| 109 |
+
"lidar_horz_anchors": 12,
|
| 110 |
+
"detect_boxes": true,
|
| 111 |
+
"data_bb_static_types_white_list": [
|
| 112 |
+
"static.prop.constructioncone",
|
| 113 |
+
"static.prop.trafficwarning"
|
| 114 |
+
],
|
| 115 |
+
"bb_confidence_threshold": 0.3,
|
| 116 |
+
"max_num_bbs": 90,
|
| 117 |
+
"num_dir_bins": 12,
|
| 118 |
+
"top_k_center_keypoints": 100,
|
| 119 |
+
"center_net_max_pooling_kernel": 3,
|
| 120 |
+
"bb_input_channel": 64,
|
| 121 |
+
"car_open_door_extra_width": 1.2,
|
| 122 |
+
"num_bb_classes": 8,
|
| 123 |
+
"use_discrete_command": true,
|
| 124 |
+
"discrete_command_dim": 6,
|
| 125 |
+
"use_noisy_tp": false,
|
| 126 |
+
"use_kalman_filter_for_gps": true,
|
| 127 |
+
"use_velocity": true,
|
| 128 |
+
"max_speed": 25.0,
|
| 129 |
+
"use_acceleration": false,
|
| 130 |
+
"max_acceleration": 10.0,
|
| 131 |
+
"use_previous_tp": true,
|
| 132 |
+
"use_next_tp": true,
|
| 133 |
+
"use_tp": true,
|
| 134 |
+
"target_points_normalization_constants": [
|
| 135 |
+
[
|
| 136 |
+
200.0,
|
| 137 |
+
50.0
|
| 138 |
+
]
|
| 139 |
+
],
|
| 140 |
+
"tp_pop_distance": 3.25,
|
| 141 |
+
"use_past_positions": false,
|
| 142 |
+
"use_past_speeds": false,
|
| 143 |
+
"num_past_samples_used": 0,
|
| 144 |
+
"transfuser_num_bev_cross_attention_layers": 6,
|
| 145 |
+
"transfuser_num_bev_cross_attention_heads": 8,
|
| 146 |
+
"transfuser_token_dim": 256,
|
| 147 |
+
"predict_target_speed": true,
|
| 148 |
+
"predict_spatial_path": true,
|
| 149 |
+
"predict_temporal_spatial_waypoints": true,
|
| 150 |
+
"use_planning_decoder": false,
|
| 151 |
+
"target_speed_classes": [
|
| 152 |
+
0.0,
|
| 153 |
+
4.0,
|
| 154 |
+
8.0,
|
| 155 |
+
10.0,
|
| 156 |
+
13.88888888,
|
| 157 |
+
16.0,
|
| 158 |
+
17.77777777,
|
| 159 |
+
20.0
|
| 160 |
+
],
|
| 161 |
+
"target_speeds": [
|
| 162 |
+
0.0,
|
| 163 |
+
4.0,
|
| 164 |
+
8.0,
|
| 165 |
+
10.0,
|
| 166 |
+
13.88888888,
|
| 167 |
+
16.0,
|
| 168 |
+
17.77777777,
|
| 169 |
+
20.0
|
| 170 |
+
],
|
| 171 |
+
"smooth_route": true,
|
| 172 |
+
"num_route_points_smoothing": 20,
|
| 173 |
+
"num_route_points_prediction": 10,
|
| 174 |
+
"max_distance_future_waypoint": 10.0,
|
| 175 |
+
"num_way_points_prediction": 8,
|
| 176 |
+
"waypoints_spacing": 5,
|
| 177 |
+
"horizontal_fov_reduction": 0,
|
| 178 |
+
"crop_height": 0,
|
| 179 |
+
"carla_crop_height_type": 3,
|
| 180 |
+
"image_width_before_camera_subselection": 1152,
|
| 181 |
+
"final_image_width": 1152,
|
| 182 |
+
"final_image_height": 384,
|
| 183 |
+
"img_vert_anchors": 12,
|
| 184 |
+
"img_horz_anchors": 36,
|
| 185 |
+
"freeze_backbone": false,
|
| 186 |
+
"image_architecture": "resnet34",
|
| 187 |
+
"lidar_architecture": "resnet34",
|
| 188 |
+
"LTF": false,
|
| 189 |
+
"block_exp": 4,
|
| 190 |
+
"n_layer": 2,
|
| 191 |
+
"n_head": 4,
|
| 192 |
+
"embd_pdrop": 0.1,
|
| 193 |
+
"resid_pdrop": 0.1,
|
| 194 |
+
"attn_pdrop": 0.1,
|
| 195 |
+
"gpt_linear_layer_init_mean": 0.0,
|
| 196 |
+
"gpt_linear_layer_init_std": 0.02,
|
| 197 |
+
"gpt_layer_norm_init_weight": 1.0,
|
| 198 |
+
"use_semantic": true,
|
| 199 |
+
"num_semantic_classes": 10,
|
| 200 |
+
"perspective_downsample_factor": 1,
|
| 201 |
+
"deconv_channel_num_0": 128,
|
| 202 |
+
"deconv_channel_num_1": 64,
|
| 203 |
+
"deconv_channel_num_2": 32,
|
| 204 |
+
"deconv_scale_factor_0": 4,
|
| 205 |
+
"deconv_scale_factor_1": 8,
|
| 206 |
+
"use_bev_semantic": true,
|
| 207 |
+
"num_bev_semantic_classes": 13,
|
| 208 |
+
"scale_pedestrian_bev_semantic_size": 2.5,
|
| 209 |
+
"pedestrian_bev_min_extent": 0.4,
|
| 210 |
+
"bev_features_chanels": 64,
|
| 211 |
+
"bev_down_sample_factor": 4,
|
| 212 |
+
"bev_upsample_factor": 2,
|
| 213 |
+
"use_carla_data": true,
|
| 214 |
+
"carla_num_samples": -1,
|
| 215 |
+
"use_navsim_data": false,
|
| 216 |
+
"navsim_data_root": "data/navsim_training_cache/navtrain",
|
| 217 |
+
"navsim_num_samples": -1,
|
| 218 |
+
"schedule_carla_num_samples": false,
|
| 219 |
+
"use_waymo_e2e_data": false,
|
| 220 |
+
"waymo_e2e_num_training_samples": -1,
|
| 221 |
+
"waymo_e2e_training_data_root": "data/waymo_open_dataset_end_to_end_camera_v_1_0_0_training",
|
| 222 |
+
"waymo_e2e_val_data_root": "data/waymo_open_dataset_end_to_end_camera_v_1_0_0_val_rfm",
|
| 223 |
+
"waymo_e2e_test_data_root": "data/waymo_open_dataset_end_to_end_camera_v_1_0_0_test_submission",
|
| 224 |
+
"waymo_e2e_subsample_factor": 5,
|
| 225 |
+
"navsim_num_bev_semantic_classes": 7,
|
| 226 |
+
"navsim_num_bb_classes": 1,
|
| 227 |
+
"mixed_data_training": false,
|
| 228 |
+
"carla_leaderboard_mode": true,
|
| 229 |
+
"log_scalars_frequency": 50,
|
| 230 |
+
"log_images_frequency": 5000,
|
| 231 |
+
"log_wandb": true,
|
| 232 |
+
"rank": 0,
|
| 233 |
+
"world_size": 2,
|
| 234 |
+
"local_rank": 0,
|
| 235 |
+
"assigned_cpu_cores": 8,
|
| 236 |
+
"workers_per_cpu_cores": 1
|
| 237 |
+
}
|
pretrain/events.out.tfevents.1776907229.047f74d76241.302462.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:97c10d47d6a5ef08b58cce04d8c5e476f905ac14b0970a42972f71be54061c24
|
| 3 |
+
size 88
|
pretrain/events.out.tfevents.1776907336.047f74d76241.304586.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ac636bf2549798b8d726a71c4cae112e63e9fdaf42dcb5e212a57dd5bc3a3b9f
|
| 3 |
+
size 88
|
pretrain/events.out.tfevents.1776909527.047f74d76241.309272.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b85fb82793494441031552cbefb5b739caa67ee8b4685e2dd1c2e11d49ca2e77
|
| 3 |
+
size 88
|
pretrain/events.out.tfevents.1776910803.047f74d76241.313106.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cb34e3b1361d6b1ff7a9d1a30cee96aefd8a9a7ef60fae96b945d732d001b396
|
| 3 |
+
size 88
|
pretrain/events.out.tfevents.1776911725.047f74d76241.316403.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ada15b4b7432aab26c2e1eaa3f8c5d9ca09ecf1c508982dfe3e110439b030634
|
| 3 |
+
size 88
|
pretrain/events.out.tfevents.1776911987.047f74d76241.317785.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:602ea8a2a69d0167af96978d73b4fe9c138e901c974d101cc2c6b21886a84911
|
| 3 |
+
size 88
|
pretrain/events.out.tfevents.1776912176.047f74d76241.319180.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6bfce6076a261ac3de747d49cfd4e5ebd6efea6fe36d5e5eee47723d08b34a96
|
| 3 |
+
size 22408152
|
pretrain/gradient_steps_skipped_0030.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
0
|
pretrain/model_0030.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8307c5271a43c4ae8b29ab6a8ed45df56c2cd2c54e4b14f23525ba4f9880a2fa
|
| 3 |
+
size 236922078
|
pretrain/optimizer_0030.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ae606607171fbe6e68b690ea9c88b97add6f62f5012fb1da56db6150caa22e61
|
| 3 |
+
size 710111654
|
pretrain/scaler_0030.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e5f881f1527bc81147a2213cbad06ff07ed5b0de3143c791692f655baacfdba5
|
| 3 |
+
size 880
|
pretrain/scheduler_0030.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4fcd4c0770698b3d3d58198f2a3333111cfe06d6e78ce4ea5ad9134a09bede93
|
| 3 |
+
size 1084
|
pretrain/wandb/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
pretrain/wandb/debug.log
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 02:42:56,952 INFO MainThread:319180 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
|
| 2 |
+
2026-04-23 02:42:56,953 INFO MainThread:319180 [wandb_setup.py:_flush():81] Configure stats pid to 319180
|
| 3 |
+
2026-04-23 02:42:56,953 INFO MainThread:319180 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 02:42:56,953 INFO MainThread:319180 [wandb_init.py:setup_run_log_directory():721] Logging user logs to outputs/local_training/pretrain/wandb/run-20260423_024256-2u07ixdc/logs/debug.log
|
| 5 |
+
2026-04-23 02:42:56,953 INFO MainThread:319180 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to outputs/local_training/pretrain/wandb/run-20260423_024256-2u07ixdc/logs/debug-internal.log
|
| 6 |
+
2026-04-23 02:42:56,953 INFO MainThread:319180 [wandb_init.py:init():848] calling init triggers
|
| 7 |
+
2026-04-23 02:42:56,954 INFO MainThread:319180 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'target_dataset': <TargetDataset.CARLA_LEADERBOARD2_3CAMERAS: 1>, 'num_available_cameras': 3, 'used_cameras': [True, True, True], 'num_used_cameras': 3, 'pixels_per_meter': 4.0, 'min_x_meter': -32, 'max_x_meter': 64, 'min_y_meter': -40, 'max_y_meter': 40, 'lidar_width_pixel': 384, 'lidar_height_pixel': 320, 'lidar_width_meter': 96, 'lidar_height_meter': 80, 'visualize_dataset': False, 'visualize_failed_scenarios': False, 'load_bev_3rd_person_images': False, 'seed': 0, 'wandb_id': None, 'wandb_resume': 'never', 'wandb_project_name': 'lead_pretrain', 'description': 'An example experiment description.', 'visualize_training': True, 'id': 'Experiment 1', 'load_file': None, 'continue_failed_training': False, 'epoch_checkpoints_keep': [], 'use_training_session_cache': True, 'use_persistent_cache': True, 'force_rebuild_data_cache': False, 'carla_cache_path': ('1152', '384', '-32', '64', '-40', '40', 'True', 'True', 'True', 'True', 'False', '10'), 'training_session_cache_path': '/tmp/divyanshu', 'carla_root': 'data/carla_leaderboard2', 'carla_data': 'data/carla_leaderboard2/data', 'logdir': 'outputs/local_training/pretrain', 'training_png_compression_level': 6, 'vehicle_min_num_lidar_points': 1, 'vehicle_min_num_visible_pixels': 1, 'pedestrian_min_num_lidar_points': 5, 'pedestrian_min_num_visible_pixels': 15, 'parking_vehicle_min_num_lidar_points': 3, 'parking_vehicle_min_num_visible_pixels': 5, 'grad_scaler_init_scale': 1024, 'grad_scaler_growth_factor': 2, 'grad_scaler_backoff_factor': 0.5, 'grad_scaler_growth_interval': 256, 'grad_scaler_max_grad_scale': 65536, 'sync_batchnorm': False, 'epochs': 31, 'batch_size': 128, 'torch_float_type': torch.bfloat16, 'use_mixed_precision_training': True, 'need_grad_scaler': False, 'use_zero_redundancy': False, 'save_model_checkpoint': True, 'is_pretraining': True, 'prefetch_factor': 16, 'compile': True, 'channel_last': True, 'lr': 0.0003, 'skip_first': 1, 'skip_last': 1, 'radar_detection': True, 'use_radar_detection': True, 'num_radar_points_per_sensor': 75, 'num_radar_queries': 20, 'radar_hidden_dim_tokenizer': 1024, 'radar_token_dim': 256, 'radar_tf_dim_ff': 1024, 'radar_dropout': 0.1, 'radar_num_heads': 8, 'radar_num_layers': 4, 'radar_hidden_dim_decoder': 1024, 'radar_classification_loss_weight': 1.0, 'radar_regression_loss_weight': 5.0, 'num_radar_sensors': 4, 'force_rebuild_bucket': False, 'randomize_route_order': False, 'hold_out_town13_routes': False, 'carla_bucket_collection': <class 'lead.data_buckets.full_pretrain_bucket_collection.FullPretrainBucketCollection'>, 'bucket_collection_path': 'data/carla_leaderboard2/buckets', 'use_sensor_perburtation': True, 'use_sensor_perburtation_prob': 0.5, 'use_color_aug': True, 'use_color_aug_prob': 0.2, 'weight_decay': 0.01, 'use_cosine_annealing_with_restarts': True, 'use_depth': True, 'training_used_lidar_steps': 10, 'min_z': -4, 'max_z': 4, 'hist_max_per_pixel': 5, 'lidar_vert_anchors': 10, 'lidar_horz_anchors': 12, 'detect_boxes': True, 'data_bb_static_types_white_list': ['static.prop.constructioncone', 'static.prop.trafficwarning'], 'bb_confidence_threshold': 0.3, 'max_num_bbs': 90, 'num_dir_bins': 12, 'top_k_center_keypoints': 100, 'center_net_max_pooling_kernel': 3, 'bb_input_channel': 64, 'car_open_door_extra_width': 1.2, 'num_bb_classes': 8, 'use_discrete_command': True, 'discrete_command_dim': 6, 'use_noisy_tp': False, 'use_kalman_filter_for_gps': True, 'use_velocity': True, 'max_speed': 25.0, 'use_acceleration': False, 'max_acceleration': 10.0, 'use_previous_tp': True, 'use_next_tp': True, 'use_tp': True, 'target_points_normalization_constants': [[200.0, 50.0]], 'tp_pop_distance': 3.25, 'use_past_positions': False, 'use_past_speeds': False, 'num_past_samples_used': 0, 'transfuser_num_bev_cross_attention_layers': 6, 'transfuser_num_bev_cross_attention_heads': 8, 'transfuser_token_dim': 256, 'predict_target_speed': True, 'predict_spatial_path': True, 'predict_temporal_spatial_waypoints': True, 'use_planning_decoder': False, 'target_speed_classes': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'target_speeds': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'smooth_route': True, 'num_route_points_smoothing': 20, 'num_route_points_prediction': 10, 'max_distance_future_waypoint': 10.0, 'num_way_points_prediction': 8, 'waypoints_spacing': 5, 'horizontal_fov_reduction': 0, 'crop_height': 0, 'carla_crop_height_type': <CarlaImageCroppingType.NONE: 3>, 'image_width_before_camera_subselection': 1152, 'final_image_width': 1152, 'final_image_height': 384, 'img_vert_anchors': 12, 'img_horz_anchors': 36, 'freeze_backbone': False, 'image_architecture': 'resnet34', 'lidar_architecture': 'resnet34', 'LTF': False, 'block_exp': 4, 'n_layer': 2, 'n_head': 4, 'embd_pdrop': 0.1, 'resid_pdrop': 0.1, 'attn_pdrop': 0.1, 'gpt_linear_layer_init_mean': 0.0, 'gpt_linear_layer_init_std': 0.02, 'gpt_layer_norm_init_weight': 1.0, 'use_semantic': True, 'num_semantic_classes': 10, 'perspective_downsample_factor': 1, 'deconv_channel_num_0': 128, 'deconv_channel_num_1': 64, 'deconv_channel_num_2': 32, 'deconv_scale_factor_0': 4, 'deconv_scale_factor_1': 8, 'use_bev_semantic': True, 'num_bev_semantic_classes': 13, 'scale_pedestrian_bev_semantic_size': 2.5, 'pedestrian_bev_min_extent': 0.4, 'bev_features_chanels': 64, 'bev_down_sample_factor': 4, 'bev_upsample_factor': 2, 'use_carla_data': True, 'carla_num_samples': -1, 'use_navsim_data': False, 'navsim_data_root': 'data/navsim_training_cache/navtrain', 'navsim_num_samples': -1, 'schedule_carla_num_samples': False, 'use_waymo_e2e_data': False, 'waymo_e2e_num_training_samples': -1, 'waymo_e2e_training_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_training', 'waymo_e2e_val_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_val_rfm', 'waymo_e2e_test_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_test_submission', 'waymo_e2e_subsample_factor': 5, 'navsim_num_bev_semantic_classes': 7, 'navsim_num_bb_classes': 1, 'mixed_data_training': False, 'carla_leaderboard_mode': True, 'log_scalars_frequency': 50, 'log_images_frequency': 5000, 'log_wandb': True, 'rank': 0, 'world_size': 2, 'local_rank': 0, 'device': device(type='cuda', index=0), 'assigned_cpu_cores': 8, 'workers_per_cpu_cores': 1, '_loaded_config': {'logdir': 'outputs/local_training/pretrain'}, '_wandb': {}}
|
| 9 |
+
2026-04-23 02:42:56,954 INFO MainThread:319180 [wandb_init.py:init():896] starting backend
|
| 10 |
+
2026-04-23 02:42:57,192 INFO MainThread:319180 [wandb_init.py:init():911] sending inform_init request
|
| 11 |
+
2026-04-23 02:42:57,204 INFO MainThread:319180 [wandb_init.py:init():919] backend started and connected
|
| 12 |
+
2026-04-23 02:42:57,215 INFO MainThread:319180 [wandb_init.py:init():989] updated telemetry
|
| 13 |
+
2026-04-23 02:42:57,269 INFO MainThread:319180 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 02:43:00,183 INFO MainThread:319180 [wandb_init.py:init():1058] starting run threads in backend
|
| 15 |
+
2026-04-23 02:43:00,478 INFO MainThread:319180 [wandb_run.py:_console_start():2542] atexit reg
|
| 16 |
+
2026-04-23 02:43:00,478 INFO MainThread:319180 [wandb_run.py:_redirect():2391] redirect: wrap_raw
|
| 17 |
+
2026-04-23 02:43:00,478 INFO MainThread:319180 [wandb_run.py:_redirect():2460] Wrapping output streams.
|
| 18 |
+
2026-04-23 02:43:00,478 INFO MainThread:319180 [wandb_run.py:_redirect():2483] Redirects installed.
|
| 19 |
+
2026-04-23 02:43:00,483 INFO MainThread:319180 [wandb_init.py:init():1098] run started, returning control to user process
|
| 20 |
+
2026-04-25 15:24:52,806 INFO MainThread:319180 [wandb_run.py:_finish():2309] finishing run divyanshu_div-penn/lead_pretrain/2u07ixdc
|
| 21 |
+
2026-04-25 15:24:52,817 INFO MainThread:319180 [wandb_run.py:_atexit_cleanup():2508] got exitcode: 0
|
| 22 |
+
2026-04-25 15:24:52,821 INFO MainThread:319180 [wandb_run.py:_restore():2490] restore
|
| 23 |
+
2026-04-25 15:24:52,822 INFO MainThread:319180 [wandb_run.py:_restore():2496] restore done
|
| 24 |
+
2026-04-25 15:24:54,064 INFO MainThread:319180 [wandb_run.py:_footer_sync_info():3886] logging synced files
|
pretrain/wandb/run-20260423_012244-al9a0z6q/files/output.log
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[34m[1mwandb[0m: Detected [agents] in use.
|
| 2 |
+
[34m[1mwandb[0m: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
|
| 3 |
+
[34m[1mwandb[0m: For more information, check out the docs at: https://weave-docs.wandb.ai
|
| 4 |
+
2026-04-23 01:22:50 [[94mINFO[0m] [lead/training/logger.py:70] WandB logger will log scalar every 50 steps
|
| 5 |
+
2026-04-23 01:22:50 [[94mINFO[0m] [lead/training/logger.py:73] WandB logger will log images every 5000 steps
|
| 6 |
+
2026-04-23 01:22:54 [[94mINFO[0m] [lead/data_loader/carla_dataset.py:1111] Loaded 1090754 images from 1 buckets.
|
| 7 |
+
0%| | 0/17042 [00:09<?, ?it/s]
|
| 8 |
+
Traceback (most recent call last):
|
| 9 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 363, in <module>
|
| 10 |
+
main()
|
| 11 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 12 |
+
return f(*args, **kwargs)
|
| 13 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 358, in main
|
| 14 |
+
Trainer().train_loop()
|
| 15 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 145, in train_loop
|
| 16 |
+
rfm_score = self.train()
|
| 17 |
+
File "<@beartype(__main__.Trainer.train) at 0x7fe35f661900>", line 12, in train
|
| 18 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 181, in train
|
| 19 |
+
predictions = self.model_wrapper(data=data)
|
| 20 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 21 |
+
return self._call_impl(*args, **kwargs)
|
| 22 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 23 |
+
return forward_call(*args, **kwargs)
|
| 24 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward
|
| 25 |
+
else self._run_ddp_forward(*inputs, **kwargs)
|
| 26 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward
|
| 27 |
+
return self.module(*inputs, **kwargs) # type: ignore[index]
|
| 28 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 29 |
+
return self._call_impl(*args, **kwargs)
|
| 30 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 31 |
+
return forward_call(*args, **kwargs)
|
| 32 |
+
File "<@beartype(lead.tfv6.tfv6.TFv6.forward) at 0x7fe3610f9990>", line 40, in forward
|
| 33 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/tfv6.py", line 117, in forward
|
| 34 |
+
bev_features, image_features = self.backbone(data)
|
| 35 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 36 |
+
return self._call_impl(*args, **kwargs)
|
| 37 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 38 |
+
return forward_call(*args, **kwargs)
|
| 39 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 211, in forward
|
| 40 |
+
return self._forward(rgb, lidar)
|
| 41 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 42 |
+
return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 43 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 44 |
+
out = fn(*args, **kwargs)
|
| 45 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 272, in _forward
|
| 46 |
+
image_features, lidar_features = self.fuse_features(
|
| 47 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 48 |
+
return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 49 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 50 |
+
out = fn(*args, **kwargs)
|
| 51 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 345, in fuse_features
|
| 52 |
+
image_features = image_features + image_features_layer
|
| 53 |
+
torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 216.00 MiB. GPU 0 has a total capacity of 79.18 GiB of which 190.00 MiB is free. Including non-PyTorch memory, this process has 6.69 GiB memory in use. Process 304587 has 450.00 MiB memory in use. Of the allocated memory 5.80 GiB is allocated by PyTorch, and 22.06 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
| 54 |
+
[rank0]: Traceback (most recent call last):
|
| 55 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 363, in <module>
|
| 56 |
+
[rank0]: main()
|
| 57 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 58 |
+
[rank0]: return f(*args, **kwargs)
|
| 59 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 358, in main
|
| 60 |
+
[rank0]: Trainer().train_loop()
|
| 61 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 145, in train_loop
|
| 62 |
+
[rank0]: rfm_score = self.train()
|
| 63 |
+
[rank0]: File "<@beartype(__main__.Trainer.train) at 0x7fe35f661900>", line 12, in train
|
| 64 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 181, in train
|
| 65 |
+
[rank0]: predictions = self.model_wrapper(data=data)
|
| 66 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 67 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 68 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 69 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 70 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward
|
| 71 |
+
[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
|
| 72 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward
|
| 73 |
+
[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
|
| 74 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 75 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 76 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 77 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 78 |
+
[rank0]: File "<@beartype(lead.tfv6.tfv6.TFv6.forward) at 0x7fe3610f9990>", line 40, in forward
|
| 79 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/tfv6.py", line 117, in forward
|
| 80 |
+
[rank0]: bev_features, image_features = self.backbone(data)
|
| 81 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 82 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 83 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 84 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 85 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 211, in forward
|
| 86 |
+
[rank0]: return self._forward(rgb, lidar)
|
| 87 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 88 |
+
[rank0]: return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 89 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 90 |
+
[rank0]: out = fn(*args, **kwargs)
|
| 91 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 272, in _forward
|
| 92 |
+
[rank0]: image_features, lidar_features = self.fuse_features(
|
| 93 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 94 |
+
[rank0]: return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 95 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 96 |
+
[rank0]: out = fn(*args, **kwargs)
|
| 97 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 345, in fuse_features
|
| 98 |
+
[rank0]: image_features = image_features + image_features_layer
|
| 99 |
+
[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 216.00 MiB. GPU 0 has a total capacity of 79.18 GiB of which 190.00 MiB is free. Including non-PyTorch memory, this process has 6.69 GiB memory in use. Process 304587 has 450.00 MiB memory in use. Of the allocated memory 5.80 GiB is allocated by PyTorch, and 22.06 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
| 100 |
+
Exception in thread Thread-3 (_pin_memory_loop):
|
| 101 |
+
Traceback (most recent call last):
|
| 102 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
|
| 103 |
+
self.run()
|
| 104 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/threading.py", line 953, in run
|
| 105 |
+
self._target(*self._args, **self._kwargs)
|
| 106 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 59, in _pin_memory_loop
|
| 107 |
+
do_one_step()
|
| 108 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 35, in do_one_step
|
| 109 |
+
r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL)
|
| 110 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/queues.py", line 122, in get
|
| 111 |
+
return _ForkingPickler.loads(res)
|
| 112 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/multiprocessing/reductions.py", line 541, in rebuild_storage_fd
|
| 113 |
+
fd = df.detach()
|
| 114 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/resource_sharer.py", line 57, in detach
|
| 115 |
+
with _resource_sharer.get_connection(self._id) as conn:
|
| 116 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/resource_sharer.py", line 86, in get_connection
|
| 117 |
+
c = Client(address, authkey=process.current_process().authkey)
|
| 118 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/connection.py", line 502, in Client
|
| 119 |
+
c = SocketClient(address)
|
| 120 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/connection.py", line 630, in SocketClient
|
| 121 |
+
s.connect(address)
|
| 122 |
+
ConnectionRefusedError: [Errno 111] Connection refused
|
| 123 |
+
Exception ignored in atexit callback: <function _MultiProcessingDataLoaderIter._clean_up_worker at 0x7fe38d8a9a20>
|
| 124 |
+
Traceback (most recent call last):
|
| 125 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1598, in _clean_up_worker
|
| 126 |
+
w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)
|
| 127 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/process.py", line 149, in join
|
| 128 |
+
res = self._popen.wait(timeout)
|
| 129 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/popen_fork.py", line 40, in wait
|
| 130 |
+
if not wait([self.sentinel], timeout):
|
| 131 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/connection.py", line 931, in wait
|
| 132 |
+
ready = selector.select(timeout)
|
| 133 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/selectors.py", line 416, in select
|
| 134 |
+
fd_event_list = self._selector.poll(timeout)
|
| 135 |
+
KeyboardInterrupt:
|
pretrain/wandb/run-20260423_012244-al9a0z6q/files/requirements.txt
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
lead==0.0.1
|
| 2 |
+
annotated-doc==0.0.4
|
| 3 |
+
wheel==0.46.3
|
| 4 |
+
pip==26.0.1
|
| 5 |
+
vhacdx==0.0.10
|
| 6 |
+
typer==0.24.1
|
| 7 |
+
zstandard==0.25.0
|
| 8 |
+
pandocfilters==1.5.1
|
| 9 |
+
annotated-types==0.7.0
|
| 10 |
+
propcache==0.4.1
|
| 11 |
+
cffi==2.0.0
|
| 12 |
+
rfc3986-validator==0.1.1
|
| 13 |
+
six==1.16.0
|
| 14 |
+
decorator==5.2.1
|
| 15 |
+
simple_watchdog_timer==0.1.1
|
| 16 |
+
webencodings==0.5.1
|
| 17 |
+
h5py==3.7.0
|
| 18 |
+
cycler==0.12.1
|
| 19 |
+
hf-xet==1.4.3
|
| 20 |
+
pycparser==3.0
|
| 21 |
+
iniconfig==2.3.0
|
| 22 |
+
json5==0.14.0
|
| 23 |
+
argon2-cffi-bindings==25.1.0
|
| 24 |
+
hydra-colorlog==1.2.0
|
| 25 |
+
comm==0.2.3
|
| 26 |
+
dictor==0.1.12
|
| 27 |
+
addict==2.4.0
|
| 28 |
+
jaxtyping==0.3.2
|
| 29 |
+
shellingham==1.5.4
|
| 30 |
+
manifold3d==3.4.1
|
| 31 |
+
blinker==1.9.0
|
| 32 |
+
zipp==3.23.1
|
| 33 |
+
appdirs==1.4.4
|
| 34 |
+
py-spy==0.4.1
|
| 35 |
+
argon2-cffi==25.1.0
|
| 36 |
+
rfc3987-syntax==1.1.0
|
| 37 |
+
absl-py==2.4.0
|
| 38 |
+
tifffile==2025.5.10
|
| 39 |
+
charset-normalizer==3.4.7
|
| 40 |
+
jsonpointer==3.1.1
|
| 41 |
+
terminado==0.18.1
|
| 42 |
+
multidict==6.7.1
|
| 43 |
+
python-json-logger==4.1.0
|
| 44 |
+
ephem==4.1.5
|
| 45 |
+
async-lru==2.3.0
|
| 46 |
+
uri-template==1.3.0
|
| 47 |
+
nvidia-nvtx-cu12==12.4.127
|
| 48 |
+
typing-inspection==0.4.2
|
| 49 |
+
isoduration==20.11.0
|
| 50 |
+
pure_eval==0.2.3
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nest-asyncio==1.6.0
|
| 53 |
+
nodeenv==1.10.0
|
| 54 |
+
typing_extensions==4.15.0
|
| 55 |
+
timm==1.0.19
|
| 56 |
+
Send2Trash==2.1.0
|
| 57 |
+
Jinja2==3.1.6
|
| 58 |
+
omegaconf==2.3.0
|
| 59 |
+
matplotlib==3.10.3
|
| 60 |
+
mdurl==0.1.2
|
| 61 |
+
xmlformatter==0.2.8
|
| 62 |
+
aiosignal==1.4.0
|
| 63 |
+
jupyter_server_terminals==0.5.4
|
| 64 |
+
shapely==2.0.4
|
| 65 |
+
matplotlib-inline==0.2.1
|
| 66 |
+
opentelemetry-semantic-conventions==0.62b0
|
| 67 |
+
attrs==26.1.0
|
| 68 |
+
jupyter_client==8.8.0
|
| 69 |
+
tensorboard-data-server==0.7.2
|
| 70 |
+
prometheus_client==0.25.0
|
| 71 |
+
antlr4-python3-runtime==4.9.3
|
| 72 |
+
itsdangerous==2.2.0
|
| 73 |
+
wadler_lindig==0.1.7
|
| 74 |
+
cachetools==7.0.5
|
| 75 |
+
coverage==7.13.5
|
| 76 |
+
jupyter==1.1.1
|
| 77 |
+
virtualenv==21.2.3
|
| 78 |
+
imgaug==0.4.0
|
| 79 |
+
async-timeout==5.0.1
|
| 80 |
+
wcwidth==0.6.0
|
| 81 |
+
beautifulsoup4==4.14.3
|
| 82 |
+
colorlog==6.10.1
|
| 83 |
+
traitlets==5.14.3
|
| 84 |
+
opencensus-context==0.1.3
|
| 85 |
+
fsspec==2026.3.0
|
| 86 |
+
lxml==5.1.0
|
| 87 |
+
websockets==16.0
|
| 88 |
+
lazy-loader==0.5
|
| 89 |
+
aiohttp==3.13.5
|
| 90 |
+
retrying==1.4.2
|
| 91 |
+
h11==0.16.0
|
| 92 |
+
filterpy==1.4.5
|
| 93 |
+
pillow==10.2.0
|
| 94 |
+
jupyter-lsp==2.3.1
|
| 95 |
+
PyYAML==6.0.3
|
| 96 |
+
pycollada==0.9.3
|
| 97 |
+
pexpect==4.9.0
|
| 98 |
+
embreex==2.17.7.post7
|
| 99 |
+
DracoPy==2.0.0
|
| 100 |
+
torchvision==0.20.0+cu124
|
| 101 |
+
lazrs==0.6.3
|
| 102 |
+
ptyprocess==0.7.0
|
| 103 |
+
python-dateutil==2.9.0.post0
|
| 104 |
+
Pygments==2.20.0
|
| 105 |
+
xxhash==3.6.0
|
| 106 |
+
msgpack==1.1.2
|
| 107 |
+
tomli==2.4.1
|
| 108 |
+
colorful==0.5.8
|
| 109 |
+
jsonpickle==3.0.3
|
| 110 |
+
gitdb==4.0.12
|
| 111 |
+
soupsieve==2.8.3
|
| 112 |
+
docker-pycreds==0.4.0
|
| 113 |
+
wrapt==2.1.2
|
| 114 |
+
nbstripout==0.8.2
|
| 115 |
+
pyquaternion==0.9.9
|
| 116 |
+
exceptiongroup==1.3.1
|
| 117 |
+
psutil==5.9.8
|
| 118 |
+
ipython==8.39.0
|
| 119 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 120 |
+
defusedxml==0.7.1
|
| 121 |
+
jupyterlab_server==2.28.0
|
| 122 |
+
cfgv==3.5.0
|
| 123 |
+
Farama-Notifications==0.0.4
|
| 124 |
+
jsonschema==4.26.0
|
| 125 |
+
rfc3339-validator==0.1.4
|
| 126 |
+
jsonschema-specifications==2025.9.1
|
| 127 |
+
google-auth==2.49.2
|
| 128 |
+
mapbox_earcut==2.0.0
|
| 129 |
+
overrides==7.7.0
|
| 130 |
+
fqdn==1.5.1
|
| 131 |
+
svg.path==7.0
|
| 132 |
+
aiohappyeyeballs==2.6.1
|
| 133 |
+
jupyter-events==0.12.0
|
| 134 |
+
cloudpickle==1.3.0
|
| 135 |
+
jupyter-console==6.6.3
|
| 136 |
+
arrow==1.4.0
|
| 137 |
+
triton==3.1.0
|
| 138 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 139 |
+
jupyterlab_widgets==3.0.16
|
| 140 |
+
py-trees==0.8.3
|
| 141 |
+
torchaudio==2.5.0+cu124
|
| 142 |
+
grpcio==1.80.0
|
| 143 |
+
pydot==4.0.1
|
| 144 |
+
google-api-core==2.30.3
|
| 145 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 146 |
+
notebook_shim==0.2.4
|
| 147 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 148 |
+
bleach==6.3.0
|
| 149 |
+
pyglet==1.5.0
|
| 150 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 151 |
+
tqdm==4.66.1
|
| 152 |
+
lead==0.0.1
|
| 153 |
+
yourdfpy==0.0.60
|
| 154 |
+
diskcache==5.4.0
|
| 155 |
+
rpds-py==0.30.0
|
| 156 |
+
ujson==5.9.0
|
| 157 |
+
tabulate==0.9.0
|
| 158 |
+
tinycss2==1.4.0
|
| 159 |
+
asttokens==3.0.1
|
| 160 |
+
pyproj==3.7.1
|
| 161 |
+
platformdirs==4.9.6
|
| 162 |
+
httpx==0.28.1
|
| 163 |
+
safetensors==0.7.0
|
| 164 |
+
msgspec==0.21.1
|
| 165 |
+
setproctitle==1.3.7
|
| 166 |
+
jupyter_core==5.9.1
|
| 167 |
+
stack-data==0.6.3
|
| 168 |
+
ConfigArgParse==1.7.5
|
| 169 |
+
pytest-cov==7.0.0
|
| 170 |
+
carla==0.9.15
|
| 171 |
+
jupyterlab_pygments==0.3.0
|
| 172 |
+
elementpath==1.3.3
|
| 173 |
+
python-discovery==1.2.2
|
| 174 |
+
pytest==9.0.2
|
| 175 |
+
MarkupSafe==3.0.3
|
| 176 |
+
smmap==5.0.3
|
| 177 |
+
nvidia-nccl-cu12==2.21.5
|
| 178 |
+
threadpoolctl==3.6.0
|
| 179 |
+
fastjsonschema==2.21.2
|
| 180 |
+
identify==2.6.18
|
| 181 |
+
filelock==3.25.2
|
| 182 |
+
yarl==1.23.0
|
| 183 |
+
webcolors==25.10.0
|
| 184 |
+
kiwisolver==1.5.0
|
| 185 |
+
mistune==3.2.0
|
| 186 |
+
ruff==0.12.4
|
| 187 |
+
joblib==1.5.3
|
| 188 |
+
Markdown==3.10.2
|
| 189 |
+
Flask==3.1.1
|
| 190 |
+
llvmlite==0.44.0
|
| 191 |
+
opencv-python==4.6.0.66
|
| 192 |
+
parso==0.8.6
|
| 193 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 194 |
+
ipywidgets==8.1.8
|
| 195 |
+
pyogrio==0.12.1
|
| 196 |
+
markdown-it-py==4.0.0
|
| 197 |
+
pyasn1_modules==0.4.2
|
| 198 |
+
viser==1.0.26
|
| 199 |
+
opentelemetry-exporter-prometheus==0.62b0
|
| 200 |
+
httpcore==1.0.9
|
| 201 |
+
mpmath==1.3.0
|
| 202 |
+
referencing==0.37.0
|
| 203 |
+
PyQt5_sip==12.18.0
|
| 204 |
+
docstring_parser_fork==0.0.14
|
| 205 |
+
smart_open==7.6.0
|
| 206 |
+
nbclient==0.10.4
|
| 207 |
+
executing==2.2.1
|
| 208 |
+
widgetsnbextension==4.0.15
|
| 209 |
+
contourpy==1.3.2
|
| 210 |
+
pre_commit==4.2.0
|
| 211 |
+
urllib3==2.6.3
|
| 212 |
+
distlib==0.4.0
|
| 213 |
+
nvidia-curand-cu12==10.3.5.147
|
| 214 |
+
importlib_metadata==8.7.1
|
| 215 |
+
anyio==4.13.0
|
| 216 |
+
aiohttp-cors==0.8.1
|
| 217 |
+
pyasn1==0.6.3
|
| 218 |
+
pyparsing==3.3.2
|
| 219 |
+
ipykernel==7.2.0
|
| 220 |
+
transforms3d==0.4.1
|
| 221 |
+
hydra-core==1.3.2
|
| 222 |
+
laspy==2.5.4
|
| 223 |
+
proto-plus==1.27.2
|
| 224 |
+
googleapis-common-protos==1.74.0
|
| 225 |
+
pluggy==1.6.0
|
| 226 |
+
narwhals==2.19.0
|
| 227 |
+
opencensus==0.11.4
|
| 228 |
+
notebook==7.5.5
|
| 229 |
+
ImageIO==2.37.3
|
| 230 |
+
nbformat==5.10.4
|
| 231 |
+
Werkzeug==3.1.8
|
| 232 |
+
prompt_toolkit==3.0.52
|
| 233 |
+
tornado==6.5.5
|
| 234 |
+
nbconvert==7.16.6
|
| 235 |
+
basedpyright==1.31.3
|
| 236 |
+
rich==14.3.4
|
| 237 |
+
torchmetrics==0.11.0
|
| 238 |
+
pyzmq==27.1.0
|
| 239 |
+
sentry-sdk==2.58.0
|
| 240 |
+
tzdata==2026.1
|
| 241 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 242 |
+
trimesh==4.11.5
|
| 243 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 244 |
+
easydict==1.13
|
| 245 |
+
rtree==1.4.1
|
| 246 |
+
websocket-client==1.9.0
|
| 247 |
+
pydoclint==0.6.10
|
| 248 |
+
fonttools==4.62.1
|
| 249 |
+
lark==1.3.1
|
| 250 |
+
tensorboard==2.20.0
|
| 251 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 252 |
+
opentelemetry-proto==1.27.0
|
| 253 |
+
opentelemetry-api==1.41.0
|
| 254 |
+
jupyter_server==2.17.0
|
| 255 |
+
opentelemetry-sdk==1.41.0
|
| 256 |
+
geopandas==1.1.3
|
| 257 |
+
debugpy==1.8.20
|
| 258 |
+
gymnasium==1.2.3
|
| 259 |
+
jupyterlab==4.5.6
|
| 260 |
+
scikit-learn==1.7.2
|
| 261 |
+
numba==0.61.2
|
| 262 |
+
PyQt5-Qt5==5.15.18
|
| 263 |
+
future==1.0.0
|
| 264 |
+
networkx==3.4.2
|
| 265 |
+
py123d==0.2.1
|
| 266 |
+
PyQt5==5.15.11
|
| 267 |
+
babel==2.18.0
|
| 268 |
+
scikit-image==0.25.2
|
| 269 |
+
sympy==1.13.1
|
| 270 |
+
dash==4.1.0
|
| 271 |
+
pytz==2026.1.post1
|
| 272 |
+
gym==0.17.2
|
| 273 |
+
pyarrow==23.0.1
|
| 274 |
+
cryptography==46.0.7
|
| 275 |
+
xmlschema==1.0.18
|
| 276 |
+
huggingface_hub==1.10.2
|
| 277 |
+
beartype==0.21.0
|
| 278 |
+
numpy==1.26.0
|
| 279 |
+
setuptools==81.0.0
|
| 280 |
+
scipy==1.14.1
|
| 281 |
+
pandas==2.3.3
|
| 282 |
+
jedi==0.19.2
|
| 283 |
+
plotly==6.7.0
|
| 284 |
+
open3d==0.19.0
|
| 285 |
+
ray==2.54.1
|
| 286 |
+
nodejs-wheel-binaries==24.14.1
|
| 287 |
+
torch==2.5.0+cu124
|
| 288 |
+
certifi==2026.4.22
|
| 289 |
+
pydantic_core==2.46.3
|
| 290 |
+
click==8.3.3
|
| 291 |
+
idna==3.13
|
| 292 |
+
requests==2.33.1
|
| 293 |
+
packaging==26.1
|
| 294 |
+
pydantic==2.13.3
|
| 295 |
+
GitPython==3.1.47
|
| 296 |
+
protobuf==7.34.1
|
| 297 |
+
wandb==0.26.0
|
| 298 |
+
importlib_metadata==8.7.1
|
| 299 |
+
jaraco.context==6.1.0
|
| 300 |
+
jaraco.functools==4.4.0
|
| 301 |
+
more-itertools==10.8.0
|
| 302 |
+
packaging==26.0
|
| 303 |
+
platformdirs==4.4.0
|
| 304 |
+
tomli==2.4.0
|
| 305 |
+
wheel==0.46.3
|
| 306 |
+
zipp==3.23.0
|
| 307 |
+
autocommand==2.2.2
|
| 308 |
+
backports.tarfile==1.2.0
|
| 309 |
+
jaraco.text==4.0.0
|
pretrain/wandb/run-20260423_012244-al9a0z6q/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-174-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-04-23T01:22:44.005409Z",
|
| 5 |
+
"program": "/home/divyanshu/carla/lead/lead/training/train.py",
|
| 6 |
+
"codePath": "lead/training/train.py",
|
| 7 |
+
"codePathLocal": "lead/training/train.py",
|
| 8 |
+
"git": {
|
| 9 |
+
"remote": "https://github.com/kesai-labs/lead.git",
|
| 10 |
+
"commit": "3bb726b2fb815fabbec3dd02a1af38cf224475c0"
|
| 11 |
+
},
|
| 12 |
+
"email": "divyanshu_div@outlook.com",
|
| 13 |
+
"root": "outputs/local_training/pretrain",
|
| 14 |
+
"host": "047f74d76241",
|
| 15 |
+
"executable": "/home/divyanshu/miniconda3/envs/lead/bin/python3",
|
| 16 |
+
"cpu_count": 64,
|
| 17 |
+
"cpu_count_logical": 128,
|
| 18 |
+
"gpu": "NVIDIA H100 PCIe",
|
| 19 |
+
"gpu_count": 10,
|
| 20 |
+
"disk": {
|
| 21 |
+
"/": {
|
| 22 |
+
"total": "242002626064384",
|
| 23 |
+
"used": "97431935545344"
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"memory": {
|
| 27 |
+
"total": "1081814073344"
|
| 28 |
+
},
|
| 29 |
+
"gpu_nvidia": [
|
| 30 |
+
{
|
| 31 |
+
"name": "NVIDIA H100 PCIe",
|
| 32 |
+
"memoryTotal": "85520809984",
|
| 33 |
+
"cudaCores": 14592,
|
| 34 |
+
"architecture": "Hopper",
|
| 35 |
+
"uuid": "GPU-45ed8fd7-e8d9-c54f-035d-dc6c397436ea"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "NVIDIA H100 PCIe",
|
| 39 |
+
"memoryTotal": "85520809984",
|
| 40 |
+
"cudaCores": 14592,
|
| 41 |
+
"architecture": "Hopper",
|
| 42 |
+
"uuid": "GPU-cb08c411-16b7-332b-55ff-1d78f490c2b9"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"name": "NVIDIA H100 PCIe",
|
| 46 |
+
"memoryTotal": "85520809984",
|
| 47 |
+
"cudaCores": 14592,
|
| 48 |
+
"architecture": "Hopper",
|
| 49 |
+
"uuid": "GPU-1deddac1-db6b-f27a-0fe6-de85d14a33f3"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"name": "NVIDIA H100 PCIe",
|
| 53 |
+
"memoryTotal": "85520809984",
|
| 54 |
+
"cudaCores": 14592,
|
| 55 |
+
"architecture": "Hopper",
|
| 56 |
+
"uuid": "GPU-414c781f-8a00-3cf7-282f-1374d279deb8"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"name": "NVIDIA H100 PCIe",
|
| 60 |
+
"memoryTotal": "85520809984",
|
| 61 |
+
"cudaCores": 14592,
|
| 62 |
+
"architecture": "Hopper",
|
| 63 |
+
"uuid": "GPU-4a960d0d-8131-5df1-8ba4-018d8fe3753e"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"name": "NVIDIA H100 PCIe",
|
| 67 |
+
"memoryTotal": "85520809984",
|
| 68 |
+
"cudaCores": 14592,
|
| 69 |
+
"architecture": "Hopper",
|
| 70 |
+
"uuid": "GPU-d6f8e4e5-4ee5-3455-e170-b5b88eba5dc7"
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"name": "NVIDIA H100 PCIe",
|
| 74 |
+
"memoryTotal": "85520809984",
|
| 75 |
+
"cudaCores": 14592,
|
| 76 |
+
"architecture": "Hopper",
|
| 77 |
+
"uuid": "GPU-753cd243-6b33-cd2c-38a3-6fb4a0788d9e"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "NVIDIA H100 PCIe",
|
| 81 |
+
"memoryTotal": "85520809984",
|
| 82 |
+
"cudaCores": 14592,
|
| 83 |
+
"architecture": "Hopper",
|
| 84 |
+
"uuid": "GPU-376fd74b-ce7d-cd02-9397-bc498fe8e1dd"
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"name": "NVIDIA H100 PCIe",
|
| 88 |
+
"memoryTotal": "85520809984",
|
| 89 |
+
"cudaCores": 14592,
|
| 90 |
+
"architecture": "Hopper",
|
| 91 |
+
"uuid": "GPU-d895b8a1-fb4b-7ca2-a6d3-432b67919f99"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"name": "NVIDIA H100 PCIe",
|
| 95 |
+
"memoryTotal": "85520809984",
|
| 96 |
+
"cudaCores": 14592,
|
| 97 |
+
"architecture": "Hopper",
|
| 98 |
+
"uuid": "GPU-af7f65d2-531d-0de1-a15d-8b1dc5070660"
|
| 99 |
+
}
|
| 100 |
+
],
|
| 101 |
+
"cudaVersion": "13.0",
|
| 102 |
+
"writerId": "xe18bvmvzm5i0w6vew13xyi0qk6km4tk"
|
| 103 |
+
}
|
pretrain/wandb/run-20260423_012244-al9a0z6q/logs/debug-core.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T01:22:44.114611774Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpiial6nw_/port-304586.txt","pid":304586,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T01:22:44.116854912Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":304586}
|
| 3 |
+
{"time":"2026-04-23T01:22:44.116773346Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-304586-306115-161398387/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T01:22:44.25307474Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T01:22:44.269945524Z","level":"INFO","msg":"handleInformInit: received","streamId":"al9a0z6q","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T01:22:45.195681939Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"al9a0z6q","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T01:23:18.307088662Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4kgsm1mbcof4"}
|
| 8 |
+
{"time":"2026-04-23T01:25:10.771104236Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
pretrain/wandb/run-20260423_012244-al9a0z6q/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T01:22:44.270262538Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-23T01:22:44.273547699Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
|
| 3 |
+
{"time":"2026-04-23T01:22:45.195251603Z","level":"INFO","msg":"stream: created new stream","id":"al9a0z6q"}
|
| 4 |
+
{"time":"2026-04-23T01:22:45.195393209Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-23T01:22:45.195659054Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-23T01:22:45.195762508Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-04-23T01:22:45.195754389Z","level":"INFO","msg":"writer: started","stream_id":"al9a0z6q"}
|
| 8 |
+
{"time":"2026-04-23T01:22:50.218669103Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-23T01:22:50.509552429Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-23T01:23:05.218865607Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":5,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-04-23T01:23:05.712577084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-23T01:23:20.218781634Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
|
| 13 |
+
{"time":"2026-04-23T01:23:20.558191693Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-04-23T01:23:35.219704024Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
|
| 15 |
+
{"time":"2026-04-23T01:23:35.491459013Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-23T01:23:50.218915227Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
|
| 17 |
+
{"time":"2026-04-23T01:23:50.712401959Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-04-23T01:24:05.219860699Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":6,"console_lines":93}
|
| 19 |
+
{"time":"2026-04-23T01:24:05.644768963Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-04-23T01:24:20.219195198Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":99,"console_lines":23}
|
| 21 |
+
{"time":"2026-04-23T01:24:20.586646009Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 22 |
+
{"time":"2026-04-23T01:24:35.219382249Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
|
| 23 |
+
{"time":"2026-04-23T01:24:35.589645995Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 24 |
+
{"time":"2026-04-23T01:24:50.218775535Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":122,"console_lines":13}
|
| 25 |
+
{"time":"2026-04-23T01:24:50.624727505Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 26 |
+
{"time":"2026-04-23T01:25:05.219280967Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
|
| 27 |
+
{"time":"2026-04-23T01:25:05.690492896Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
pretrain/wandb/run-20260423_012244-al9a0z6q/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 01:22:44,010 INFO MainThread:304586 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
|
| 2 |
+
2026-04-23 01:22:44,010 INFO MainThread:304586 [wandb_setup.py:_flush():81] Configure stats pid to 304586
|
| 3 |
+
2026-04-23 01:22:44,010 INFO MainThread:304586 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 01:22:44,010 INFO MainThread:304586 [wandb_init.py:setup_run_log_directory():721] Logging user logs to outputs/local_training/pretrain/wandb/run-20260423_012244-al9a0z6q/logs/debug.log
|
| 5 |
+
2026-04-23 01:22:44,010 INFO MainThread:304586 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to outputs/local_training/pretrain/wandb/run-20260423_012244-al9a0z6q/logs/debug-internal.log
|
| 6 |
+
2026-04-23 01:22:44,011 INFO MainThread:304586 [wandb_init.py:init():848] calling init triggers
|
| 7 |
+
2026-04-23 01:22:44,011 INFO MainThread:304586 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'target_dataset': <TargetDataset.CARLA_LEADERBOARD2_3CAMERAS: 1>, 'num_available_cameras': 3, 'used_cameras': [True, True, True], 'num_used_cameras': 3, 'pixels_per_meter': 4.0, 'min_x_meter': -32, 'max_x_meter': 64, 'min_y_meter': -40, 'max_y_meter': 40, 'lidar_width_pixel': 384, 'lidar_height_pixel': 320, 'lidar_width_meter': 96, 'lidar_height_meter': 80, 'visualize_dataset': False, 'visualize_failed_scenarios': False, 'load_bev_3rd_person_images': False, 'seed': 0, 'wandb_id': None, 'wandb_resume': 'never', 'wandb_project_name': 'lead_pretrain', 'description': 'An example experiment description.', 'visualize_training': True, 'id': 'Experiment 1', 'load_file': None, 'continue_failed_training': False, 'epoch_checkpoints_keep': [], 'use_training_session_cache': True, 'use_persistent_cache': True, 'force_rebuild_data_cache': False, 'carla_cache_path': ('1152', '384', '-32', '64', '-40', '40', 'True', 'True', 'True', 'True', 'False', '10'), 'training_session_cache_path': '/tmp/divyanshu', 'carla_root': 'data/carla_leaderboard2', 'carla_data': 'data/carla_leaderboard2/data', 'logdir': 'outputs/local_training/pretrain', 'training_png_compression_level': 6, 'vehicle_min_num_lidar_points': 1, 'vehicle_min_num_visible_pixels': 1, 'pedestrian_min_num_lidar_points': 5, 'pedestrian_min_num_visible_pixels': 15, 'parking_vehicle_min_num_lidar_points': 3, 'parking_vehicle_min_num_visible_pixels': 5, 'grad_scaler_init_scale': 1024, 'grad_scaler_growth_factor': 2, 'grad_scaler_backoff_factor': 0.5, 'grad_scaler_growth_interval': 256, 'grad_scaler_max_grad_scale': 65536, 'sync_batchnorm': False, 'epochs': 31, 'batch_size': 64, 'torch_float_type': torch.bfloat16, 'use_mixed_precision_training': True, 'need_grad_scaler': False, 'use_zero_redundancy': False, 'save_model_checkpoint': True, 'is_pretraining': True, 'prefetch_factor': 16, 'compile': True, 'channel_last': True, 'lr': 0.0003, 'skip_first': 1, 'skip_last': 1, 'radar_detection': True, 'use_radar_detection': True, 'num_radar_points_per_sensor': 75, 'num_radar_queries': 20, 'radar_hidden_dim_tokenizer': 1024, 'radar_token_dim': 256, 'radar_tf_dim_ff': 1024, 'radar_dropout': 0.1, 'radar_num_heads': 8, 'radar_num_layers': 4, 'radar_hidden_dim_decoder': 1024, 'radar_classification_loss_weight': 1.0, 'radar_regression_loss_weight': 5.0, 'num_radar_sensors': 4, 'force_rebuild_bucket': False, 'randomize_route_order': False, 'hold_out_town13_routes': False, 'carla_bucket_collection': <class 'lead.data_buckets.full_pretrain_bucket_collection.FullPretrainBucketCollection'>, 'bucket_collection_path': 'data/carla_leaderboard2/buckets', 'use_sensor_perburtation': True, 'use_sensor_perburtation_prob': 0.5, 'use_color_aug': True, 'use_color_aug_prob': 0.2, 'weight_decay': 0.01, 'use_cosine_annealing_with_restarts': True, 'use_depth': True, 'training_used_lidar_steps': 10, 'min_z': -4, 'max_z': 4, 'hist_max_per_pixel': 5, 'lidar_vert_anchors': 10, 'lidar_horz_anchors': 12, 'detect_boxes': True, 'data_bb_static_types_white_list': ['static.prop.constructioncone', 'static.prop.trafficwarning'], 'bb_confidence_threshold': 0.3, 'max_num_bbs': 90, 'num_dir_bins': 12, 'top_k_center_keypoints': 100, 'center_net_max_pooling_kernel': 3, 'bb_input_channel': 64, 'car_open_door_extra_width': 1.2, 'num_bb_classes': 8, 'use_discrete_command': True, 'discrete_command_dim': 6, 'use_noisy_tp': False, 'use_kalman_filter_for_gps': True, 'use_velocity': True, 'max_speed': 25.0, 'use_acceleration': False, 'max_acceleration': 10.0, 'use_previous_tp': True, 'use_next_tp': True, 'use_tp': True, 'target_points_normalization_constants': [[200.0, 50.0]], 'tp_pop_distance': 3.25, 'use_past_positions': False, 'use_past_speeds': False, 'num_past_samples_used': 0, 'transfuser_num_bev_cross_attention_layers': 6, 'transfuser_num_bev_cross_attention_heads': 8, 'transfuser_token_dim': 256, 'predict_target_speed': True, 'predict_spatial_path': True, 'predict_temporal_spatial_waypoints': True, 'use_planning_decoder': False, 'target_speed_classes': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'target_speeds': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'smooth_route': True, 'num_route_points_smoothing': 20, 'num_route_points_prediction': 10, 'max_distance_future_waypoint': 10.0, 'num_way_points_prediction': 8, 'waypoints_spacing': 5, 'horizontal_fov_reduction': 0, 'crop_height': 0, 'carla_crop_height_type': <CarlaImageCroppingType.NONE: 3>, 'image_width_before_camera_subselection': 1152, 'final_image_width': 1152, 'final_image_height': 384, 'img_vert_anchors': 12, 'img_horz_anchors': 36, 'freeze_backbone': False, 'image_architecture': 'resnet34', 'lidar_architecture': 'resnet34', 'LTF': False, 'block_exp': 4, 'n_layer': 2, 'n_head': 4, 'embd_pdrop': 0.1, 'resid_pdrop': 0.1, 'attn_pdrop': 0.1, 'gpt_linear_layer_init_mean': 0.0, 'gpt_linear_layer_init_std': 0.02, 'gpt_layer_norm_init_weight': 1.0, 'use_semantic': True, 'num_semantic_classes': 10, 'perspective_downsample_factor': 1, 'deconv_channel_num_0': 128, 'deconv_channel_num_1': 64, 'deconv_channel_num_2': 32, 'deconv_scale_factor_0': 4, 'deconv_scale_factor_1': 8, 'use_bev_semantic': True, 'num_bev_semantic_classes': 13, 'scale_pedestrian_bev_semantic_size': 2.5, 'pedestrian_bev_min_extent': 0.4, 'bev_features_chanels': 64, 'bev_down_sample_factor': 4, 'bev_upsample_factor': 2, 'use_carla_data': True, 'carla_num_samples': -1, 'use_navsim_data': False, 'navsim_data_root': 'data/navsim_training_cache/navtrain', 'navsim_num_samples': -1, 'schedule_carla_num_samples': False, 'use_waymo_e2e_data': False, 'waymo_e2e_num_training_samples': -1, 'waymo_e2e_training_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_training', 'waymo_e2e_val_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_val_rfm', 'waymo_e2e_test_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_test_submission', 'waymo_e2e_subsample_factor': 5, 'navsim_num_bev_semantic_classes': 7, 'navsim_num_bb_classes': 1, 'mixed_data_training': False, 'carla_leaderboard_mode': True, 'log_scalars_frequency': 50, 'log_images_frequency': 5000, 'log_wandb': True, 'rank': 0, 'world_size': 2, 'local_rank': 0, 'device': device(type='cuda', index=0), 'assigned_cpu_cores': 32, 'workers_per_cpu_cores': 1, '_loaded_config': {'logdir': 'outputs/local_training/pretrain'}, '_wandb': {}}
|
| 9 |
+
2026-04-23 01:22:44,011 INFO MainThread:304586 [wandb_init.py:init():896] starting backend
|
| 10 |
+
2026-04-23 01:22:44,253 INFO MainThread:304586 [wandb_init.py:init():911] sending inform_init request
|
| 11 |
+
2026-04-23 01:22:44,265 INFO MainThread:304586 [wandb_init.py:init():919] backend started and connected
|
| 12 |
+
2026-04-23 01:22:44,282 INFO MainThread:304586 [wandb_init.py:init():989] updated telemetry
|
| 13 |
+
2026-04-23 01:22:44,323 INFO MainThread:304586 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 01:22:46,252 INFO MainThread:304586 [wandb_init.py:init():1058] starting run threads in backend
|
| 15 |
+
2026-04-23 01:22:50,192 INFO MainThread:304586 [wandb_run.py:_console_start():2542] atexit reg
|
| 16 |
+
2026-04-23 01:22:50,192 INFO MainThread:304586 [wandb_run.py:_redirect():2391] redirect: wrap_raw
|
| 17 |
+
2026-04-23 01:22:50,192 INFO MainThread:304586 [wandb_run.py:_redirect():2460] Wrapping output streams.
|
| 18 |
+
2026-04-23 01:22:50,192 INFO MainThread:304586 [wandb_run.py:_redirect():2483] Redirects installed.
|
| 19 |
+
2026-04-23 01:22:50,206 INFO MainThread:304586 [wandb_init.py:init():1098] run started, returning control to user process
|
pretrain/wandb/run-20260423_012244-al9a0z6q/run-al9a0z6q.wandb
ADDED
|
Binary file (98.3 kB). View file
|
|
|
pretrain/wandb/run-20260423_015848-rau44hjb/files/config.yaml
ADDED
|
@@ -0,0 +1,530 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_loaded_config:
|
| 2 |
+
value:
|
| 3 |
+
logdir: outputs/local_training/pretrain
|
| 4 |
+
_wandb:
|
| 5 |
+
value:
|
| 6 |
+
cli_version: 0.26.0
|
| 7 |
+
e:
|
| 8 |
+
o1y2727p9npdbgm7jk2dg618az4wqcxy:
|
| 9 |
+
codePath: lead/training/train.py
|
| 10 |
+
codePathLocal: lead/training/train.py
|
| 11 |
+
cpu_count: 64
|
| 12 |
+
cpu_count_logical: 128
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "242002626064384"
|
| 17 |
+
used: "97432535457792"
|
| 18 |
+
email: divyanshu_div@outlook.com
|
| 19 |
+
executable: /home/divyanshu/miniconda3/envs/lead/bin/python3
|
| 20 |
+
git:
|
| 21 |
+
commit: 3bb726b2fb815fabbec3dd02a1af38cf224475c0
|
| 22 |
+
remote: https://github.com/kesai-labs/lead.git
|
| 23 |
+
gpu: NVIDIA H100 PCIe
|
| 24 |
+
gpu_count: 10
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Hopper
|
| 27 |
+
cudaCores: 14592
|
| 28 |
+
memoryTotal: "85520809984"
|
| 29 |
+
name: NVIDIA H100 PCIe
|
| 30 |
+
uuid: GPU-45ed8fd7-e8d9-c54f-035d-dc6c397436ea
|
| 31 |
+
- architecture: Hopper
|
| 32 |
+
cudaCores: 14592
|
| 33 |
+
memoryTotal: "85520809984"
|
| 34 |
+
name: NVIDIA H100 PCIe
|
| 35 |
+
uuid: GPU-cb08c411-16b7-332b-55ff-1d78f490c2b9
|
| 36 |
+
- architecture: Hopper
|
| 37 |
+
cudaCores: 14592
|
| 38 |
+
memoryTotal: "85520809984"
|
| 39 |
+
name: NVIDIA H100 PCIe
|
| 40 |
+
uuid: GPU-1deddac1-db6b-f27a-0fe6-de85d14a33f3
|
| 41 |
+
- architecture: Hopper
|
| 42 |
+
cudaCores: 14592
|
| 43 |
+
memoryTotal: "85520809984"
|
| 44 |
+
name: NVIDIA H100 PCIe
|
| 45 |
+
uuid: GPU-414c781f-8a00-3cf7-282f-1374d279deb8
|
| 46 |
+
- architecture: Hopper
|
| 47 |
+
cudaCores: 14592
|
| 48 |
+
memoryTotal: "85520809984"
|
| 49 |
+
name: NVIDIA H100 PCIe
|
| 50 |
+
uuid: GPU-4a960d0d-8131-5df1-8ba4-018d8fe3753e
|
| 51 |
+
- architecture: Hopper
|
| 52 |
+
cudaCores: 14592
|
| 53 |
+
memoryTotal: "85520809984"
|
| 54 |
+
name: NVIDIA H100 PCIe
|
| 55 |
+
uuid: GPU-d6f8e4e5-4ee5-3455-e170-b5b88eba5dc7
|
| 56 |
+
- architecture: Hopper
|
| 57 |
+
cudaCores: 14592
|
| 58 |
+
memoryTotal: "85520809984"
|
| 59 |
+
name: NVIDIA H100 PCIe
|
| 60 |
+
uuid: GPU-753cd243-6b33-cd2c-38a3-6fb4a0788d9e
|
| 61 |
+
- architecture: Hopper
|
| 62 |
+
cudaCores: 14592
|
| 63 |
+
memoryTotal: "85520809984"
|
| 64 |
+
name: NVIDIA H100 PCIe
|
| 65 |
+
uuid: GPU-376fd74b-ce7d-cd02-9397-bc498fe8e1dd
|
| 66 |
+
- architecture: Hopper
|
| 67 |
+
cudaCores: 14592
|
| 68 |
+
memoryTotal: "85520809984"
|
| 69 |
+
name: NVIDIA H100 PCIe
|
| 70 |
+
uuid: GPU-d895b8a1-fb4b-7ca2-a6d3-432b67919f99
|
| 71 |
+
- architecture: Hopper
|
| 72 |
+
cudaCores: 14592
|
| 73 |
+
memoryTotal: "85520809984"
|
| 74 |
+
name: NVIDIA H100 PCIe
|
| 75 |
+
uuid: GPU-af7f65d2-531d-0de1-a15d-8b1dc5070660
|
| 76 |
+
host: 047f74d76241
|
| 77 |
+
memory:
|
| 78 |
+
total: "1081814073344"
|
| 79 |
+
os: Linux-5.15.0-174-generic-x86_64-with-glibc2.35
|
| 80 |
+
program: /home/divyanshu/carla/lead/lead/training/train.py
|
| 81 |
+
python: CPython 3.10.20
|
| 82 |
+
root: outputs/local_training/pretrain
|
| 83 |
+
startedAt: "2026-04-23T01:58:48.502634Z"
|
| 84 |
+
writerId: o1y2727p9npdbgm7jk2dg618az4wqcxy
|
| 85 |
+
m: []
|
| 86 |
+
python_version: 3.10.20
|
| 87 |
+
t:
|
| 88 |
+
"1":
|
| 89 |
+
- 1
|
| 90 |
+
- 41
|
| 91 |
+
- 49
|
| 92 |
+
- 63
|
| 93 |
+
"2":
|
| 94 |
+
- 1
|
| 95 |
+
- 41
|
| 96 |
+
- 49
|
| 97 |
+
- 63
|
| 98 |
+
"3":
|
| 99 |
+
- 13
|
| 100 |
+
- 16
|
| 101 |
+
- 41
|
| 102 |
+
"4": 3.10.20
|
| 103 |
+
"5": 0.26.0
|
| 104 |
+
"12": 0.26.0
|
| 105 |
+
"13": linux-x86_64
|
| 106 |
+
LTF:
|
| 107 |
+
value: false
|
| 108 |
+
assigned_cpu_cores:
|
| 109 |
+
value: 32
|
| 110 |
+
attn_pdrop:
|
| 111 |
+
value: 0.1
|
| 112 |
+
batch_size:
|
| 113 |
+
value: 64
|
| 114 |
+
bb_confidence_threshold:
|
| 115 |
+
value: 0.3
|
| 116 |
+
bb_input_channel:
|
| 117 |
+
value: 64
|
| 118 |
+
bev_down_sample_factor:
|
| 119 |
+
value: 4
|
| 120 |
+
bev_features_chanels:
|
| 121 |
+
value: 64
|
| 122 |
+
bev_upsample_factor:
|
| 123 |
+
value: 2
|
| 124 |
+
block_exp:
|
| 125 |
+
value: 4
|
| 126 |
+
bucket_collection_path:
|
| 127 |
+
value: data/carla_leaderboard2/buckets
|
| 128 |
+
car_open_door_extra_width:
|
| 129 |
+
value: 1.2
|
| 130 |
+
carla_bucket_collection:
|
| 131 |
+
value: lead.data_buckets.full_pretrain_bucket_collection.FullPretrainBucketCollection
|
| 132 |
+
carla_cache_path:
|
| 133 |
+
value:
|
| 134 |
+
- "1152"
|
| 135 |
+
- "384"
|
| 136 |
+
- "-32"
|
| 137 |
+
- "64"
|
| 138 |
+
- "-40"
|
| 139 |
+
- "40"
|
| 140 |
+
- "True"
|
| 141 |
+
- "True"
|
| 142 |
+
- "True"
|
| 143 |
+
- "True"
|
| 144 |
+
- "False"
|
| 145 |
+
- "10"
|
| 146 |
+
carla_crop_height_type:
|
| 147 |
+
value: NONE
|
| 148 |
+
carla_data:
|
| 149 |
+
value: data/carla_leaderboard2/data
|
| 150 |
+
carla_leaderboard_mode:
|
| 151 |
+
value: true
|
| 152 |
+
carla_num_samples:
|
| 153 |
+
value: -1
|
| 154 |
+
carla_root:
|
| 155 |
+
value: data/carla_leaderboard2
|
| 156 |
+
center_net_max_pooling_kernel:
|
| 157 |
+
value: 3
|
| 158 |
+
channel_last:
|
| 159 |
+
value: true
|
| 160 |
+
compile:
|
| 161 |
+
value: true
|
| 162 |
+
continue_failed_training:
|
| 163 |
+
value: false
|
| 164 |
+
crop_height:
|
| 165 |
+
value: 0
|
| 166 |
+
data_bb_static_types_white_list:
|
| 167 |
+
value:
|
| 168 |
+
- static.prop.constructioncone
|
| 169 |
+
- static.prop.trafficwarning
|
| 170 |
+
deconv_channel_num_0:
|
| 171 |
+
value: 128
|
| 172 |
+
deconv_channel_num_1:
|
| 173 |
+
value: 64
|
| 174 |
+
deconv_channel_num_2:
|
| 175 |
+
value: 32
|
| 176 |
+
deconv_scale_factor_0:
|
| 177 |
+
value: 4
|
| 178 |
+
deconv_scale_factor_1:
|
| 179 |
+
value: 8
|
| 180 |
+
description:
|
| 181 |
+
value: An example experiment description.
|
| 182 |
+
detect_boxes:
|
| 183 |
+
value: true
|
| 184 |
+
device:
|
| 185 |
+
value: cuda:0
|
| 186 |
+
discrete_command_dim:
|
| 187 |
+
value: 6
|
| 188 |
+
embd_pdrop:
|
| 189 |
+
value: 0.1
|
| 190 |
+
epoch_checkpoints_keep:
|
| 191 |
+
value: []
|
| 192 |
+
epochs:
|
| 193 |
+
value: 31
|
| 194 |
+
final_image_height:
|
| 195 |
+
value: 384
|
| 196 |
+
final_image_width:
|
| 197 |
+
value: 1152
|
| 198 |
+
force_rebuild_bucket:
|
| 199 |
+
value: false
|
| 200 |
+
force_rebuild_data_cache:
|
| 201 |
+
value: false
|
| 202 |
+
freeze_backbone:
|
| 203 |
+
value: false
|
| 204 |
+
gpt_layer_norm_init_weight:
|
| 205 |
+
value: 1
|
| 206 |
+
gpt_linear_layer_init_mean:
|
| 207 |
+
value: 0
|
| 208 |
+
gpt_linear_layer_init_std:
|
| 209 |
+
value: 0.02
|
| 210 |
+
grad_scaler_backoff_factor:
|
| 211 |
+
value: 0.5
|
| 212 |
+
grad_scaler_growth_factor:
|
| 213 |
+
value: 2
|
| 214 |
+
grad_scaler_growth_interval:
|
| 215 |
+
value: 256
|
| 216 |
+
grad_scaler_init_scale:
|
| 217 |
+
value: 1024
|
| 218 |
+
grad_scaler_max_grad_scale:
|
| 219 |
+
value: 65536
|
| 220 |
+
hist_max_per_pixel:
|
| 221 |
+
value: 5
|
| 222 |
+
hold_out_town13_routes:
|
| 223 |
+
value: false
|
| 224 |
+
horizontal_fov_reduction:
|
| 225 |
+
value: 0
|
| 226 |
+
id:
|
| 227 |
+
value: Experiment 1
|
| 228 |
+
image_architecture:
|
| 229 |
+
value: resnet34
|
| 230 |
+
image_width_before_camera_subselection:
|
| 231 |
+
value: 1152
|
| 232 |
+
img_horz_anchors:
|
| 233 |
+
value: 36
|
| 234 |
+
img_vert_anchors:
|
| 235 |
+
value: 12
|
| 236 |
+
is_pretraining:
|
| 237 |
+
value: true
|
| 238 |
+
lidar_architecture:
|
| 239 |
+
value: resnet34
|
| 240 |
+
lidar_height_meter:
|
| 241 |
+
value: 80
|
| 242 |
+
lidar_height_pixel:
|
| 243 |
+
value: 320
|
| 244 |
+
lidar_horz_anchors:
|
| 245 |
+
value: 12
|
| 246 |
+
lidar_vert_anchors:
|
| 247 |
+
value: 10
|
| 248 |
+
lidar_width_meter:
|
| 249 |
+
value: 96
|
| 250 |
+
lidar_width_pixel:
|
| 251 |
+
value: 384
|
| 252 |
+
load_bev_3rd_person_images:
|
| 253 |
+
value: false
|
| 254 |
+
load_file:
|
| 255 |
+
value: null
|
| 256 |
+
local_rank:
|
| 257 |
+
value: 0
|
| 258 |
+
log_images_frequency:
|
| 259 |
+
value: 5000
|
| 260 |
+
log_scalars_frequency:
|
| 261 |
+
value: 50
|
| 262 |
+
log_wandb:
|
| 263 |
+
value: true
|
| 264 |
+
logdir:
|
| 265 |
+
value: outputs/local_training/pretrain
|
| 266 |
+
lr:
|
| 267 |
+
value: 0.0003
|
| 268 |
+
max_acceleration:
|
| 269 |
+
value: 10
|
| 270 |
+
max_distance_future_waypoint:
|
| 271 |
+
value: 10
|
| 272 |
+
max_num_bbs:
|
| 273 |
+
value: 90
|
| 274 |
+
max_speed:
|
| 275 |
+
value: 25
|
| 276 |
+
max_x_meter:
|
| 277 |
+
value: 64
|
| 278 |
+
max_y_meter:
|
| 279 |
+
value: 40
|
| 280 |
+
max_z:
|
| 281 |
+
value: 4
|
| 282 |
+
min_x_meter:
|
| 283 |
+
value: -32
|
| 284 |
+
min_y_meter:
|
| 285 |
+
value: -40
|
| 286 |
+
min_z:
|
| 287 |
+
value: -4
|
| 288 |
+
mixed_data_training:
|
| 289 |
+
value: false
|
| 290 |
+
n_head:
|
| 291 |
+
value: 4
|
| 292 |
+
n_layer:
|
| 293 |
+
value: 2
|
| 294 |
+
navsim_data_root:
|
| 295 |
+
value: data/navsim_training_cache/navtrain
|
| 296 |
+
navsim_num_bb_classes:
|
| 297 |
+
value: 1
|
| 298 |
+
navsim_num_bev_semantic_classes:
|
| 299 |
+
value: 7
|
| 300 |
+
navsim_num_samples:
|
| 301 |
+
value: -1
|
| 302 |
+
need_grad_scaler:
|
| 303 |
+
value: false
|
| 304 |
+
num_available_cameras:
|
| 305 |
+
value: 3
|
| 306 |
+
num_bb_classes:
|
| 307 |
+
value: 8
|
| 308 |
+
num_bev_semantic_classes:
|
| 309 |
+
value: 13
|
| 310 |
+
num_dir_bins:
|
| 311 |
+
value: 12
|
| 312 |
+
num_past_samples_used:
|
| 313 |
+
value: 0
|
| 314 |
+
num_radar_points_per_sensor:
|
| 315 |
+
value: 75
|
| 316 |
+
num_radar_queries:
|
| 317 |
+
value: 20
|
| 318 |
+
num_radar_sensors:
|
| 319 |
+
value: 4
|
| 320 |
+
num_route_points_prediction:
|
| 321 |
+
value: 10
|
| 322 |
+
num_route_points_smoothing:
|
| 323 |
+
value: 20
|
| 324 |
+
num_semantic_classes:
|
| 325 |
+
value: 10
|
| 326 |
+
num_used_cameras:
|
| 327 |
+
value: 3
|
| 328 |
+
num_way_points_prediction:
|
| 329 |
+
value: 8
|
| 330 |
+
parking_vehicle_min_num_lidar_points:
|
| 331 |
+
value: 3
|
| 332 |
+
parking_vehicle_min_num_visible_pixels:
|
| 333 |
+
value: 5
|
| 334 |
+
pedestrian_bev_min_extent:
|
| 335 |
+
value: 0.4
|
| 336 |
+
pedestrian_min_num_lidar_points:
|
| 337 |
+
value: 5
|
| 338 |
+
pedestrian_min_num_visible_pixels:
|
| 339 |
+
value: 15
|
| 340 |
+
perspective_downsample_factor:
|
| 341 |
+
value: 1
|
| 342 |
+
pixels_per_meter:
|
| 343 |
+
value: 4
|
| 344 |
+
predict_spatial_path:
|
| 345 |
+
value: true
|
| 346 |
+
predict_target_speed:
|
| 347 |
+
value: true
|
| 348 |
+
predict_temporal_spatial_waypoints:
|
| 349 |
+
value: true
|
| 350 |
+
prefetch_factor:
|
| 351 |
+
value: 16
|
| 352 |
+
radar_classification_loss_weight:
|
| 353 |
+
value: 1
|
| 354 |
+
radar_detection:
|
| 355 |
+
value: true
|
| 356 |
+
radar_dropout:
|
| 357 |
+
value: 0.1
|
| 358 |
+
radar_hidden_dim_decoder:
|
| 359 |
+
value: 1024
|
| 360 |
+
radar_hidden_dim_tokenizer:
|
| 361 |
+
value: 1024
|
| 362 |
+
radar_num_heads:
|
| 363 |
+
value: 8
|
| 364 |
+
radar_num_layers:
|
| 365 |
+
value: 4
|
| 366 |
+
radar_regression_loss_weight:
|
| 367 |
+
value: 5
|
| 368 |
+
radar_tf_dim_ff:
|
| 369 |
+
value: 1024
|
| 370 |
+
radar_token_dim:
|
| 371 |
+
value: 256
|
| 372 |
+
randomize_route_order:
|
| 373 |
+
value: false
|
| 374 |
+
rank:
|
| 375 |
+
value: 0
|
| 376 |
+
resid_pdrop:
|
| 377 |
+
value: 0.1
|
| 378 |
+
save_model_checkpoint:
|
| 379 |
+
value: true
|
| 380 |
+
scale_pedestrian_bev_semantic_size:
|
| 381 |
+
value: 2.5
|
| 382 |
+
schedule_carla_num_samples:
|
| 383 |
+
value: false
|
| 384 |
+
seed:
|
| 385 |
+
value: 0
|
| 386 |
+
skip_first:
|
| 387 |
+
value: 1
|
| 388 |
+
skip_last:
|
| 389 |
+
value: 1
|
| 390 |
+
smooth_route:
|
| 391 |
+
value: true
|
| 392 |
+
sync_batchnorm:
|
| 393 |
+
value: false
|
| 394 |
+
target_dataset:
|
| 395 |
+
value: CARLA_LEADERBOARD2_3CAMERAS
|
| 396 |
+
target_points_normalization_constants:
|
| 397 |
+
value:
|
| 398 |
+
- - 200
|
| 399 |
+
- 50
|
| 400 |
+
target_speed_classes:
|
| 401 |
+
value:
|
| 402 |
+
- 0
|
| 403 |
+
- 4
|
| 404 |
+
- 8
|
| 405 |
+
- 10
|
| 406 |
+
- 13.88888888
|
| 407 |
+
- 16
|
| 408 |
+
- 17.77777777
|
| 409 |
+
- 20
|
| 410 |
+
target_speeds:
|
| 411 |
+
value:
|
| 412 |
+
- 0
|
| 413 |
+
- 4
|
| 414 |
+
- 8
|
| 415 |
+
- 10
|
| 416 |
+
- 13.88888888
|
| 417 |
+
- 16
|
| 418 |
+
- 17.77777777
|
| 419 |
+
- 20
|
| 420 |
+
top_k_center_keypoints:
|
| 421 |
+
value: 100
|
| 422 |
+
torch_float_type:
|
| 423 |
+
value: torch.bfloat16
|
| 424 |
+
tp_pop_distance:
|
| 425 |
+
value: 3.25
|
| 426 |
+
training_png_compression_level:
|
| 427 |
+
value: 6
|
| 428 |
+
training_session_cache_path:
|
| 429 |
+
value: /tmp/divyanshu
|
| 430 |
+
training_used_lidar_steps:
|
| 431 |
+
value: 10
|
| 432 |
+
transfuser_num_bev_cross_attention_heads:
|
| 433 |
+
value: 8
|
| 434 |
+
transfuser_num_bev_cross_attention_layers:
|
| 435 |
+
value: 6
|
| 436 |
+
transfuser_token_dim:
|
| 437 |
+
value: 256
|
| 438 |
+
use_acceleration:
|
| 439 |
+
value: false
|
| 440 |
+
use_bev_semantic:
|
| 441 |
+
value: true
|
| 442 |
+
use_carla_data:
|
| 443 |
+
value: true
|
| 444 |
+
use_color_aug:
|
| 445 |
+
value: true
|
| 446 |
+
use_color_aug_prob:
|
| 447 |
+
value: 0.2
|
| 448 |
+
use_cosine_annealing_with_restarts:
|
| 449 |
+
value: true
|
| 450 |
+
use_depth:
|
| 451 |
+
value: true
|
| 452 |
+
use_discrete_command:
|
| 453 |
+
value: true
|
| 454 |
+
use_kalman_filter_for_gps:
|
| 455 |
+
value: true
|
| 456 |
+
use_mixed_precision_training:
|
| 457 |
+
value: true
|
| 458 |
+
use_navsim_data:
|
| 459 |
+
value: false
|
| 460 |
+
use_next_tp:
|
| 461 |
+
value: true
|
| 462 |
+
use_noisy_tp:
|
| 463 |
+
value: false
|
| 464 |
+
use_past_positions:
|
| 465 |
+
value: false
|
| 466 |
+
use_past_speeds:
|
| 467 |
+
value: false
|
| 468 |
+
use_persistent_cache:
|
| 469 |
+
value: true
|
| 470 |
+
use_planning_decoder:
|
| 471 |
+
value: false
|
| 472 |
+
use_previous_tp:
|
| 473 |
+
value: true
|
| 474 |
+
use_radar_detection:
|
| 475 |
+
value: true
|
| 476 |
+
use_semantic:
|
| 477 |
+
value: true
|
| 478 |
+
use_sensor_perburtation:
|
| 479 |
+
value: true
|
| 480 |
+
use_sensor_perburtation_prob:
|
| 481 |
+
value: 0.5
|
| 482 |
+
use_tp:
|
| 483 |
+
value: true
|
| 484 |
+
use_training_session_cache:
|
| 485 |
+
value: true
|
| 486 |
+
use_velocity:
|
| 487 |
+
value: true
|
| 488 |
+
use_waymo_e2e_data:
|
| 489 |
+
value: false
|
| 490 |
+
use_zero_redundancy:
|
| 491 |
+
value: false
|
| 492 |
+
used_cameras:
|
| 493 |
+
value:
|
| 494 |
+
- true
|
| 495 |
+
- true
|
| 496 |
+
- true
|
| 497 |
+
vehicle_min_num_lidar_points:
|
| 498 |
+
value: 1
|
| 499 |
+
vehicle_min_num_visible_pixels:
|
| 500 |
+
value: 1
|
| 501 |
+
visualize_dataset:
|
| 502 |
+
value: false
|
| 503 |
+
visualize_failed_scenarios:
|
| 504 |
+
value: false
|
| 505 |
+
visualize_training:
|
| 506 |
+
value: true
|
| 507 |
+
wandb_id:
|
| 508 |
+
value: null
|
| 509 |
+
wandb_project_name:
|
| 510 |
+
value: lead_pretrain
|
| 511 |
+
wandb_resume:
|
| 512 |
+
value: never
|
| 513 |
+
waymo_e2e_num_training_samples:
|
| 514 |
+
value: -1
|
| 515 |
+
waymo_e2e_subsample_factor:
|
| 516 |
+
value: 5
|
| 517 |
+
waymo_e2e_test_data_root:
|
| 518 |
+
value: data/waymo_open_dataset_end_to_end_camera_v_1_0_0_test_submission
|
| 519 |
+
waymo_e2e_training_data_root:
|
| 520 |
+
value: data/waymo_open_dataset_end_to_end_camera_v_1_0_0_training
|
| 521 |
+
waymo_e2e_val_data_root:
|
| 522 |
+
value: data/waymo_open_dataset_end_to_end_camera_v_1_0_0_val_rfm
|
| 523 |
+
waypoints_spacing:
|
| 524 |
+
value: 5
|
| 525 |
+
weight_decay:
|
| 526 |
+
value: 0.01
|
| 527 |
+
workers_per_cpu_cores:
|
| 528 |
+
value: 1
|
| 529 |
+
world_size:
|
| 530 |
+
value: 2
|
pretrain/wandb/run-20260423_015848-rau44hjb/files/output.log
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[34m[1mwandb[0m: Detected [agents] in use.
|
| 2 |
+
[34m[1mwandb[0m: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
|
| 3 |
+
[34m[1mwandb[0m: For more information, check out the docs at: https://weave-docs.wandb.ai
|
| 4 |
+
2026-04-23 01:58:51 [[94mINFO[0m] [lead/training/logger.py:70] WandB logger will log scalar every 50 steps
|
| 5 |
+
2026-04-23 01:58:51 [[94mINFO[0m] [lead/training/logger.py:73] WandB logger will log images every 5000 steps
|
| 6 |
+
2026-04-23 01:58:54 [[94mINFO[0m] [lead/data_loader/carla_dataset.py:1111] Loaded 1090754 images from 1 buckets.
|
| 7 |
+
0%| | 0/17042 [00:00<?, ?it/s]Exception in thread Thread-3 (_pin_memory_loop):
|
| 8 |
+
Traceback (most recent call last):
|
| 9 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
|
| 10 |
+
self.run()
|
| 11 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/threading.py", line 953, in run
|
| 12 |
+
self._target(*self._args, **self._kwargs)
|
| 13 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 59, in _pin_memory_loop
|
| 14 |
+
do_one_step()
|
| 15 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 35, in do_one_step
|
| 16 |
+
r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL)
|
| 17 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/queues.py", line 122, in get
|
| 18 |
+
return _ForkingPickler.loads(res)
|
| 19 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/multiprocessing/reductions.py", line 541, in rebuild_storage_fd
|
| 20 |
+
fd = df.detach()
|
| 21 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/resource_sharer.py", line 57, in detach
|
| 22 |
+
with _resource_sharer.get_connection(self._id) as conn:
|
| 23 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/resource_sharer.py", line 86, in get_connection
|
| 24 |
+
c = Client(address, authkey=process.current_process().authkey)
|
| 25 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/connection.py", line 508, in Client
|
| 26 |
+
answer_challenge(c, authkey)
|
| 27 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/connection.py", line 752, in answer_challenge
|
| 28 |
+
message = connection.recv_bytes(256) # reject large message
|
| 29 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/connection.py", line 216, in recv_bytes
|
| 30 |
+
buf = self._recv_bytes(maxlength)
|
| 31 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/connection.py", line 414, in _recv_bytes
|
| 32 |
+
buf = self._recv(4)
|
| 33 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/multiprocessing/connection.py", line 379, in _recv
|
| 34 |
+
chunk = read(handle, remaining)
|
| 35 |
+
ConnectionResetError: [Errno 104] Connection reset by peer
|
| 36 |
+
0%| | 0/17042 [00:14<?, ?it/s]
|
| 37 |
+
Traceback (most recent call last):
|
| 38 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 363, in <module>
|
| 39 |
+
main()
|
| 40 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 41 |
+
return f(*args, **kwargs)
|
| 42 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 358, in main
|
| 43 |
+
Trainer().train_loop()
|
| 44 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 145, in train_loop
|
| 45 |
+
rfm_score = self.train()
|
| 46 |
+
File "<@beartype(__main__.Trainer.train) at 0x7fd41aab32e0>", line 12, in train
|
| 47 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 181, in train
|
| 48 |
+
predictions = self.model_wrapper(data=data)
|
| 49 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 50 |
+
return self._call_impl(*args, **kwargs)
|
| 51 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 52 |
+
return forward_call(*args, **kwargs)
|
| 53 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward
|
| 54 |
+
else self._run_ddp_forward(*inputs, **kwargs)
|
| 55 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward
|
| 56 |
+
return self.module(*inputs, **kwargs) # type: ignore[index]
|
| 57 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 58 |
+
return self._call_impl(*args, **kwargs)
|
| 59 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 60 |
+
return forward_call(*args, **kwargs)
|
| 61 |
+
File "<@beartype(lead.tfv6.tfv6.TFv6.forward) at 0x7fd41c52f520>", line 40, in forward
|
| 62 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/tfv6.py", line 117, in forward
|
| 63 |
+
bev_features, image_features = self.backbone(data)
|
| 64 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 65 |
+
return self._call_impl(*args, **kwargs)
|
| 66 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 67 |
+
return forward_call(*args, **kwargs)
|
| 68 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 211, in forward
|
| 69 |
+
return self._forward(rgb, lidar)
|
| 70 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 71 |
+
return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 72 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 73 |
+
out = fn(*args, **kwargs)
|
| 74 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 272, in _forward
|
| 75 |
+
image_features, lidar_features = self.fuse_features(
|
| 76 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 77 |
+
return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 78 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 79 |
+
out = fn(*args, **kwargs)
|
| 80 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 324, in fuse_features
|
| 81 |
+
image_features_layer, lidar_features_layer = self.transformers[layer_idx](
|
| 82 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 83 |
+
return self._call_impl(*args, **kwargs)
|
| 84 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 85 |
+
return forward_call(*args, **kwargs)
|
| 86 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 87 |
+
return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 88 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 89 |
+
out = fn(*args, **kwargs)
|
| 90 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 450, in forward
|
| 91 |
+
x = self.blocks(x) # (B, an * T, C)
|
| 92 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 93 |
+
return self._call_impl(*args, **kwargs)
|
| 94 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 95 |
+
return forward_call(*args, **kwargs)
|
| 96 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward
|
| 97 |
+
input = module(input)
|
| 98 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 99 |
+
return self._call_impl(*args, **kwargs)
|
| 100 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 101 |
+
return forward_call(*args, **kwargs)
|
| 102 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 103 |
+
return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 104 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 105 |
+
out = fn(*args, **kwargs)
|
| 106 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 520, in forward
|
| 107 |
+
x = x + self.attn(self.ln1(x))
|
| 108 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 109 |
+
return self._call_impl(*args, **kwargs)
|
| 110 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 111 |
+
return forward_call(*args, **kwargs)
|
| 112 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 113 |
+
return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 114 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 115 |
+
out = fn(*args, **kwargs)
|
| 116 |
+
File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 595, in forward
|
| 117 |
+
y = torch.nn.functional.scaled_dot_product_attention(
|
| 118 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/utils/data/_utils/signal_handling.py", line 73, in handler
|
| 119 |
+
_error_if_any_worker_fails()
|
| 120 |
+
RuntimeError: DataLoader worker (pid 310905) is killed by signal: Bus error. It is possible that dataloader's workers are out of shared memory. Please try to raise your shared memory limit.
|
| 121 |
+
[rank0]: Traceback (most recent call last):
|
| 122 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 363, in <module>
|
| 123 |
+
[rank0]: main()
|
| 124 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 125 |
+
[rank0]: return f(*args, **kwargs)
|
| 126 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 358, in main
|
| 127 |
+
[rank0]: Trainer().train_loop()
|
| 128 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 145, in train_loop
|
| 129 |
+
[rank0]: rfm_score = self.train()
|
| 130 |
+
[rank0]: File "<@beartype(__main__.Trainer.train) at 0x7fd41aab32e0>", line 12, in train
|
| 131 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 181, in train
|
| 132 |
+
[rank0]: predictions = self.model_wrapper(data=data)
|
| 133 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 134 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 135 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 136 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 137 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward
|
| 138 |
+
[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
|
| 139 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward
|
| 140 |
+
[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
|
| 141 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 142 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 143 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 144 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 145 |
+
[rank0]: File "<@beartype(lead.tfv6.tfv6.TFv6.forward) at 0x7fd41c52f520>", line 40, in forward
|
| 146 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/tfv6.py", line 117, in forward
|
| 147 |
+
[rank0]: bev_features, image_features = self.backbone(data)
|
| 148 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 149 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 150 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 151 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 152 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 211, in forward
|
| 153 |
+
[rank0]: return self._forward(rgb, lidar)
|
| 154 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 155 |
+
[rank0]: return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 156 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 157 |
+
[rank0]: out = fn(*args, **kwargs)
|
| 158 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 272, in _forward
|
| 159 |
+
[rank0]: image_features, lidar_features = self.fuse_features(
|
| 160 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 161 |
+
[rank0]: return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 162 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 163 |
+
[rank0]: out = fn(*args, **kwargs)
|
| 164 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 324, in fuse_features
|
| 165 |
+
[rank0]: image_features_layer, lidar_features_layer = self.transformers[layer_idx](
|
| 166 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 167 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 168 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 169 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 170 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 171 |
+
[rank0]: return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 172 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 173 |
+
[rank0]: out = fn(*args, **kwargs)
|
| 174 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 450, in forward
|
| 175 |
+
[rank0]: x = self.blocks(x) # (B, an * T, C)
|
| 176 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 177 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 178 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 179 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 180 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward
|
| 181 |
+
[rank0]: input = module(input)
|
| 182 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 183 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 184 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 185 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 186 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 187 |
+
[rank0]: return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 188 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 189 |
+
[rank0]: out = fn(*args, **kwargs)
|
| 190 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 520, in forward
|
| 191 |
+
[rank0]: x = x + self.attn(self.ln1(x))
|
| 192 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
| 193 |
+
[rank0]: return self._call_impl(*args, **kwargs)
|
| 194 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
| 195 |
+
[rank0]: return forward_call(*args, **kwargs)
|
| 196 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
|
| 197 |
+
[rank0]: return wrapped_fn_impl(args, kwargs, bound, memos)
|
| 198 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
|
| 199 |
+
[rank0]: out = fn(*args, **kwargs)
|
| 200 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/tfv6/transfuser_backbone.py", line 595, in forward
|
| 201 |
+
[rank0]: y = torch.nn.functional.scaled_dot_product_attention(
|
| 202 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/utils/data/_utils/signal_handling.py", line 73, in handler
|
| 203 |
+
[rank0]: _error_if_any_worker_fails()
|
| 204 |
+
[rank0]: RuntimeError: DataLoader worker (pid 310905) is killed by signal: Bus error. It is possible that dataloader's workers are out of shared memory. Please try to raise your shared memory limit.
|
pretrain/wandb/run-20260423_015848-rau44hjb/files/requirements.txt
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
lead==0.0.1
|
| 2 |
+
annotated-doc==0.0.4
|
| 3 |
+
wheel==0.46.3
|
| 4 |
+
pip==26.0.1
|
| 5 |
+
vhacdx==0.0.10
|
| 6 |
+
typer==0.24.1
|
| 7 |
+
zstandard==0.25.0
|
| 8 |
+
pandocfilters==1.5.1
|
| 9 |
+
annotated-types==0.7.0
|
| 10 |
+
propcache==0.4.1
|
| 11 |
+
cffi==2.0.0
|
| 12 |
+
rfc3986-validator==0.1.1
|
| 13 |
+
six==1.16.0
|
| 14 |
+
decorator==5.2.1
|
| 15 |
+
simple_watchdog_timer==0.1.1
|
| 16 |
+
webencodings==0.5.1
|
| 17 |
+
h5py==3.7.0
|
| 18 |
+
cycler==0.12.1
|
| 19 |
+
hf-xet==1.4.3
|
| 20 |
+
pycparser==3.0
|
| 21 |
+
iniconfig==2.3.0
|
| 22 |
+
json5==0.14.0
|
| 23 |
+
argon2-cffi-bindings==25.1.0
|
| 24 |
+
hydra-colorlog==1.2.0
|
| 25 |
+
comm==0.2.3
|
| 26 |
+
dictor==0.1.12
|
| 27 |
+
addict==2.4.0
|
| 28 |
+
jaxtyping==0.3.2
|
| 29 |
+
shellingham==1.5.4
|
| 30 |
+
manifold3d==3.4.1
|
| 31 |
+
blinker==1.9.0
|
| 32 |
+
zipp==3.23.1
|
| 33 |
+
appdirs==1.4.4
|
| 34 |
+
py-spy==0.4.1
|
| 35 |
+
argon2-cffi==25.1.0
|
| 36 |
+
rfc3987-syntax==1.1.0
|
| 37 |
+
absl-py==2.4.0
|
| 38 |
+
tifffile==2025.5.10
|
| 39 |
+
charset-normalizer==3.4.7
|
| 40 |
+
jsonpointer==3.1.1
|
| 41 |
+
terminado==0.18.1
|
| 42 |
+
multidict==6.7.1
|
| 43 |
+
python-json-logger==4.1.0
|
| 44 |
+
ephem==4.1.5
|
| 45 |
+
async-lru==2.3.0
|
| 46 |
+
uri-template==1.3.0
|
| 47 |
+
nvidia-nvtx-cu12==12.4.127
|
| 48 |
+
typing-inspection==0.4.2
|
| 49 |
+
isoduration==20.11.0
|
| 50 |
+
pure_eval==0.2.3
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nest-asyncio==1.6.0
|
| 53 |
+
nodeenv==1.10.0
|
| 54 |
+
typing_extensions==4.15.0
|
| 55 |
+
timm==1.0.19
|
| 56 |
+
Send2Trash==2.1.0
|
| 57 |
+
Jinja2==3.1.6
|
| 58 |
+
omegaconf==2.3.0
|
| 59 |
+
matplotlib==3.10.3
|
| 60 |
+
mdurl==0.1.2
|
| 61 |
+
xmlformatter==0.2.8
|
| 62 |
+
aiosignal==1.4.0
|
| 63 |
+
jupyter_server_terminals==0.5.4
|
| 64 |
+
shapely==2.0.4
|
| 65 |
+
matplotlib-inline==0.2.1
|
| 66 |
+
opentelemetry-semantic-conventions==0.62b0
|
| 67 |
+
attrs==26.1.0
|
| 68 |
+
jupyter_client==8.8.0
|
| 69 |
+
tensorboard-data-server==0.7.2
|
| 70 |
+
prometheus_client==0.25.0
|
| 71 |
+
antlr4-python3-runtime==4.9.3
|
| 72 |
+
itsdangerous==2.2.0
|
| 73 |
+
wadler_lindig==0.1.7
|
| 74 |
+
cachetools==7.0.5
|
| 75 |
+
coverage==7.13.5
|
| 76 |
+
jupyter==1.1.1
|
| 77 |
+
virtualenv==21.2.3
|
| 78 |
+
imgaug==0.4.0
|
| 79 |
+
async-timeout==5.0.1
|
| 80 |
+
wcwidth==0.6.0
|
| 81 |
+
beautifulsoup4==4.14.3
|
| 82 |
+
colorlog==6.10.1
|
| 83 |
+
traitlets==5.14.3
|
| 84 |
+
opencensus-context==0.1.3
|
| 85 |
+
fsspec==2026.3.0
|
| 86 |
+
lxml==5.1.0
|
| 87 |
+
websockets==16.0
|
| 88 |
+
lazy-loader==0.5
|
| 89 |
+
aiohttp==3.13.5
|
| 90 |
+
retrying==1.4.2
|
| 91 |
+
h11==0.16.0
|
| 92 |
+
filterpy==1.4.5
|
| 93 |
+
pillow==10.2.0
|
| 94 |
+
jupyter-lsp==2.3.1
|
| 95 |
+
PyYAML==6.0.3
|
| 96 |
+
pycollada==0.9.3
|
| 97 |
+
pexpect==4.9.0
|
| 98 |
+
embreex==2.17.7.post7
|
| 99 |
+
DracoPy==2.0.0
|
| 100 |
+
torchvision==0.20.0+cu124
|
| 101 |
+
lazrs==0.6.3
|
| 102 |
+
ptyprocess==0.7.0
|
| 103 |
+
python-dateutil==2.9.0.post0
|
| 104 |
+
Pygments==2.20.0
|
| 105 |
+
xxhash==3.6.0
|
| 106 |
+
msgpack==1.1.2
|
| 107 |
+
tomli==2.4.1
|
| 108 |
+
colorful==0.5.8
|
| 109 |
+
jsonpickle==3.0.3
|
| 110 |
+
gitdb==4.0.12
|
| 111 |
+
soupsieve==2.8.3
|
| 112 |
+
docker-pycreds==0.4.0
|
| 113 |
+
wrapt==2.1.2
|
| 114 |
+
nbstripout==0.8.2
|
| 115 |
+
pyquaternion==0.9.9
|
| 116 |
+
exceptiongroup==1.3.1
|
| 117 |
+
psutil==5.9.8
|
| 118 |
+
ipython==8.39.0
|
| 119 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 120 |
+
defusedxml==0.7.1
|
| 121 |
+
jupyterlab_server==2.28.0
|
| 122 |
+
cfgv==3.5.0
|
| 123 |
+
Farama-Notifications==0.0.4
|
| 124 |
+
jsonschema==4.26.0
|
| 125 |
+
rfc3339-validator==0.1.4
|
| 126 |
+
jsonschema-specifications==2025.9.1
|
| 127 |
+
google-auth==2.49.2
|
| 128 |
+
mapbox_earcut==2.0.0
|
| 129 |
+
overrides==7.7.0
|
| 130 |
+
fqdn==1.5.1
|
| 131 |
+
svg.path==7.0
|
| 132 |
+
aiohappyeyeballs==2.6.1
|
| 133 |
+
jupyter-events==0.12.0
|
| 134 |
+
cloudpickle==1.3.0
|
| 135 |
+
jupyter-console==6.6.3
|
| 136 |
+
arrow==1.4.0
|
| 137 |
+
triton==3.1.0
|
| 138 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 139 |
+
jupyterlab_widgets==3.0.16
|
| 140 |
+
py-trees==0.8.3
|
| 141 |
+
torchaudio==2.5.0+cu124
|
| 142 |
+
grpcio==1.80.0
|
| 143 |
+
pydot==4.0.1
|
| 144 |
+
google-api-core==2.30.3
|
| 145 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 146 |
+
notebook_shim==0.2.4
|
| 147 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 148 |
+
bleach==6.3.0
|
| 149 |
+
pyglet==1.5.0
|
| 150 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 151 |
+
tqdm==4.66.1
|
| 152 |
+
lead==0.0.1
|
| 153 |
+
yourdfpy==0.0.60
|
| 154 |
+
diskcache==5.4.0
|
| 155 |
+
rpds-py==0.30.0
|
| 156 |
+
ujson==5.9.0
|
| 157 |
+
tabulate==0.9.0
|
| 158 |
+
tinycss2==1.4.0
|
| 159 |
+
asttokens==3.0.1
|
| 160 |
+
pyproj==3.7.1
|
| 161 |
+
platformdirs==4.9.6
|
| 162 |
+
httpx==0.28.1
|
| 163 |
+
safetensors==0.7.0
|
| 164 |
+
msgspec==0.21.1
|
| 165 |
+
setproctitle==1.3.7
|
| 166 |
+
jupyter_core==5.9.1
|
| 167 |
+
stack-data==0.6.3
|
| 168 |
+
ConfigArgParse==1.7.5
|
| 169 |
+
pytest-cov==7.0.0
|
| 170 |
+
carla==0.9.15
|
| 171 |
+
jupyterlab_pygments==0.3.0
|
| 172 |
+
elementpath==1.3.3
|
| 173 |
+
python-discovery==1.2.2
|
| 174 |
+
pytest==9.0.2
|
| 175 |
+
MarkupSafe==3.0.3
|
| 176 |
+
smmap==5.0.3
|
| 177 |
+
nvidia-nccl-cu12==2.21.5
|
| 178 |
+
threadpoolctl==3.6.0
|
| 179 |
+
fastjsonschema==2.21.2
|
| 180 |
+
identify==2.6.18
|
| 181 |
+
filelock==3.25.2
|
| 182 |
+
yarl==1.23.0
|
| 183 |
+
webcolors==25.10.0
|
| 184 |
+
kiwisolver==1.5.0
|
| 185 |
+
mistune==3.2.0
|
| 186 |
+
ruff==0.12.4
|
| 187 |
+
joblib==1.5.3
|
| 188 |
+
Markdown==3.10.2
|
| 189 |
+
Flask==3.1.1
|
| 190 |
+
llvmlite==0.44.0
|
| 191 |
+
opencv-python==4.6.0.66
|
| 192 |
+
parso==0.8.6
|
| 193 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 194 |
+
ipywidgets==8.1.8
|
| 195 |
+
pyogrio==0.12.1
|
| 196 |
+
markdown-it-py==4.0.0
|
| 197 |
+
pyasn1_modules==0.4.2
|
| 198 |
+
viser==1.0.26
|
| 199 |
+
opentelemetry-exporter-prometheus==0.62b0
|
| 200 |
+
httpcore==1.0.9
|
| 201 |
+
mpmath==1.3.0
|
| 202 |
+
referencing==0.37.0
|
| 203 |
+
PyQt5_sip==12.18.0
|
| 204 |
+
docstring_parser_fork==0.0.14
|
| 205 |
+
smart_open==7.6.0
|
| 206 |
+
nbclient==0.10.4
|
| 207 |
+
executing==2.2.1
|
| 208 |
+
widgetsnbextension==4.0.15
|
| 209 |
+
contourpy==1.3.2
|
| 210 |
+
pre_commit==4.2.0
|
| 211 |
+
urllib3==2.6.3
|
| 212 |
+
distlib==0.4.0
|
| 213 |
+
nvidia-curand-cu12==10.3.5.147
|
| 214 |
+
importlib_metadata==8.7.1
|
| 215 |
+
anyio==4.13.0
|
| 216 |
+
aiohttp-cors==0.8.1
|
| 217 |
+
pyasn1==0.6.3
|
| 218 |
+
pyparsing==3.3.2
|
| 219 |
+
ipykernel==7.2.0
|
| 220 |
+
transforms3d==0.4.1
|
| 221 |
+
hydra-core==1.3.2
|
| 222 |
+
laspy==2.5.4
|
| 223 |
+
proto-plus==1.27.2
|
| 224 |
+
googleapis-common-protos==1.74.0
|
| 225 |
+
pluggy==1.6.0
|
| 226 |
+
narwhals==2.19.0
|
| 227 |
+
opencensus==0.11.4
|
| 228 |
+
notebook==7.5.5
|
| 229 |
+
ImageIO==2.37.3
|
| 230 |
+
nbformat==5.10.4
|
| 231 |
+
Werkzeug==3.1.8
|
| 232 |
+
prompt_toolkit==3.0.52
|
| 233 |
+
tornado==6.5.5
|
| 234 |
+
nbconvert==7.16.6
|
| 235 |
+
basedpyright==1.31.3
|
| 236 |
+
rich==14.3.4
|
| 237 |
+
torchmetrics==0.11.0
|
| 238 |
+
pyzmq==27.1.0
|
| 239 |
+
sentry-sdk==2.58.0
|
| 240 |
+
tzdata==2026.1
|
| 241 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 242 |
+
trimesh==4.11.5
|
| 243 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 244 |
+
easydict==1.13
|
| 245 |
+
rtree==1.4.1
|
| 246 |
+
websocket-client==1.9.0
|
| 247 |
+
pydoclint==0.6.10
|
| 248 |
+
fonttools==4.62.1
|
| 249 |
+
lark==1.3.1
|
| 250 |
+
tensorboard==2.20.0
|
| 251 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 252 |
+
opentelemetry-proto==1.27.0
|
| 253 |
+
opentelemetry-api==1.41.0
|
| 254 |
+
jupyter_server==2.17.0
|
| 255 |
+
opentelemetry-sdk==1.41.0
|
| 256 |
+
geopandas==1.1.3
|
| 257 |
+
debugpy==1.8.20
|
| 258 |
+
gymnasium==1.2.3
|
| 259 |
+
jupyterlab==4.5.6
|
| 260 |
+
scikit-learn==1.7.2
|
| 261 |
+
numba==0.61.2
|
| 262 |
+
PyQt5-Qt5==5.15.18
|
| 263 |
+
future==1.0.0
|
| 264 |
+
networkx==3.4.2
|
| 265 |
+
py123d==0.2.1
|
| 266 |
+
PyQt5==5.15.11
|
| 267 |
+
babel==2.18.0
|
| 268 |
+
scikit-image==0.25.2
|
| 269 |
+
sympy==1.13.1
|
| 270 |
+
dash==4.1.0
|
| 271 |
+
pytz==2026.1.post1
|
| 272 |
+
gym==0.17.2
|
| 273 |
+
pyarrow==23.0.1
|
| 274 |
+
cryptography==46.0.7
|
| 275 |
+
xmlschema==1.0.18
|
| 276 |
+
huggingface_hub==1.10.2
|
| 277 |
+
beartype==0.21.0
|
| 278 |
+
numpy==1.26.0
|
| 279 |
+
setuptools==81.0.0
|
| 280 |
+
scipy==1.14.1
|
| 281 |
+
pandas==2.3.3
|
| 282 |
+
jedi==0.19.2
|
| 283 |
+
plotly==6.7.0
|
| 284 |
+
open3d==0.19.0
|
| 285 |
+
ray==2.54.1
|
| 286 |
+
nodejs-wheel-binaries==24.14.1
|
| 287 |
+
torch==2.5.0+cu124
|
| 288 |
+
certifi==2026.4.22
|
| 289 |
+
pydantic_core==2.46.3
|
| 290 |
+
click==8.3.3
|
| 291 |
+
idna==3.13
|
| 292 |
+
requests==2.33.1
|
| 293 |
+
packaging==26.1
|
| 294 |
+
pydantic==2.13.3
|
| 295 |
+
GitPython==3.1.47
|
| 296 |
+
protobuf==7.34.1
|
| 297 |
+
wandb==0.26.0
|
| 298 |
+
importlib_metadata==8.7.1
|
| 299 |
+
jaraco.context==6.1.0
|
| 300 |
+
jaraco.functools==4.4.0
|
| 301 |
+
more-itertools==10.8.0
|
| 302 |
+
packaging==26.0
|
| 303 |
+
platformdirs==4.4.0
|
| 304 |
+
tomli==2.4.0
|
| 305 |
+
wheel==0.46.3
|
| 306 |
+
zipp==3.23.0
|
| 307 |
+
autocommand==2.2.2
|
| 308 |
+
backports.tarfile==1.2.0
|
| 309 |
+
jaraco.text==4.0.0
|
pretrain/wandb/run-20260423_015848-rau44hjb/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-174-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-04-23T01:58:48.502634Z",
|
| 5 |
+
"program": "/home/divyanshu/carla/lead/lead/training/train.py",
|
| 6 |
+
"codePath": "lead/training/train.py",
|
| 7 |
+
"codePathLocal": "lead/training/train.py",
|
| 8 |
+
"git": {
|
| 9 |
+
"remote": "https://github.com/kesai-labs/lead.git",
|
| 10 |
+
"commit": "3bb726b2fb815fabbec3dd02a1af38cf224475c0"
|
| 11 |
+
},
|
| 12 |
+
"email": "divyanshu_div@outlook.com",
|
| 13 |
+
"root": "outputs/local_training/pretrain",
|
| 14 |
+
"host": "047f74d76241",
|
| 15 |
+
"executable": "/home/divyanshu/miniconda3/envs/lead/bin/python3",
|
| 16 |
+
"cpu_count": 64,
|
| 17 |
+
"cpu_count_logical": 128,
|
| 18 |
+
"gpu": "NVIDIA H100 PCIe",
|
| 19 |
+
"gpu_count": 10,
|
| 20 |
+
"disk": {
|
| 21 |
+
"/": {
|
| 22 |
+
"total": "242002626064384",
|
| 23 |
+
"used": "97432535457792"
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"memory": {
|
| 27 |
+
"total": "1081814073344"
|
| 28 |
+
},
|
| 29 |
+
"gpu_nvidia": [
|
| 30 |
+
{
|
| 31 |
+
"name": "NVIDIA H100 PCIe",
|
| 32 |
+
"memoryTotal": "85520809984",
|
| 33 |
+
"cudaCores": 14592,
|
| 34 |
+
"architecture": "Hopper",
|
| 35 |
+
"uuid": "GPU-45ed8fd7-e8d9-c54f-035d-dc6c397436ea"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "NVIDIA H100 PCIe",
|
| 39 |
+
"memoryTotal": "85520809984",
|
| 40 |
+
"cudaCores": 14592,
|
| 41 |
+
"architecture": "Hopper",
|
| 42 |
+
"uuid": "GPU-cb08c411-16b7-332b-55ff-1d78f490c2b9"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"name": "NVIDIA H100 PCIe",
|
| 46 |
+
"memoryTotal": "85520809984",
|
| 47 |
+
"cudaCores": 14592,
|
| 48 |
+
"architecture": "Hopper",
|
| 49 |
+
"uuid": "GPU-1deddac1-db6b-f27a-0fe6-de85d14a33f3"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"name": "NVIDIA H100 PCIe",
|
| 53 |
+
"memoryTotal": "85520809984",
|
| 54 |
+
"cudaCores": 14592,
|
| 55 |
+
"architecture": "Hopper",
|
| 56 |
+
"uuid": "GPU-414c781f-8a00-3cf7-282f-1374d279deb8"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"name": "NVIDIA H100 PCIe",
|
| 60 |
+
"memoryTotal": "85520809984",
|
| 61 |
+
"cudaCores": 14592,
|
| 62 |
+
"architecture": "Hopper",
|
| 63 |
+
"uuid": "GPU-4a960d0d-8131-5df1-8ba4-018d8fe3753e"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"name": "NVIDIA H100 PCIe",
|
| 67 |
+
"memoryTotal": "85520809984",
|
| 68 |
+
"cudaCores": 14592,
|
| 69 |
+
"architecture": "Hopper",
|
| 70 |
+
"uuid": "GPU-d6f8e4e5-4ee5-3455-e170-b5b88eba5dc7"
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"name": "NVIDIA H100 PCIe",
|
| 74 |
+
"memoryTotal": "85520809984",
|
| 75 |
+
"cudaCores": 14592,
|
| 76 |
+
"architecture": "Hopper",
|
| 77 |
+
"uuid": "GPU-753cd243-6b33-cd2c-38a3-6fb4a0788d9e"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "NVIDIA H100 PCIe",
|
| 81 |
+
"memoryTotal": "85520809984",
|
| 82 |
+
"cudaCores": 14592,
|
| 83 |
+
"architecture": "Hopper",
|
| 84 |
+
"uuid": "GPU-376fd74b-ce7d-cd02-9397-bc498fe8e1dd"
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"name": "NVIDIA H100 PCIe",
|
| 88 |
+
"memoryTotal": "85520809984",
|
| 89 |
+
"cudaCores": 14592,
|
| 90 |
+
"architecture": "Hopper",
|
| 91 |
+
"uuid": "GPU-d895b8a1-fb4b-7ca2-a6d3-432b67919f99"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"name": "NVIDIA H100 PCIe",
|
| 95 |
+
"memoryTotal": "85520809984",
|
| 96 |
+
"cudaCores": 14592,
|
| 97 |
+
"architecture": "Hopper",
|
| 98 |
+
"uuid": "GPU-af7f65d2-531d-0de1-a15d-8b1dc5070660"
|
| 99 |
+
}
|
| 100 |
+
],
|
| 101 |
+
"cudaVersion": "13.0",
|
| 102 |
+
"writerId": "o1y2727p9npdbgm7jk2dg618az4wqcxy"
|
| 103 |
+
}
|
pretrain/wandb/run-20260423_015848-rau44hjb/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_wandb":{"runtime":67},"_runtime":67}
|
pretrain/wandb/run-20260423_015848-rau44hjb/logs/debug-core.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T01:58:48.585811705Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpmutdotec/port-309272.txt","pid":309272,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T01:58:48.58795729Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":309272}
|
| 3 |
+
{"time":"2026-04-23T01:58:48.587947891Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-309272-309570-1604598625/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T01:58:48.747320006Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T01:58:48.762370425Z","level":"INFO","msg":"handleInformInit: received","streamId":"rau44hjb","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T01:58:49.701542706Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rau44hjb","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T01:58:57.735764884Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"l3q5y022c0zu"}
|
| 8 |
+
{"time":"2026-04-23T01:59:58.693485774Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-23T01:59:58.693595874Z","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2026-04-23T01:59:58.693716689Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-23T01:59:58.69514826Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-23T01:59:58.695201351Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-309272-309570-1604598625/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-23T01:59:59.493658394Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-23T01:59:59.493726789Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 15 |
+
{"time":"2026-04-23T01:59:59.493757703Z","level":"INFO","msg":"server is closed"}
|
pretrain/wandb/run-20260423_015848-rau44hjb/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T01:58:48.763463793Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-23T01:58:48.765948661Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
|
| 3 |
+
{"time":"2026-04-23T01:58:49.698788728Z","level":"INFO","msg":"stream: created new stream","id":"rau44hjb"}
|
| 4 |
+
{"time":"2026-04-23T01:58:49.701209721Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-23T01:58:49.701519686Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-23T01:58:49.70156629Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-04-23T01:58:49.70155736Z","level":"INFO","msg":"writer: started","stream_id":"rau44hjb"}
|
| 8 |
+
{"time":"2026-04-23T01:58:51.240755546Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-23T01:58:51.697645037Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-23T01:59:06.240817566Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":6,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-04-23T01:59:06.546093421Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-23T01:59:13.650072868Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":157}
|
| 13 |
+
{"time":"2026-04-23T01:59:13.656018236Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":11}
|
| 14 |
+
{"time":"2026-04-23T01:59:21.242644407Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1,"events_lines":3,"console_offset":6,"console_lines":198}
|
| 15 |
+
{"time":"2026-04-23T01:59:21.549608639Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-23T01:59:36.24069877Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":1}
|
| 17 |
+
{"time":"2026-04-23T01:59:36.612122313Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-04-23T01:59:51.241924692Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":5,"events_lines":2}
|
| 19 |
+
{"time":"2026-04-23T01:59:51.601109173Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-04-23T01:59:58.693601453Z","level":"INFO","msg":"stream: closing"}
|
| 21 |
+
{"time":"2026-04-23T01:59:59.004890188Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 22 |
+
{"time":"2026-04-23T01:59:59.005285122Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":7,"events_lines":1,"uploaded_len":3,"complete":true,"exit_code":1}
|
| 23 |
+
{"time":"2026-04-23T01:59:59.492664056Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 24 |
+
{"time":"2026-04-23T01:59:59.492949541Z","level":"INFO","msg":"handler: closed"}
|
| 25 |
+
{"time":"2026-04-23T01:59:59.493067101Z","level":"INFO","msg":"sender: closed"}
|
| 26 |
+
{"time":"2026-04-23T01:59:59.493085771Z","level":"INFO","msg":"stream: closed"}
|
pretrain/wandb/run-20260423_015848-rau44hjb/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 01:58:48,507 INFO MainThread:309272 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
|
| 2 |
+
2026-04-23 01:58:48,507 INFO MainThread:309272 [wandb_setup.py:_flush():81] Configure stats pid to 309272
|
| 3 |
+
2026-04-23 01:58:48,507 INFO MainThread:309272 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 01:58:48,507 INFO MainThread:309272 [wandb_init.py:setup_run_log_directory():721] Logging user logs to outputs/local_training/pretrain/wandb/run-20260423_015848-rau44hjb/logs/debug.log
|
| 5 |
+
2026-04-23 01:58:48,507 INFO MainThread:309272 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to outputs/local_training/pretrain/wandb/run-20260423_015848-rau44hjb/logs/debug-internal.log
|
| 6 |
+
2026-04-23 01:58:48,508 INFO MainThread:309272 [wandb_init.py:init():848] calling init triggers
|
| 7 |
+
2026-04-23 01:58:48,508 INFO MainThread:309272 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'target_dataset': <TargetDataset.CARLA_LEADERBOARD2_3CAMERAS: 1>, 'num_available_cameras': 3, 'used_cameras': [True, True, True], 'num_used_cameras': 3, 'pixels_per_meter': 4.0, 'min_x_meter': -32, 'max_x_meter': 64, 'min_y_meter': -40, 'max_y_meter': 40, 'lidar_width_pixel': 384, 'lidar_height_pixel': 320, 'lidar_width_meter': 96, 'lidar_height_meter': 80, 'visualize_dataset': False, 'visualize_failed_scenarios': False, 'load_bev_3rd_person_images': False, 'seed': 0, 'wandb_id': None, 'wandb_resume': 'never', 'wandb_project_name': 'lead_pretrain', 'description': 'An example experiment description.', 'visualize_training': True, 'id': 'Experiment 1', 'load_file': None, 'continue_failed_training': False, 'epoch_checkpoints_keep': [], 'use_training_session_cache': True, 'use_persistent_cache': True, 'force_rebuild_data_cache': False, 'carla_cache_path': ('1152', '384', '-32', '64', '-40', '40', 'True', 'True', 'True', 'True', 'False', '10'), 'training_session_cache_path': '/tmp/divyanshu', 'carla_root': 'data/carla_leaderboard2', 'carla_data': 'data/carla_leaderboard2/data', 'logdir': 'outputs/local_training/pretrain', 'training_png_compression_level': 6, 'vehicle_min_num_lidar_points': 1, 'vehicle_min_num_visible_pixels': 1, 'pedestrian_min_num_lidar_points': 5, 'pedestrian_min_num_visible_pixels': 15, 'parking_vehicle_min_num_lidar_points': 3, 'parking_vehicle_min_num_visible_pixels': 5, 'grad_scaler_init_scale': 1024, 'grad_scaler_growth_factor': 2, 'grad_scaler_backoff_factor': 0.5, 'grad_scaler_growth_interval': 256, 'grad_scaler_max_grad_scale': 65536, 'sync_batchnorm': False, 'epochs': 31, 'batch_size': 64, 'torch_float_type': torch.bfloat16, 'use_mixed_precision_training': True, 'need_grad_scaler': False, 'use_zero_redundancy': False, 'save_model_checkpoint': True, 'is_pretraining': True, 'prefetch_factor': 16, 'compile': True, 'channel_last': True, 'lr': 0.0003, 'skip_first': 1, 'skip_last': 1, 'radar_detection': True, 'use_radar_detection': True, 'num_radar_points_per_sensor': 75, 'num_radar_queries': 20, 'radar_hidden_dim_tokenizer': 1024, 'radar_token_dim': 256, 'radar_tf_dim_ff': 1024, 'radar_dropout': 0.1, 'radar_num_heads': 8, 'radar_num_layers': 4, 'radar_hidden_dim_decoder': 1024, 'radar_classification_loss_weight': 1.0, 'radar_regression_loss_weight': 5.0, 'num_radar_sensors': 4, 'force_rebuild_bucket': False, 'randomize_route_order': False, 'hold_out_town13_routes': False, 'carla_bucket_collection': <class 'lead.data_buckets.full_pretrain_bucket_collection.FullPretrainBucketCollection'>, 'bucket_collection_path': 'data/carla_leaderboard2/buckets', 'use_sensor_perburtation': True, 'use_sensor_perburtation_prob': 0.5, 'use_color_aug': True, 'use_color_aug_prob': 0.2, 'weight_decay': 0.01, 'use_cosine_annealing_with_restarts': True, 'use_depth': True, 'training_used_lidar_steps': 10, 'min_z': -4, 'max_z': 4, 'hist_max_per_pixel': 5, 'lidar_vert_anchors': 10, 'lidar_horz_anchors': 12, 'detect_boxes': True, 'data_bb_static_types_white_list': ['static.prop.constructioncone', 'static.prop.trafficwarning'], 'bb_confidence_threshold': 0.3, 'max_num_bbs': 90, 'num_dir_bins': 12, 'top_k_center_keypoints': 100, 'center_net_max_pooling_kernel': 3, 'bb_input_channel': 64, 'car_open_door_extra_width': 1.2, 'num_bb_classes': 8, 'use_discrete_command': True, 'discrete_command_dim': 6, 'use_noisy_tp': False, 'use_kalman_filter_for_gps': True, 'use_velocity': True, 'max_speed': 25.0, 'use_acceleration': False, 'max_acceleration': 10.0, 'use_previous_tp': True, 'use_next_tp': True, 'use_tp': True, 'target_points_normalization_constants': [[200.0, 50.0]], 'tp_pop_distance': 3.25, 'use_past_positions': False, 'use_past_speeds': False, 'num_past_samples_used': 0, 'transfuser_num_bev_cross_attention_layers': 6, 'transfuser_num_bev_cross_attention_heads': 8, 'transfuser_token_dim': 256, 'predict_target_speed': True, 'predict_spatial_path': True, 'predict_temporal_spatial_waypoints': True, 'use_planning_decoder': False, 'target_speed_classes': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'target_speeds': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'smooth_route': True, 'num_route_points_smoothing': 20, 'num_route_points_prediction': 10, 'max_distance_future_waypoint': 10.0, 'num_way_points_prediction': 8, 'waypoints_spacing': 5, 'horizontal_fov_reduction': 0, 'crop_height': 0, 'carla_crop_height_type': <CarlaImageCroppingType.NONE: 3>, 'image_width_before_camera_subselection': 1152, 'final_image_width': 1152, 'final_image_height': 384, 'img_vert_anchors': 12, 'img_horz_anchors': 36, 'freeze_backbone': False, 'image_architecture': 'resnet34', 'lidar_architecture': 'resnet34', 'LTF': False, 'block_exp': 4, 'n_layer': 2, 'n_head': 4, 'embd_pdrop': 0.1, 'resid_pdrop': 0.1, 'attn_pdrop': 0.1, 'gpt_linear_layer_init_mean': 0.0, 'gpt_linear_layer_init_std': 0.02, 'gpt_layer_norm_init_weight': 1.0, 'use_semantic': True, 'num_semantic_classes': 10, 'perspective_downsample_factor': 1, 'deconv_channel_num_0': 128, 'deconv_channel_num_1': 64, 'deconv_channel_num_2': 32, 'deconv_scale_factor_0': 4, 'deconv_scale_factor_1': 8, 'use_bev_semantic': True, 'num_bev_semantic_classes': 13, 'scale_pedestrian_bev_semantic_size': 2.5, 'pedestrian_bev_min_extent': 0.4, 'bev_features_chanels': 64, 'bev_down_sample_factor': 4, 'bev_upsample_factor': 2, 'use_carla_data': True, 'carla_num_samples': -1, 'use_navsim_data': False, 'navsim_data_root': 'data/navsim_training_cache/navtrain', 'navsim_num_samples': -1, 'schedule_carla_num_samples': False, 'use_waymo_e2e_data': False, 'waymo_e2e_num_training_samples': -1, 'waymo_e2e_training_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_training', 'waymo_e2e_val_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_val_rfm', 'waymo_e2e_test_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_test_submission', 'waymo_e2e_subsample_factor': 5, 'navsim_num_bev_semantic_classes': 7, 'navsim_num_bb_classes': 1, 'mixed_data_training': False, 'carla_leaderboard_mode': True, 'log_scalars_frequency': 50, 'log_images_frequency': 5000, 'log_wandb': True, 'rank': 0, 'world_size': 2, 'local_rank': 0, 'device': device(type='cuda', index=0), 'assigned_cpu_cores': 32, 'workers_per_cpu_cores': 1, '_loaded_config': {'logdir': 'outputs/local_training/pretrain'}, '_wandb': {}}
|
| 9 |
+
2026-04-23 01:58:48,508 INFO MainThread:309272 [wandb_init.py:init():896] starting backend
|
| 10 |
+
2026-04-23 01:58:48,747 INFO MainThread:309272 [wandb_init.py:init():911] sending inform_init request
|
| 11 |
+
2026-04-23 01:58:48,758 INFO MainThread:309272 [wandb_init.py:init():919] backend started and connected
|
| 12 |
+
2026-04-23 01:58:48,768 INFO MainThread:309272 [wandb_init.py:init():989] updated telemetry
|
| 13 |
+
2026-04-23 01:58:48,835 INFO MainThread:309272 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 01:58:50,935 INFO MainThread:309272 [wandb_init.py:init():1058] starting run threads in backend
|
| 15 |
+
2026-04-23 01:58:51,232 INFO MainThread:309272 [wandb_run.py:_console_start():2542] atexit reg
|
| 16 |
+
2026-04-23 01:58:51,233 INFO MainThread:309272 [wandb_run.py:_redirect():2391] redirect: wrap_raw
|
| 17 |
+
2026-04-23 01:58:51,233 INFO MainThread:309272 [wandb_run.py:_redirect():2460] Wrapping output streams.
|
| 18 |
+
2026-04-23 01:58:51,233 INFO MainThread:309272 [wandb_run.py:_redirect():2483] Redirects installed.
|
| 19 |
+
2026-04-23 01:58:51,238 INFO MainThread:309272 [wandb_init.py:init():1098] run started, returning control to user process
|
| 20 |
+
2026-04-23 01:59:58,694 INFO wandb-AsyncioManager-main:309272 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-23 01:59:58,695 INFO wandb-AsyncioManager-main:309272 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
pretrain/wandb/run-20260423_015848-rau44hjb/run-rau44hjb.wandb
ADDED
|
Binary file (68.3 kB). View file
|
|
|
pretrain/wandb/run-20260423_022003-8rqq7dwf/files/output.log
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[34m[1mwandb[0m: Detected [agents] in use.
|
| 2 |
+
[34m[1mwandb[0m: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
|
| 3 |
+
[34m[1mwandb[0m: For more information, check out the docs at: https://weave-docs.wandb.ai
|
| 4 |
+
2026-04-23 02:20:06 [[94mINFO[0m] [lead/training/logger.py:70] WandB logger will log scalar every 50 steps
|
| 5 |
+
2026-04-23 02:20:06 [[94mINFO[0m] [lead/training/logger.py:73] WandB logger will log images every 5000 steps
|
| 6 |
+
2026-04-23 02:20:10 [[94mINFO[0m] [lead/data_loader/carla_dataset.py:1111] Loaded 1090754 images from 1 buckets.
|
| 7 |
+
0%| | 0/17042 [00:33<?, ?it/s]
|
| 8 |
+
Traceback (most recent call last):
|
| 9 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 363, in <module>
|
| 10 |
+
main()
|
| 11 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 12 |
+
return f(*args, **kwargs)
|
| 13 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 358, in main
|
| 14 |
+
Trainer().train_loop()
|
| 15 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 145, in train_loop
|
| 16 |
+
rfm_score = self.train()
|
| 17 |
+
File "<@beartype(__main__.Trainer.train) at 0x7fd02e99f2e0>", line 12, in train
|
| 18 |
+
File "/home/divyanshu/carla/lead/lead/training/train.py", line 198, in train
|
| 19 |
+
self.scaler.scale(loss).backward()
|
| 20 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/_tensor.py", line 581, in backward
|
| 21 |
+
torch.autograd.backward(
|
| 22 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 23 |
+
_engine_run_backward(
|
| 24 |
+
File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/autograd/graph.py", line 825, in _engine_run_backward
|
| 25 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 26 |
+
UserWarning: cuDNN SDPA backward got grad_output.strides() != output.strides(), attempting to materialize a grad_output with matching strides... (Triggered internally at ../aten/src/ATen/native/cudnn/MHA.cpp:674.)
|
| 27 |
+
[rank0]: Traceback (most recent call last):
|
| 28 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 363, in <module>
|
| 29 |
+
[rank0]: main()
|
| 30 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 31 |
+
[rank0]: return f(*args, **kwargs)
|
| 32 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 358, in main
|
| 33 |
+
[rank0]: Trainer().train_loop()
|
| 34 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 145, in train_loop
|
| 35 |
+
[rank0]: rfm_score = self.train()
|
| 36 |
+
[rank0]: File "<@beartype(__main__.Trainer.train) at 0x7fd02e99f2e0>", line 12, in train
|
| 37 |
+
[rank0]: File "/home/divyanshu/carla/lead/lead/training/train.py", line 198, in train
|
| 38 |
+
[rank0]: self.scaler.scale(loss).backward()
|
| 39 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/_tensor.py", line 581, in backward
|
| 40 |
+
[rank0]: torch.autograd.backward(
|
| 41 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 42 |
+
[rank0]: _engine_run_backward(
|
| 43 |
+
[rank0]: File "/home/divyanshu/miniconda3/envs/lead/lib/python3.10/site-packages/torch/autograd/graph.py", line 825, in _engine_run_backward
|
| 44 |
+
[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 45 |
+
[rank0]: UserWarning: cuDNN SDPA backward got grad_output.strides() != output.strides(), attempting to materialize a grad_output with matching strides... (Triggered internally at ../aten/src/ATen/native/cudnn/MHA.cpp:674.)
|
pretrain/wandb/run-20260423_022003-8rqq7dwf/files/requirements.txt
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
lead==0.0.1
|
| 2 |
+
annotated-doc==0.0.4
|
| 3 |
+
wheel==0.46.3
|
| 4 |
+
pip==26.0.1
|
| 5 |
+
vhacdx==0.0.10
|
| 6 |
+
typer==0.24.1
|
| 7 |
+
zstandard==0.25.0
|
| 8 |
+
pandocfilters==1.5.1
|
| 9 |
+
annotated-types==0.7.0
|
| 10 |
+
propcache==0.4.1
|
| 11 |
+
cffi==2.0.0
|
| 12 |
+
rfc3986-validator==0.1.1
|
| 13 |
+
six==1.16.0
|
| 14 |
+
decorator==5.2.1
|
| 15 |
+
simple_watchdog_timer==0.1.1
|
| 16 |
+
webencodings==0.5.1
|
| 17 |
+
h5py==3.7.0
|
| 18 |
+
cycler==0.12.1
|
| 19 |
+
hf-xet==1.4.3
|
| 20 |
+
pycparser==3.0
|
| 21 |
+
iniconfig==2.3.0
|
| 22 |
+
json5==0.14.0
|
| 23 |
+
argon2-cffi-bindings==25.1.0
|
| 24 |
+
hydra-colorlog==1.2.0
|
| 25 |
+
comm==0.2.3
|
| 26 |
+
dictor==0.1.12
|
| 27 |
+
addict==2.4.0
|
| 28 |
+
jaxtyping==0.3.2
|
| 29 |
+
shellingham==1.5.4
|
| 30 |
+
manifold3d==3.4.1
|
| 31 |
+
blinker==1.9.0
|
| 32 |
+
zipp==3.23.1
|
| 33 |
+
appdirs==1.4.4
|
| 34 |
+
py-spy==0.4.1
|
| 35 |
+
argon2-cffi==25.1.0
|
| 36 |
+
rfc3987-syntax==1.1.0
|
| 37 |
+
absl-py==2.4.0
|
| 38 |
+
tifffile==2025.5.10
|
| 39 |
+
charset-normalizer==3.4.7
|
| 40 |
+
jsonpointer==3.1.1
|
| 41 |
+
terminado==0.18.1
|
| 42 |
+
multidict==6.7.1
|
| 43 |
+
python-json-logger==4.1.0
|
| 44 |
+
ephem==4.1.5
|
| 45 |
+
async-lru==2.3.0
|
| 46 |
+
uri-template==1.3.0
|
| 47 |
+
nvidia-nvtx-cu12==12.4.127
|
| 48 |
+
typing-inspection==0.4.2
|
| 49 |
+
isoduration==20.11.0
|
| 50 |
+
pure_eval==0.2.3
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nest-asyncio==1.6.0
|
| 53 |
+
nodeenv==1.10.0
|
| 54 |
+
typing_extensions==4.15.0
|
| 55 |
+
timm==1.0.19
|
| 56 |
+
Send2Trash==2.1.0
|
| 57 |
+
Jinja2==3.1.6
|
| 58 |
+
omegaconf==2.3.0
|
| 59 |
+
matplotlib==3.10.3
|
| 60 |
+
mdurl==0.1.2
|
| 61 |
+
xmlformatter==0.2.8
|
| 62 |
+
aiosignal==1.4.0
|
| 63 |
+
jupyter_server_terminals==0.5.4
|
| 64 |
+
shapely==2.0.4
|
| 65 |
+
matplotlib-inline==0.2.1
|
| 66 |
+
opentelemetry-semantic-conventions==0.62b0
|
| 67 |
+
attrs==26.1.0
|
| 68 |
+
jupyter_client==8.8.0
|
| 69 |
+
tensorboard-data-server==0.7.2
|
| 70 |
+
prometheus_client==0.25.0
|
| 71 |
+
antlr4-python3-runtime==4.9.3
|
| 72 |
+
itsdangerous==2.2.0
|
| 73 |
+
wadler_lindig==0.1.7
|
| 74 |
+
cachetools==7.0.5
|
| 75 |
+
coverage==7.13.5
|
| 76 |
+
jupyter==1.1.1
|
| 77 |
+
virtualenv==21.2.3
|
| 78 |
+
imgaug==0.4.0
|
| 79 |
+
async-timeout==5.0.1
|
| 80 |
+
wcwidth==0.6.0
|
| 81 |
+
beautifulsoup4==4.14.3
|
| 82 |
+
colorlog==6.10.1
|
| 83 |
+
traitlets==5.14.3
|
| 84 |
+
opencensus-context==0.1.3
|
| 85 |
+
fsspec==2026.3.0
|
| 86 |
+
lxml==5.1.0
|
| 87 |
+
websockets==16.0
|
| 88 |
+
lazy-loader==0.5
|
| 89 |
+
aiohttp==3.13.5
|
| 90 |
+
retrying==1.4.2
|
| 91 |
+
h11==0.16.0
|
| 92 |
+
filterpy==1.4.5
|
| 93 |
+
pillow==10.2.0
|
| 94 |
+
jupyter-lsp==2.3.1
|
| 95 |
+
PyYAML==6.0.3
|
| 96 |
+
pycollada==0.9.3
|
| 97 |
+
pexpect==4.9.0
|
| 98 |
+
embreex==2.17.7.post7
|
| 99 |
+
DracoPy==2.0.0
|
| 100 |
+
torchvision==0.20.0+cu124
|
| 101 |
+
lazrs==0.6.3
|
| 102 |
+
ptyprocess==0.7.0
|
| 103 |
+
python-dateutil==2.9.0.post0
|
| 104 |
+
Pygments==2.20.0
|
| 105 |
+
xxhash==3.6.0
|
| 106 |
+
msgpack==1.1.2
|
| 107 |
+
tomli==2.4.1
|
| 108 |
+
colorful==0.5.8
|
| 109 |
+
jsonpickle==3.0.3
|
| 110 |
+
gitdb==4.0.12
|
| 111 |
+
soupsieve==2.8.3
|
| 112 |
+
docker-pycreds==0.4.0
|
| 113 |
+
wrapt==2.1.2
|
| 114 |
+
nbstripout==0.8.2
|
| 115 |
+
pyquaternion==0.9.9
|
| 116 |
+
exceptiongroup==1.3.1
|
| 117 |
+
psutil==5.9.8
|
| 118 |
+
ipython==8.39.0
|
| 119 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 120 |
+
defusedxml==0.7.1
|
| 121 |
+
jupyterlab_server==2.28.0
|
| 122 |
+
cfgv==3.5.0
|
| 123 |
+
Farama-Notifications==0.0.4
|
| 124 |
+
jsonschema==4.26.0
|
| 125 |
+
rfc3339-validator==0.1.4
|
| 126 |
+
jsonschema-specifications==2025.9.1
|
| 127 |
+
google-auth==2.49.2
|
| 128 |
+
mapbox_earcut==2.0.0
|
| 129 |
+
overrides==7.7.0
|
| 130 |
+
fqdn==1.5.1
|
| 131 |
+
svg.path==7.0
|
| 132 |
+
aiohappyeyeballs==2.6.1
|
| 133 |
+
jupyter-events==0.12.0
|
| 134 |
+
cloudpickle==1.3.0
|
| 135 |
+
jupyter-console==6.6.3
|
| 136 |
+
arrow==1.4.0
|
| 137 |
+
triton==3.1.0
|
| 138 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 139 |
+
jupyterlab_widgets==3.0.16
|
| 140 |
+
py-trees==0.8.3
|
| 141 |
+
torchaudio==2.5.0+cu124
|
| 142 |
+
grpcio==1.80.0
|
| 143 |
+
pydot==4.0.1
|
| 144 |
+
google-api-core==2.30.3
|
| 145 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 146 |
+
notebook_shim==0.2.4
|
| 147 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 148 |
+
bleach==6.3.0
|
| 149 |
+
pyglet==1.5.0
|
| 150 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 151 |
+
tqdm==4.66.1
|
| 152 |
+
lead==0.0.1
|
| 153 |
+
yourdfpy==0.0.60
|
| 154 |
+
diskcache==5.4.0
|
| 155 |
+
rpds-py==0.30.0
|
| 156 |
+
ujson==5.9.0
|
| 157 |
+
tabulate==0.9.0
|
| 158 |
+
tinycss2==1.4.0
|
| 159 |
+
asttokens==3.0.1
|
| 160 |
+
pyproj==3.7.1
|
| 161 |
+
platformdirs==4.9.6
|
| 162 |
+
httpx==0.28.1
|
| 163 |
+
safetensors==0.7.0
|
| 164 |
+
msgspec==0.21.1
|
| 165 |
+
setproctitle==1.3.7
|
| 166 |
+
jupyter_core==5.9.1
|
| 167 |
+
stack-data==0.6.3
|
| 168 |
+
ConfigArgParse==1.7.5
|
| 169 |
+
pytest-cov==7.0.0
|
| 170 |
+
carla==0.9.15
|
| 171 |
+
jupyterlab_pygments==0.3.0
|
| 172 |
+
elementpath==1.3.3
|
| 173 |
+
python-discovery==1.2.2
|
| 174 |
+
pytest==9.0.2
|
| 175 |
+
MarkupSafe==3.0.3
|
| 176 |
+
smmap==5.0.3
|
| 177 |
+
nvidia-nccl-cu12==2.21.5
|
| 178 |
+
threadpoolctl==3.6.0
|
| 179 |
+
fastjsonschema==2.21.2
|
| 180 |
+
identify==2.6.18
|
| 181 |
+
filelock==3.25.2
|
| 182 |
+
yarl==1.23.0
|
| 183 |
+
webcolors==25.10.0
|
| 184 |
+
kiwisolver==1.5.0
|
| 185 |
+
mistune==3.2.0
|
| 186 |
+
ruff==0.12.4
|
| 187 |
+
joblib==1.5.3
|
| 188 |
+
Markdown==3.10.2
|
| 189 |
+
Flask==3.1.1
|
| 190 |
+
llvmlite==0.44.0
|
| 191 |
+
opencv-python==4.6.0.66
|
| 192 |
+
parso==0.8.6
|
| 193 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 194 |
+
ipywidgets==8.1.8
|
| 195 |
+
pyogrio==0.12.1
|
| 196 |
+
markdown-it-py==4.0.0
|
| 197 |
+
pyasn1_modules==0.4.2
|
| 198 |
+
viser==1.0.26
|
| 199 |
+
opentelemetry-exporter-prometheus==0.62b0
|
| 200 |
+
httpcore==1.0.9
|
| 201 |
+
mpmath==1.3.0
|
| 202 |
+
referencing==0.37.0
|
| 203 |
+
PyQt5_sip==12.18.0
|
| 204 |
+
docstring_parser_fork==0.0.14
|
| 205 |
+
smart_open==7.6.0
|
| 206 |
+
nbclient==0.10.4
|
| 207 |
+
executing==2.2.1
|
| 208 |
+
widgetsnbextension==4.0.15
|
| 209 |
+
contourpy==1.3.2
|
| 210 |
+
pre_commit==4.2.0
|
| 211 |
+
urllib3==2.6.3
|
| 212 |
+
distlib==0.4.0
|
| 213 |
+
nvidia-curand-cu12==10.3.5.147
|
| 214 |
+
importlib_metadata==8.7.1
|
| 215 |
+
anyio==4.13.0
|
| 216 |
+
aiohttp-cors==0.8.1
|
| 217 |
+
pyasn1==0.6.3
|
| 218 |
+
pyparsing==3.3.2
|
| 219 |
+
ipykernel==7.2.0
|
| 220 |
+
transforms3d==0.4.1
|
| 221 |
+
hydra-core==1.3.2
|
| 222 |
+
laspy==2.5.4
|
| 223 |
+
proto-plus==1.27.2
|
| 224 |
+
googleapis-common-protos==1.74.0
|
| 225 |
+
pluggy==1.6.0
|
| 226 |
+
narwhals==2.19.0
|
| 227 |
+
opencensus==0.11.4
|
| 228 |
+
notebook==7.5.5
|
| 229 |
+
ImageIO==2.37.3
|
| 230 |
+
nbformat==5.10.4
|
| 231 |
+
Werkzeug==3.1.8
|
| 232 |
+
prompt_toolkit==3.0.52
|
| 233 |
+
tornado==6.5.5
|
| 234 |
+
nbconvert==7.16.6
|
| 235 |
+
basedpyright==1.31.3
|
| 236 |
+
rich==14.3.4
|
| 237 |
+
torchmetrics==0.11.0
|
| 238 |
+
pyzmq==27.1.0
|
| 239 |
+
sentry-sdk==2.58.0
|
| 240 |
+
tzdata==2026.1
|
| 241 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 242 |
+
trimesh==4.11.5
|
| 243 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 244 |
+
easydict==1.13
|
| 245 |
+
rtree==1.4.1
|
| 246 |
+
websocket-client==1.9.0
|
| 247 |
+
pydoclint==0.6.10
|
| 248 |
+
fonttools==4.62.1
|
| 249 |
+
lark==1.3.1
|
| 250 |
+
tensorboard==2.20.0
|
| 251 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 252 |
+
opentelemetry-proto==1.27.0
|
| 253 |
+
opentelemetry-api==1.41.0
|
| 254 |
+
jupyter_server==2.17.0
|
| 255 |
+
opentelemetry-sdk==1.41.0
|
| 256 |
+
geopandas==1.1.3
|
| 257 |
+
debugpy==1.8.20
|
| 258 |
+
gymnasium==1.2.3
|
| 259 |
+
jupyterlab==4.5.6
|
| 260 |
+
scikit-learn==1.7.2
|
| 261 |
+
numba==0.61.2
|
| 262 |
+
PyQt5-Qt5==5.15.18
|
| 263 |
+
future==1.0.0
|
| 264 |
+
networkx==3.4.2
|
| 265 |
+
py123d==0.2.1
|
| 266 |
+
PyQt5==5.15.11
|
| 267 |
+
babel==2.18.0
|
| 268 |
+
scikit-image==0.25.2
|
| 269 |
+
sympy==1.13.1
|
| 270 |
+
dash==4.1.0
|
| 271 |
+
pytz==2026.1.post1
|
| 272 |
+
gym==0.17.2
|
| 273 |
+
pyarrow==23.0.1
|
| 274 |
+
cryptography==46.0.7
|
| 275 |
+
xmlschema==1.0.18
|
| 276 |
+
huggingface_hub==1.10.2
|
| 277 |
+
beartype==0.21.0
|
| 278 |
+
numpy==1.26.0
|
| 279 |
+
setuptools==81.0.0
|
| 280 |
+
scipy==1.14.1
|
| 281 |
+
pandas==2.3.3
|
| 282 |
+
jedi==0.19.2
|
| 283 |
+
plotly==6.7.0
|
| 284 |
+
open3d==0.19.0
|
| 285 |
+
ray==2.54.1
|
| 286 |
+
nodejs-wheel-binaries==24.14.1
|
| 287 |
+
torch==2.5.0+cu124
|
| 288 |
+
certifi==2026.4.22
|
| 289 |
+
pydantic_core==2.46.3
|
| 290 |
+
click==8.3.3
|
| 291 |
+
idna==3.13
|
| 292 |
+
requests==2.33.1
|
| 293 |
+
packaging==26.1
|
| 294 |
+
pydantic==2.13.3
|
| 295 |
+
GitPython==3.1.47
|
| 296 |
+
protobuf==7.34.1
|
| 297 |
+
wandb==0.26.0
|
| 298 |
+
importlib_metadata==8.7.1
|
| 299 |
+
jaraco.context==6.1.0
|
| 300 |
+
jaraco.functools==4.4.0
|
| 301 |
+
more-itertools==10.8.0
|
| 302 |
+
packaging==26.0
|
| 303 |
+
platformdirs==4.4.0
|
| 304 |
+
tomli==2.4.0
|
| 305 |
+
wheel==0.46.3
|
| 306 |
+
zipp==3.23.0
|
| 307 |
+
autocommand==2.2.2
|
| 308 |
+
backports.tarfile==1.2.0
|
| 309 |
+
jaraco.text==4.0.0
|
pretrain/wandb/run-20260423_022003-8rqq7dwf/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-174-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-04-23T02:20:03.659543Z",
|
| 5 |
+
"program": "/home/divyanshu/carla/lead/lead/training/train.py",
|
| 6 |
+
"codePath": "lead/training/train.py",
|
| 7 |
+
"codePathLocal": "lead/training/train.py",
|
| 8 |
+
"git": {
|
| 9 |
+
"remote": "https://github.com/kesai-labs/lead.git",
|
| 10 |
+
"commit": "3bb726b2fb815fabbec3dd02a1af38cf224475c0"
|
| 11 |
+
},
|
| 12 |
+
"email": "divyanshu_div@outlook.com",
|
| 13 |
+
"root": "outputs/local_training/pretrain",
|
| 14 |
+
"host": "047f74d76241",
|
| 15 |
+
"executable": "/home/divyanshu/miniconda3/envs/lead/bin/python3",
|
| 16 |
+
"cpu_count": 64,
|
| 17 |
+
"cpu_count_logical": 128,
|
| 18 |
+
"gpu": "NVIDIA H100 PCIe",
|
| 19 |
+
"gpu_count": 10,
|
| 20 |
+
"disk": {
|
| 21 |
+
"/": {
|
| 22 |
+
"total": "242002626064384",
|
| 23 |
+
"used": "97432595591168"
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"memory": {
|
| 27 |
+
"total": "1081814073344"
|
| 28 |
+
},
|
| 29 |
+
"gpu_nvidia": [
|
| 30 |
+
{
|
| 31 |
+
"name": "NVIDIA H100 PCIe",
|
| 32 |
+
"memoryTotal": "85520809984",
|
| 33 |
+
"cudaCores": 14592,
|
| 34 |
+
"architecture": "Hopper",
|
| 35 |
+
"uuid": "GPU-45ed8fd7-e8d9-c54f-035d-dc6c397436ea"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "NVIDIA H100 PCIe",
|
| 39 |
+
"memoryTotal": "85520809984",
|
| 40 |
+
"cudaCores": 14592,
|
| 41 |
+
"architecture": "Hopper",
|
| 42 |
+
"uuid": "GPU-cb08c411-16b7-332b-55ff-1d78f490c2b9"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"name": "NVIDIA H100 PCIe",
|
| 46 |
+
"memoryTotal": "85520809984",
|
| 47 |
+
"cudaCores": 14592,
|
| 48 |
+
"architecture": "Hopper",
|
| 49 |
+
"uuid": "GPU-1deddac1-db6b-f27a-0fe6-de85d14a33f3"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"name": "NVIDIA H100 PCIe",
|
| 53 |
+
"memoryTotal": "85520809984",
|
| 54 |
+
"cudaCores": 14592,
|
| 55 |
+
"architecture": "Hopper",
|
| 56 |
+
"uuid": "GPU-414c781f-8a00-3cf7-282f-1374d279deb8"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"name": "NVIDIA H100 PCIe",
|
| 60 |
+
"memoryTotal": "85520809984",
|
| 61 |
+
"cudaCores": 14592,
|
| 62 |
+
"architecture": "Hopper",
|
| 63 |
+
"uuid": "GPU-4a960d0d-8131-5df1-8ba4-018d8fe3753e"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"name": "NVIDIA H100 PCIe",
|
| 67 |
+
"memoryTotal": "85520809984",
|
| 68 |
+
"cudaCores": 14592,
|
| 69 |
+
"architecture": "Hopper",
|
| 70 |
+
"uuid": "GPU-d6f8e4e5-4ee5-3455-e170-b5b88eba5dc7"
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"name": "NVIDIA H100 PCIe",
|
| 74 |
+
"memoryTotal": "85520809984",
|
| 75 |
+
"cudaCores": 14592,
|
| 76 |
+
"architecture": "Hopper",
|
| 77 |
+
"uuid": "GPU-753cd243-6b33-cd2c-38a3-6fb4a0788d9e"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "NVIDIA H100 PCIe",
|
| 81 |
+
"memoryTotal": "85520809984",
|
| 82 |
+
"cudaCores": 14592,
|
| 83 |
+
"architecture": "Hopper",
|
| 84 |
+
"uuid": "GPU-376fd74b-ce7d-cd02-9397-bc498fe8e1dd"
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"name": "NVIDIA H100 PCIe",
|
| 88 |
+
"memoryTotal": "85520809984",
|
| 89 |
+
"cudaCores": 14592,
|
| 90 |
+
"architecture": "Hopper",
|
| 91 |
+
"uuid": "GPU-d895b8a1-fb4b-7ca2-a6d3-432b67919f99"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"name": "NVIDIA H100 PCIe",
|
| 95 |
+
"memoryTotal": "85520809984",
|
| 96 |
+
"cudaCores": 14592,
|
| 97 |
+
"architecture": "Hopper",
|
| 98 |
+
"uuid": "GPU-af7f65d2-531d-0de1-a15d-8b1dc5070660"
|
| 99 |
+
}
|
| 100 |
+
],
|
| 101 |
+
"cudaVersion": "13.0",
|
| 102 |
+
"writerId": "3gzz5lxazkqkol4759yumw0dvid94m46"
|
| 103 |
+
}
|
pretrain/wandb/run-20260423_022003-8rqq7dwf/logs/debug-core.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T02:20:03.756623736Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpjrvsbk6z/port-313106.txt","pid":313106,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T02:20:03.760292032Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":313106}
|
| 3 |
+
{"time":"2026-04-23T02:20:03.760256762Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-313106-313363-1800131010/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T02:20:03.919907723Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T02:20:03.937905127Z","level":"INFO","msg":"handleInformInit: received","streamId":"8rqq7dwf","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T02:20:05.095000298Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"8rqq7dwf","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T02:20:12.74062152Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k1c3uwxd9l8z"}
|
| 8 |
+
{"time":"2026-04-23T02:21:23.999422407Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
pretrain/wandb/run-20260423_022003-8rqq7dwf/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T02:20:03.938205918Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-23T02:20:03.940722318Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
|
| 3 |
+
{"time":"2026-04-23T02:20:05.092606777Z","level":"INFO","msg":"stream: created new stream","id":"8rqq7dwf"}
|
| 4 |
+
{"time":"2026-04-23T02:20:05.094667883Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-23T02:20:05.094976927Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-23T02:20:05.095025932Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-04-23T02:20:05.095017293Z","level":"INFO","msg":"writer: started","stream_id":"8rqq7dwf"}
|
| 8 |
+
{"time":"2026-04-23T02:20:06.326269806Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":2}
|
| 9 |
+
{"time":"2026-04-23T02:20:06.589722906Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-23T02:20:21.326238713Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":6,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-04-23T02:20:21.631955415Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-23T02:20:36.32704294Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
|
| 13 |
+
{"time":"2026-04-23T02:20:36.543724597Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-04-23T02:20:51.327601937Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":3,"events_lines":3,"console_offset":6,"console_lines":39}
|
| 15 |
+
{"time":"2026-04-23T02:20:51.587822246Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-23T02:21:06.327048258Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":1}
|
| 17 |
+
{"time":"2026-04-23T02:21:06.762082714Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-04-23T02:21:21.327160672Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":7,"events_lines":3}
|
| 19 |
+
{"time":"2026-04-23T02:21:21.580247027Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
pretrain/wandb/run-20260423_022003-8rqq7dwf/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 02:20:03,670 INFO MainThread:313106 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
|
| 2 |
+
2026-04-23 02:20:03,670 INFO MainThread:313106 [wandb_setup.py:_flush():81] Configure stats pid to 313106
|
| 3 |
+
2026-04-23 02:20:03,670 INFO MainThread:313106 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 02:20:03,670 INFO MainThread:313106 [wandb_init.py:setup_run_log_directory():721] Logging user logs to outputs/local_training/pretrain/wandb/run-20260423_022003-8rqq7dwf/logs/debug.log
|
| 5 |
+
2026-04-23 02:20:03,670 INFO MainThread:313106 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to outputs/local_training/pretrain/wandb/run-20260423_022003-8rqq7dwf/logs/debug-internal.log
|
| 6 |
+
2026-04-23 02:20:03,671 INFO MainThread:313106 [wandb_init.py:init():848] calling init triggers
|
| 7 |
+
2026-04-23 02:20:03,671 INFO MainThread:313106 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'target_dataset': <TargetDataset.CARLA_LEADERBOARD2_3CAMERAS: 1>, 'num_available_cameras': 3, 'used_cameras': [True, True, True], 'num_used_cameras': 3, 'pixels_per_meter': 4.0, 'min_x_meter': -32, 'max_x_meter': 64, 'min_y_meter': -40, 'max_y_meter': 40, 'lidar_width_pixel': 384, 'lidar_height_pixel': 320, 'lidar_width_meter': 96, 'lidar_height_meter': 80, 'visualize_dataset': False, 'visualize_failed_scenarios': False, 'load_bev_3rd_person_images': False, 'seed': 0, 'wandb_id': None, 'wandb_resume': 'never', 'wandb_project_name': 'lead_pretrain', 'description': 'An example experiment description.', 'visualize_training': True, 'id': 'Experiment 1', 'load_file': None, 'continue_failed_training': False, 'epoch_checkpoints_keep': [], 'use_training_session_cache': True, 'use_persistent_cache': True, 'force_rebuild_data_cache': False, 'carla_cache_path': ('1152', '384', '-32', '64', '-40', '40', 'True', 'True', 'True', 'True', 'False', '10'), 'training_session_cache_path': '/tmp/divyanshu', 'carla_root': 'data/carla_leaderboard2', 'carla_data': 'data/carla_leaderboard2/data', 'logdir': 'outputs/local_training/pretrain', 'training_png_compression_level': 6, 'vehicle_min_num_lidar_points': 1, 'vehicle_min_num_visible_pixels': 1, 'pedestrian_min_num_lidar_points': 5, 'pedestrian_min_num_visible_pixels': 15, 'parking_vehicle_min_num_lidar_points': 3, 'parking_vehicle_min_num_visible_pixels': 5, 'grad_scaler_init_scale': 1024, 'grad_scaler_growth_factor': 2, 'grad_scaler_backoff_factor': 0.5, 'grad_scaler_growth_interval': 256, 'grad_scaler_max_grad_scale': 65536, 'sync_batchnorm': False, 'epochs': 31, 'batch_size': 64, 'torch_float_type': torch.bfloat16, 'use_mixed_precision_training': True, 'need_grad_scaler': False, 'use_zero_redundancy': False, 'save_model_checkpoint': True, 'is_pretraining': True, 'prefetch_factor': 16, 'compile': True, 'channel_last': True, 'lr': 0.0003, 'skip_first': 1, 'skip_last': 1, 'radar_detection': True, 'use_radar_detection': True, 'num_radar_points_per_sensor': 75, 'num_radar_queries': 20, 'radar_hidden_dim_tokenizer': 1024, 'radar_token_dim': 256, 'radar_tf_dim_ff': 1024, 'radar_dropout': 0.1, 'radar_num_heads': 8, 'radar_num_layers': 4, 'radar_hidden_dim_decoder': 1024, 'radar_classification_loss_weight': 1.0, 'radar_regression_loss_weight': 5.0, 'num_radar_sensors': 4, 'force_rebuild_bucket': False, 'randomize_route_order': False, 'hold_out_town13_routes': False, 'carla_bucket_collection': <class 'lead.data_buckets.full_pretrain_bucket_collection.FullPretrainBucketCollection'>, 'bucket_collection_path': 'data/carla_leaderboard2/buckets', 'use_sensor_perburtation': True, 'use_sensor_perburtation_prob': 0.5, 'use_color_aug': True, 'use_color_aug_prob': 0.2, 'weight_decay': 0.01, 'use_cosine_annealing_with_restarts': True, 'use_depth': True, 'training_used_lidar_steps': 10, 'min_z': -4, 'max_z': 4, 'hist_max_per_pixel': 5, 'lidar_vert_anchors': 10, 'lidar_horz_anchors': 12, 'detect_boxes': True, 'data_bb_static_types_white_list': ['static.prop.constructioncone', 'static.prop.trafficwarning'], 'bb_confidence_threshold': 0.3, 'max_num_bbs': 90, 'num_dir_bins': 12, 'top_k_center_keypoints': 100, 'center_net_max_pooling_kernel': 3, 'bb_input_channel': 64, 'car_open_door_extra_width': 1.2, 'num_bb_classes': 8, 'use_discrete_command': True, 'discrete_command_dim': 6, 'use_noisy_tp': False, 'use_kalman_filter_for_gps': True, 'use_velocity': True, 'max_speed': 25.0, 'use_acceleration': False, 'max_acceleration': 10.0, 'use_previous_tp': True, 'use_next_tp': True, 'use_tp': True, 'target_points_normalization_constants': [[200.0, 50.0]], 'tp_pop_distance': 3.25, 'use_past_positions': False, 'use_past_speeds': False, 'num_past_samples_used': 0, 'transfuser_num_bev_cross_attention_layers': 6, 'transfuser_num_bev_cross_attention_heads': 8, 'transfuser_token_dim': 256, 'predict_target_speed': True, 'predict_spatial_path': True, 'predict_temporal_spatial_waypoints': True, 'use_planning_decoder': False, 'target_speed_classes': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'target_speeds': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'smooth_route': True, 'num_route_points_smoothing': 20, 'num_route_points_prediction': 10, 'max_distance_future_waypoint': 10.0, 'num_way_points_prediction': 8, 'waypoints_spacing': 5, 'horizontal_fov_reduction': 0, 'crop_height': 0, 'carla_crop_height_type': <CarlaImageCroppingType.NONE: 3>, 'image_width_before_camera_subselection': 1152, 'final_image_width': 1152, 'final_image_height': 384, 'img_vert_anchors': 12, 'img_horz_anchors': 36, 'freeze_backbone': False, 'image_architecture': 'resnet34', 'lidar_architecture': 'resnet34', 'LTF': False, 'block_exp': 4, 'n_layer': 2, 'n_head': 4, 'embd_pdrop': 0.1, 'resid_pdrop': 0.1, 'attn_pdrop': 0.1, 'gpt_linear_layer_init_mean': 0.0, 'gpt_linear_layer_init_std': 0.02, 'gpt_layer_norm_init_weight': 1.0, 'use_semantic': True, 'num_semantic_classes': 10, 'perspective_downsample_factor': 1, 'deconv_channel_num_0': 128, 'deconv_channel_num_1': 64, 'deconv_channel_num_2': 32, 'deconv_scale_factor_0': 4, 'deconv_scale_factor_1': 8, 'use_bev_semantic': True, 'num_bev_semantic_classes': 13, 'scale_pedestrian_bev_semantic_size': 2.5, 'pedestrian_bev_min_extent': 0.4, 'bev_features_chanels': 64, 'bev_down_sample_factor': 4, 'bev_upsample_factor': 2, 'use_carla_data': True, 'carla_num_samples': -1, 'use_navsim_data': False, 'navsim_data_root': 'data/navsim_training_cache/navtrain', 'navsim_num_samples': -1, 'schedule_carla_num_samples': False, 'use_waymo_e2e_data': False, 'waymo_e2e_num_training_samples': -1, 'waymo_e2e_training_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_training', 'waymo_e2e_val_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_val_rfm', 'waymo_e2e_test_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_test_submission', 'waymo_e2e_subsample_factor': 5, 'navsim_num_bev_semantic_classes': 7, 'navsim_num_bb_classes': 1, 'mixed_data_training': False, 'carla_leaderboard_mode': True, 'log_scalars_frequency': 50, 'log_images_frequency': 5000, 'log_wandb': True, 'rank': 0, 'world_size': 2, 'local_rank': 0, 'device': device(type='cuda', index=0), 'assigned_cpu_cores': 16, 'workers_per_cpu_cores': 1, '_loaded_config': {'logdir': 'outputs/local_training/pretrain'}, '_wandb': {}}
|
| 9 |
+
2026-04-23 02:20:03,671 INFO MainThread:313106 [wandb_init.py:init():896] starting backend
|
| 10 |
+
2026-04-23 02:20:03,920 INFO MainThread:313106 [wandb_init.py:init():911] sending inform_init request
|
| 11 |
+
2026-04-23 02:20:03,933 INFO MainThread:313106 [wandb_init.py:init():919] backend started and connected
|
| 12 |
+
2026-04-23 02:20:03,945 INFO MainThread:313106 [wandb_init.py:init():989] updated telemetry
|
| 13 |
+
2026-04-23 02:20:04,115 INFO MainThread:313106 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 02:20:06,018 INFO MainThread:313106 [wandb_init.py:init():1058] starting run threads in backend
|
| 15 |
+
2026-04-23 02:20:06,315 INFO MainThread:313106 [wandb_run.py:_console_start():2542] atexit reg
|
| 16 |
+
2026-04-23 02:20:06,315 INFO MainThread:313106 [wandb_run.py:_redirect():2391] redirect: wrap_raw
|
| 17 |
+
2026-04-23 02:20:06,316 INFO MainThread:313106 [wandb_run.py:_redirect():2460] Wrapping output streams.
|
| 18 |
+
2026-04-23 02:20:06,316 INFO MainThread:313106 [wandb_run.py:_redirect():2483] Redirects installed.
|
| 19 |
+
2026-04-23 02:20:06,321 INFO MainThread:313106 [wandb_init.py:init():1098] run started, returning control to user process
|
pretrain/wandb/run-20260423_022003-8rqq7dwf/run-8rqq7dwf.wandb
ADDED
|
Binary file (32.8 kB). View file
|
|
|
pretrain/wandb/run-20260423_023526-gi92or6f/files/output.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[34m[1mwandb[0m: Detected [agents] in use.
|
| 2 |
+
[34m[1mwandb[0m: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
|
| 3 |
+
[34m[1mwandb[0m: For more information, check out the docs at: https://weave-docs.wandb.ai
|
| 4 |
+
2026-04-23 02:35:29 [[94mINFO[0m] [lead/training/logger.py:70] WandB logger will log scalar every 50 steps
|
| 5 |
+
2026-04-23 02:35:29 [[94mINFO[0m] [lead/training/logger.py:73] WandB logger will log images every 5000 steps
|
| 6 |
+
2026-04-23 02:35:32 [[94mINFO[0m] [lead/data_loader/carla_dataset.py:1111] Loaded 1090754 images from 1 buckets.
|
| 7 |
+
0%| | 0/17042 [00:20<?, ?it/s]
|
pretrain/wandb/run-20260423_023526-gi92or6f/files/requirements.txt
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
lead==0.0.1
|
| 2 |
+
annotated-doc==0.0.4
|
| 3 |
+
wheel==0.46.3
|
| 4 |
+
pip==26.0.1
|
| 5 |
+
vhacdx==0.0.10
|
| 6 |
+
typer==0.24.1
|
| 7 |
+
zstandard==0.25.0
|
| 8 |
+
pandocfilters==1.5.1
|
| 9 |
+
annotated-types==0.7.0
|
| 10 |
+
propcache==0.4.1
|
| 11 |
+
cffi==2.0.0
|
| 12 |
+
rfc3986-validator==0.1.1
|
| 13 |
+
six==1.16.0
|
| 14 |
+
decorator==5.2.1
|
| 15 |
+
simple_watchdog_timer==0.1.1
|
| 16 |
+
webencodings==0.5.1
|
| 17 |
+
h5py==3.7.0
|
| 18 |
+
cycler==0.12.1
|
| 19 |
+
hf-xet==1.4.3
|
| 20 |
+
pycparser==3.0
|
| 21 |
+
iniconfig==2.3.0
|
| 22 |
+
json5==0.14.0
|
| 23 |
+
argon2-cffi-bindings==25.1.0
|
| 24 |
+
hydra-colorlog==1.2.0
|
| 25 |
+
comm==0.2.3
|
| 26 |
+
dictor==0.1.12
|
| 27 |
+
addict==2.4.0
|
| 28 |
+
jaxtyping==0.3.2
|
| 29 |
+
shellingham==1.5.4
|
| 30 |
+
manifold3d==3.4.1
|
| 31 |
+
blinker==1.9.0
|
| 32 |
+
zipp==3.23.1
|
| 33 |
+
appdirs==1.4.4
|
| 34 |
+
py-spy==0.4.1
|
| 35 |
+
argon2-cffi==25.1.0
|
| 36 |
+
rfc3987-syntax==1.1.0
|
| 37 |
+
absl-py==2.4.0
|
| 38 |
+
tifffile==2025.5.10
|
| 39 |
+
charset-normalizer==3.4.7
|
| 40 |
+
jsonpointer==3.1.1
|
| 41 |
+
terminado==0.18.1
|
| 42 |
+
multidict==6.7.1
|
| 43 |
+
python-json-logger==4.1.0
|
| 44 |
+
ephem==4.1.5
|
| 45 |
+
async-lru==2.3.0
|
| 46 |
+
uri-template==1.3.0
|
| 47 |
+
nvidia-nvtx-cu12==12.4.127
|
| 48 |
+
typing-inspection==0.4.2
|
| 49 |
+
isoduration==20.11.0
|
| 50 |
+
pure_eval==0.2.3
|
| 51 |
+
frozenlist==1.8.0
|
| 52 |
+
nest-asyncio==1.6.0
|
| 53 |
+
nodeenv==1.10.0
|
| 54 |
+
typing_extensions==4.15.0
|
| 55 |
+
timm==1.0.19
|
| 56 |
+
Send2Trash==2.1.0
|
| 57 |
+
Jinja2==3.1.6
|
| 58 |
+
omegaconf==2.3.0
|
| 59 |
+
matplotlib==3.10.3
|
| 60 |
+
mdurl==0.1.2
|
| 61 |
+
xmlformatter==0.2.8
|
| 62 |
+
aiosignal==1.4.0
|
| 63 |
+
jupyter_server_terminals==0.5.4
|
| 64 |
+
shapely==2.0.4
|
| 65 |
+
matplotlib-inline==0.2.1
|
| 66 |
+
opentelemetry-semantic-conventions==0.62b0
|
| 67 |
+
attrs==26.1.0
|
| 68 |
+
jupyter_client==8.8.0
|
| 69 |
+
tensorboard-data-server==0.7.2
|
| 70 |
+
prometheus_client==0.25.0
|
| 71 |
+
antlr4-python3-runtime==4.9.3
|
| 72 |
+
itsdangerous==2.2.0
|
| 73 |
+
wadler_lindig==0.1.7
|
| 74 |
+
cachetools==7.0.5
|
| 75 |
+
coverage==7.13.5
|
| 76 |
+
jupyter==1.1.1
|
| 77 |
+
virtualenv==21.2.3
|
| 78 |
+
imgaug==0.4.0
|
| 79 |
+
async-timeout==5.0.1
|
| 80 |
+
wcwidth==0.6.0
|
| 81 |
+
beautifulsoup4==4.14.3
|
| 82 |
+
colorlog==6.10.1
|
| 83 |
+
traitlets==5.14.3
|
| 84 |
+
opencensus-context==0.1.3
|
| 85 |
+
fsspec==2026.3.0
|
| 86 |
+
lxml==5.1.0
|
| 87 |
+
websockets==16.0
|
| 88 |
+
lazy-loader==0.5
|
| 89 |
+
aiohttp==3.13.5
|
| 90 |
+
retrying==1.4.2
|
| 91 |
+
h11==0.16.0
|
| 92 |
+
filterpy==1.4.5
|
| 93 |
+
pillow==10.2.0
|
| 94 |
+
jupyter-lsp==2.3.1
|
| 95 |
+
PyYAML==6.0.3
|
| 96 |
+
pycollada==0.9.3
|
| 97 |
+
pexpect==4.9.0
|
| 98 |
+
embreex==2.17.7.post7
|
| 99 |
+
DracoPy==2.0.0
|
| 100 |
+
torchvision==0.20.0+cu124
|
| 101 |
+
lazrs==0.6.3
|
| 102 |
+
ptyprocess==0.7.0
|
| 103 |
+
python-dateutil==2.9.0.post0
|
| 104 |
+
Pygments==2.20.0
|
| 105 |
+
xxhash==3.6.0
|
| 106 |
+
msgpack==1.1.2
|
| 107 |
+
tomli==2.4.1
|
| 108 |
+
colorful==0.5.8
|
| 109 |
+
jsonpickle==3.0.3
|
| 110 |
+
gitdb==4.0.12
|
| 111 |
+
soupsieve==2.8.3
|
| 112 |
+
docker-pycreds==0.4.0
|
| 113 |
+
wrapt==2.1.2
|
| 114 |
+
nbstripout==0.8.2
|
| 115 |
+
pyquaternion==0.9.9
|
| 116 |
+
exceptiongroup==1.3.1
|
| 117 |
+
psutil==5.9.8
|
| 118 |
+
ipython==8.39.0
|
| 119 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 120 |
+
defusedxml==0.7.1
|
| 121 |
+
jupyterlab_server==2.28.0
|
| 122 |
+
cfgv==3.5.0
|
| 123 |
+
Farama-Notifications==0.0.4
|
| 124 |
+
jsonschema==4.26.0
|
| 125 |
+
rfc3339-validator==0.1.4
|
| 126 |
+
jsonschema-specifications==2025.9.1
|
| 127 |
+
google-auth==2.49.2
|
| 128 |
+
mapbox_earcut==2.0.0
|
| 129 |
+
overrides==7.7.0
|
| 130 |
+
fqdn==1.5.1
|
| 131 |
+
svg.path==7.0
|
| 132 |
+
aiohappyeyeballs==2.6.1
|
| 133 |
+
jupyter-events==0.12.0
|
| 134 |
+
cloudpickle==1.3.0
|
| 135 |
+
jupyter-console==6.6.3
|
| 136 |
+
arrow==1.4.0
|
| 137 |
+
triton==3.1.0
|
| 138 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 139 |
+
jupyterlab_widgets==3.0.16
|
| 140 |
+
py-trees==0.8.3
|
| 141 |
+
torchaudio==2.5.0+cu124
|
| 142 |
+
grpcio==1.80.0
|
| 143 |
+
pydot==4.0.1
|
| 144 |
+
google-api-core==2.30.3
|
| 145 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 146 |
+
notebook_shim==0.2.4
|
| 147 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 148 |
+
bleach==6.3.0
|
| 149 |
+
pyglet==1.5.0
|
| 150 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 151 |
+
tqdm==4.66.1
|
| 152 |
+
lead==0.0.1
|
| 153 |
+
yourdfpy==0.0.60
|
| 154 |
+
diskcache==5.4.0
|
| 155 |
+
rpds-py==0.30.0
|
| 156 |
+
ujson==5.9.0
|
| 157 |
+
tabulate==0.9.0
|
| 158 |
+
tinycss2==1.4.0
|
| 159 |
+
asttokens==3.0.1
|
| 160 |
+
pyproj==3.7.1
|
| 161 |
+
platformdirs==4.9.6
|
| 162 |
+
httpx==0.28.1
|
| 163 |
+
safetensors==0.7.0
|
| 164 |
+
msgspec==0.21.1
|
| 165 |
+
setproctitle==1.3.7
|
| 166 |
+
jupyter_core==5.9.1
|
| 167 |
+
stack-data==0.6.3
|
| 168 |
+
ConfigArgParse==1.7.5
|
| 169 |
+
pytest-cov==7.0.0
|
| 170 |
+
carla==0.9.15
|
| 171 |
+
jupyterlab_pygments==0.3.0
|
| 172 |
+
elementpath==1.3.3
|
| 173 |
+
python-discovery==1.2.2
|
| 174 |
+
pytest==9.0.2
|
| 175 |
+
MarkupSafe==3.0.3
|
| 176 |
+
smmap==5.0.3
|
| 177 |
+
nvidia-nccl-cu12==2.21.5
|
| 178 |
+
threadpoolctl==3.6.0
|
| 179 |
+
fastjsonschema==2.21.2
|
| 180 |
+
identify==2.6.18
|
| 181 |
+
filelock==3.25.2
|
| 182 |
+
yarl==1.23.0
|
| 183 |
+
webcolors==25.10.0
|
| 184 |
+
kiwisolver==1.5.0
|
| 185 |
+
mistune==3.2.0
|
| 186 |
+
ruff==0.12.4
|
| 187 |
+
joblib==1.5.3
|
| 188 |
+
Markdown==3.10.2
|
| 189 |
+
Flask==3.1.1
|
| 190 |
+
llvmlite==0.44.0
|
| 191 |
+
opencv-python==4.6.0.66
|
| 192 |
+
parso==0.8.6
|
| 193 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 194 |
+
ipywidgets==8.1.8
|
| 195 |
+
pyogrio==0.12.1
|
| 196 |
+
markdown-it-py==4.0.0
|
| 197 |
+
pyasn1_modules==0.4.2
|
| 198 |
+
viser==1.0.26
|
| 199 |
+
opentelemetry-exporter-prometheus==0.62b0
|
| 200 |
+
httpcore==1.0.9
|
| 201 |
+
mpmath==1.3.0
|
| 202 |
+
referencing==0.37.0
|
| 203 |
+
PyQt5_sip==12.18.0
|
| 204 |
+
docstring_parser_fork==0.0.14
|
| 205 |
+
smart_open==7.6.0
|
| 206 |
+
nbclient==0.10.4
|
| 207 |
+
executing==2.2.1
|
| 208 |
+
widgetsnbextension==4.0.15
|
| 209 |
+
contourpy==1.3.2
|
| 210 |
+
pre_commit==4.2.0
|
| 211 |
+
urllib3==2.6.3
|
| 212 |
+
distlib==0.4.0
|
| 213 |
+
nvidia-curand-cu12==10.3.5.147
|
| 214 |
+
importlib_metadata==8.7.1
|
| 215 |
+
anyio==4.13.0
|
| 216 |
+
aiohttp-cors==0.8.1
|
| 217 |
+
pyasn1==0.6.3
|
| 218 |
+
pyparsing==3.3.2
|
| 219 |
+
ipykernel==7.2.0
|
| 220 |
+
transforms3d==0.4.1
|
| 221 |
+
hydra-core==1.3.2
|
| 222 |
+
laspy==2.5.4
|
| 223 |
+
proto-plus==1.27.2
|
| 224 |
+
googleapis-common-protos==1.74.0
|
| 225 |
+
pluggy==1.6.0
|
| 226 |
+
narwhals==2.19.0
|
| 227 |
+
opencensus==0.11.4
|
| 228 |
+
notebook==7.5.5
|
| 229 |
+
ImageIO==2.37.3
|
| 230 |
+
nbformat==5.10.4
|
| 231 |
+
Werkzeug==3.1.8
|
| 232 |
+
prompt_toolkit==3.0.52
|
| 233 |
+
tornado==6.5.5
|
| 234 |
+
nbconvert==7.16.6
|
| 235 |
+
basedpyright==1.31.3
|
| 236 |
+
rich==14.3.4
|
| 237 |
+
torchmetrics==0.11.0
|
| 238 |
+
pyzmq==27.1.0
|
| 239 |
+
sentry-sdk==2.58.0
|
| 240 |
+
tzdata==2026.1
|
| 241 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 242 |
+
trimesh==4.11.5
|
| 243 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 244 |
+
easydict==1.13
|
| 245 |
+
rtree==1.4.1
|
| 246 |
+
websocket-client==1.9.0
|
| 247 |
+
pydoclint==0.6.10
|
| 248 |
+
fonttools==4.62.1
|
| 249 |
+
lark==1.3.1
|
| 250 |
+
tensorboard==2.20.0
|
| 251 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 252 |
+
opentelemetry-proto==1.27.0
|
| 253 |
+
opentelemetry-api==1.41.0
|
| 254 |
+
jupyter_server==2.17.0
|
| 255 |
+
opentelemetry-sdk==1.41.0
|
| 256 |
+
geopandas==1.1.3
|
| 257 |
+
debugpy==1.8.20
|
| 258 |
+
gymnasium==1.2.3
|
| 259 |
+
jupyterlab==4.5.6
|
| 260 |
+
scikit-learn==1.7.2
|
| 261 |
+
numba==0.61.2
|
| 262 |
+
PyQt5-Qt5==5.15.18
|
| 263 |
+
future==1.0.0
|
| 264 |
+
networkx==3.4.2
|
| 265 |
+
py123d==0.2.1
|
| 266 |
+
PyQt5==5.15.11
|
| 267 |
+
babel==2.18.0
|
| 268 |
+
scikit-image==0.25.2
|
| 269 |
+
sympy==1.13.1
|
| 270 |
+
dash==4.1.0
|
| 271 |
+
pytz==2026.1.post1
|
| 272 |
+
gym==0.17.2
|
| 273 |
+
pyarrow==23.0.1
|
| 274 |
+
cryptography==46.0.7
|
| 275 |
+
xmlschema==1.0.18
|
| 276 |
+
huggingface_hub==1.10.2
|
| 277 |
+
beartype==0.21.0
|
| 278 |
+
numpy==1.26.0
|
| 279 |
+
setuptools==81.0.0
|
| 280 |
+
scipy==1.14.1
|
| 281 |
+
pandas==2.3.3
|
| 282 |
+
jedi==0.19.2
|
| 283 |
+
plotly==6.7.0
|
| 284 |
+
open3d==0.19.0
|
| 285 |
+
ray==2.54.1
|
| 286 |
+
nodejs-wheel-binaries==24.14.1
|
| 287 |
+
torch==2.5.0+cu124
|
| 288 |
+
certifi==2026.4.22
|
| 289 |
+
pydantic_core==2.46.3
|
| 290 |
+
click==8.3.3
|
| 291 |
+
idna==3.13
|
| 292 |
+
requests==2.33.1
|
| 293 |
+
packaging==26.1
|
| 294 |
+
pydantic==2.13.3
|
| 295 |
+
GitPython==3.1.47
|
| 296 |
+
protobuf==7.34.1
|
| 297 |
+
wandb==0.26.0
|
| 298 |
+
importlib_metadata==8.7.1
|
| 299 |
+
jaraco.context==6.1.0
|
| 300 |
+
jaraco.functools==4.4.0
|
| 301 |
+
more-itertools==10.8.0
|
| 302 |
+
packaging==26.0
|
| 303 |
+
platformdirs==4.4.0
|
| 304 |
+
tomli==2.4.0
|
| 305 |
+
wheel==0.46.3
|
| 306 |
+
zipp==3.23.0
|
| 307 |
+
autocommand==2.2.2
|
| 308 |
+
backports.tarfile==1.2.0
|
| 309 |
+
jaraco.text==4.0.0
|
pretrain/wandb/run-20260423_023526-gi92or6f/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-174-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-04-23T02:35:26.403830Z",
|
| 5 |
+
"program": "/home/divyanshu/carla/lead/lead/training/train.py",
|
| 6 |
+
"codePath": "lead/training/train.py",
|
| 7 |
+
"codePathLocal": "lead/training/train.py",
|
| 8 |
+
"git": {
|
| 9 |
+
"remote": "https://github.com/kesai-labs/lead.git",
|
| 10 |
+
"commit": "3bb726b2fb815fabbec3dd02a1af38cf224475c0"
|
| 11 |
+
},
|
| 12 |
+
"email": "divyanshu_div@outlook.com",
|
| 13 |
+
"root": "outputs/local_training/pretrain",
|
| 14 |
+
"host": "047f74d76241",
|
| 15 |
+
"executable": "/home/divyanshu/miniconda3/envs/lead/bin/python3",
|
| 16 |
+
"cpu_count": 64,
|
| 17 |
+
"cpu_count_logical": 128,
|
| 18 |
+
"gpu": "NVIDIA H100 PCIe",
|
| 19 |
+
"gpu_count": 10,
|
| 20 |
+
"disk": {
|
| 21 |
+
"/": {
|
| 22 |
+
"total": "242002626064384",
|
| 23 |
+
"used": "97450585673728"
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"memory": {
|
| 27 |
+
"total": "1081814073344"
|
| 28 |
+
},
|
| 29 |
+
"gpu_nvidia": [
|
| 30 |
+
{
|
| 31 |
+
"name": "NVIDIA H100 PCIe",
|
| 32 |
+
"memoryTotal": "85520809984",
|
| 33 |
+
"cudaCores": 14592,
|
| 34 |
+
"architecture": "Hopper",
|
| 35 |
+
"uuid": "GPU-45ed8fd7-e8d9-c54f-035d-dc6c397436ea"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "NVIDIA H100 PCIe",
|
| 39 |
+
"memoryTotal": "85520809984",
|
| 40 |
+
"cudaCores": 14592,
|
| 41 |
+
"architecture": "Hopper",
|
| 42 |
+
"uuid": "GPU-cb08c411-16b7-332b-55ff-1d78f490c2b9"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"name": "NVIDIA H100 PCIe",
|
| 46 |
+
"memoryTotal": "85520809984",
|
| 47 |
+
"cudaCores": 14592,
|
| 48 |
+
"architecture": "Hopper",
|
| 49 |
+
"uuid": "GPU-1deddac1-db6b-f27a-0fe6-de85d14a33f3"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"name": "NVIDIA H100 PCIe",
|
| 53 |
+
"memoryTotal": "85520809984",
|
| 54 |
+
"cudaCores": 14592,
|
| 55 |
+
"architecture": "Hopper",
|
| 56 |
+
"uuid": "GPU-414c781f-8a00-3cf7-282f-1374d279deb8"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"name": "NVIDIA H100 PCIe",
|
| 60 |
+
"memoryTotal": "85520809984",
|
| 61 |
+
"cudaCores": 14592,
|
| 62 |
+
"architecture": "Hopper",
|
| 63 |
+
"uuid": "GPU-4a960d0d-8131-5df1-8ba4-018d8fe3753e"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"name": "NVIDIA H100 PCIe",
|
| 67 |
+
"memoryTotal": "85520809984",
|
| 68 |
+
"cudaCores": 14592,
|
| 69 |
+
"architecture": "Hopper",
|
| 70 |
+
"uuid": "GPU-d6f8e4e5-4ee5-3455-e170-b5b88eba5dc7"
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"name": "NVIDIA H100 PCIe",
|
| 74 |
+
"memoryTotal": "85520809984",
|
| 75 |
+
"cudaCores": 14592,
|
| 76 |
+
"architecture": "Hopper",
|
| 77 |
+
"uuid": "GPU-753cd243-6b33-cd2c-38a3-6fb4a0788d9e"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "NVIDIA H100 PCIe",
|
| 81 |
+
"memoryTotal": "85520809984",
|
| 82 |
+
"cudaCores": 14592,
|
| 83 |
+
"architecture": "Hopper",
|
| 84 |
+
"uuid": "GPU-376fd74b-ce7d-cd02-9397-bc498fe8e1dd"
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"name": "NVIDIA H100 PCIe",
|
| 88 |
+
"memoryTotal": "85520809984",
|
| 89 |
+
"cudaCores": 14592,
|
| 90 |
+
"architecture": "Hopper",
|
| 91 |
+
"uuid": "GPU-d895b8a1-fb4b-7ca2-a6d3-432b67919f99"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"name": "NVIDIA H100 PCIe",
|
| 95 |
+
"memoryTotal": "85520809984",
|
| 96 |
+
"cudaCores": 14592,
|
| 97 |
+
"architecture": "Hopper",
|
| 98 |
+
"uuid": "GPU-af7f65d2-531d-0de1-a15d-8b1dc5070660"
|
| 99 |
+
}
|
| 100 |
+
],
|
| 101 |
+
"cudaVersion": "13.0",
|
| 102 |
+
"writerId": "f8ijgc05blh874wyyhebvmarq8lc39b7"
|
| 103 |
+
}
|
pretrain/wandb/run-20260423_023526-gi92or6f/logs/debug-core.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T02:35:26.497628839Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_3uhf7v9/port-316403.txt","pid":316403,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T02:35:26.498595648Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":316403}
|
| 3 |
+
{"time":"2026-04-23T02:35:26.498581528Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-316403-316660-1574962329/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T02:35:26.649197317Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T02:35:26.664719783Z","level":"INFO","msg":"handleInformInit: received","streamId":"gi92or6f","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T02:35:27.806660011Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gi92or6f","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T02:35:35.567134211Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"gahhohs04zyj"}
|
| 8 |
+
{"time":"2026-04-23T02:36:01.911397077Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
pretrain/wandb/run-20260423_023526-gi92or6f/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T02:35:26.665164408Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-23T02:35:26.667730833Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
|
| 3 |
+
{"time":"2026-04-23T02:35:27.806270496Z","level":"INFO","msg":"stream: created new stream","id":"gi92or6f"}
|
| 4 |
+
{"time":"2026-04-23T02:35:27.806415096Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-23T02:35:27.806640341Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-23T02:35:27.806692181Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-04-23T02:35:27.806683772Z","level":"INFO","msg":"writer: started","stream_id":"gi92or6f"}
|
| 8 |
+
{"time":"2026-04-23T02:35:29.129920333Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":2}
|
| 9 |
+
{"time":"2026-04-23T02:35:29.398193325Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-23T02:35:44.12992423Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":6,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-04-23T02:35:44.46942495Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-23T02:35:59.131272193Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1,"events_lines":2,"console_offset":6,"console_lines":39}
|
| 13 |
+
{"time":"2026-04-23T02:35:59.418157811Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
pretrain/wandb/run-20260423_023526-gi92or6f/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 02:35:26,408 INFO MainThread:316403 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
|
| 2 |
+
2026-04-23 02:35:26,408 INFO MainThread:316403 [wandb_setup.py:_flush():81] Configure stats pid to 316403
|
| 3 |
+
2026-04-23 02:35:26,408 INFO MainThread:316403 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 02:35:26,408 INFO MainThread:316403 [wandb_init.py:setup_run_log_directory():721] Logging user logs to outputs/local_training/pretrain/wandb/run-20260423_023526-gi92or6f/logs/debug.log
|
| 5 |
+
2026-04-23 02:35:26,409 INFO MainThread:316403 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to outputs/local_training/pretrain/wandb/run-20260423_023526-gi92or6f/logs/debug-internal.log
|
| 6 |
+
2026-04-23 02:35:26,409 INFO MainThread:316403 [wandb_init.py:init():848] calling init triggers
|
| 7 |
+
2026-04-23 02:35:26,409 INFO MainThread:316403 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'target_dataset': <TargetDataset.CARLA_LEADERBOARD2_3CAMERAS: 1>, 'num_available_cameras': 3, 'used_cameras': [True, True, True], 'num_used_cameras': 3, 'pixels_per_meter': 4.0, 'min_x_meter': -32, 'max_x_meter': 64, 'min_y_meter': -40, 'max_y_meter': 40, 'lidar_width_pixel': 384, 'lidar_height_pixel': 320, 'lidar_width_meter': 96, 'lidar_height_meter': 80, 'visualize_dataset': False, 'visualize_failed_scenarios': False, 'load_bev_3rd_person_images': False, 'seed': 0, 'wandb_id': None, 'wandb_resume': 'never', 'wandb_project_name': 'lead_pretrain', 'description': 'An example experiment description.', 'visualize_training': True, 'id': 'Experiment 1', 'load_file': None, 'continue_failed_training': False, 'epoch_checkpoints_keep': [], 'use_training_session_cache': True, 'use_persistent_cache': True, 'force_rebuild_data_cache': False, 'carla_cache_path': ('1152', '384', '-32', '64', '-40', '40', 'True', 'True', 'True', 'True', 'False', '10'), 'training_session_cache_path': '/tmp/divyanshu', 'carla_root': 'data/carla_leaderboard2', 'carla_data': 'data/carla_leaderboard2/data', 'logdir': 'outputs/local_training/pretrain', 'training_png_compression_level': 6, 'vehicle_min_num_lidar_points': 1, 'vehicle_min_num_visible_pixels': 1, 'pedestrian_min_num_lidar_points': 5, 'pedestrian_min_num_visible_pixels': 15, 'parking_vehicle_min_num_lidar_points': 3, 'parking_vehicle_min_num_visible_pixels': 5, 'grad_scaler_init_scale': 1024, 'grad_scaler_growth_factor': 2, 'grad_scaler_backoff_factor': 0.5, 'grad_scaler_growth_interval': 256, 'grad_scaler_max_grad_scale': 65536, 'sync_batchnorm': False, 'epochs': 31, 'batch_size': 64, 'torch_float_type': torch.bfloat16, 'use_mixed_precision_training': True, 'need_grad_scaler': False, 'use_zero_redundancy': False, 'save_model_checkpoint': True, 'is_pretraining': True, 'prefetch_factor': 16, 'compile': True, 'channel_last': True, 'lr': 0.0003, 'skip_first': 1, 'skip_last': 1, 'radar_detection': True, 'use_radar_detection': True, 'num_radar_points_per_sensor': 75, 'num_radar_queries': 20, 'radar_hidden_dim_tokenizer': 1024, 'radar_token_dim': 256, 'radar_tf_dim_ff': 1024, 'radar_dropout': 0.1, 'radar_num_heads': 8, 'radar_num_layers': 4, 'radar_hidden_dim_decoder': 1024, 'radar_classification_loss_weight': 1.0, 'radar_regression_loss_weight': 5.0, 'num_radar_sensors': 4, 'force_rebuild_bucket': False, 'randomize_route_order': False, 'hold_out_town13_routes': False, 'carla_bucket_collection': <class 'lead.data_buckets.full_pretrain_bucket_collection.FullPretrainBucketCollection'>, 'bucket_collection_path': 'data/carla_leaderboard2/buckets', 'use_sensor_perburtation': True, 'use_sensor_perburtation_prob': 0.5, 'use_color_aug': True, 'use_color_aug_prob': 0.2, 'weight_decay': 0.01, 'use_cosine_annealing_with_restarts': True, 'use_depth': True, 'training_used_lidar_steps': 10, 'min_z': -4, 'max_z': 4, 'hist_max_per_pixel': 5, 'lidar_vert_anchors': 10, 'lidar_horz_anchors': 12, 'detect_boxes': True, 'data_bb_static_types_white_list': ['static.prop.constructioncone', 'static.prop.trafficwarning'], 'bb_confidence_threshold': 0.3, 'max_num_bbs': 90, 'num_dir_bins': 12, 'top_k_center_keypoints': 100, 'center_net_max_pooling_kernel': 3, 'bb_input_channel': 64, 'car_open_door_extra_width': 1.2, 'num_bb_classes': 8, 'use_discrete_command': True, 'discrete_command_dim': 6, 'use_noisy_tp': False, 'use_kalman_filter_for_gps': True, 'use_velocity': True, 'max_speed': 25.0, 'use_acceleration': False, 'max_acceleration': 10.0, 'use_previous_tp': True, 'use_next_tp': True, 'use_tp': True, 'target_points_normalization_constants': [[200.0, 50.0]], 'tp_pop_distance': 3.25, 'use_past_positions': False, 'use_past_speeds': False, 'num_past_samples_used': 0, 'transfuser_num_bev_cross_attention_layers': 6, 'transfuser_num_bev_cross_attention_heads': 8, 'transfuser_token_dim': 256, 'predict_target_speed': True, 'predict_spatial_path': True, 'predict_temporal_spatial_waypoints': True, 'use_planning_decoder': False, 'target_speed_classes': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'target_speeds': [0.0, 4.0, 8.0, 10.0, 13.88888888, 16.0, 17.77777777, 20.0], 'smooth_route': True, 'num_route_points_smoothing': 20, 'num_route_points_prediction': 10, 'max_distance_future_waypoint': 10.0, 'num_way_points_prediction': 8, 'waypoints_spacing': 5, 'horizontal_fov_reduction': 0, 'crop_height': 0, 'carla_crop_height_type': <CarlaImageCroppingType.NONE: 3>, 'image_width_before_camera_subselection': 1152, 'final_image_width': 1152, 'final_image_height': 384, 'img_vert_anchors': 12, 'img_horz_anchors': 36, 'freeze_backbone': False, 'image_architecture': 'resnet34', 'lidar_architecture': 'resnet34', 'LTF': False, 'block_exp': 4, 'n_layer': 2, 'n_head': 4, 'embd_pdrop': 0.1, 'resid_pdrop': 0.1, 'attn_pdrop': 0.1, 'gpt_linear_layer_init_mean': 0.0, 'gpt_linear_layer_init_std': 0.02, 'gpt_layer_norm_init_weight': 1.0, 'use_semantic': True, 'num_semantic_classes': 10, 'perspective_downsample_factor': 1, 'deconv_channel_num_0': 128, 'deconv_channel_num_1': 64, 'deconv_channel_num_2': 32, 'deconv_scale_factor_0': 4, 'deconv_scale_factor_1': 8, 'use_bev_semantic': True, 'num_bev_semantic_classes': 13, 'scale_pedestrian_bev_semantic_size': 2.5, 'pedestrian_bev_min_extent': 0.4, 'bev_features_chanels': 64, 'bev_down_sample_factor': 4, 'bev_upsample_factor': 2, 'use_carla_data': True, 'carla_num_samples': -1, 'use_navsim_data': False, 'navsim_data_root': 'data/navsim_training_cache/navtrain', 'navsim_num_samples': -1, 'schedule_carla_num_samples': False, 'use_waymo_e2e_data': False, 'waymo_e2e_num_training_samples': -1, 'waymo_e2e_training_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_training', 'waymo_e2e_val_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_val_rfm', 'waymo_e2e_test_data_root': 'data/waymo_open_dataset_end_to_end_camera_v_1_0_0_test_submission', 'waymo_e2e_subsample_factor': 5, 'navsim_num_bev_semantic_classes': 7, 'navsim_num_bb_classes': 1, 'mixed_data_training': False, 'carla_leaderboard_mode': True, 'log_scalars_frequency': 50, 'log_images_frequency': 5000, 'log_wandb': True, 'rank': 0, 'world_size': 2, 'local_rank': 0, 'device': device(type='cuda', index=0), 'assigned_cpu_cores': 2, 'workers_per_cpu_cores': 1, '_loaded_config': {'logdir': 'outputs/local_training/pretrain'}, '_wandb': {}}
|
| 9 |
+
2026-04-23 02:35:26,409 INFO MainThread:316403 [wandb_init.py:init():896] starting backend
|
| 10 |
+
2026-04-23 02:35:26,649 INFO MainThread:316403 [wandb_init.py:init():911] sending inform_init request
|
| 11 |
+
2026-04-23 02:35:26,660 INFO MainThread:316403 [wandb_init.py:init():919] backend started and connected
|
| 12 |
+
2026-04-23 02:35:26,670 INFO MainThread:316403 [wandb_init.py:init():989] updated telemetry
|
| 13 |
+
2026-04-23 02:35:26,847 INFO MainThread:316403 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 02:35:28,899 INFO MainThread:316403 [wandb_init.py:init():1058] starting run threads in backend
|
| 15 |
+
2026-04-23 02:35:29,123 INFO MainThread:316403 [wandb_run.py:_console_start():2542] atexit reg
|
| 16 |
+
2026-04-23 02:35:29,123 INFO MainThread:316403 [wandb_run.py:_redirect():2391] redirect: wrap_raw
|
| 17 |
+
2026-04-23 02:35:29,123 INFO MainThread:316403 [wandb_run.py:_redirect():2460] Wrapping output streams.
|
| 18 |
+
2026-04-23 02:35:29,123 INFO MainThread:316403 [wandb_run.py:_redirect():2483] Redirects installed.
|
| 19 |
+
2026-04-23 02:35:29,127 INFO MainThread:316403 [wandb_init.py:init():1098] run started, returning control to user process
|
pretrain/wandb/run-20260423_023526-gi92or6f/run-gi92or6f.wandb
ADDED
|
Binary file (7 Bytes). View file
|
|
|
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/train_viz/feature_maps_0_4e1f0beb5dc171629abe.png
ADDED
|
Git LFS Details
|
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/train_viz/feature_maps_0_a4bfb42de805ac3cdd19.png
ADDED
|
Git LFS Details
|
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/viz/_train_gt_0_3d3ff6913bfd3113bd14.png
ADDED
|
Git LFS Details
|
pretrain/wandb/run-20260423_023947-6w8i5xh1/files/media/images/viz/_train_gt_0_96988caced64988329d4.png
ADDED
|
Git LFS Details
|