sync run artifacts: vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e173a3e93f86a39c26ad135b2c2127d6a1435f88a18d51f9e41a456b94c6f3e8
+size 6188235489

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank0.log

@@ -0,0 +1,29 @@
+Rank 0 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank1.log

@@ -0,0 +1,29 @@
+Rank 1 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank10.log

@@ -0,0 +1,29 @@
+Rank 10 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank11.log

@@ -0,0 +1,29 @@
+Rank 11 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank12.log

@@ -0,0 +1,29 @@
+Rank 12 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank13.log

@@ -0,0 +1,29 @@
+Rank 13 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank14.log

@@ -0,0 +1,29 @@
+Rank 14 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank15.log

@@ -0,0 +1,29 @@
+Rank 15 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank2.log

@@ -0,0 +1,29 @@
+Rank 2 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank3.log

@@ -0,0 +1,29 @@
+Rank 3 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank4.log

@@ -0,0 +1,29 @@
+Rank 4 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank5.log

@@ -0,0 +1,29 @@
+Rank 5 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank6.log

@@ -0,0 +1,29 @@
+Rank 6 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank7.log

@@ -0,0 +1,29 @@
+Rank 7 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank8.log

@@ -0,0 +1,29 @@
+Rank 8 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank9.log

@@ -0,0 +1,29 @@
+Rank 9 crashed at epoch 15, itr 250
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/git-info.txt

@@ -0,0 +1,2 @@
+branch: main
+commit: 4f75caae343937a59636f9600dd815d5d6b06a6f

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_0_log.err

@@ -0,0 +1,77 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+[rank0]:[W510 12:20:58.732173736 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+wandb: Currently logged in as: dgcnz (uvjepa) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 4zt31gid
+wandb: Tracking run with wandb version 0.23.1
+wandb: Run data is saved locally in /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/wandb/run-20260510_122107-4zt31gid
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run c002_vitl_k16_simple_cross_stable
+wandb: ⭐️ View project at https://wandb.ai/uvjepa/vjepa_ujepaside
+wandb: 🚀 View run at https://wandb.ai/uvjepa/vjepa_ujepaside/runs/4zt31gid
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 16:19:44,648) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank0]:[W510 16:19:47.642195666 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+srun: error: gcn87: task 15: Exited with exit code 1
+srun: Terminating StepId=22614766.0
+[2026-05-10T16:19:56.527] error: *** STEP 22614766.0 ON gcn80 CANCELLED AT 2026-05-10T16:19:56 DUE TO TASK FAILURE ***
+srun: error: gcn80: task 1: Exited with exit code 1
+srun: error: gcn85: task 10: Exited with exit code 1
+srun: error: gcn82: task 6: Exited with exit code 1
+srun: error: gcn87: task 14: Exited with exit code 1
+srun: error: gcn85: task 8: Exited with exit code 1
+srun: error: gcn80: task 3: Exited with exit code 1
+srun: error: gcn82: task 7: Exited with exit code 1
+srun: error: gcn80: task 2: Exited with exit code 1
+srun: error: gcn80: task 0: Exited with exit code 1
+srun: error: gcn82: task 5: Exited with exit code 1
+srun: error: gcn85: task 11: Exited with exit code 1
+srun: error: gcn87: task 12: Exited with exit code 1
+srun: error: gcn82: task 4: Exited with exit code 1
+srun: error: gcn87: task 13: Exited with exit code 1
+srun: error: gcn85: task 9: Exited with exit code 1

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_10_log.err

@@ -0,0 +1,68 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank10]:[W510 12:20:01.953233658 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank10]:[W510 16:19:49.713092222 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48690, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x150c7f339fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x150cc34c825d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x150cc34c61f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x150c805a8eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x150dbfa12164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x150dd048419a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x150dd0509100 in /lib64/libc.so.6)
+
+[rank10]:[W510 16:19:49.756854865 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 10] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+submitit ERROR (2026-05-10 16:19:50,186) - Submitted job triggered an exception
+[rank10]:[W510 16:19:50.757056654 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48690, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x150c7f339fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x150cc34c76d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x150cc34c61cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x150c805a8eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x150dbfa12164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x150dd048419a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x150dd0509100 in /lib64/libc.so.6)
+
+[rank10]:[W510 16:19:50.760593632 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 10] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank10]:[W510 16:19:51.760775392 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48690, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x150c7f339fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x150cc34c76d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x150cc34c61cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x150c805a8eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x150dbfa12164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x150dd048419a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x150dd0509100 in /lib64/libc.so.6)
+
+[rank10]:[W510 16:19:51.763251976 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 10] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank10]:[W510 16:19:51.202554786 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_10_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 12:14:34,049) - Starting with JobEnvironment(job_id=22614766, hostname=gcn85.local.snellius.surf.nl, local_rank=2(4), node=2(4), global_rank=10(16))
+submitit INFO (2026-05-10 12:14:34,049) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:57][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 10/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:57][root                ][stage_datasets           ] [local_rank 2/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:57][root                ][_stage_targz_parts       ] [rank 2] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:11][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:25][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:39][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:53][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:07][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:21][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:35][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:49][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:04][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:18][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:32][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:46][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:20:01][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:20:01][root                ][stage_datasets           ] [local_rank 2/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:51][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:52][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:52][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:52][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:52][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:52][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:52][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:23:52][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:23:52][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 10 / 16
+[INFO    ][2026-05-10 12:23:52][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:23:52][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:23:52][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 10)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:50,186) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:50][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:51][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank10.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_11_log.err

@@ -0,0 +1,158 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank11]:[W510 12:19:56.602814346 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank11]:[W510 16:19:49.713098182 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x147d5bfe125d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x147d5bfdf1f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:49.756860915 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank11]:[W510 16:19:50.757031945 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:50.760781021 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank11]:[W510 16:19:51.760872672 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:51.763308266 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank11]:[W510 16:19:52.763400316 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:52.765625912 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank11]:[W510 16:19:53.765744111 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:53.768030807 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank11]:[W510 16:19:54.768144006 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:54.770513821 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank11]:[W510 16:19:55.770599811 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:55.772834467 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank11]:[W510 16:19:56.772941577 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:56.775308622 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank11]:[W510 16:19:57.775405062 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:57.777864156 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+submitit WARNING (2026-05-10 16:19:58,222) - Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:19:58,234) - Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:19:58,234) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank11]:[W510 16:19:58.778051455 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:58.780439720 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank11]:[W510 16:19:58.283183608 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank11]:[W510 16:19:59.780555439 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48702, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147d17e52fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x147d5bfe06d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x147d5bfdf1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x147d190c1eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x147e5852b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x147e68f9d19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x147e69022100 in /lib64/libc.so.6)
+
+[rank11]:[W510 16:19:59.782896125 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 11] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_11_log.out

@@ -0,0 +1,280 @@
+submitit INFO (2026-05-10 12:14:34,049) - Starting with JobEnvironment(job_id=22614766, hostname=gcn85.local.snellius.surf.nl, local_rank=3(4), node=2(4), global_rank=11(16))
+submitit INFO (2026-05-10 12:14:34,049) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:57][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 11/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:57][root                ][stage_datasets           ] [local_rank 3/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:57][root                ][_stage_targz_parts       ] [rank 3] Extracting 25/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:11][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 2/25 parts
+[INFO    ][2026-05-10 12:17:25][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 4/25 parts
+[INFO    ][2026-05-10 12:17:39][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 6/25 parts
+[INFO    ][2026-05-10 12:17:53][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 8/25 parts
+[INFO    ][2026-05-10 12:18:07][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 10/25 parts
+[INFO    ][2026-05-10 12:18:22][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 12/25 parts
+[INFO    ][2026-05-10 12:18:36][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 14/25 parts
+[INFO    ][2026-05-10 12:18:51][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 16/25 parts
+[INFO    ][2026-05-10 12:19:04][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 18/25 parts
+[INFO    ][2026-05-10 12:19:19][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 20/25 parts
+[INFO    ][2026-05-10 12:19:33][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 22/25 parts
+[INFO    ][2026-05-10 12:19:47][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 24/25 parts
+[INFO    ][2026-05-10 12:19:55][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 25/25 parts
+[INFO    ][2026-05-10 12:19:55][root                ][stage_datasets           ] [local_rank 3/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:55][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:23:56][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:23:56][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 11 / 16
+[INFO    ][2026-05-10 12:23:56][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:23:56][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:23:56][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 11)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit WARNING (2026-05-10 16:19:58,222) - Bypassing signal SIGTERM
+[WARNING ][2026-05-10 16:19:58][submitit            ][bypass                   ] Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:19:58,234) - Bypassing signal SIGCONT
+[WARNING ][2026-05-10 16:19:58][submitit            ][bypass                   ] Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:19:58,234) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:58][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:58][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank11.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_12_log.err

@@ -0,0 +1,158 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank12]:[W510 12:20:52.172098656 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank12]:[W510 16:19:49.920378344 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x14a561ded25d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x14a561deb1f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:49.926036927 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank12]:[W510 16:19:50.926190511 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:50.929892860 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank12]:[W510 16:19:51.930033624 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:51.932343947 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank12]:[W510 16:19:52.932472232 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:52.934789205 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank12]:[W510 16:19:53.934942109 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:53.937294752 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank12]:[W510 16:19:54.937423426 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:54.939693100 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank12]:[W510 16:19:55.939816784 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:55.942163637 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank12]:[W510 16:19:56.942310751 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:56.944697424 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank12]:[W510 16:19:57.944832968 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:57.947139571 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+submitit WARNING (2026-05-10 16:19:58,222) - Bypassing signal SIGTERM
+[rank12]:[W510 16:19:58.947284656 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:58.949610339 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+submitit WARNING (2026-05-10 16:19:58,948) - Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:19:58,949) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank12]:[W510 16:19:59.949764342 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=4, addr=[gcn87.local.snellius.surf.nl]:50944, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14a51dc5efdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14a561dec6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14a561deb1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14a51eecdeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14a65e337164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14a66eda919a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14a66ee2e100 in /lib64/libc.so.6)
+
+[rank12]:[W510 16:19:59.952134326 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 12] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank12]:[W510 16:19:59.180205490 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_12_log.out

@@ -0,0 +1,284 @@
+submitit INFO (2026-05-10 12:14:34,055) - Starting with JobEnvironment(job_id=22614766, hostname=gcn87.local.snellius.surf.nl, local_rank=0(4), node=3(4), global_rank=12(16))
+submitit INFO (2026-05-10 12:14:34,056) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 12/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:54][root                ][stage_datasets           ] [local_rank 0/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:54][root                ][_stage_targz_parts       ] [rank 0] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:08][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:22][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:36][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:50][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:04][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:19][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:33][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:47][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:01][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:16][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:30][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:44][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:19:58][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:19:58][root                ][stage_datasets           ] [local_rank 0/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:19:58][root                ][_stage_multipart_tar     ] [rank 0] Extracting multipart tar (2 files) to /scratch-node/dcanez.22614766/ssv2
+[INFO    ][2026-05-10 12:21:02][root                ][stage_datasets           ] Data staging completed in 248.1s (4.1min)
+[INFO    ][2026-05-10 12:21:02][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22614766/kinetics_240/train.csv (239789 entries)
+[INFO    ][2026-05-10 12:21:03][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22614766/ssv2/train.csv (168913 entries)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:57][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:08][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:08][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 12 / 16
+[INFO    ][2026-05-10 12:24:08][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 12)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit WARNING (2026-05-10 16:19:58,222) - Bypassing signal SIGTERM
+[WARNING ][2026-05-10 16:19:58][submitit            ][bypass                   ] Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:19:58,948) - Bypassing signal SIGCONT
+[WARNING ][2026-05-10 16:19:58][submitit            ][bypass                   ] Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:19:58,949) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:58][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:58][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank12.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_13_log.err

@@ -0,0 +1,180 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank13]:[W510 12:19:58.813234569 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank13]:[W510 16:19:49.920365474 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x1477cd06025d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x1477cd05e1f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:49.926025477 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank13]:[W510 16:19:50.926173311 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:50.929575721 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:51.929706165 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:51.932054678 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:52.932179413 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:52.934418546 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:53.934561220 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:53.936917383 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:54.937063907 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:54.939398820 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:55.939534535 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:55.941842188 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:56.941987682 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:56.944320255 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:57.944470919 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:57.946740642 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:58.946891777 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:58.949185960 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank13]:[W510 16:19:59.949324054 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:19:59.951693287 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+submitit WARNING (2026-05-10 16:20:00,106) - Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:20:00,196) - Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:20:00,196) - Submitted job triggered an exception
+[rank13]:[W510 16:20:00.951837531 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:20:00.954199564 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank13]:[W510 16:20:01.632183756 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank13]:[W510 16:20:01.954349608 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50950, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x147788ed1fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x1477cd05f6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x1477cd05e1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14778a140eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x1478c95aa164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1478da01c19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x1478da0a1100 in /lib64/libc.so.6)
+
+[rank13]:[W510 16:20:01.956608451 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 13] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_13_log.out

@@ -0,0 +1,280 @@
+submitit INFO (2026-05-10 12:14:34,055) - Starting with JobEnvironment(job_id=22614766, hostname=gcn87.local.snellius.surf.nl, local_rank=1(4), node=3(4), global_rank=13(16))
+submitit INFO (2026-05-10 12:14:34,056) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:34][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:34][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:34][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:56][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 13/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:56][root                ][stage_datasets           ] [local_rank 1/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:56][root                ][_stage_targz_parts       ] [rank 1] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:08][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:22][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:36][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:50][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:04][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:19][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:32][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:46][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:00][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:14][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:29][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:43][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:19:58][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:19:58][root                ][stage_datasets           ] [local_rank 1/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:57][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:08][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:08][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 13 / 16
+[INFO    ][2026-05-10 12:24:08][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 13)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit WARNING (2026-05-10 16:20:00,106) - Bypassing signal SIGTERM
+[WARNING ][2026-05-10 16:20:00][submitit            ][bypass                   ] Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:20:00,196) - Bypassing signal SIGCONT
+[WARNING ][2026-05-10 16:20:00][submitit            ][bypass                   ] Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:20:00,196) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:20:00][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:20:00][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank13.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_14_log.err

@@ -0,0 +1,79 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank14]:[W510 12:20:03.054704675 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank14]:[W510 16:19:49.920375084 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:34352, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x1496358b8fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x149679a4725d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x149679a451f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x149636b27eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x149775f91164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x149786a0319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x149786a88100 in /lib64/libc.so.6)
+
+[rank14]:[W510 16:19:49.926028947 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 14] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+submitit ERROR (2026-05-10 16:19:50,186) - Submitted job triggered an exception
+[rank14]:[W510 16:19:50.926154361 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:34352, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x1496358b8fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x149679a466d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x149679a451cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x149636b27eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x149775f91164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x149786a0319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x149786a88100 in /lib64/libc.so.6)
+
+[rank14]:[W510 16:19:50.929831610 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 14] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank14]:[W510 16:19:51.929981635 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:34352, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x1496358b8fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x149679a466d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x149679a451cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x149636b27eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x149775f91164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x149786a0319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x149786a88100 in /lib64/libc.so.6)
+
+[rank14]:[W510 16:19:51.932269128 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 14] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank14]:[W510 16:19:51.445846354 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank14]:[W510 16:19:52.932408362 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:34352, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x1496358b8fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x149679a466d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x149679a451cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x149636b27eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x149775f91164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x149786a0319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x149786a88100 in /lib64/libc.so.6)
+
+[rank14]:[W510 16:19:52.934668395 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 14] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_14_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 12:14:34,056) - Starting with JobEnvironment(job_id=22614766, hostname=gcn87.local.snellius.surf.nl, local_rank=2(4), node=3(4), global_rank=14(16))
+submitit INFO (2026-05-10 12:14:34,056) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:17:00][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 14/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:17:00][root                ][stage_datasets           ] [local_rank 2/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:17:00][root                ][_stage_targz_parts       ] [rank 2] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:14][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:27][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:42][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:56][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:10][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:24][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:38][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:52][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:06][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:20][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:35][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:49][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:20:03][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:20:03][root                ][stage_datasets           ] [local_rank 2/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:57][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:08][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:08][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 14 / 16
+[INFO    ][2026-05-10 12:24:08][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 14)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:50,186) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:50][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:51][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank14.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_15_log.err

@@ -0,0 +1,68 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank15]:[W510 12:19:56.808669295 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 16:19:48,628) - Submitted job triggered an exception
+[rank15]:[W510 16:19:49.920372704 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50954, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x1523c078ffdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x15240491e25d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x15240491c1f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x1523c19feeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x152500e68164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1525118da19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x15251195f100 in /lib64/libc.so.6)
+
+[rank15]:[W510 16:19:49.926026027 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 15] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank15]:[W510 16:19:50.926168251 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50954, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x1523c078ffdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x15240491d6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x15240491c1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x1523c19feeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x152500e68164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1525118da19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x15251195f100 in /lib64/libc.so.6)
+
+[rank15]:[W510 16:19:50.929624471 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 15] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank15]:[W510 16:19:51.929784985 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn87.local.snellius.surf.nl]:50954, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x1523c078ffdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x15240491d6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x15240491c1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x1523c19feeec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x152500e68164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x1525118da19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x15251195f100 in /lib64/libc.so.6)
+
+[rank15]:[W510 16:19:51.932123188 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 15] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank15]:[W510 16:19:51.445334286 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_15_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 12:14:34,055) - Starting with JobEnvironment(job_id=22614766, hostname=gcn87.local.snellius.surf.nl, local_rank=3(4), node=3(4), global_rank=15(16))
+submitit INFO (2026-05-10 12:14:34,056) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:33][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:57][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 15/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:57][root                ][stage_datasets           ] [local_rank 3/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:57][root                ][_stage_targz_parts       ] [rank 3] Extracting 25/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:11][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 2/25 parts
+[INFO    ][2026-05-10 12:17:25][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 4/25 parts
+[INFO    ][2026-05-10 12:17:39][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 6/25 parts
+[INFO    ][2026-05-10 12:17:53][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 8/25 parts
+[INFO    ][2026-05-10 12:18:08][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 10/25 parts
+[INFO    ][2026-05-10 12:18:22][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 12/25 parts
+[INFO    ][2026-05-10 12:18:36][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 14/25 parts
+[INFO    ][2026-05-10 12:18:51][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 16/25 parts
+[INFO    ][2026-05-10 12:19:04][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 18/25 parts
+[INFO    ][2026-05-10 12:19:19][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 20/25 parts
+[INFO    ][2026-05-10 12:19:33][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 22/25 parts
+[INFO    ][2026-05-10 12:19:47][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 24/25 parts
+[INFO    ][2026-05-10 12:19:54][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 25/25 parts
+[INFO    ][2026-05-10 12:19:54][root                ][stage_datasets           ] [local_rank 3/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:57][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:08][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:08][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 15 / 16
+[INFO    ][2026-05-10 12:24:08][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 15)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:48,628) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:48][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:51][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank15.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_1_log.err

@@ -0,0 +1,41 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank1]:[W510 12:19:58.977747785 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 16:19:44,648) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank1]:[W510 16:19:45.445536181 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_1_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 12:14:34,051) - Starting with JobEnvironment(job_id=22614766, hostname=gcn80.local.snellius.surf.nl, local_rank=1(4), node=0(4), global_rank=1(16))
+submitit INFO (2026-05-10 12:14:34,051) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:55][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 1/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:55][root                ][stage_datasets           ] [local_rank 1/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:55][root                ][_stage_targz_parts       ] [rank 1] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:08][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:22][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:36][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:50][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:04][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:19][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:32][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:46][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:00][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:15][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:29][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:43][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:19:57][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:19:57][root                ][stage_datasets           ] [local_rank 1/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:57][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:07][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 1 / 16
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 1)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:44,648) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:44][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:44][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank1.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_2_log.err

@@ -0,0 +1,41 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank2]:[W510 12:20:01.138964796 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 16:19:44,648) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank2]:[W510 16:19:45.446521046 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_2_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 12:14:34,051) - Starting with JobEnvironment(job_id=22614766, hostname=gcn80.local.snellius.surf.nl, local_rank=2(4), node=0(4), global_rank=2(16))
+submitit INFO (2026-05-10 12:14:34,051) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 2/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:55][root                ][stage_datasets           ] [local_rank 2/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:55][root                ][_stage_targz_parts       ] [rank 2] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:09][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:23][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:38][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:53][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:07][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:21][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:35][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:49][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:04][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:18][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:32][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:46][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:20:01][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:20:01][root                ][stage_datasets           ] [local_rank 2/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:57][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:07][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 2 / 16
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 2)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:44,648) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:44][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:44][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank2.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_3_log.err

@@ -0,0 +1,41 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank3]:[W510 12:19:56.018259668 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 16:19:44,648) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank3]:[W510 16:19:45.445996938 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_3_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 12:14:34,050) - Starting with JobEnvironment(job_id=22614766, hostname=gcn80.local.snellius.surf.nl, local_rank=3(4), node=0(4), global_rank=3(16))
+submitit INFO (2026-05-10 12:14:34,051) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:56][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 3/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:56][root                ][stage_datasets           ] [local_rank 3/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:56][root                ][_stage_targz_parts       ] [rank 3] Extracting 25/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:10][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 2/25 parts
+[INFO    ][2026-05-10 12:17:25][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 4/25 parts
+[INFO    ][2026-05-10 12:17:39][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 6/25 parts
+[INFO    ][2026-05-10 12:17:53][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 8/25 parts
+[INFO    ][2026-05-10 12:18:07][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 10/25 parts
+[INFO    ][2026-05-10 12:18:22][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 12/25 parts
+[INFO    ][2026-05-10 12:18:36][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 14/25 parts
+[INFO    ][2026-05-10 12:18:50][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 16/25 parts
+[INFO    ][2026-05-10 12:19:04][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 18/25 parts
+[INFO    ][2026-05-10 12:19:19][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 20/25 parts
+[INFO    ][2026-05-10 12:19:33][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 22/25 parts
+[INFO    ][2026-05-10 12:19:47][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 24/25 parts
+[INFO    ][2026-05-10 12:19:54][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 25/25 parts
+[INFO    ][2026-05-10 12:19:54][root                ][stage_datasets           ] [local_rank 3/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:57][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:57][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:07][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 3 / 16
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 3)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:44,648) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:44][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:44][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank3.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_4_log.err

@@ -0,0 +1,180 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank4]:[W510 12:20:52.615848849 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank4]:[W510 16:19:49.193287972 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x14e5438c625d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x14e5438c41f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:49.250674818 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank4]:[W510 16:19:50.250831182 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:50.254593415 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:51.254722169 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:51.256917393 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:52.257044298 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:52.259240942 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:53.259363187 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:53.261586531 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:54.261712946 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:54.263938389 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:55.264077644 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:55.266358357 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:56.266502282 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:56.268782175 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:57.268924191 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:57.271270323 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:58.271408538 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:58.273590672 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:19:59.273737087 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:19:59.276033990 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+submitit WARNING (2026-05-10 16:20:00,106) - Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:20:00,118) - Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:20:00,118) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank4]:[W510 16:20:00.276182975 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:20:00.278511248 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank4]:[W510 16:20:00.717128829 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank4]:[W510 16:20:01.278653903 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43904, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14e4ff737fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14e5438c56d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14e5438c41cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14e5009a6eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14e63fe10164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14e65088219a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14e650907100 in /lib64/libc.so.6)
+
+[rank4]:[W510 16:20:01.280841577 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 4] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_4_log.out

@@ -0,0 +1,284 @@
+submitit INFO (2026-05-10 12:14:34,049) - Starting with JobEnvironment(job_id=22614766, hostname=gcn82.local.snellius.surf.nl, local_rank=0(4), node=1(4), global_rank=4(16))
+submitit INFO (2026-05-10 12:14:34,049) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:55][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 4/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:55][root                ][stage_datasets           ] [local_rank 0/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:55][root                ][_stage_targz_parts       ] [rank 0] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:09][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:23][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:37][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:51][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:05][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:19][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:33][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:47][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:02][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:16][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:30][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:44][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:19:58][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:19:58][root                ][stage_datasets           ] [local_rank 0/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:19:58][root                ][_stage_multipart_tar     ] [rank 0] Extracting multipart tar (2 files) to /scratch-node/dcanez.22614766/ssv2
+[INFO    ][2026-05-10 12:21:02][root                ][stage_datasets           ] Data staging completed in 246.4s (4.1min)
+[INFO    ][2026-05-10 12:21:02][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22614766/kinetics_240/train.csv (239789 entries)
+[INFO    ][2026-05-10 12:21:03][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22614766/ssv2/train.csv (168913 entries)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:56][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:07][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 4 / 16
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 4)...
+[INFO    ][2026-05-10 12:24:09][app.vjepa.train     ][main                     ] Initializing loader...
+submitit WARNING (2026-05-10 16:20:00,106) - Bypassing signal SIGTERM
+[WARNING ][2026-05-10 16:20:00][submitit            ][bypass                   ] Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:20:00,118) - Bypassing signal SIGCONT
+[WARNING ][2026-05-10 16:20:00][submitit            ][bypass                   ] Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:20:00,118) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:20:00][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:20:00][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank4.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_5_log.err

@@ -0,0 +1,158 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank5]:[W510 12:19:58.114218222 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank5]:[W510 16:19:49.194232775 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x14bccfd8b25d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x14bccfd891f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:49.250678388 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank5]:[W510 16:19:50.250824642 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:50.254503495 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank5]:[W510 16:19:51.254625920 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:51.256900143 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank5]:[W510 16:19:52.257043938 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:52.259270682 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank5]:[W510 16:19:53.259384977 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:53.261599871 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank5]:[W510 16:19:54.261725616 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:54.264077558 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank5]:[W510 16:19:55.264218893 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:55.266477376 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank5]:[W510 16:19:56.266606102 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:56.268802325 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank5]:[W510 16:19:57.268933681 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:57.271200114 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+submitit WARNING (2026-05-10 16:19:58,222) - Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:19:58,230) - Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:19:58,230) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank5]:[W510 16:19:58.271351009 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:58.273646922 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank5]:[W510 16:19:58.864755947 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank5]:[W510 16:19:59.273790727 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43910, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14bc8bbfcfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14bccfd8a6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14bccfd891cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14bc8ce6beec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14bdcc2d5164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14bddcd4719a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14bddcdcc100 in /lib64/libc.so.6)
+
+[rank5]:[W510 16:19:59.276053390 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 5] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_5_log.out

@@ -0,0 +1,280 @@
+submitit INFO (2026-05-10 12:14:34,049) - Starting with JobEnvironment(job_id=22614766, hostname=gcn82.local.snellius.surf.nl, local_rank=1(4), node=1(4), global_rank=5(16))
+submitit INFO (2026-05-10 12:14:34,049) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:56][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 5/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:56][root                ][stage_datasets           ] [local_rank 1/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:56][root                ][_stage_targz_parts       ] [rank 1] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:08][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:22][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:36][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:50][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:04][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:19][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:32][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:46][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:00][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:15][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:29][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:43][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:19:57][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:19:57][root                ][stage_datasets           ] [local_rank 1/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:56][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:07][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 5 / 16
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 5)...
+[INFO    ][2026-05-10 12:24:09][app.vjepa.train     ][main                     ] Initializing loader...
+submitit WARNING (2026-05-10 16:19:58,222) - Bypassing signal SIGTERM
+[WARNING ][2026-05-10 16:19:58][submitit            ][bypass                   ] Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:19:58,230) - Bypassing signal SIGCONT
+[WARNING ][2026-05-10 16:19:58][submitit            ][bypass                   ] Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:19:58,230) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:58][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:58][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank5.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_6_log.err

@@ -0,0 +1,68 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank6]:[W510 12:20:01.354617374 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 16:19:48,628) - Submitted job triggered an exception
+[rank6]:[W510 16:19:49.534536952 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43922, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14620edb2fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x146252f4125d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x146252f3f1f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x146210021eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14634f48b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14635fefd19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14635ff82100 in /lib64/libc.so.6)
+
+[rank6]:[W510 16:19:49.536929964 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 6] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank6]:[W510 16:19:50.537085079 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43922, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14620edb2fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x146252f406d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x146252f3f1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x146210021eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14634f48b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14635fefd19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14635ff82100 in /lib64/libc.so.6)
+
+[rank6]:[W510 16:19:50.540863871 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 6] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank6]:[W510 16:19:51.541031985 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43922, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14620edb2fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x146252f406d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x146252f3f1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x146210021eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14634f48b164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14635fefd19a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14635ff82100 in /lib64/libc.so.6)
+
+[rank6]:[W510 16:19:51.543425278 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 6] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank6]:[W510 16:19:51.813700522 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_6_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 12:14:34,049) - Starting with JobEnvironment(job_id=22614766, hostname=gcn82.local.snellius.surf.nl, local_rank=2(4), node=1(4), global_rank=6(16))
+submitit INFO (2026-05-10 12:14:34,049) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:56][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 6/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:56][root                ][stage_datasets           ] [local_rank 2/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:56][root                ][_stage_targz_parts       ] [rank 2] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:10][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:23][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:38][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:53][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:07][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:21][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:35][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:49][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:03][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:18][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:32][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:46][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:20:01][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:20:01][root                ][stage_datasets           ] [local_rank 2/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:56][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:07][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 6 / 16
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 6)...
+[INFO    ][2026-05-10 12:24:09][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:48,628) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:48][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:51][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank6.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_7_log.err

@@ -0,0 +1,68 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank7]:[W510 12:19:56.166705489 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank7]:[W510 16:19:49.193338512 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43918, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14f93fe48fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x14f983fd725d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x14f983fd51f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14f9410b7eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14fa80521164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14fa90f9319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14fa91018100 in /lib64/libc.so.6)
+
+[rank7]:[W510 16:19:49.250685538 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 7] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+submitit ERROR (2026-05-10 16:19:50,186) - Submitted job triggered an exception
+[rank7]:[W510 16:19:50.250857942 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43918, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14f93fe48fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14f983fd66d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14f983fd51cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14f9410b7eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14fa80521164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14fa90f9319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14fa91018100 in /lib64/libc.so.6)
+
+[rank7]:[W510 16:19:50.254608224 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 7] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank7]:[W510 16:19:51.254784989 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn82.local.snellius.surf.nl]:43918, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14f93fe48fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14f983fd66d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14f983fd51cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14f9410b7eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14fa80521164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14fa90f9319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14fa91018100 in /lib64/libc.so.6)
+
+[rank7]:[W510 16:19:51.257267151 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 7] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank7]:[W510 16:19:51.813923330 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_7_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 12:14:34,049) - Starting with JobEnvironment(job_id=22614766, hostname=gcn82.local.snellius.surf.nl, local_rank=3(4), node=1(4), global_rank=7(16))
+submitit INFO (2026-05-10 12:14:34,049) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:53][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:57][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 7/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:57][root                ][stage_datasets           ] [local_rank 3/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:57][root                ][_stage_targz_parts       ] [rank 3] Extracting 25/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:11][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 2/25 parts
+[INFO    ][2026-05-10 12:17:25][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 4/25 parts
+[INFO    ][2026-05-10 12:17:39][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 6/25 parts
+[INFO    ][2026-05-10 12:17:53][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 8/25 parts
+[INFO    ][2026-05-10 12:18:07][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 10/25 parts
+[INFO    ][2026-05-10 12:18:22][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 12/25 parts
+[INFO    ][2026-05-10 12:18:36][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 14/25 parts
+[INFO    ][2026-05-10 12:18:51][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 16/25 parts
+[INFO    ][2026-05-10 12:19:04][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 18/25 parts
+[INFO    ][2026-05-10 12:19:19][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 20/25 parts
+[INFO    ][2026-05-10 12:19:33][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 22/25 parts
+[INFO    ][2026-05-10 12:19:47][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 24/25 parts
+[INFO    ][2026-05-10 12:19:55][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 25/25 parts
+[INFO    ][2026-05-10 12:19:55][root                ][stage_datasets           ] [local_rank 3/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:56][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:56][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:24:07][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 7 / 16
+[INFO    ][2026-05-10 12:24:07][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:24:07][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 7)...
+[INFO    ][2026-05-10 12:24:09][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:50,186) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:50][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:51][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank7.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_8_log.err

@@ -0,0 +1,68 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank8]:[W510 12:20:54.584744077 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 16:19:48,627) - Submitted job triggered an exception
+[rank8]:[W510 16:19:49.713101052 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48716, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14ec81298fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x14ecc542725d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x14ecc54251f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14ec82507eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14edc1971164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14edd23e319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14edd2468100 in /lib64/libc.so.6)
+
+[rank8]:[W510 16:19:49.756860305 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 8] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank8]:[W510 16:19:50.757024745 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48716, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14ec81298fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14ecc54266d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14ecc54251cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14ec82507eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14edc1971164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14edd23e319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14edd2468100 in /lib64/libc.so.6)
+
+[rank8]:[W510 16:19:50.760610632 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 8] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank8]:[W510 16:19:51.760810102 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48716, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x14ec81298fdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14ecc54266d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14ecc54251cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x14ec82507eec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14edc1971164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14edd23e319a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14edd2468100 in /lib64/libc.so.6)
+
+[rank8]:[W510 16:19:51.763104707 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 8] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank8]:[W510 16:19:51.202810034 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_8_log.out

@@ -0,0 +1,280 @@
+submitit INFO (2026-05-10 12:14:34,049) - Starting with JobEnvironment(job_id=22614766, hostname=gcn85.local.snellius.surf.nl, local_rank=0(4), node=2(4), global_rank=8(16))
+submitit INFO (2026-05-10 12:14:34,049) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:56][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 8/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:56][root                ][stage_datasets           ] [local_rank 0/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:56][root                ][_stage_targz_parts       ] [rank 0] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:10][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:23][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:37][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:51][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:05][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:19][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:33][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:47][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:01][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:16][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:30][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:44][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:19:58][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:19:58][root                ][stage_datasets           ] [local_rank 0/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:19:58][root                ][_stage_multipart_tar     ] [rank 0] Extracting multipart tar (2 files) to /scratch-node/dcanez.22614766/ssv2
+[INFO    ][2026-05-10 12:21:02][root                ][stage_datasets           ] Data staging completed in 245.8s (4.1min)
+[INFO    ][2026-05-10 12:21:02][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22614766/kinetics_240/train.csv (239789 entries)
+[INFO    ][2026-05-10 12:21:03][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22614766/ssv2/train.csv (168913 entries)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:55][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:55][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:23:56][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:23:56][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 8 / 16
+[INFO    ][2026-05-10 12:23:56][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:23:56][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:23:56][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 8)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 16:19:48,627) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:48][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:19:51][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank8.log

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_9_log.err

@@ -0,0 +1,191 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank9]:[W510 12:19:58.520496896 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+[rank9]:[W510 16:19:49.713098522 TCPStore.cpp:125] [c10d] recvValue failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:682 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a3325d (0x14902df0c25d in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x273 (0x14902df0a1f3 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:49.756857925 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Failed to recv, got 0 bytes. Connection was likely closed. Did the remote server shutdown or crash?
+[rank9]:[W510 16:19:50.756994415 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:50.760565092 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:51.760703343 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:51.762962548 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:52.763081018 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:52.765328454 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:53.765437783 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:53.767857408 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:54.767966367 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:54.770239623 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:55.770352823 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:55.772627358 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:56.772765518 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:56.775215702 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:57.775350242 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:57.777628537 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:58.777757757 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:58.779960573 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:19:59.780087782 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:19:59.782502357 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+submitit WARNING (2026-05-10 16:20:00,106) - Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:20:00,176) - Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:20:00,176) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 102, in __call__
+    app_main(app, args=params, resume_preempt=resume_preempt)
+    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank9]:[W510 16:20:00.782633087 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:20:00.784970142 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:20:00.236003417 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank9]:[W510 16:20:01.785104182 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:20:01.787343787 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe
+[rank9]:[W510 16:20:02.787484377 TCPStore.cpp:106] [c10d] sendBytes failed on SocketImpl(fd=3, addr=[gcn85.local.snellius.surf.nl]:48682, remote=[gcn80.local.snellius.surf.nl]:37129): Broken pipe
+Exception raised from sendBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:653 (most recent call first):
+frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x9d (0x148fe9d7dfdd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libc10.so)
+frame #1: <unknown function> + 0x6a326d1 (0x14902df0b6d1 in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #2: c10d::TCPStore::check(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) + 0x24d (0x14902df0a1cd in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cpu.so)
+frame #3: c10d::ProcessGroupNCCL::HeartbeatMonitor::runLoop() + 0x44c (0x148feafeceec in /scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/lib/libtorch_cuda.so)
+frame #4: <unknown function> + 0xed164 (0x14912a456164 in /sw/arch/RHEL9/EB_production/2025/software/GCCcore/14.2.0/lib64/libstdc++.so.6)
+frame #5: <unknown function> + 0x8a19a (0x14913aec819a in /lib64/libc.so.6)
+frame #6: <unknown function> + 0x10f100 (0x14913af4d100 in /lib64/libc.so.6)
+
+[rank9]:[W510 16:20:02.789916262 ProcessGroupNCCL.cpp:1802] [PG ID 0 PG GUID 0(default_pg) Rank 9] Failed to check the "should dump" flag on TCPStore, (maybe TCPStore server has shut down too early), with error: Broken pipe

vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_9_log.out

@@ -0,0 +1,280 @@
+submitit INFO (2026-05-10 12:14:34,049) - Starting with JobEnvironment(job_id=22614766, hostname=gcn85.local.snellius.surf.nl, local_rank=1(4), node=2(4), global_rank=9(16))
+submitit INFO (2026-05-10 12:14:34,049) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/job_22614766/22614766_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 16,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 12:16:54][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 12:16:56][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 9/16, tasks_per_node=4
+[INFO    ][2026-05-10 12:16:56][root                ][stage_datasets           ] [local_rank 1/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 12:16:56][root                ][_stage_targz_parts       ] [rank 1] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22614766/kinetics_240
+[INFO    ][2026-05-10 12:17:08][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 2/26 parts
+[INFO    ][2026-05-10 12:17:22][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 4/26 parts
+[INFO    ][2026-05-10 12:17:36][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 6/26 parts
+[INFO    ][2026-05-10 12:17:50][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 8/26 parts
+[INFO    ][2026-05-10 12:18:04][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 10/26 parts
+[INFO    ][2026-05-10 12:18:19][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 12/26 parts
+[INFO    ][2026-05-10 12:18:32][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 14/26 parts
+[INFO    ][2026-05-10 12:18:46][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 16/26 parts
+[INFO    ][2026-05-10 12:19:00][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 18/26 parts
+[INFO    ][2026-05-10 12:19:14][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 20/26 parts
+[INFO    ][2026-05-10 12:19:29][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 22/26 parts
+[INFO    ][2026-05-10 12:19:43][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 24/26 parts
+[INFO    ][2026-05-10 12:19:57][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 26/26 parts
+[INFO    ][2026-05-10 12:19:57][root                ][stage_datasets           ] [local_rank 1/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 12:21:03][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22614766/kinetics_240/train.csv', '/scratch-node/dcanez.22614766/ssv2/train.csv']
+[INFO    ][2026-05-10 12:23:31][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 12:23:45][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 12:23:46][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 12:23:46][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 12:23:46][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 12:23:46][root                ][init_video_model         ] Encoder number of parameters: 302399488
+[INFO    ][2026-05-10 12:23:46][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 12:23:46][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 12:23:49][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 12:23:49][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 9 / 16
+[INFO    ][2026-05-10 12:23:49][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 12:23:49][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 12:23:49][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 9)...
+[INFO    ][2026-05-10 12:24:08][app.vjepa.train     ][main                     ] Initializing loader...
+submitit WARNING (2026-05-10 16:20:00,106) - Bypassing signal SIGTERM
+[WARNING ][2026-05-10 16:20:00][submitit            ][bypass                   ] Bypassing signal SIGTERM
+submitit WARNING (2026-05-10 16:20:00,176) - Bypassing signal SIGCONT
+[WARNING ][2026-05-10 16:20:00][submitit            ][bypass                   ] Bypassing signal SIGCONT
+submitit ERROR (2026-05-10 16:20:00,176) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:20:00][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 16:20:00][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c002_vitl_k16_simple_cross_stable/crash_rank9.log